├── Few_Shot_Learning ├── tests │ ├── test_video_reader.py │ └── test_models.py ├── scripts │ ├── examples.md │ ├── shrink_dataset.py │ ├── shrink_dataset_ssv2.py │ ├── shrink_dataset_ucf.py │ ├── extract_ssv2.py │ ├── extract_hmdb.py │ ├── extract_ucf.py │ └── pretrain_backbone.py ├── README.md ├── soft_dtw.py ├── runfull.py └── splits │ └── ssv2_OTAM │ └── val.txt ├── README.md └── Supervised_Learning ├── README.md ├── dataset.py ├── OPA_supervised_hm.py ├── OPA_supervised_binomial.py ├── OPA_supervised_lifted.py ├── main_supervised_hm.py ├── main_supervised_binomial.py └── main_supervised_lifted.py /Few_Shot_Learning/tests/test_video_reader.py: -------------------------------------------------------------------------------- 1 | from re import A 2 | import pytest 3 | import torch 4 | 5 | from video_reader import VideoDataset 6 | 7 | @pytest.fixture(scope='module') 8 | def args(): 9 | class Object(object): 10 | pass 11 | args = Object() 12 | args.dataset = "data/ssv2small" 13 | args.seq_len = 8 14 | args.img_size = 224 15 | args.way = 5 16 | args.shot = 3 17 | args.query_per_class = 2 18 | 19 | yield args 20 | 21 | def test_num_vids_read(args): 22 | vd = VideoDataset(args) 23 | assert len(vd.train_split.videos) == 50 24 | 25 | def test_check_returned_data_size(args): 26 | vd = VideoDataset(args) 27 | assert vd[0]['support_set'].shape == torch.Size([vd.args.seq_len * vd.args.way * vd.args.shot, 3, vd.args.img_size, vd.args.img_size]) 28 | 29 | def test_single_video_mode(args): 30 | vd = VideoDataset(args, meta_batches=False) 31 | vid, gt = vd[0] 32 | assert vid.shape == torch.Size([vd.args.seq_len, 3, vd.args.img_size, vd.args.img_size]) 33 | 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Temporal Alignment Prediction for Supervised Representation Learning and Few-Shot Sequence Classification 2 | 3 | 1. Introduction. 4 | 5 | This package includes the python codes for implementing Temporal Alignment Prediction (TAP) in supervised learning and few-shot action recognition, described in 6 | 7 | Bing Su and Ji-Rong Wen. "Temporal Alignment Prediction for Supervised Representation Learning and Few-Shot Sequence Classification" ICLR, 2022. 8 | 9 | 10 | 2. License & disclaimer. 11 | 12 | The codes can be used for research purposes only. This package is strictly for non-commercial academic use only. 13 | 14 | 15 | 3. Usage 16 | 17 | The "Supervised_Learning" folder contains codes for using TAP in supervised distance learning for sequence data. 18 | 19 | The "Few_Shot_Learning" folder contains codes for using TAP in few-shot action recognition. 20 | 21 | Please refer to the ReadMe files in the folds. 22 | 23 | 24 | 4. Citations 25 | 26 | Please cite the following paper if you use the codes: 27 | 28 | Bing Su and Ji-Rong Wen, "Temporal Alignment Prediction for Supervised Representation Learning and Few-Shot Sequence Classification", ICLR, 2022. -------------------------------------------------------------------------------- /Few_Shot_Learning/scripts/examples.md: -------------------------------------------------------------------------------- 1 | TSN baseline for 50000 iterations on SSv2 2 | 3 | python run.py -c /work/tp8961/tmpfsar/ssv2_tsn --dataset /work/tp8961/video_datasets/data/somethingsomethingv2_256x256q5_7l8 --method tsn --tasks_per_batch 1 -i 50000 4 | 5 | TRX for 150000 iterations on SSv2, take best val model 6 | 7 | python run.py -c /work/tp8961/tmpfsar/ssv2_trx --dataset /work/tp8961/video_datasets/data/somethingsomethingv2_256x256q5_7l8 --method trx -i 150000 --val_iters 50000 60000 70000 80000 90000 100000 110000 120000 130000 140000 150000 8 | 9 | OTAM for 50000 iterations on SSv2 10 | 11 | python run.py -c /work/tp8961/tmpfsar/ssv2_otam --dataset /work/tp8961/video_datasets/data/somethingsomethingv2_256x256q5_7l8 --method otam --tasks_per_batch 1 -i 50000 12 | 13 | PAL on SSv2. First pretrain the backbone, then fine-tune the head. 14 | 15 | python pretrain_backbone.py /work/tp8961/pretrained_models/ssv27tsn --workers 8 --dataset /work/tp8961/video_datasets/data/somethingsomethingv2_256x256q5_7l8 16 | python run.py -c /work/tp8961/tmpfsar/ssv2_pal --dataset /work/tp8961/video_datasets/data/somethingsomethingv2_256x256q5_7l8 --method pal --tasks_per_batch 1 -pt /work/tp8961/pretrained_models/ssv27tsn/epoch70.pth.tar -lr 0.0001 --test_iters 7000 --sch 3000 6000 17 | 18 | -------------------------------------------------------------------------------- /Few_Shot_Learning/scripts/shrink_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import shutil 4 | 5 | import numpy as np 6 | 7 | max_seq_len = 8 8 | 9 | # old_dir = "data/ssv2_256x256q5" 10 | # new_dir = "data/ssv2_256x256q5_l{}".format(max_seq_len) 11 | 12 | old_dir = "data/ucf_256x256q5" 13 | new_dir = "data/ucf101_l{}".format(max_seq_len) 14 | 15 | classes = [j for j in glob.glob(os.path.join(old_dir, "*"))] 16 | 17 | print(classes) 18 | 19 | os.mkdir(new_dir) 20 | 21 | src_jpgs = 0 22 | tgt_jpgs = 0 23 | 24 | for class_folder in classes: 25 | c = os.path.split(class_folder)[-1] 26 | new_c = os.path.join(new_dir, c) 27 | print(new_c) 28 | os.mkdir(new_c) 29 | for video_folder in glob.glob(os.path.join(class_folder, "*")): 30 | v = os.path.split(video_folder)[-1] 31 | print(v) 32 | new_v = os.path.join(new_c, v) 33 | 34 | jpgs = [j for j in glob.glob(os.path.join(video_folder, "*.jpg"))] 35 | 36 | n_jpgs = len(jpgs) 37 | print(n_jpgs) 38 | src_jpgs += n_jpgs 39 | 40 | if n_jpgs <= max_seq_len: 41 | tgt_jpgs += n_jpgs 42 | shutil.copytree(video_folder, new_v) 43 | else: 44 | os.mkdir(new_v) 45 | jpgs.sort() 46 | 47 | idx_f = np.linspace(0, n_jpgs-1, num=max_seq_len) 48 | idxs = [int(f) for f in idx_f] 49 | 50 | for i in range(len(idxs)): 51 | src = jpgs[idxs[i]] 52 | tgt = os.path.join(new_v, "{:08d}.jpg".format(i+1)) 53 | shutil.copy(src, tgt) 54 | tgt_jpgs += 1 55 | 56 | print("Reduced {} to {} jpgs".format(src_jpgs, tgt_jpgs)) -------------------------------------------------------------------------------- /Few_Shot_Learning/scripts/shrink_dataset_ssv2.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import shutil 4 | 5 | import numpy as np 6 | 7 | max_seq_len = 8 8 | 9 | old_dir = "data/ssv2_256x256q5/val" 10 | new_dir = "data/ssv2_l{}/val".format(max_seq_len) 11 | 12 | # old_dir = "data/ucf_256x256q5/val" 13 | # new_dir = "data/ucf101_l{}/val".format(max_seq_len) 14 | 15 | # old_dir = "data/hmdb_256x256q5/train" 16 | # new_dir = "data/hmdb_l{}/train".format(max_seq_len) 17 | 18 | classes = [j for j in glob.glob(os.path.join(old_dir, "*"))] 19 | 20 | print(classes) 21 | 22 | os.mkdir(new_dir) 23 | 24 | src_jpgs = 0 25 | tgt_jpgs = 0 26 | 27 | for class_folder in classes: 28 | c = os.path.split(class_folder)[-1] 29 | new_c = os.path.join(new_dir, c) 30 | print(new_c) 31 | os.mkdir(new_c) 32 | for video_folder in glob.glob(os.path.join(class_folder, "*")): 33 | v = os.path.split(video_folder)[-1] 34 | print(v) 35 | new_v = os.path.join(new_c, v) 36 | 37 | jpgs = [j for j in glob.glob(os.path.join(video_folder, "*.jpg"))] 38 | 39 | n_jpgs = len(jpgs) 40 | print(n_jpgs) 41 | src_jpgs += n_jpgs 42 | 43 | if n_jpgs <= max_seq_len: 44 | tgt_jpgs += n_jpgs 45 | shutil.copytree(video_folder, new_v) 46 | else: 47 | os.mkdir(new_v) 48 | jpgs.sort() 49 | 50 | idx_f = np.linspace(0, n_jpgs-1, num=max_seq_len) 51 | idxs = [int(f) for f in idx_f] 52 | 53 | for i in range(len(idxs)): 54 | src = jpgs[idxs[i]] 55 | tgt = os.path.join(new_v, "{:08d}.jpg".format(i+1)) 56 | shutil.copy(src, tgt) 57 | tgt_jpgs += 1 58 | 59 | print("Reduced {} to {} jpgs".format(src_jpgs, tgt_jpgs)) -------------------------------------------------------------------------------- /Few_Shot_Learning/scripts/shrink_dataset_ucf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import shutil 4 | 5 | import numpy as np 6 | 7 | max_seq_len = 8 8 | 9 | # old_dir = "data/ssv2_256x256q5" 10 | # new_dir = "data/ssv2_256x256q5_l{}".format(max_seq_len) 11 | 12 | # old_dir = "data/ucf_256x256q5/val" 13 | # new_dir = "data/ucf101_l{}/val".format(max_seq_len) 14 | 15 | old_dir = "data/hmdb_256x256q5/train" 16 | new_dir = "data/hmdb_l{}/train".format(max_seq_len) 17 | 18 | classes = [j for j in glob.glob(os.path.join(old_dir, "*"))] 19 | 20 | print(classes) 21 | 22 | os.mkdir(new_dir) 23 | 24 | src_jpgs = 0 25 | tgt_jpgs = 0 26 | 27 | for class_folder in classes: 28 | c = os.path.split(class_folder)[-1] 29 | new_c = os.path.join(new_dir, c) 30 | print(new_c) 31 | os.mkdir(new_c) 32 | for video_folder in glob.glob(os.path.join(class_folder, "*")): 33 | v = os.path.split(video_folder)[-1] 34 | print(v) 35 | new_v = os.path.join(new_c, v) 36 | 37 | jpgs = [j for j in glob.glob(os.path.join(video_folder, "*.jpg"))] 38 | 39 | n_jpgs = len(jpgs) 40 | print(n_jpgs) 41 | src_jpgs += n_jpgs 42 | 43 | if n_jpgs <= max_seq_len: 44 | tgt_jpgs += n_jpgs 45 | shutil.copytree(video_folder, new_v) 46 | else: 47 | os.mkdir(new_v) 48 | jpgs.sort() 49 | 50 | idx_f = np.linspace(0, n_jpgs-1, num=max_seq_len) 51 | idxs = [int(f) for f in idx_f] 52 | 53 | for i in range(len(idxs)): 54 | src = jpgs[idxs[i]] 55 | tgt = os.path.join(new_v, "{:08d}.jpg".format(i+1)) 56 | shutil.copy(src, tgt) 57 | tgt_jpgs += 1 58 | 59 | print("Reduced {} to {} jpgs".format(src_jpgs, tgt_jpgs)) -------------------------------------------------------------------------------- /Few_Shot_Learning/scripts/extract_ssv2.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import json 4 | from glob import glob 5 | from joblib import Parallel, delayed 6 | 7 | out_h = 256 8 | out_w = 256 9 | #in_folder = 'data/ssv2/' 10 | in_folder = '/home/bing_su/SSv2/smth-smth-v2/videos/' 11 | out_folder = 'data/ssv2_{}x{}q5'.format(out_w,out_h) 12 | 13 | split_dir = "splits/ssv2_OTAM" 14 | 15 | wc = os.path.join(split_dir, "*.txt") 16 | 17 | def run_cmd(cmd): 18 | try: 19 | os.mkdir(cmd[1]) 20 | subprocess.call(cmd[0]) 21 | except: 22 | pass 23 | 24 | try: 25 | os.mkdir(out_folder) 26 | except: 27 | pass 28 | 29 | 30 | for fn in glob(wc): 31 | classes = [] 32 | vids = [] 33 | 34 | print(fn) 35 | if "train" in fn: 36 | cur_split = "train" 37 | elif "val" in fn: 38 | cur_split = "val" 39 | elif "test" in fn: 40 | cur_split = "test" 41 | 42 | with open(fn, "r") as f: 43 | data = f.readlines() 44 | c = [x.split(os.sep)[-2].strip() for x in data] 45 | v = [x.split(os.sep)[-1].strip() for x in data] 46 | vids.extend(v) 47 | classes.extend(c) 48 | 49 | try: 50 | os.mkdir(os.path.join(out_folder, cur_split)) 51 | except: 52 | pass 53 | 54 | for c in list(set(classes)): 55 | try: 56 | os.mkdir(os.path.join(out_folder, cur_split, c)) 57 | except: 58 | pass 59 | 60 | cmds = [] 61 | for v, c in zip(vids, classes): 62 | source_vid = os.path.join(in_folder, "{}.webm".format(v)) 63 | extract_dir = os.path.join(out_folder, cur_split, c, v) 64 | 65 | if os.path.exists(extract_dir): 66 | continue 67 | 68 | out_wc = os.path.join(extract_dir, '%08d.jpg') 69 | 70 | print(source_vid, out_wc) 71 | 72 | scale_string = 'scale={}:{}'.format(out_w, out_h) 73 | os.mkdir(extract_dir) 74 | try: 75 | cmd = ['ffmpeg', '-i', source_vid, '-vf', scale_string, '-q:v', '5', out_wc] 76 | 77 | cmds.append((cmd, extract_dir)) 78 | subprocess.call(cmd) 79 | except: 80 | pass 81 | #Parallel(n_jobs=8, require='sharedmem')(delayed(run_cmd)(cmds[i]) for i in range(0, len(cmds))) -------------------------------------------------------------------------------- /Few_Shot_Learning/scripts/extract_hmdb.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import json 4 | from glob import glob 5 | from joblib import Parallel, delayed 6 | 7 | out_h = 256 8 | out_w = 256 9 | #in_folder = 'data/hmdb51_org/' 10 | in_folder = 'DATASET/HMDB51/' 11 | out_folder = 'data/hmdb_{}x{}q5'.format(out_w,out_h) 12 | 13 | split_dir = "splits/hmdb_ARN" 14 | 15 | wc = os.path.join(split_dir, "*.txt") 16 | 17 | def run_cmd(cmd): 18 | try: 19 | os.mkdir(cmd[1]) 20 | subprocess.call(cmd[0]) 21 | except: 22 | pass 23 | 24 | try: 25 | os.mkdir(out_folder) 26 | except: 27 | pass 28 | 29 | 30 | 31 | 32 | for fn in glob(wc): 33 | classes = [] 34 | vids = [] 35 | 36 | print(fn) 37 | if "train" in fn: 38 | cur_split = "train" 39 | elif "val" in fn: 40 | cur_split = "val" 41 | elif "test" in fn: 42 | cur_split = "test" 43 | 44 | with open(fn, "r") as f: 45 | data = f.readlines() 46 | c = [x.split(os.sep)[-2].strip() for x in data] 47 | v = [x.split(os.sep)[-1].strip() for x in data] 48 | vids.extend(v) 49 | classes.extend(c) 50 | 51 | try: 52 | os.mkdir(os.path.join(out_folder, cur_split)) 53 | except: 54 | pass 55 | 56 | for c in list(set(classes)): 57 | try: 58 | os.mkdir(os.path.join(out_folder, cur_split, c)) 59 | except: 60 | pass 61 | 62 | 63 | 64 | cmds = [] 65 | 66 | for v, c in zip(vids, classes): 67 | source_vid = os.path.join(in_folder, c, "{}.avi".format(v)) 68 | extract_dir = os.path.join(out_folder, cur_split, c, v) 69 | 70 | if os.path.exists(extract_dir): 71 | continue 72 | 73 | out_wc = os.path.join(extract_dir, '%08d.jpg') 74 | 75 | print(source_vid, out_wc) 76 | 77 | scale_string = 'scale={}:{}'.format(out_w, out_h) 78 | os.mkdir(extract_dir) 79 | try: 80 | cmd = ['ffmpeg', '-i', source_vid, '-vf', scale_string, '-q:v', '5', out_wc] 81 | 82 | cmds.append((cmd, extract_dir)) 83 | subprocess.call(cmd) 84 | except: 85 | pass 86 | #Parallel(n_jobs=8, require='sharedmem')(delayed(run_cmd)(cmds[i]) for i in range(0, len(cmds))) 87 | -------------------------------------------------------------------------------- /Few_Shot_Learning/scripts/extract_ucf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import json 4 | from glob import glob 5 | from joblib import Parallel, delayed 6 | 7 | out_h = 256 8 | out_w = 256 9 | # in_folder = 'data/UCF-101/' 10 | # out_folder = 'data/ucf_{}x{}q5'.format(out_w,out_h) 11 | 12 | in_folder = 'DATASET/UCF101/UCF-101/' 13 | out_folder = 'data/ucf_{}x{}q5'.format(out_w,out_h) 14 | 15 | split_dir = "splits/ucf_ARN" 16 | 17 | wc = os.path.join(split_dir, "*.txt") 18 | 19 | def run_cmd(cmd): 20 | try: 21 | os.mkdir(cmd[1]) 22 | subprocess.call(cmd[0]) 23 | except: 24 | pass 25 | 26 | try: 27 | os.mkdir(out_folder) 28 | except: 29 | pass 30 | 31 | 32 | for fn in glob(wc): 33 | classes = [] 34 | vids = [] 35 | 36 | print(fn) 37 | if "train" in fn: 38 | cur_split = "train" 39 | elif "val" in fn: 40 | cur_split = "val" 41 | elif "test" in fn: 42 | cur_split = "test" 43 | 44 | with open(fn, "r") as f: 45 | data = f.readlines() 46 | c = [x.split(os.sep)[-2].strip() for x in data] 47 | v = [x.split(os.sep)[-1].strip() for x in data] 48 | vids.extend(v) 49 | classes.extend(c) 50 | 51 | try: 52 | os.mkdir(os.path.join(out_folder, cur_split)) 53 | except: 54 | pass 55 | 56 | for c in list(set(classes)): 57 | try: 58 | os.mkdir(os.path.join(out_folder, cur_split, c)) 59 | except: 60 | pass 61 | 62 | cmds = [] 63 | for v, c in zip(vids, classes): 64 | source_vid = os.path.join(in_folder, c, "{}.avi".format(v)) 65 | extract_dir = os.path.join(out_folder, cur_split, c, v) 66 | 67 | if os.path.exists(extract_dir): 68 | continue 69 | 70 | out_wc = os.path.join(extract_dir, '%08d.jpg') 71 | 72 | print(source_vid, out_wc) 73 | 74 | scale_string = 'scale={}:{}'.format(out_w, out_h) 75 | os.mkdir(extract_dir) 76 | try: 77 | cmd = ['ffmpeg', '-i', source_vid, '-vf', scale_string, '-q:v', '5', out_wc] 78 | 79 | cmds.append((cmd, extract_dir)) 80 | subprocess.call(cmd) 81 | except: 82 | pass 83 | #Parallel(n_jobs=8, require='sharedmem')(delayed(run_cmd)(cmds[i]) for i in range(0, len(cmds))) -------------------------------------------------------------------------------- /Few_Shot_Learning/tests/test_models.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import torch 3 | import numpy as np 4 | 5 | from model import CNN_TRX, CNN_OTAM, CNN_TSN, CNN_PAL 6 | 7 | @pytest.fixture(scope='module') 8 | def args(): 9 | class ArgsObject(object): 10 | def __init__(self): 11 | self.trans_linear_in_dim = 512 12 | self.trans_linear_out_dim = 128 13 | 14 | self.way = 3 15 | self.shot = 2 16 | self.query_per_class = 1 17 | self.trans_dropout = 0.1 18 | self.seq_len = 4 19 | self.img_size = 84 20 | self.backbone = "resnet18" 21 | self.num_gpus = 1 22 | self.temp_set = [2,3] 23 | self.pretrained_backbone=None 24 | args = ArgsObject() 25 | return args 26 | 27 | @pytest.fixture(scope='module') 28 | def task_dict(args): 29 | device = 'cpu' 30 | support_imgs = torch.rand(args.way * args.shot * args.seq_len,3, args.img_size, args.img_size).to(device) 31 | target_imgs = torch.rand(args.way * args.query_per_class * args.seq_len ,3, args.img_size, args.img_size).to(device) 32 | support_labels = torch.tensor([n for n in range(args.way)] * args.shot).to(device) 33 | target_labels = torch.tensor([n for n in range(args.way)] * args.query_per_class).to(device) 34 | 35 | task_dict = {} 36 | task_dict["support_set"] = support_imgs 37 | task_dict["support_labels"] = support_labels 38 | task_dict["target_set"] = target_imgs 39 | task_dict["target_labels"] = target_labels 40 | 41 | return task_dict 42 | 43 | @pytest.mark.parametrize("arch", [CNN_TRX, CNN_OTAM, CNN_TSN, CNN_PAL]) 44 | def test_model_return_shapes(arch, task_dict, args): 45 | device = 'cpu' 46 | model = arch(args).to(device) 47 | model_dict = model(task_dict["support_set"], task_dict["support_labels"], task_dict["target_set"]) 48 | assert model_dict["logits"].shape == torch.Size([args.query_per_class * args.way, args.way]) 49 | 50 | 51 | @pytest.mark.parametrize("arch", [CNN_TRX, CNN_OTAM, CNN_TSN, CNN_PAL]) 52 | def test_backbone_trains(arch, task_dict, args): 53 | device = 'cpu' 54 | model = arch(args).to(device) 55 | opt = torch.optim.SGD(model.parameters(), lr=1.0) 56 | 57 | init_weights = model.backbone[-2][-1].conv2.weight 58 | for _ in range(3): 59 | model_dict = model(task_dict["support_set"], task_dict["support_labels"], task_dict["target_set"]) 60 | loss = model.loss(task_dict, model_dict) 61 | loss.backward(retain_graph=False) 62 | opt.step() 63 | opt.zero_grad() 64 | final_weights = model.backbone[-2][-1].conv2.weight 65 | 66 | assert not np.array_equal(init_weights, final_weights) 67 | 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /Few_Shot_Learning/README.md: -------------------------------------------------------------------------------- 1 | # Temporal Alignment Prediction for Supervised Representation Learning and Few-Shot Sequence Classification 2 | 3 | Pytorch implementation of using TAP for few-shot action recognition. 4 | 5 | # Acknowledgments 6 | We based our code on [Few Shot Action Recognition Library] which is publicly available at https://github.com/tobyperrett/few-shot-action-recognition. 7 | Please also check the license and usage there if you want to make use of this code. 8 | 9 | 10 | # Usage 11 | Please check the README file in https://github.com/tobyperrett/few-shot-action-recognition. Here we quote and slightly adapt a part of the descriptions for quick start and running as follows: 12 | ############################################################################### 13 | 14 | ### Requirements 15 | 16 | - python >= 3.6 17 | - pytorch >= 1.8 18 | - einops 19 | - ffmpeg (for extracting data) 20 | - soft_dtw (https://github.com/Sleepwalking/pytorch-softdtw) 21 | 22 | 23 | ### Data preparation 24 | 25 | Download the datasets from their original locations: 26 | 27 | - [Something-Something V2](https://20bn.com/datasets/something-something#download) ([splits from OTAM](https://openaccess.thecvf.com/content_CVPR_2020/papers/Cao_Few-Shot_Video_Classification_via_Temporal_Alignment_CVPR_2020_paper.pdf)) 28 | - [UCF101](https://www.crcv.ucf.edu/data/UCF101.php) ([splits from ARN](https://www.ecva.net/papers/eccv_2020/papers_ECCV/papers/123500511.pdf)) 29 | - [HMDB51](https://serre-lab.clps.brown.edu/resource/hmdb-a-large-human-motion-database/#Downloads) ([splits from ARN](https://www.ecva.net/papers/eccv_2020/papers_ECCV/papers/123500511.pdf)) 30 | 31 | Once you've downloaded the datasets, you can use the extract scripts to extract frames and put them in train/val/test folders. You'll need to modify the paths at the top of the scripts. 32 | To remove unnecessary frames and save space (e.g. just leave 8 uniformly sampled frames), you can use shrink_dataset.py. Again, modify the paths at the top of the sctipt. 33 | 34 | 35 | ### Running examples 36 | 37 | Training TAP-CNN on the UCF101 dataset, 5-way 1-shot: 38 | ```bash 39 | python runfull.py -c ./checkpoints/ucfoap --dataset ./data/ucf101_l8 --method oap --shot 1 --tasks_per_batch 1 -i 35000 --val_iters 5000 10000 15000 20000 25000 30000 35000 40 | ``` 41 | 42 | Training SA-TAP-CNN on the UCF101 dataset, 5-way 5-shot: 43 | ```bash 44 | python runfull.py -c ./checkpoints/ucfsaoap5shot --dataset ./data/ucf101_l8 --method saoap --shot 5 --tasks_per_batch 1 -i 35000 --val_iters 5000 10000 15000 20000 25000 30000 35000 45 | ``` 46 | 47 | Training TAP-CNN on the SSv2 dataset, 5-way 5-shot: 48 | ```bash 49 | python runfull.py -c ./checkpoints/ssv2oap5shot --dataset ./data/ssv2_l8 --method oap --shot 5 --tasks_per_batch 1 -i 50000 --val_iters 5000 10000 15000 20000 25000 30000 35000 40000 45000 50000 50 | ``` 51 | 52 | For training TAP-TL in any case, please in uncomment line 587 and comment line 588 in "model.py" (the "__init__" function of class "CNN_OAP"). 53 | -------------------------------------------------------------------------------- /Supervised_Learning/README.md: -------------------------------------------------------------------------------- 1 | # Temporal Alignment Prediction for Supervised Representation Learning and Few-Shot Sequence Classification 2 | 3 | Pytorch implementation of the supervised learning of TAP for sequence data. 4 | 5 | 6 | # Acknowledgments 7 | Part of the coda is adapted from the PyTorch implementation of MoCo which is publicly available at https://github.com/facebookresearch/moco. 8 | Please also check the license and usage there if you want to make use of this code. 9 | 10 | 11 | ############################################################################### 12 | ## Quick Start 13 | 14 | ### 1. Requirements 15 | 16 | Python3, Pytorch, numpy, scipy, sklearn 17 | 18 | ### 2. Prepare Data 19 | 20 | We reorganize sequence data into the ".mat" format by Matlab. 21 | Training sequence samples and statistics are stored in "trainseqfull.mat"; 22 | Testing sequence samples and statistics are stored in "testseqfull.mat". 23 | 24 | #### "trainseqfull.mat" contains the following three variables: 25 | 26 | "trainseqfull" is a matrix of size N*T*d, where N is the number of training sequences, T is the length of the longest sequence in the dataset, d the dimension of the element-wise features. It contains all training sequences, sequences shorter than T are padded. 27 | 28 | “trainlengthseqfull” is a matrix of size N*1, it contains the lengths of the corresponding sequences in "trainseqfull". 29 | 30 | "trainlabelseqfull" is a matrix of size N*1, it contains the class labels of the corresponding sequences in "trainseqfull". 31 | 32 | #### "testseqfull.mat" contains the following three variables: 33 | 34 | "testseqfull" is a matrix of size N*T*d, where N is the number of testing sequences, T is the length of the longest sequence in the dataset, d the dimension of the element-wise features. It contains all testing sequences, sequences shorter than T are padded. 35 | 36 | “testlengthseqfull” is a matrix of size N*1, it contains the lengths of the corresponding sequences in "testseqfull". 37 | 38 | "testlabelseqfull" is a matrix of size N*1, it contains the class labels of the corresponding sequences in "testseqfull". 39 | 40 | For example, if the processed data of the ChaLearn dataset can be saved at 41 | 42 | ``` 43 | /datamat 44 | /ChaLearn 45 | trainseqfull.mat 46 | testseqfull.mat 47 | ``` 48 | 49 | ### 3. Usage 50 | Please modify to your own data and checkpoint paths. Please set the arguments according to your own data. 51 | 52 | main_supervised_hm.py: the code for training TAP using the triplet loss with hard minming 53 | OPA_supervised_hm.py: the code for the TAP model with the triplet loss, where the prediction network is implemented by a 3-layer CNN 54 | 55 | main_supervised_lifted.py: the code for training TAP using the lifted structured loss 56 | OPA_supervised_lifted.py: the code for the TAP model with the lifted structured loss, where the prediction network is implemented by a 3-layer CNN 57 | 58 | main_supervised_binomial.py: the code for training TAP using the binomial deviance loss 59 | OPA_supervised_binomial.py: the code for the TAP model with the binomial deviance loss, where the prediction network is implemented by a 3-layer CNN 60 | 61 | 62 | #### Supervised training 63 | Please refer to the beginning of main_supervised_binomial.py for the arguments. 64 | 65 | ```bash 66 | python main_supervised_binomial.py -data ./datamat/ChaLearn/ -savedir ./checkpoints/ChaLearn --inputdim 100 -b 64 67 | ``` 68 | 69 | #### Continue training 70 | ```bash 71 | python main_supervised_binomial.py -data ./datamat/ChaLearn/ -savedir ./checkpoints/ChaLearn --inputdim 100 --resume ./checkpoints/ChaLearn/checkpoint_0030.pth.tar 72 | ``` -------------------------------------------------------------------------------- /Few_Shot_Learning/soft_dtw.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from numba import jit 4 | from torch.autograd import Function 5 | 6 | @jit(nopython = True) 7 | def compute_softdtw(D, gamma): 8 | B = D.shape[0] 9 | N = D.shape[1] 10 | M = D.shape[2] 11 | R = np.ones((B, N + 2, M + 2)) * np.inf 12 | R[:, 0, 0] = 0 13 | for k in range(B): 14 | for j in range(1, M + 1): 15 | for i in range(1, N + 1): 16 | r0 = -R[k, i - 1, j - 1] / gamma 17 | r1 = -R[k, i - 1, j] / gamma 18 | r2 = -R[k, i, j - 1] / gamma 19 | rmax = max(max(r0, r1), r2) 20 | rsum = np.exp(r0 - rmax) + np.exp(r1 - rmax) + np.exp(r2 - rmax) 21 | softmin = - gamma * (np.log(rsum) + rmax) 22 | R[k, i, j] = D[k, i - 1, j - 1] + softmin 23 | return R 24 | 25 | @jit(nopython = True) 26 | def compute_softdtw_backward(D_, R, gamma): 27 | B = D_.shape[0] 28 | N = D_.shape[1] 29 | M = D_.shape[2] 30 | D = np.zeros((B, N + 2, M + 2)) 31 | E = np.zeros((B, N + 2, M + 2)) 32 | D[:, 1:N + 1, 1:M + 1] = D_ 33 | E[:, -1, -1] = 1 34 | R[:, : , -1] = -np.inf 35 | R[:, -1, :] = -np.inf 36 | R[:, -1, -1] = R[:, -2, -2] 37 | for k in range(B): 38 | for j in range(M, 0, -1): 39 | for i in range(N, 0, -1): 40 | a0 = (R[k, i + 1, j] - R[k, i, j] - D[k, i + 1, j]) / gamma 41 | b0 = (R[k, i, j + 1] - R[k, i, j] - D[k, i, j + 1]) / gamma 42 | c0 = (R[k, i + 1, j + 1] - R[k, i, j] - D[k, i + 1, j + 1]) / gamma 43 | a = np.exp(a0) 44 | b = np.exp(b0) 45 | c = np.exp(c0) 46 | E[k, i, j] = E[k, i + 1, j] * a + E[k, i, j + 1] * b + E[k, i + 1, j + 1] * c 47 | return E[:, 1:N + 1, 1:M + 1] 48 | 49 | class _SoftDTW(Function): 50 | @staticmethod 51 | def forward(ctx, D, gamma): 52 | dev = D.device 53 | dtype = D.dtype 54 | gamma = torch.Tensor([gamma]).to(dev).type(dtype) # dtype fixed 55 | D_ = D.detach().cpu().numpy() 56 | g_ = gamma.item() 57 | R = torch.Tensor(compute_softdtw(D_, g_)).to(dev).type(dtype) 58 | ctx.save_for_backward(D, R, gamma) 59 | return R[:, -2, -2] 60 | 61 | @staticmethod 62 | def backward(ctx, grad_output): 63 | dev = grad_output.device 64 | dtype = grad_output.dtype 65 | D, R, gamma = ctx.saved_tensors 66 | D_ = D.detach().cpu().numpy() 67 | R_ = R.detach().cpu().numpy() 68 | g_ = gamma.item() 69 | E = torch.Tensor(compute_softdtw_backward(D_, R_, g_)).to(dev).type(dtype) 70 | return grad_output.view(-1, 1, 1).expand_as(E) * E, None 71 | 72 | class SoftDTW(torch.nn.Module): 73 | def __init__(self, gamma=1.0, normalize=False): 74 | super(SoftDTW, self).__init__() 75 | self.normalize = normalize 76 | self.gamma=gamma 77 | self.func_dtw = _SoftDTW.apply 78 | 79 | def calc_distance_matrix(self, x, y): 80 | n = x.size(1) 81 | m = y.size(1) 82 | d = x.size(2) 83 | x = x.unsqueeze(2).expand(-1, n, m, d) 84 | y = y.unsqueeze(1).expand(-1, n, m, d) 85 | dist = torch.pow(x - y, 2).sum(3) 86 | return dist 87 | 88 | def forward(self, x, y): 89 | assert len(x.shape) == len(y.shape) 90 | squeeze = False 91 | if len(x.shape) < 3: 92 | x = x.unsqueeze(0) 93 | y = y.unsqueeze(0) 94 | squeeze = True 95 | if self.normalize: 96 | D_xy = self.calc_distance_matrix(x, y) 97 | out_xy = self.func_dtw(D_xy, self.gamma) 98 | D_xx = self.calc_distance_matrix(x, x) 99 | out_xx = self.func_dtw(D_xx, self.gamma) 100 | D_yy = self.calc_distance_matrix(y, y) 101 | out_yy = self.func_dtw(D_yy, self.gamma) 102 | result = out_xy - 1/2 * (out_xx + out_yy) # distance 103 | else: 104 | D_xy = self.calc_distance_matrix(x, y) 105 | out_xy = self.func_dtw(D_xy, self.gamma) 106 | result = out_xy # discrepancy 107 | return result.squeeze(0) if squeeze else result 108 | -------------------------------------------------------------------------------- /Supervised_Learning/dataset.py: -------------------------------------------------------------------------------- 1 | 2 | import torch 3 | import torch.utils.data as data 4 | import numpy as np 5 | #from PIL import Image 6 | 7 | import os 8 | #from DataSet import transforms 9 | from collections import defaultdict 10 | 11 | import scipy.io as scio 12 | 13 | 14 | class TrainData(data.Dataset): 15 | def __init__(self, root=None, data_mat=None): 16 | datapath = os.path.join(root,data_mat) 17 | Dismatdata = scio.loadmat(datapath) 18 | sequence_data = Dismatdata['trainseqfull'] 19 | sequence_data = sequence_data.copy(order='C') 20 | sequence_data = sequence_data.astype(np.float32) 21 | sequence_label = Dismatdata['trainlabelseqfull'] 22 | sequence_label = sequence_label.copy(order='C') 23 | #sequence_label = sequence_label.astype(np.float32) 24 | sequence_length = Dismatdata['trainlengthseqfull'] 25 | sequence_length = sequence_length.copy(order='C') 26 | #sequence_length = sequence_length.astype(np.float32) 27 | 28 | sequences = [] 29 | for i in range(sequence_data.shape[0]): 30 | sequences.append(sequence_data[i,:,:]) 31 | labels = [] 32 | intlabels = [] 33 | for i in range(sequence_label.shape[0]): 34 | labels.append(sequence_label[i,:]) 35 | intlabels.append(int(sequence_label[i,sequence_label.shape[1]-1])) 36 | lengths = [] 37 | for i in range(sequence_length.shape[0]): 38 | lengths.append(int(sequence_length[i,sequence_length.shape[1]-1])) 39 | #print(sequences) 40 | print(sequences[0].shape) 41 | print(len(intlabels)) 42 | print(len(lengths)) 43 | 44 | classes = list(set(intlabels)) 45 | 46 | # Generate Index Dictionary for every class 47 | Index = defaultdict(list) 48 | for i, label in enumerate(intlabels): 49 | Index[label].append(i) 50 | 51 | # Initialization Done 52 | self.root = root 53 | self.sequences = sequences 54 | self.labels = labels 55 | self.classes = classes 56 | self.intlabels = intlabels 57 | self.lengths = lengths 58 | #self.transform = transform 59 | self.Index = Index 60 | #self.loader = loader 61 | 62 | def __getitem__(self, index): 63 | sequence, intlabel, length = self.sequences[index], self.intlabels[index], self.lengths[index] 64 | return sequence, intlabel, length 65 | 66 | def __len__(self): 67 | return len(self.sequences) 68 | 69 | 70 | class TestData(data.Dataset): 71 | def __init__(self, root=None, data_mat=None): 72 | datapath = os.path.join(root,data_mat) 73 | Dismatdata = scio.loadmat(datapath) 74 | sequence_data = Dismatdata['testseqfull'] 75 | sequence_data = sequence_data.copy(order='C') 76 | sequence_data = sequence_data.astype(np.float32) 77 | sequence_label = Dismatdata['testlabelseqfull'] 78 | sequence_label = sequence_label.copy(order='C') 79 | sequence_length = Dismatdata['testlengthseqfull'] 80 | sequence_length = sequence_length.copy(order='C') 81 | #sequence_length = sequence_length.astype(np.float32) 82 | 83 | sequences = [] 84 | for i in range(sequence_data.shape[0]): 85 | sequences.append(sequence_data[i,:,:]) 86 | labels = [] 87 | intlabels = [] 88 | for i in range(sequence_label.shape[0]): 89 | labels.append(sequence_label[i,:]) 90 | intlabels.append(int(sequence_label[i,sequence_label.shape[1]-1])) 91 | lengths = [] 92 | for i in range(sequence_length.shape[0]): 93 | lengths.append(int(sequence_length[i,sequence_length.shape[1]-1])) 94 | #print(sequences) 95 | print(sequences[0].shape) 96 | print(len(intlabels)) 97 | print(len(lengths)) 98 | 99 | classes = list(set(intlabels)) 100 | 101 | # Generate Index Dictionary for every class 102 | Index = defaultdict(list) 103 | for i, label in enumerate(intlabels): 104 | Index[label].append(i) 105 | 106 | # Initialization Done 107 | self.root = root 108 | self.sequences = sequences 109 | self.labels = labels 110 | self.classes = classes 111 | self.intlabels = intlabels 112 | self.lengths = lengths 113 | #self.transform = transform 114 | self.Index = Index 115 | #self.loader = loader 116 | 117 | def __getitem__(self, index): 118 | sequence, intlabel, length = self.sequences[index], self.intlabels[index], self.lengths[index] 119 | return sequence, intlabel, length 120 | 121 | def __len__(self): 122 | return len(self.sequences) 123 | 124 | 125 | class SequenceDataset: 126 | def __init__(self, root=None): 127 | train_mat = "trainseqfull.mat" 128 | test_mat = "testseqfull.mat" 129 | if root is None: 130 | root = './' 131 | self.traindata = TrainData(root, data_mat=train_mat) 132 | self.testdata = TestData(root, data_mat=test_mat) 133 | 134 | # if train_flag: 135 | # self.seqdata = TrainData(root, data_mat=train_mat) 136 | # else: 137 | # self.seqdata = TestData(root, data_mat=test_mat) 138 | 139 | 140 | def testsequence(): 141 | print(SequenceDataset.__name__) 142 | data = SequenceDataset() 143 | print(len(data.testdata)) 144 | print(len(data.traindata)) 145 | print(data.traindata[1]) 146 | 147 | 148 | if __name__ == "__main__": 149 | testsequence() 150 | 151 | 152 | -------------------------------------------------------------------------------- /Few_Shot_Learning/runfull.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | import argparse 4 | import os 5 | import pickle 6 | from utils import print_and_log, get_log_files, TestAccuracies, aggregate_accuracy, verify_checkpoint_dir 7 | from model import CNN_TRX, CNN_OTAM, CNN_TSN, CNN_PAL, CNN_OAP, CNN_SoftDTW, CNN_CA_OAP, CNN_SA_OAP 8 | 9 | from torch.optim.lr_scheduler import MultiStepLR 10 | from torch.utils.tensorboard import SummaryWriter 11 | import torchvision 12 | import video_reader 13 | import random 14 | 15 | # torch.autograd.set_detect_anomaly(True) 16 | 17 | class Learner: 18 | def __init__(self): 19 | self.args = self.parse_command_line() 20 | 21 | self.checkpoint_dir, self.logfile, self.checkpoint_path_validation, self.checkpoint_path_final \ 22 | = get_log_files(self.args.checkpoint_dir, self.args.resume_from_checkpoint, False) 23 | 24 | print_and_log(self.logfile, "Options: %s\n" % self.args) 25 | print_and_log(self.logfile, "Checkpoint Directory: %s\n" % self.checkpoint_dir) 26 | 27 | self.writer = SummaryWriter() 28 | 29 | self.device = torch.device('cuda' if (torch.cuda.is_available() and self.args.num_gpus > 0) else 'cpu') 30 | self.model = self.init_model() 31 | 32 | self.video_dataset = video_reader.VideoDataset(self.args) 33 | self.video_loader = torch.utils.data.DataLoader(self.video_dataset, batch_size=1, num_workers=self.args.num_workers) 34 | self.val_accuracies = TestAccuracies([self.args.dataset]) 35 | 36 | self.accuracy_fn = aggregate_accuracy 37 | 38 | if self.args.opt == "adam": 39 | self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.args.learning_rate) 40 | elif self.args.opt == "sgd": 41 | self.optimizer = torch.optim.SGD(self.model.parameters(), lr=self.args.learning_rate) 42 | 43 | self.scheduler = MultiStepLR(self.optimizer, milestones=self.args.sch, gamma=0.1) 44 | 45 | self.start_iteration = 0 46 | if self.args.resume_from_checkpoint: 47 | self.load_checkpoint() 48 | self.optimizer.zero_grad() 49 | 50 | def init_model(self): 51 | if self.args.method == "trx": 52 | model = CNN_TRX(self.args) 53 | elif self.args.method == "otam": 54 | model = CNN_OTAM(self.args) 55 | elif self.args.method == "tsn": 56 | model = CNN_TSN(self.args) 57 | elif self.args.method == "pal": 58 | model = CNN_PAL(self.args) 59 | elif self.args.method == "softdtw": 60 | model = CNN_SoftDTW(self.args) 61 | elif self.args.method == "oap": 62 | model = CNN_OAP(self.args) 63 | elif self.args.method == "saoap": 64 | model = CNN_SA_OAP(self.args) 65 | 66 | model = model.to(self.device) 67 | 68 | if torch.cuda.is_available() and self.args.num_gpus > 1: 69 | model.distribute_model() 70 | return model 71 | 72 | 73 | def parse_command_line(self): 74 | parser = argparse.ArgumentParser() 75 | 76 | parser.add_argument("--dataset", type=str, default="data/ssv2small", help="Path to dataset") 77 | parser.add_argument("--learning_rate", "-lr", type=float, default=0.001, help="Learning rate.") 78 | parser.add_argument("--tasks_per_batch", type=int, default=16, help="Number of tasks between parameter optimizations.") 79 | parser.add_argument("--checkpoint_dir", "-c", default=None, help="Directory to save checkpoint to.") 80 | parser.add_argument("--test_model_name", "-m", default="checkpoint_best_val.pt", help="Path to model to load and test.") 81 | parser.add_argument("--training_iterations", "-i", type=int, default=60000, help="Number of meta-training iterations.") 82 | parser.add_argument("--resume_from_checkpoint", "-r", dest="resume_from_checkpoint", default=False, action="store_true", help="Restart from latest checkpoint.") 83 | parser.add_argument("--way", type=int, default=5, help="Way of each task.") 84 | parser.add_argument("--shot", type=int, default=5, help="Shots per class.") 85 | parser.add_argument("--query_per_class", "-qpc", type=int, default=5, help="Target samples (i.e. queries) per class used for training.") 86 | parser.add_argument("--query_per_class_test", "-qpct", type=int, default=1, help="Target samples (i.e. queries) per class used for testing.") 87 | parser.add_argument('--val_iters', nargs='+', type=int, help='iterations to val at.', default=[]) 88 | parser.add_argument("--num_val_tasks", type=int, default=1000, help="number of random tasks to val on.") 89 | parser.add_argument("--num_test_tasks", type=int, default=10000, help="number of random tasks to test on.") 90 | parser.add_argument("--print_freq", type=int, default=1000, help="print and log every n iterations.") 91 | parser.add_argument("--seq_len", type=int, default=8, help="Frames per video.") 92 | parser.add_argument("--num_workers", type=int, default=8, help="Num dataloader workers.") 93 | parser.add_argument("--backbone", choices=["resnet18", "resnet34", "resnet50"], default="resnet50", help="backbone") 94 | parser.add_argument("--opt", choices=["adam", "sgd"], default="sgd", help="Optimizer") 95 | parser.add_argument("--save_freq", type=int, default=5000, help="Number of iterations between checkpoint saves.") 96 | parser.add_argument("--img_size", type=int, default=224, help="Input image size to the CNN after cropping.") 97 | parser.add_argument("--num_gpus", type=int, default=2, help="Number of GPUs to split the ResNet over") 98 | parser.add_argument('--sch', nargs='+', type=int, help='iters to drop learning rate', default=[1000000]) 99 | parser.add_argument("--method", choices=["trx", "otam", "tsn", "pal", "softdtw", "oap", "caoap", "saoap"], default="trx", help="few-shot method to use") 100 | parser.add_argument("--pretrained_backbone", "-pt", type=str, default=None, help="pretrained backbone path, used by PAL") 101 | parser.add_argument("--val_on_test", default=False, action="store_true", help="Danger: Validate on the test set, not the validation set. Use for debugging or checking overfitting on test set. Not good practice to use when developing, hyperparameter tuning or training models.") 102 | 103 | args = parser.parse_args() 104 | 105 | if args.checkpoint_dir == None: 106 | print("need to specify a checkpoint dir") 107 | exit(1) 108 | 109 | if args.backbone == "resnet50": 110 | args.trans_linear_in_dim = 2048 111 | else: 112 | args.trans_linear_in_dim = 512 113 | 114 | return args 115 | 116 | def run(self): 117 | train_accuracies = [] 118 | losses = [] 119 | total_iterations = self.args.training_iterations 120 | 121 | iteration = self.start_iteration 122 | 123 | #val_accuraies = [0] * 5 124 | val_accuraies = [] 125 | test_accuraies = [] 126 | best_val_accuracy = 0 127 | 128 | for task_dict in self.video_loader: 129 | if iteration >= total_iterations: 130 | break 131 | iteration += 1 132 | torch.set_grad_enabled(True) 133 | 134 | task_loss, task_accuracy = self.train_task(task_dict) 135 | train_accuracies.append(task_accuracy) 136 | losses.append(task_loss) 137 | 138 | # optimize 139 | if ((iteration + 1) % self.args.tasks_per_batch == 0) or (iteration == (total_iterations - 1)): 140 | self.optimizer.step() 141 | self.optimizer.zero_grad() 142 | self.scheduler.step() 143 | 144 | # print training stats 145 | if (iteration + 1) % self.args.print_freq == 0: 146 | print_and_log(self.logfile,'Task [{}/{}], Train Loss: {:.7f}, Train Accuracy: {:.7f}' 147 | .format(iteration + 1, total_iterations, torch.Tensor(losses).mean().item(), 148 | torch.Tensor(train_accuracies).mean().item())) 149 | train_accuracies = [] 150 | losses = [] 151 | 152 | # save checkpoint 153 | if ((iteration + 1) % self.args.save_freq == 0) and (iteration + 1) != total_iterations: 154 | self.save_checkpoint(iteration + 1) 155 | 156 | # validate 157 | if ((iteration + 1) in self.args.val_iters) and (iteration + 1) != total_iterations: 158 | 159 | # accuracy_dict2 = self.evaluate("test") 160 | # iter_acc = accuracy_dict2[self.args.dataset]["accuracy"] 161 | # #print(iter_acc) 162 | # test_accuraies.append(iter_acc) 163 | # self.val_accuracies.print(self.logfile, accuracy_dict2, mode="test") 164 | 165 | accuracy_dict = self.evaluate("val") 166 | iter_acc = accuracy_dict[self.args.dataset]["accuracy"] 167 | val_accuraies.append(iter_acc) 168 | self.val_accuracies.print(self.logfile, accuracy_dict, mode="val") 169 | 170 | # save checkpoint if best validation score 171 | if iter_acc > best_val_accuracy: 172 | best_val_accuracy = iter_acc 173 | self.save_checkpoint(iteration + 1, "checkpoint_best_val.pt") 174 | 175 | # if self.args.val_on_test: 176 | # accuracy_dict = self.evaluate("test") 177 | # self.val_accuracies.print(self.logfile, accuracy_dict, mode="test") 178 | 179 | # # get out if best accuracy was two validations ago 180 | # if val_accuraies[-1] < val_accuraies[-3]: 181 | # break 182 | 183 | # save the final model 184 | self.save_checkpoint(iteration + 1, "checkpoint_final.pt") 185 | 186 | 187 | # evaluate best validation model if it exists, otherwise evaluate the final model. 188 | try: 189 | self.load_checkpoint("checkpoint_best_val.pt") 190 | except: 191 | self.load_checkpoint("checkpoint_final.pt") 192 | 193 | accuracy_dict = self.evaluate("test") 194 | self.val_accuracies.print(self.logfile, accuracy_dict, mode="test") 195 | 196 | self.logfile.close() 197 | 198 | def train_task(self, task_dict): 199 | """ 200 | For one task, runs forward, calculates the loss and accuracy and backprops 201 | """ 202 | task_dict = self.prepare_task(task_dict) 203 | model_dict = self.model(task_dict['support_set'], task_dict['support_labels'], task_dict['target_set']) 204 | target_logits = model_dict['logits'] 205 | 206 | task_loss = self.model.loss(task_dict, model_dict) / self.args.tasks_per_batch 207 | task_accuracy = self.accuracy_fn(target_logits, task_dict['target_labels']) 208 | 209 | task_loss.backward(retain_graph=False) 210 | 211 | return task_loss, task_accuracy 212 | 213 | def evaluate(self, mode="val"): 214 | self.model.eval() 215 | with torch.no_grad(): 216 | 217 | self.video_loader.dataset.split = mode 218 | if mode == "val": 219 | n_tasks = self.args.num_val_tasks 220 | elif mode == "test": 221 | n_tasks = self.args.num_test_tasks 222 | 223 | accuracy_dict ={} 224 | accuracies = [] 225 | iteration = 0 226 | item = self.args.dataset 227 | for task_dict in self.video_loader: 228 | if iteration >= n_tasks: 229 | break 230 | iteration += 1 231 | 232 | task_dict = self.prepare_task(task_dict) 233 | model_dict = self.model(task_dict['support_set'], task_dict['support_labels'], task_dict['target_set']) 234 | target_logits = model_dict['logits'] 235 | accuracy = self.accuracy_fn(target_logits, task_dict['target_labels']) 236 | accuracies.append(accuracy.item()) 237 | del target_logits 238 | 239 | accuracy = np.array(accuracies).mean() * 100.0 240 | # 95% confidence interval 241 | confidence = (196.0 * np.array(accuracies).std()) / np.sqrt(len(accuracies)) 242 | 243 | accuracy_dict[item] = {"accuracy": accuracy, "confidence": confidence} 244 | self.video_loader.dataset.split = "train" 245 | self.model.train() 246 | 247 | return accuracy_dict 248 | 249 | 250 | def prepare_task(self, task_dict): 251 | """ 252 | Remove first batch dimension (as we only ever use a batch size of 1) and move data to device. 253 | """ 254 | for k in task_dict.keys(): 255 | task_dict[k] = task_dict[k][0].to(self.device) 256 | return task_dict 257 | 258 | def save_checkpoint(self, iteration, name="checkpoint.pt"): 259 | d = {'iteration': iteration, 260 | 'model_state_dict': self.model.state_dict(), 261 | 'optimizer_state_dict': self.optimizer.state_dict(), 262 | 'scheduler': self.scheduler.state_dict()} 263 | 264 | torch.save(d, os.path.join(self.checkpoint_dir, 'checkpoint{}.pt'.format(iteration))) 265 | torch.save(d, os.path.join(self.checkpoint_dir, name)) 266 | 267 | def load_checkpoint(self, name="checkpoint.pt"): 268 | checkpoint = torch.load(os.path.join(self.checkpoint_dir, name)) 269 | self.start_iteration = checkpoint['iteration'] 270 | self.model.load_state_dict(checkpoint['model_state_dict']) 271 | self.optimizer.load_state_dict(checkpoint['optimizer_state_dict']) 272 | self.scheduler.load_state_dict(checkpoint['scheduler']) 273 | 274 | def main(): 275 | learner = Learner() 276 | learner.run() 277 | 278 | if __name__ == "__main__": 279 | main() 280 | -------------------------------------------------------------------------------- /Supervised_Learning/OPA_supervised_hm.py: -------------------------------------------------------------------------------- 1 | import math 2 | import random 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | import numpy as np 7 | from torch.autograd import Variable 8 | 9 | 10 | def initialize_weight(x): 11 | nn.init.xavier_uniform_(x.weight) 12 | if x.bias is not None: 13 | nn.init.constant_(x.bias, 0) 14 | 15 | def tensor_shift(t, dim, shift): 16 | """ 17 | t (tensor): tensor to be shifted. 18 | dim (int): the dimension apply shift. 19 | shift (int): shift distance. 20 | """ 21 | assert 0 <= shift <= t.size(dim), "shift distance should be smaller than or equal to the dim length." 22 | 23 | overflow = t.index_select(dim, torch.arange(t.size(dim)-shift, t.size(dim)).cuda()) 24 | remain = t.index_select(dim, torch.arange(t.size(dim)-shift).cuda()) 25 | 26 | return torch.cat((overflow, remain),dim=dim) 27 | 28 | class SequenceInception(nn.Module): 29 | def __init__(self, in_dim,middle_dim,out_dim, normalized=1): 30 | super(SequenceInception, self).__init__() 31 | self.in_dim = in_dim 32 | self.middle_dim = middle_dim 33 | self.out_dim = out_dim 34 | self.normalized = normalized 35 | self.layer1 = nn.Linear(in_dim, middle_dim) 36 | self.relu = nn.ReLU() #nn.Tanh() 37 | self.layer3 = nn.Linear(middle_dim, out_dim) 38 | self.layer2 = nn.Linear(middle_dim, middle_dim) 39 | 40 | def forward(self, input): 41 | batch_size = input.size(0) 42 | T = input.size(1) 43 | dim = input.size(2) 44 | input = input.view(batch_size*T,dim) 45 | out1 = self.layer1(input) 46 | out1 = self.relu(out1) 47 | out1 = self.layer2(out1) 48 | out1 = self.relu(out1) 49 | output = self.layer3(out1) 50 | 51 | if self.normalized == 1: 52 | norm = output.norm(dim=1, p=2, keepdim=True) 53 | #if norm>0: 54 | output = output.div(norm.expand_as(output)) 55 | output = torch.where(torch.isnan(output), torch.full_like(output, 0), output) 56 | output = torch.where(torch.isinf(output), torch.full_like(output, 0), output) 57 | 58 | output = output.view(batch_size,T,self.out_dim) 59 | return output 60 | 61 | class PredictionInception(nn.Module): 62 | def __init__(self, in_dim,middle_dim,out_dim, normalized=1): 63 | super(PredictionInception, self).__init__() 64 | self.in_dim = in_dim 65 | self.middle_dim = middle_dim 66 | self.out_dim = out_dim 67 | self.normalized = normalized 68 | #inplace = True 69 | 70 | self.layer1 = nn.Linear(in_dim, middle_dim) 71 | self.relu = nn.ReLU() 72 | self.layer2 = nn.Linear(middle_dim, out_dim) 73 | initialize_weight(self.layer1) 74 | initialize_weight(self.layer2) 75 | 76 | def forward(self, input): 77 | batch_size = input.size(0) 78 | T = input.size(1) 79 | dim = input.size(2) 80 | input = input.view(-1,dim) 81 | out1 = self.layer1(input) 82 | out1 = self.relu(out1) 83 | output = self.layer2(out1) 84 | 85 | if self.normalized == 1: 86 | norm = output.norm(dim=1, p=2, keepdim=True) 87 | #if norm>0: 88 | output = output.div(norm.expand_as(output)) 89 | output = torch.where(torch.isnan(output), torch.full_like(output, 0), output) 90 | output = torch.where(torch.isinf(output), torch.full_like(output, 0), output) 91 | 92 | output = output.view(batch_size,T,self.out_dim) 93 | return output 94 | 95 | 96 | class OPA(nn.Module): 97 | def __init__(self, input_dim): 98 | super(OPA, self).__init__() 99 | #self.lam = lam 100 | self.input_dim = input_dim 101 | self.att_size = att_size = 30 102 | self.att_size2 = att_size2 = 100 103 | #self.att_size3 = att_size3 = 30 104 | self.scale = att_size ** -0.5 105 | self.scale2 = att_size2 ** -0.5 106 | 107 | self.conv1 = nn.Conv2d(2, 30, 5, padding=2) 108 | self.conv2 = nn.Conv2d(30, 30, 5, padding=2) 109 | self.conv3 = nn.Conv2d(30, 1, 3, padding=1) 110 | 111 | def forward(self, seq_q, seq_k, len_q, len_k): 112 | 113 | len_q = len_q.detach() 114 | len_k = len_k.detach() 115 | T_q = seq_q.size(1) 116 | T_k = seq_k.size(1) 117 | 118 | batch_size = seq_q.size(0) 119 | assert batch_size==1 120 | 121 | D = torch.cdist(seq_q.view(batch_size, T_q, self.input_dim), seq_k.view(batch_size, T_k, self.input_dim), 2) 122 | 123 | P = torch.cdist(len_q.view(batch_size,T_q,1), len_k.view(batch_size,T_k,1), 2) 124 | 125 | A = torch.cat([D, P], 0).unsqueeze(0) 126 | A = F.relu(self.conv1(A)) 127 | A = F.relu(self.conv2(A)) 128 | A = self.conv3(A).squeeze(1) + D 129 | A = torch.softmax(-A, dim=2) 130 | 131 | At = A.view(batch_size,T_q*T_k,1) 132 | norm = At.norm(dim=1, p=1, keepdim=True) 133 | At = At.div(norm.expand_as(At)) 134 | 135 | dis = torch.matmul(D.view(batch_size,1,T_q*T_k),At) 136 | return A, dis 137 | 138 | 139 | 140 | class BlurGenerationPair(nn.Module): 141 | def __init__(self): 142 | super(BlurGenerationPair, self).__init__() 143 | 144 | def forward(self, seq, len_seq): 145 | batch_size = seq.size(0) 146 | T = seq.size(1) 147 | dim = seq.size(2) 148 | 149 | SeqtoBlur = torch.zeros( size=(batch_size,T,T),dtype=torch.float32 ).cuda() 150 | avged_seq = torch.zeros_like(seq).cuda() 151 | avged_len = torch.zeros_like(len_seq).cuda() 152 | R = torch.zeros(batch_size,T).cuda() 153 | avged_R = torch.zeros(batch_size,T).cuda() 154 | 155 | for b in range(batch_size): 156 | Tc = len_seq[b].item() 157 | R[b,0:Tc] = torch.tensor([float(t+1)/float(Tc) for t in range(Tc)]) 158 | 159 | if Tc>4: 160 | randseed = random.randint(1,10) 161 | if randseed>5: 162 | tempseq = seq[b,0:Tc,:] 163 | tempseq = tempseq.view(1,1,Tc,dim) 164 | if randseed>7: 165 | GaussKer = torch.tensor([[[[0.1, 0.8, 0.1]]]]) 166 | midnum = random.uniform(0.55,0.9) 167 | sidenum = (1.-midnum)/2. 168 | GaussKer[0,0,0,0] = sidenum 169 | GaussKer[0,0,0,2] = sidenum 170 | GaussKer[0,0,0,1] = midnum 171 | GaussKer = GaussKer.transpose(2,3).cuda() 172 | stride = random.randint(1,3) 173 | outTc = int((Tc-3)/stride) + 1 174 | blured_seq = F.conv2d(tempseq,GaussKer,stride=[stride,1]) 175 | avged_seq[b,0:outTc,:] = blured_seq.squeeze(1) 176 | startf = 0 177 | for i in range(outTc): 178 | endf = startf + 3 179 | SeqtoBlur[b,startf:endf,i] = GaussKer.squeeze() 180 | startf = startf + stride 181 | #SeqtoBlur[b,1:Tc-1,0:outTc] = torch.eye(Tc-2) 182 | avged_R[b,0:outTc] = torch.tensor([float(t+1)/float(outTc) for t in range(outTc)]) 183 | avged_len[b] = outTc 184 | 185 | else: 186 | GaussKer = torch.tensor([[[[0.1, 0.15, 0.5, 0.15, 0.1]]]]) 187 | midnum = random.uniform(0.33,0.5) 188 | sidenum = (1.-midnum)/2. 189 | sidenum1 = random.uniform(0.16,sidenum) 190 | sidenum2 = sidenum - sidenum1 191 | GaussKer[0,0,0,0] = sidenum2 192 | GaussKer[0,0,0,1] = sidenum1 193 | GaussKer[0,0,0,2] = midnum 194 | GaussKer[0,0,0,3] = sidenum1 195 | GaussKer[0,0,0,4] = sidenum2 196 | GaussKer = GaussKer.transpose(2,3).cuda() 197 | stride = random.randint(1,3) 198 | outTc = int((Tc-5)/stride) + 1 199 | blured_seq = F.conv2d(tempseq,GaussKer,stride=[stride,1]) 200 | startf = 0 201 | for i in range(outTc): 202 | endf = startf + 5 203 | SeqtoBlur[b,startf:endf,i] = GaussKer.squeeze() 204 | startf = startf + stride 205 | avged_R[b,0:outTc] = torch.tensor([float(t+1)/float(outTc) for t in range(outTc)]) 206 | avged_len[b] = outTc 207 | else: 208 | min_len = int(0.5*Tc) 209 | if min_len<1: 210 | min_len = 1 211 | max_len = int(0.8*Tc) 212 | if max_len<1: 213 | max_len = 1 214 | if max_len>Tc: 215 | max_len = Tc 216 | cur_len = random.choice(range(min_len,max_len+1)) 217 | #print(cur_len) 218 | interval = random.sample(range(1,Tc),cur_len-1) 219 | interval.sort() 220 | interval.append(Tc) 221 | #print(interval) 222 | startf = 0 223 | for i in range(cur_len): 224 | endf = interval[i] 225 | templ = endf-startf 226 | localatt = torch.softmax(torch.tensor([random.gauss(0,1) for _ in range(templ)]).view(templ,1),dim=0).cuda() 227 | #print(localatt) 228 | tempfr = seq[b,startf:endf,:] 229 | avged_seq[b,i,:] = (tempfr*localatt).sum(dim=0) 230 | #print(localatt.shape, startf, endf) 231 | SeqtoBlur[b,startf:endf,i] = localatt.view(-1) 232 | startf = endf 233 | 234 | avged_R[b,0:cur_len] = torch.tensor([float(t+1)/float(cur_len) for t in range(cur_len)]) 235 | avged_len[b] = cur_len 236 | else: 237 | if Tc>2: 238 | tempseq = seq[b,0:Tc,:] 239 | tempseq = tempseq.view(1,1,Tc,dim) 240 | GaussKer = torch.tensor([[[[0.1, 0.8, 0.1]]]]) 241 | midnum = random.uniform(0.55,0.9) 242 | sidenum = (1.-midnum)/2. 243 | GaussKer[0,0,0,0] = sidenum 244 | GaussKer[0,0,0,2] = sidenum 245 | GaussKer[0,0,0,1] = midnum 246 | GaussKer = GaussKer.transpose(2,3).cuda() 247 | blured_seq = F.conv2d(tempseq,GaussKer) 248 | avged_seq[b,0:Tc-2,:] = blured_seq.squeeze(1) 249 | SeqtoBlur[b,1:Tc-1,0:Tc-2] = torch.eye(Tc-2) 250 | avged_R[b,0:Tc-2] = torch.tensor([float(t+1)/float(Tc-2) for t in range(Tc-2)]) 251 | avged_len[b] = Tc-2 252 | else: 253 | avged_seq[b,:,:] = seq[b,:,:] 254 | SeqtoBlur[b,0:Tc,0:Tc] = torch.eye(Tc) 255 | avged_R[b,:] = R[b,:] 256 | avged_len[b] = Tc 257 | 258 | 259 | return SeqtoBlur.detach(), avged_seq.detach(), R.detach(), avged_R.detach(), avged_len.detach() 260 | 261 | 262 | class BlurContrastiveModelPair(nn.Module): 263 | def __init__(self, input_dim, output_dim=-1, lam1=1., lam2=1.): 264 | super(BlurContrastiveModelPair, self).__init__() 265 | 266 | self.input_dim = input_dim 267 | if output_dim>0: 268 | self.output_dim = output_dim 269 | else: 270 | self.output_dim = input_dim 271 | self.middle_dim = 1024 272 | self.lsoftmax = nn.LogSoftmax(dim=1) 273 | 274 | self.alignment = OPA(self.output_dim) 275 | self.aug = BlurGenerationPair() 276 | self.encoder = SequenceInception(self.input_dim, self.middle_dim, self.output_dim) 277 | self.predictor = PredictionInception(self.input_dim, self.middle_dim, self.output_dim) 278 | #self.mse = nn.MSELoss(reduction='mean') 279 | self.lam1 = lam1 280 | self.lam2 = lam2 281 | self.mse = nn.MSELoss(reduction='mean') 282 | 283 | def forward(self, seq, len_seq, targets, train_flag = True): 284 | margin=0.1 285 | batch_size = seq.size(0) 286 | SeqtoBlur, avged_seq, R, avged_R, avged_len = self.aug(seq, len_seq) 287 | 288 | DisMat = torch.zeros([batch_size,batch_size],dtype=torch.float).cuda() 289 | Lalign = 0 290 | Lalign2 = 0 291 | Ldis = 0 292 | Ldis2 = 0 293 | 294 | 295 | for b in range(batch_size): 296 | seq0 = seq[b,0:len_seq[b],:].unsqueeze(0) 297 | R0 = R[b,0:len_seq[b]].unsqueeze(0) 298 | avged_seq0 = avged_seq[b,0:avged_len[b],:].unsqueeze(0) 299 | avged_R0 = avged_R[b,0:avged_len[b]].unsqueeze(0) 300 | seq1 = self.encoder(seq0) 301 | avged_seq1 = self.encoder(avged_seq0) 302 | Aa1b, disa1b = self.alignment(seq1, avged_seq1, R0, avged_R0) 303 | Aa2b, disa2b = self.alignment(avged_seq1, seq1, avged_R0, R0) 304 | 305 | disa1b = disa1b.view(-1) 306 | disa2b = disa2b.view(-1) 307 | DisMat[b,b] = DisMat[b,b] + (disa1b + disa2b)/2.0 308 | Ldis = Ldis + disa1b + disa2b 309 | Ldis2 = Ldis2 + self.mse(disa1b,disa2b) 310 | Lalign2 = Lalign2 + self.mse(Aa1b,Aa2b.transpose(1,2)) 311 | tempAlign = SeqtoBlur[b,0:len_seq[b],0:avged_len[b]].unsqueeze(0) 312 | #print(tempAlign.shape) 313 | Lalign = Lalign + self.mse(Aa1b,tempAlign) + self.mse(Aa2b,tempAlign.transpose(1,2)) 314 | 315 | for b2 in range(b+1,batch_size): 316 | shift_seq0 = seq[b2,0:len_seq[b2],:].unsqueeze(0) 317 | shift_R0 = R[b2,0:len_seq[b2]].unsqueeze(0) 318 | 319 | shift_seq1 = self.encoder(shift_seq0) 320 | As1b, dis1b = self.alignment(seq1, shift_seq1, R0, shift_R0) 321 | As2b, dis2b = self.alignment(shift_seq1, seq1, shift_R0, R0) 322 | 323 | dis1b = dis1b.view(-1) 324 | dis2b = dis2b.view(-1) 325 | DisMat[b,b2] = DisMat[b,b2] + dis1b 326 | DisMat[b2,b] = DisMat[b2,b] + dis2b 327 | Ldis2 = Ldis2 + self.mse(dis1b,dis2b) 328 | Lalign2 = Lalign2 + self.mse(As1b,As2b.transpose(1,2)) 329 | 330 | loss = list() 331 | for i in range(batch_size): 332 | pos_pair_ = torch.masked_select(DisMat[i], targets==targets[i]) 333 | neg_pair_ = torch.masked_select(DisMat[i], targets!=targets[i]) 334 | 335 | pos_pair_ = torch.sort(pos_pair_)[0] 336 | neg_pair_ = torch.sort(neg_pair_)[0] 337 | 338 | neg_pair = torch.masked_select(neg_pair_, neg_pair_ < pos_pair_[-1] + margin) 339 | pos_pair = torch.masked_select(pos_pair_, pos_pair_ > neg_pair_[0] - margin) 340 | if len(neg_pair) < 1: 341 | continue 342 | 343 | pos_loss = torch.mean(pos_pair) 344 | neg_loss = torch.mean(margin-neg_pair) 345 | loss.append(pos_loss + neg_loss) 346 | 347 | Ldis /= batch_size 348 | Lalign /= batch_size 349 | Ldis2 /= batch_size 350 | Lalign2 /= batch_size 351 | L = sum(loss) + Ldis/batch_size + Lalign/batch_size 352 | return L 353 | 354 | def getlen(self, seq, len_seq): 355 | batch_size = seq.size(0) 356 | T = seq.size(1) 357 | dim = seq.size(2) 358 | R = torch.zeros(batch_size,T).cuda() 359 | for b in range(batch_size): 360 | Tc = len_seq[b].item() 361 | R[b,0:Tc] = torch.tensor([float(t+1)/float(Tc) for t in range(Tc)]) 362 | return R 363 | 364 | 365 | def KNN(X, k): 366 | X = X.float() 367 | mat_square = torch.mm(mat, mat.t()) 368 | diag = torch.diagonal(mat_square) 369 | diag = diag.expand_as(mat_square) 370 | dist_mat = diag + diag.t() - 2*mat_square 371 | dist_col = dist_mat[-1, :-1] 372 | val, index = dist_col.topk(k, largest=False, sorted=True) 373 | return val, index 374 | 375 | if __name__ == "__main__": 376 | seq = torch.tensor([[[1.,2.,3.,4.,5.,6],[7.,8.,9.,10.,11.,12.],[13.,14.,15.,16.,17.,18.]],[[11.,12.,13.,41.,52.,0.],[72.,81.,93.,8.,1.,0.],[3.,4.,6.,8.,12.,0.]]]) 377 | seq = seq.transpose(1,2).contiguous() 378 | #print(seq) 379 | aug = BlurGeneration() 380 | len_seq = torch.tensor([[6],[5]]) 381 | blured_seq, SeqtoBlur, avged_seq, R, blured_R, avged_R, blured_len, avged_len = aug(seq,len_seq) 382 | print(blured_len) 383 | print(avged_len) 384 | 385 | bcm = BlurContrastiveModel(3, output_dim=3) 386 | As1, dis1, As2, dis2, Ab1, disb1, Ab2, disb2, Aa1, disa1, Aa2, disa2 = bcm(seq, len_seq) 387 | 388 | bcl = BlurContrastiveLoss() 389 | l = bcl(As1, dis1, As2, dis2, Ab1, disb1, Ab2, disb2, Aa1, disa1, Aa2, disa2) 390 | print(l) 391 | -------------------------------------------------------------------------------- /Supervised_Learning/OPA_supervised_binomial.py: -------------------------------------------------------------------------------- 1 | import math 2 | import random 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | import numpy as np 7 | from torch.autograd import Variable 8 | 9 | def initialize_weight(x): 10 | nn.init.xavier_uniform_(x.weight) 11 | if x.bias is not None: 12 | nn.init.constant_(x.bias, 0) 13 | 14 | def tensor_shift(t, dim, shift): 15 | """ 16 | t (tensor): tensor to be shifted. 17 | dim (int): the dimension apply shift. 18 | shift (int): shift distance. 19 | """ 20 | assert 0 <= shift <= t.size(dim), "shift distance should be smaller than or equal to the dim length." 21 | 22 | overflow = t.index_select(dim, torch.arange(t.size(dim)-shift, t.size(dim)).cuda()) 23 | remain = t.index_select(dim, torch.arange(t.size(dim)-shift).cuda()) 24 | 25 | return torch.cat((overflow, remain),dim=dim) 26 | 27 | class SequenceInception(nn.Module): 28 | def __init__(self, in_dim,middle_dim,out_dim, normalized=1): 29 | super(SequenceInception, self).__init__() 30 | self.in_dim = in_dim 31 | self.middle_dim = middle_dim 32 | self.out_dim = out_dim 33 | self.normalized = normalized 34 | 35 | self.layer1 = nn.Linear(in_dim, middle_dim) 36 | self.relu = nn.ReLU() #nn.Tanh() 37 | self.layer3 = nn.Linear(middle_dim, out_dim) 38 | self.layer2 = nn.Linear(middle_dim, middle_dim) 39 | 40 | def forward(self, input): 41 | batch_size = input.size(0) 42 | T = input.size(1) 43 | dim = input.size(2) 44 | input = input.view(batch_size*T,dim) 45 | out1 = self.layer1(input) 46 | out1 = self.relu(out1) 47 | out1 = self.layer2(out1) 48 | out1 = self.relu(out1) 49 | output = self.layer3(out1) 50 | 51 | if self.normalized == 1: 52 | norm = output.norm(dim=1, p=2, keepdim=True) 53 | #if norm>0: 54 | output = output.div(norm.expand_as(output)) 55 | output = torch.where(torch.isnan(output), torch.full_like(output, 0), output) 56 | output = torch.where(torch.isinf(output), torch.full_like(output, 0), output) 57 | 58 | output = output.view(batch_size,T,self.out_dim) 59 | return output 60 | 61 | class PredictionInception(nn.Module): 62 | def __init__(self, in_dim,middle_dim,out_dim, normalized=1): 63 | super(PredictionInception, self).__init__() 64 | self.in_dim = in_dim 65 | self.middle_dim = middle_dim 66 | self.out_dim = out_dim 67 | self.normalized = normalized 68 | #inplace = True 69 | 70 | self.layer1 = nn.Linear(in_dim, middle_dim) 71 | self.relu = nn.ReLU() 72 | self.layer2 = nn.Linear(middle_dim, out_dim) 73 | initialize_weight(self.layer1) 74 | initialize_weight(self.layer2) 75 | 76 | def forward(self, input): 77 | batch_size = input.size(0) 78 | T = input.size(1) 79 | dim = input.size(2) 80 | input = input.view(-1,dim) 81 | out1 = self.layer1(input) 82 | out1 = self.relu(out1) 83 | output = self.layer2(out1) 84 | 85 | if self.normalized == 1: 86 | norm = output.norm(dim=1, p=2, keepdim=True) 87 | #if norm>0: 88 | output = output.div(norm.expand_as(output)) 89 | output = torch.where(torch.isnan(output), torch.full_like(output, 0), output) 90 | output = torch.where(torch.isinf(output), torch.full_like(output, 0), output) 91 | 92 | output = output.view(batch_size,T,self.out_dim) 93 | return output 94 | 95 | 96 | class OPA(nn.Module): 97 | def __init__(self, input_dim): 98 | super(OPA, self).__init__() 99 | #self.lam = lam 100 | self.input_dim = input_dim 101 | self.att_size = att_size = 30 102 | self.att_size2 = att_size2 = 100 103 | #self.att_size3 = att_size3 = 30 104 | self.scale = att_size ** -0.5 105 | self.scale2 = att_size2 ** -0.5 106 | 107 | self.conv1 = nn.Conv2d(2, 30, 5, padding=2) 108 | self.conv2 = nn.Conv2d(30, 30, 5, padding=2) 109 | self.conv3 = nn.Conv2d(30, 1, 3, padding=1) 110 | 111 | def forward(self, seq_q, seq_k, len_q, len_k): 112 | 113 | len_q = len_q.detach() 114 | len_k = len_k.detach() 115 | T_q = seq_q.size(1) 116 | T_k = seq_k.size(1) 117 | 118 | batch_size = seq_q.size(0) 119 | assert batch_size==1 120 | 121 | D = torch.cdist(seq_q.view(batch_size, T_q, self.input_dim), seq_k.view(batch_size, T_k, self.input_dim), 2) 122 | P = torch.cdist(len_q.view(batch_size,T_q,1), len_k.view(batch_size,T_k,1), 2) 123 | 124 | A = torch.cat([D, P], 0).unsqueeze(0) 125 | A = F.relu(self.conv1(A)) 126 | A = F.relu(self.conv2(A)) 127 | A = self.conv3(A).squeeze(1) + D 128 | A = torch.softmax(-A, dim=2) 129 | 130 | At = A.view(batch_size,T_q*T_k,1) 131 | norm = At.norm(dim=1, p=1, keepdim=True) 132 | At = At.div(norm.expand_as(At)) 133 | 134 | dis = torch.matmul(D.view(batch_size,1,T_q*T_k),At) 135 | return A, dis 136 | 137 | 138 | class BlurGenerationPair(nn.Module): 139 | def __init__(self): 140 | super(BlurGenerationPair, self).__init__() 141 | 142 | def forward(self, seq, len_seq): 143 | batch_size = seq.size(0) 144 | T = seq.size(1) 145 | dim = seq.size(2) 146 | 147 | SeqtoBlur = torch.zeros( size=(batch_size,T,T),dtype=torch.float32 ).cuda() 148 | avged_seq = torch.zeros_like(seq).cuda() 149 | avged_len = torch.zeros_like(len_seq).cuda() 150 | R = torch.zeros(batch_size,T).cuda() 151 | avged_R = torch.zeros(batch_size,T).cuda() 152 | 153 | for b in range(batch_size): 154 | Tc = len_seq[b].item() 155 | R[b,0:Tc] = torch.tensor([float(t+1)/float(Tc) for t in range(Tc)]) 156 | 157 | if Tc>4: 158 | randseed = random.randint(1,10) 159 | if randseed>5: 160 | tempseq = seq[b,0:Tc,:] 161 | tempseq = tempseq.view(1,1,Tc,dim) 162 | if randseed>7: 163 | GaussKer = torch.tensor([[[[0.1, 0.8, 0.1]]]]) 164 | midnum = random.uniform(0.55,0.9) 165 | sidenum = (1.-midnum)/2. 166 | GaussKer[0,0,0,0] = sidenum 167 | GaussKer[0,0,0,2] = sidenum 168 | GaussKer[0,0,0,1] = midnum 169 | GaussKer = GaussKer.transpose(2,3).cuda() 170 | stride = random.randint(1,3) 171 | outTc = int((Tc-3)/stride) + 1 172 | blured_seq = F.conv2d(tempseq,GaussKer,stride=[stride,1]) 173 | avged_seq[b,0:outTc,:] = blured_seq.squeeze(1) 174 | startf = 0 175 | for i in range(outTc): 176 | endf = startf + 3 177 | SeqtoBlur[b,startf:endf,i] = GaussKer.squeeze() 178 | startf = startf + stride 179 | #SeqtoBlur[b,1:Tc-1,0:outTc] = torch.eye(Tc-2) 180 | avged_R[b,0:outTc] = torch.tensor([float(t+1)/float(outTc) for t in range(outTc)]) 181 | avged_len[b] = outTc 182 | 183 | else: 184 | GaussKer = torch.tensor([[[[0.1, 0.15, 0.5, 0.15, 0.1]]]]) 185 | midnum = random.uniform(0.33,0.5) 186 | sidenum = (1.-midnum)/2. 187 | sidenum1 = random.uniform(0.16,sidenum) 188 | sidenum2 = sidenum - sidenum1 189 | GaussKer[0,0,0,0] = sidenum2 190 | GaussKer[0,0,0,1] = sidenum1 191 | GaussKer[0,0,0,2] = midnum 192 | GaussKer[0,0,0,3] = sidenum1 193 | GaussKer[0,0,0,4] = sidenum2 194 | GaussKer = GaussKer.transpose(2,3).cuda() 195 | stride = random.randint(1,3) 196 | outTc = int((Tc-5)/stride) + 1 197 | blured_seq = F.conv2d(tempseq,GaussKer,stride=[stride,1]) 198 | startf = 0 199 | for i in range(outTc): 200 | endf = startf + 5 201 | SeqtoBlur[b,startf:endf,i] = GaussKer.squeeze() 202 | startf = startf + stride 203 | #SeqtoBlur[b,1:Tc-1,0:outTc] = torch.eye(Tc-2) 204 | avged_R[b,0:outTc] = torch.tensor([float(t+1)/float(outTc) for t in range(outTc)]) 205 | avged_len[b] = outTc 206 | else: 207 | min_len = int(0.5*Tc) 208 | if min_len<1: 209 | min_len = 1 210 | max_len = int(0.8*Tc) 211 | if max_len<1: 212 | max_len = 1 213 | if max_len>Tc: 214 | max_len = Tc 215 | cur_len = random.choice(range(min_len,max_len+1)) 216 | #print(cur_len) 217 | interval = random.sample(range(1,Tc),cur_len-1) 218 | interval.sort() 219 | interval.append(Tc) 220 | #print(interval) 221 | startf = 0 222 | for i in range(cur_len): 223 | endf = interval[i] 224 | templ = endf-startf 225 | localatt = torch.softmax(torch.tensor([random.gauss(0,1) for _ in range(templ)]).view(templ,1),dim=0).cuda() 226 | #print(localatt) 227 | tempfr = seq[b,startf:endf,:] 228 | avged_seq[b,i,:] = (tempfr*localatt).sum(dim=0) 229 | #print(localatt.shape, startf, endf) 230 | SeqtoBlur[b,startf:endf,i] = localatt.view(-1) 231 | startf = endf 232 | avged_R[b,0:cur_len] = torch.tensor([float(t+1)/float(cur_len) for t in range(cur_len)]) 233 | avged_len[b] = cur_len 234 | else: 235 | if Tc>2: 236 | tempseq = seq[b,0:Tc,:] 237 | tempseq = tempseq.view(1,1,Tc,dim) 238 | GaussKer = torch.tensor([[[[0.1, 0.8, 0.1]]]]) 239 | midnum = random.uniform(0.55,0.9) 240 | sidenum = (1.-midnum)/2. 241 | GaussKer[0,0,0,0] = sidenum 242 | GaussKer[0,0,0,2] = sidenum 243 | GaussKer[0,0,0,1] = midnum 244 | GaussKer = GaussKer.transpose(2,3).cuda() 245 | blured_seq = F.conv2d(tempseq,GaussKer) 246 | avged_seq[b,0:Tc-2,:] = blured_seq.squeeze(1) 247 | SeqtoBlur[b,1:Tc-1,0:Tc-2] = torch.eye(Tc-2) 248 | avged_R[b,0:Tc-2] = torch.tensor([float(t+1)/float(Tc-2) for t in range(Tc-2)]) 249 | avged_len[b] = Tc-2 250 | else: 251 | avged_seq[b,:,:] = seq[b,:,:] 252 | SeqtoBlur[b,0:Tc,0:Tc] = torch.eye(Tc) 253 | avged_R[b,:] = R[b,:] 254 | avged_len[b] = Tc 255 | 256 | 257 | return SeqtoBlur.detach(), avged_seq.detach(), R.detach(), avged_R.detach(), avged_len.detach() 258 | 259 | 260 | class BlurContrastiveModelPair(nn.Module): 261 | def __init__(self, input_dim, output_dim=-1, lam1=1., lam2=1.): 262 | super(BlurContrastiveModelPair, self).__init__() 263 | 264 | self.input_dim = input_dim 265 | if output_dim>0: 266 | self.output_dim = output_dim 267 | else: 268 | self.output_dim = input_dim 269 | self.middle_dim = 1024 270 | self.lsoftmax = nn.LogSoftmax(dim=1) 271 | 272 | self.alignment = OPA(self.output_dim) 273 | self.aug = BlurGenerationPair() 274 | self.encoder = SequenceInception(self.input_dim, self.middle_dim, self.output_dim) 275 | self.predictor = PredictionInception(self.input_dim, self.middle_dim, self.output_dim) 276 | #self.mse = nn.MSELoss(reduction='mean') 277 | self.lam1 = lam1 278 | self.lam2 = lam2 279 | self.mse = nn.MSELoss(reduction='mean') 280 | 281 | self.alpha = 40 282 | self.beta = 2 283 | self.hard_mining = True #None 284 | 285 | def forward(self, seq, len_seq, targets, train_flag = True): 286 | margin=0.5 287 | batch_size = seq.size(0) 288 | SeqtoBlur, avged_seq, R, avged_R, avged_len = self.aug(seq, len_seq) 289 | 290 | DisMat = torch.zeros([batch_size,batch_size],dtype=torch.float).cuda() 291 | Lalign = 0 292 | Lalign2 = 0 293 | Ldis = 0 294 | Ldis2 = 0 295 | 296 | 297 | for b in range(batch_size): 298 | seq0 = seq[b,0:len_seq[b],:].unsqueeze(0) 299 | R0 = R[b,0:len_seq[b]].unsqueeze(0) 300 | avged_seq0 = avged_seq[b,0:avged_len[b],:].unsqueeze(0) 301 | avged_R0 = avged_R[b,0:avged_len[b]].unsqueeze(0) 302 | seq1 = self.encoder(seq0) 303 | avged_seq1 = self.encoder(avged_seq0) 304 | Aa1b, disa1b = self.alignment(seq1, avged_seq1, R0, avged_R0) 305 | Aa2b, disa2b = self.alignment(avged_seq1, seq1, avged_R0, R0) 306 | 307 | disa1b = disa1b.view(-1) 308 | disa2b = disa2b.view(-1) 309 | DisMat[b,b] = DisMat[b,b] + (disa1b + disa2b)/2.0 310 | Ldis = Ldis + disa1b + disa2b 311 | Ldis2 = Ldis2 + self.mse(disa1b,disa2b) 312 | Lalign2 = Lalign2 + self.mse(Aa1b,Aa2b.transpose(1,2)) 313 | tempAlign = SeqtoBlur[b,0:len_seq[b],0:avged_len[b]].unsqueeze(0) 314 | #print(tempAlign.shape) 315 | Lalign = Lalign + self.mse(Aa1b,tempAlign) + self.mse(Aa2b,tempAlign.transpose(1,2)) 316 | 317 | for b2 in range(b+1,batch_size): 318 | shift_seq0 = seq[b2,0:len_seq[b2],:].unsqueeze(0) 319 | shift_R0 = R[b2,0:len_seq[b2]].unsqueeze(0) 320 | 321 | shift_seq1 = self.encoder(shift_seq0) 322 | As1b, dis1b = self.alignment(seq1, shift_seq1, R0, shift_R0) 323 | As2b, dis2b = self.alignment(shift_seq1, seq1, shift_R0, R0) 324 | 325 | dis1b = dis1b.view(-1) 326 | dis2b = dis2b.view(-1) 327 | DisMat[b,b2] = DisMat[b,b2] + dis1b 328 | DisMat[b2,b] = DisMat[b2,b] + dis2b 329 | Ldis2 = Ldis2 + self.mse(dis1b,dis2b) 330 | Lalign2 = Lalign2 + self.mse(As1b,As2b.transpose(1,2)) 331 | 332 | loss = list() 333 | c = 0 334 | for i in range(batch_size): 335 | pos_pair_ = torch.masked_select(DisMat[i], targets==targets[i]) 336 | neg_pair_ = torch.masked_select(DisMat[i], targets!=targets[i]) 337 | 338 | pos_pair_ = torch.sort(pos_pair_)[0] 339 | neg_pair_ = torch.sort(neg_pair_)[0] 340 | 341 | if self.hard_mining is not None: 342 | 343 | neg_pair = torch.masked_select(neg_pair_, neg_pair_ - 0.1 < pos_pair_[-1]) 344 | pos_pair = torch.masked_select(pos_pair_, pos_pair_ + 0.1 > neg_pair_[0]) 345 | 346 | if len(neg_pair) < 1 or len(pos_pair) < 1: 347 | c += 1 348 | continue 349 | 350 | pos_loss = 2.0/self.beta * torch.mean(torch.log(1 + torch.exp(self.beta*(pos_pair - 0.5)))) 351 | neg_loss = 2.0/self.alpha * torch.mean(torch.log(1 + torch.exp(-self.alpha*(neg_pair - 0.5)))) 352 | 353 | else: 354 | pos_pair = pos_pair_ 355 | neg_pair = neg_pair_ 356 | 357 | pos_loss = torch.mean(torch.log(1 + torch.exp(2*(pos_pair - 0.5)))) 358 | neg_loss = torch.mean(torch.log(1 + torch.exp(-self.alpha*(neg_pair - 0.5)))) 359 | 360 | if len(neg_pair) < 1: 361 | c += 1 362 | continue 363 | 364 | loss.append(pos_loss + neg_loss) 365 | 366 | Ldis /= batch_size 367 | Lalign /= batch_size 368 | Ldis2 /= batch_size 369 | Lalign2 /= batch_size 370 | L = sum(loss) + Ldis/batch_size + Lalign/batch_size 371 | return L 372 | 373 | def getlen(self, seq, len_seq): 374 | batch_size = seq.size(0) 375 | T = seq.size(1) 376 | dim = seq.size(2) 377 | R = torch.zeros(batch_size,T).cuda() 378 | for b in range(batch_size): 379 | Tc = len_seq[b].item() 380 | R[b,0:Tc] = torch.tensor([float(t+1)/float(Tc) for t in range(Tc)]) 381 | return R 382 | 383 | 384 | def KNN(X, k): 385 | X = X.float() 386 | mat_square = torch.mm(mat, mat.t()) 387 | diag = torch.diagonal(mat_square) 388 | diag = diag.expand_as(mat_square) 389 | dist_mat = diag + diag.t() - 2*mat_square 390 | dist_col = dist_mat[-1, :-1] 391 | val, index = dist_col.topk(k, largest=False, sorted=True) 392 | return val, index 393 | 394 | if __name__ == "__main__": 395 | seq = torch.tensor([[[1.,2.,3.,4.,5.,6],[7.,8.,9.,10.,11.,12.],[13.,14.,15.,16.,17.,18.]],[[11.,12.,13.,41.,52.,0.],[72.,81.,93.,8.,1.,0.],[3.,4.,6.,8.,12.,0.]]]) 396 | seq = seq.transpose(1,2).contiguous() 397 | #print(seq) 398 | aug = BlurGeneration() 399 | len_seq = torch.tensor([[6],[5]]) 400 | blured_seq, SeqtoBlur, avged_seq, R, blured_R, avged_R, blured_len, avged_len = aug(seq,len_seq) 401 | print(blured_len) 402 | print(avged_len) 403 | 404 | bcm = BlurContrastiveModel(3, output_dim=3) 405 | As1, dis1, As2, dis2, Ab1, disb1, Ab2, disb2, Aa1, disa1, Aa2, disa2 = bcm(seq, len_seq) 406 | 407 | bcl = BlurContrastiveLoss() 408 | l = bcl(As1, dis1, As2, dis2, Ab1, disb1, Ab2, disb2, Aa1, disa1, Aa2, disa2) 409 | print(l) 410 | -------------------------------------------------------------------------------- /Supervised_Learning/OPA_supervised_lifted.py: -------------------------------------------------------------------------------- 1 | import math 2 | import random 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | import numpy as np 7 | from torch.autograd import Variable 8 | 9 | 10 | def initialize_weight(x): 11 | nn.init.xavier_uniform_(x.weight) 12 | if x.bias is not None: 13 | nn.init.constant_(x.bias, 0) 14 | 15 | def tensor_shift(t, dim, shift): 16 | """ 17 | t (tensor): tensor to be shifted. 18 | dim (int): the dimension apply shift. 19 | shift (int): shift distance. 20 | """ 21 | assert 0 <= shift <= t.size(dim), "shift distance should be smaller than or equal to the dim length." 22 | 23 | overflow = t.index_select(dim, torch.arange(t.size(dim)-shift, t.size(dim)).cuda()) 24 | remain = t.index_select(dim, torch.arange(t.size(dim)-shift).cuda()) 25 | 26 | return torch.cat((overflow, remain),dim=dim) 27 | 28 | class SequenceInception(nn.Module): 29 | def __init__(self, in_dim,middle_dim,out_dim, normalized=1): 30 | super(SequenceInception, self).__init__() 31 | self.in_dim = in_dim 32 | self.middle_dim = middle_dim 33 | self.out_dim = out_dim 34 | self.normalized = normalized 35 | self.layer1 = nn.Linear(in_dim, middle_dim) 36 | self.relu = nn.ReLU() #nn.Tanh() 37 | self.layer3 = nn.Linear(middle_dim, out_dim) 38 | self.layer2 = nn.Linear(middle_dim, middle_dim) 39 | 40 | def forward(self, input): 41 | batch_size = input.size(0) 42 | T = input.size(1) 43 | dim = input.size(2) 44 | input = input.view(batch_size*T,dim) 45 | out1 = self.layer1(input) 46 | out1 = self.relu(out1) 47 | out1 = self.layer2(out1) 48 | out1 = self.relu(out1) 49 | output = self.layer3(out1) 50 | 51 | if self.normalized == 1: 52 | norm = output.norm(dim=1, p=2, keepdim=True) 53 | #if norm>0: 54 | output = output.div(norm.expand_as(output)) 55 | output = torch.where(torch.isnan(output), torch.full_like(output, 0), output) 56 | output = torch.where(torch.isinf(output), torch.full_like(output, 0), output) 57 | 58 | output = output.view(batch_size,T,self.out_dim) 59 | return output 60 | 61 | class PredictionInception(nn.Module): 62 | def __init__(self, in_dim,middle_dim,out_dim, normalized=1): 63 | super(PredictionInception, self).__init__() 64 | self.in_dim = in_dim 65 | self.middle_dim = middle_dim 66 | self.out_dim = out_dim 67 | self.normalized = normalized 68 | #inplace = True 69 | 70 | self.layer1 = nn.Linear(in_dim, middle_dim) 71 | self.relu = nn.ReLU() 72 | self.layer2 = nn.Linear(middle_dim, out_dim) 73 | initialize_weight(self.layer1) 74 | initialize_weight(self.layer2) 75 | 76 | def forward(self, input): 77 | batch_size = input.size(0) 78 | T = input.size(1) 79 | dim = input.size(2) 80 | input = input.view(-1,dim) 81 | out1 = self.layer1(input) 82 | out1 = self.relu(out1) 83 | output = self.layer2(out1) 84 | 85 | if self.normalized == 1: 86 | norm = output.norm(dim=1, p=2, keepdim=True) 87 | #if norm>0: 88 | output = output.div(norm.expand_as(output)) 89 | output = torch.where(torch.isnan(output), torch.full_like(output, 0), output) 90 | output = torch.where(torch.isinf(output), torch.full_like(output, 0), output) 91 | 92 | output = output.view(batch_size,T,self.out_dim) 93 | return output 94 | 95 | 96 | 97 | class OPA(nn.Module): 98 | def __init__(self, input_dim): 99 | super(OPA, self).__init__() 100 | #self.lam = lam 101 | self.input_dim = input_dim 102 | self.att_size = att_size = 30 103 | self.att_size2 = att_size2 = 100 104 | #self.att_size3 = att_size3 = 30 105 | self.scale = att_size ** -0.5 106 | self.scale2 = att_size2 ** -0.5 107 | 108 | self.conv1 = nn.Conv2d(2, 30, 5, padding=2) 109 | self.conv2 = nn.Conv2d(30, 30, 5, padding=2) 110 | self.conv3 = nn.Conv2d(30, 1, 3, padding=1) 111 | 112 | def forward(self, seq_q, seq_k, len_q, len_k): 113 | 114 | len_q = len_q.detach() 115 | len_k = len_k.detach() 116 | T_q = seq_q.size(1) 117 | T_k = seq_k.size(1) 118 | 119 | batch_size = seq_q.size(0) 120 | assert batch_size==1 121 | 122 | D = torch.cdist(seq_q.view(batch_size, T_q, self.input_dim), seq_k.view(batch_size, T_k, self.input_dim), 2) 123 | 124 | P = torch.cdist(len_q.view(batch_size,T_q,1), len_k.view(batch_size,T_k,1), 2) 125 | 126 | A = torch.cat([D, P], 0).unsqueeze(0) 127 | A = F.relu(self.conv1(A)) 128 | A = F.relu(self.conv2(A)) 129 | A = self.conv3(A).squeeze(1) + D 130 | A = torch.softmax(-A, dim=2) 131 | 132 | At = A.view(batch_size,T_q*T_k,1) 133 | norm = At.norm(dim=1, p=1, keepdim=True) 134 | At = At.div(norm.expand_as(At)) 135 | 136 | dis = torch.matmul(D.view(batch_size,1,T_q*T_k),At) 137 | return A, dis 138 | 139 | 140 | 141 | class BlurGenerationPair(nn.Module): 142 | def __init__(self): 143 | super(BlurGenerationPair, self).__init__() 144 | 145 | def forward(self, seq, len_seq): 146 | batch_size = seq.size(0) 147 | T = seq.size(1) 148 | dim = seq.size(2) 149 | 150 | SeqtoBlur = torch.zeros( size=(batch_size,T,T),dtype=torch.float32 ).cuda() 151 | avged_seq = torch.zeros_like(seq).cuda() 152 | avged_len = torch.zeros_like(len_seq).cuda() 153 | R = torch.zeros(batch_size,T).cuda() 154 | avged_R = torch.zeros(batch_size,T).cuda() 155 | 156 | for b in range(batch_size): 157 | Tc = len_seq[b].item() 158 | R[b,0:Tc] = torch.tensor([float(t+1)/float(Tc) for t in range(Tc)]) 159 | 160 | if Tc>4: 161 | randseed = random.randint(1,10) 162 | if randseed>5: 163 | tempseq = seq[b,0:Tc,:] 164 | tempseq = tempseq.view(1,1,Tc,dim) 165 | if randseed>7: 166 | GaussKer = torch.tensor([[[[0.1, 0.8, 0.1]]]]) 167 | midnum = random.uniform(0.55,0.9) 168 | sidenum = (1.-midnum)/2. 169 | GaussKer[0,0,0,0] = sidenum 170 | GaussKer[0,0,0,2] = sidenum 171 | GaussKer[0,0,0,1] = midnum 172 | GaussKer = GaussKer.transpose(2,3).cuda() 173 | stride = random.randint(1,3) 174 | outTc = int((Tc-3)/stride) + 1 175 | blured_seq = F.conv2d(tempseq,GaussKer,stride=[stride,1]) 176 | avged_seq[b,0:outTc,:] = blured_seq.squeeze(1) 177 | startf = 0 178 | for i in range(outTc): 179 | endf = startf + 3 180 | SeqtoBlur[b,startf:endf,i] = GaussKer.squeeze() 181 | startf = startf + stride 182 | #SeqtoBlur[b,1:Tc-1,0:outTc] = torch.eye(Tc-2) 183 | avged_R[b,0:outTc] = torch.tensor([float(t+1)/float(outTc) for t in range(outTc)]) 184 | avged_len[b] = outTc 185 | 186 | else: 187 | GaussKer = torch.tensor([[[[0.1, 0.15, 0.5, 0.15, 0.1]]]]) 188 | midnum = random.uniform(0.33,0.5) 189 | sidenum = (1.-midnum)/2. 190 | sidenum1 = random.uniform(0.16,sidenum) 191 | sidenum2 = sidenum - sidenum1 192 | GaussKer[0,0,0,0] = sidenum2 193 | GaussKer[0,0,0,1] = sidenum1 194 | GaussKer[0,0,0,2] = midnum 195 | GaussKer[0,0,0,3] = sidenum1 196 | GaussKer[0,0,0,4] = sidenum2 197 | GaussKer = GaussKer.transpose(2,3).cuda() 198 | stride = random.randint(1,3) 199 | outTc = int((Tc-5)/stride) + 1 200 | blured_seq = F.conv2d(tempseq,GaussKer,stride=[stride,1]) 201 | startf = 0 202 | for i in range(outTc): 203 | endf = startf + 5 204 | SeqtoBlur[b,startf:endf,i] = GaussKer.squeeze() 205 | startf = startf + stride 206 | avged_R[b,0:outTc] = torch.tensor([float(t+1)/float(outTc) for t in range(outTc)]) 207 | avged_len[b] = outTc 208 | else: 209 | min_len = int(0.5*Tc) 210 | if min_len<1: 211 | min_len = 1 212 | max_len = int(0.8*Tc) 213 | if max_len<1: 214 | max_len = 1 215 | if max_len>Tc: 216 | max_len = Tc 217 | cur_len = random.choice(range(min_len,max_len+1)) 218 | #print(cur_len) 219 | interval = random.sample(range(1,Tc),cur_len-1) #.sort() 220 | interval.sort() 221 | interval.append(Tc) 222 | #print(interval) 223 | startf = 0 224 | for i in range(cur_len): 225 | endf = interval[i] 226 | templ = endf-startf 227 | localatt = torch.softmax(torch.tensor([random.gauss(0,1) for _ in range(templ)]).view(templ,1),dim=0).cuda() 228 | #print(localatt) 229 | tempfr = seq[b,startf:endf,:] 230 | avged_seq[b,i,:] = (tempfr*localatt).sum(dim=0) 231 | #print(localatt.shape, startf, endf) 232 | SeqtoBlur[b,startf:endf,i] = localatt.view(-1) 233 | startf = endf 234 | 235 | avged_R[b,0:cur_len] = torch.tensor([float(t+1)/float(cur_len) for t in range(cur_len)]) 236 | avged_len[b] = cur_len 237 | else: 238 | if Tc>2: 239 | tempseq = seq[b,0:Tc,:] 240 | tempseq = tempseq.view(1,1,Tc,dim) 241 | GaussKer = torch.tensor([[[[0.1, 0.8, 0.1]]]]) 242 | midnum = random.uniform(0.55,0.9) 243 | sidenum = (1.-midnum)/2. 244 | GaussKer[0,0,0,0] = sidenum 245 | GaussKer[0,0,0,2] = sidenum 246 | GaussKer[0,0,0,1] = midnum 247 | GaussKer = GaussKer.transpose(2,3).cuda() 248 | blured_seq = F.conv2d(tempseq,GaussKer) 249 | avged_seq[b,0:Tc-2,:] = blured_seq.squeeze(1) 250 | SeqtoBlur[b,1:Tc-1,0:Tc-2] = torch.eye(Tc-2) 251 | avged_R[b,0:Tc-2] = torch.tensor([float(t+1)/float(Tc-2) for t in range(Tc-2)]) 252 | avged_len[b] = Tc-2 253 | else: 254 | avged_seq[b,:,:] = seq[b,:,:] 255 | SeqtoBlur[b,0:Tc,0:Tc] = torch.eye(Tc) 256 | avged_R[b,:] = R[b,:] 257 | avged_len[b] = Tc 258 | 259 | 260 | return SeqtoBlur.detach(), avged_seq.detach(), R.detach(), avged_R.detach(), avged_len.detach() 261 | 262 | 263 | class BlurContrastiveModelPair(nn.Module): 264 | def __init__(self, input_dim, output_dim=-1, lam1=1., lam2=1.): 265 | super(BlurContrastiveModelPair, self).__init__() 266 | 267 | self.input_dim = input_dim 268 | if output_dim>0: 269 | self.output_dim = output_dim 270 | else: 271 | self.output_dim = input_dim 272 | self.middle_dim = 1024 273 | self.lsoftmax = nn.LogSoftmax(dim=1) 274 | 275 | self.alignment = OPA(self.output_dim) 276 | self.aug = BlurGenerationPair() 277 | self.encoder = SequenceInception(self.input_dim, self.middle_dim, self.output_dim) 278 | self.predictor = PredictionInception(self.input_dim, self.middle_dim, self.output_dim) 279 | #self.mse = nn.MSELoss(reduction='mean') 280 | self.lam1 = lam1 281 | self.lam2 = lam2 282 | self.mse = nn.MSELoss(reduction='mean') 283 | 284 | self.alpha = 40 285 | self.beta = 2 286 | self.hard_mining = None 287 | 288 | def forward(self, seq, len_seq, targets, train_flag = True): 289 | margin=0.5 290 | batch_size = seq.size(0) 291 | SeqtoBlur, avged_seq, R, avged_R, avged_len = self.aug(seq, len_seq) 292 | #SeqtoBlur2, avged_seq2, R2, avged_R2, avged_len2 = self.aug(seq, len_seq) 293 | 294 | DisMat = torch.zeros([batch_size,batch_size],dtype=torch.float).cuda() 295 | Lalign = 0 296 | Lalign2 = 0 297 | Ldis = 0 298 | Ldis2 = 0 299 | 300 | for b in range(batch_size): 301 | seq0 = seq[b,0:len_seq[b],:].unsqueeze(0) 302 | R0 = R[b,0:len_seq[b]].unsqueeze(0) 303 | avged_seq0 = avged_seq[b,0:avged_len[b],:].unsqueeze(0) 304 | avged_R0 = avged_R[b,0:avged_len[b]].unsqueeze(0) 305 | 306 | seq1 = self.encoder(seq0) 307 | avged_seq1 = self.encoder(avged_seq0) 308 | Aa1b, disa1b = self.alignment(seq1, avged_seq1, R0, avged_R0) 309 | Aa2b, disa2b = self.alignment(avged_seq1, seq1, avged_R0, R0) 310 | 311 | disa1b = disa1b.view(-1) 312 | disa2b = disa2b.view(-1) 313 | DisMat[b,b] = DisMat[b,b] + (disa1b + disa2b)/2.0 314 | Ldis = Ldis + disa1b + disa2b 315 | Ldis2 = Ldis2 + self.mse(disa1b,disa2b) 316 | Lalign2 = Lalign2 + self.mse(Aa1b,Aa2b.transpose(1,2)) 317 | tempAlign = SeqtoBlur[b,0:len_seq[b],0:avged_len[b]].unsqueeze(0) 318 | #print(tempAlign.shape) 319 | Lalign = Lalign + self.mse(Aa1b,tempAlign) + self.mse(Aa2b,tempAlign.transpose(1,2)) 320 | 321 | for b2 in range(b+1,batch_size): 322 | shift_seq0 = seq[b2,0:len_seq[b2],:].unsqueeze(0) 323 | shift_R0 = R[b2,0:len_seq[b2]].unsqueeze(0) 324 | 325 | shift_seq1 = self.encoder(shift_seq0) 326 | As1b, dis1b = self.alignment(seq1, shift_seq1, R0, shift_R0) 327 | As2b, dis2b = self.alignment(shift_seq1, seq1, shift_R0, R0) 328 | 329 | dis1b = dis1b.view(-1) 330 | dis2b = dis2b.view(-1) 331 | DisMat[b,b2] = DisMat[b,b2] + dis1b 332 | DisMat[b2,b] = DisMat[b2,b] + dis2b 333 | Ldis2 = Ldis2 + self.mse(dis1b,dis2b) 334 | Lalign2 = Lalign2 + self.mse(As1b,As2b.transpose(1,2)) 335 | 336 | loss = list() 337 | c = 0 338 | for i in range(batch_size): 339 | pos_pair_ = torch.masked_select(DisMat[i], targets==targets[i]) 340 | 341 | neg_pair_ = torch.masked_select(DisMat[i], targets!=targets[i]) 342 | 343 | pos_pair_ = torch.sort(pos_pair_)[0] 344 | neg_pair_ = torch.sort(neg_pair_)[0] 345 | 346 | if self.hard_mining is not None: 347 | 348 | neg_pair = torch.masked_select(neg_pair_, neg_pair_ - 0.1 < pos_pair_[-1]) 349 | pos_pair = torch.masked_select(pos_pair_, pos_pair_ + 0.1 > neg_pair_[0]) 350 | 351 | 352 | if len(neg_pair) < 1 or len(pos_pair) < 1: 353 | c += 1 354 | continue 355 | 356 | pos_loss = 2.0/self.beta * torch.log(torch.sum(torch.exp(self.beta*pos_pair))) 357 | neg_loss = 2.0/self.alpha * torch.log(torch.sum(torch.exp(-self.alpha*neg_pair))) 358 | 359 | else: 360 | pos_pair = pos_pair_ 361 | neg_pair = neg_pair_ 362 | 363 | pos_loss = 2.0/self.beta * torch.log(torch.sum(torch.exp(self.beta*pos_pair))) 364 | neg_loss = 2.0/self.alpha * torch.log(torch.sum(torch.exp(-self.alpha*neg_pair))) 365 | 366 | if len(neg_pair) < 1: 367 | c += 1 368 | continue 369 | 370 | loss.append(pos_loss + neg_loss) 371 | 372 | Ldis /= batch_size 373 | Lalign /= batch_size 374 | Ldis2 /= batch_size 375 | Lalign2 /= batch_size 376 | L = sum(loss) + Ldis/batch_size + Lalign/batch_size 377 | return L 378 | 379 | def getlen(self, seq, len_seq): 380 | batch_size = seq.size(0) 381 | T = seq.size(1) 382 | dim = seq.size(2) 383 | R = torch.zeros(batch_size,T).cuda() 384 | for b in range(batch_size): 385 | Tc = len_seq[b].item() 386 | R[b,0:Tc] = torch.tensor([float(t+1)/float(Tc) for t in range(Tc)]) 387 | return R 388 | 389 | 390 | 391 | 392 | 393 | def KNN(X, k): 394 | X = X.float() 395 | mat_square = torch.mm(mat, mat.t()) 396 | diag = torch.diagonal(mat_square) 397 | diag = diag.expand_as(mat_square) 398 | dist_mat = diag + diag.t() - 2*mat_square 399 | dist_col = dist_mat[-1, :-1] 400 | val, index = dist_col.topk(k, largest=False, sorted=True) 401 | return val, index 402 | 403 | if __name__ == "__main__": 404 | seq = torch.tensor([[[1.,2.,3.,4.,5.,6],[7.,8.,9.,10.,11.,12.],[13.,14.,15.,16.,17.,18.]],[[11.,12.,13.,41.,52.,0.],[72.,81.,93.,8.,1.,0.],[3.,4.,6.,8.,12.,0.]]]) 405 | seq = seq.transpose(1,2).contiguous() 406 | #print(seq) 407 | aug = BlurGeneration() 408 | len_seq = torch.tensor([[6],[5]]) 409 | blured_seq, SeqtoBlur, avged_seq, R, blured_R, avged_R, blured_len, avged_len = aug(seq,len_seq) 410 | print(blured_len) 411 | print(avged_len) 412 | 413 | bcm = BlurContrastiveModel(3, output_dim=3) 414 | As1, dis1, As2, dis2, Ab1, disb1, Ab2, disb2, Aa1, disa1, Aa2, disa2 = bcm(seq, len_seq) 415 | 416 | bcl = BlurContrastiveLoss() 417 | l = bcl(As1, dis1, As2, dis2, Ab1, disb1, Ab2, disb2, Aa1, disa1, Aa2, disa2) 418 | print(l) 419 | -------------------------------------------------------------------------------- /Few_Shot_Learning/scripts/pretrain_backbone.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script to pretrain the backbone on the training set, as used in PAL. Adapted from example imagnet distributed training code. 3 | """ 4 | 5 | 6 | import argparse 7 | import os 8 | import random 9 | import shutil 10 | import time 11 | import warnings 12 | 13 | import torch 14 | import torch.nn as nn 15 | import torch.nn.parallel 16 | import torch.backends.cudnn as cudnn 17 | import torch.distributed as dist 18 | import torch.optim 19 | import torch.multiprocessing as mp 20 | import torch.utils.data 21 | import torch.utils.data.distributed 22 | import torchvision.transforms as transforms 23 | import torchvision.datasets as datasets 24 | import torchvision.models as models 25 | 26 | import video_reader 27 | 28 | model_names = sorted(name for name in models.__dict__ 29 | if name.islower() and not name.startswith("__") 30 | and callable(models.__dict__[name])) 31 | 32 | parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') 33 | # parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', 34 | # choices=model_names, 35 | # help='model architecture: ' + 36 | # ' | '.join(model_names) + 37 | # ' (default: resnet18)') 38 | parser.add_argument('-j', '--workers', default=4, type=int, metavar='N', 39 | help='number of data loading workers (default: 4)') 40 | parser.add_argument('--epochs', default=71, type=int, metavar='N', 41 | help='number of total epochs to run') 42 | parser.add_argument('--start-epoch', default=0, type=int, metavar='N', 43 | help='manual epoch number (useful on restarts)') 44 | parser.add_argument('-b', '--batch-size', default=40, type=int, 45 | metavar='N', 46 | help='mini-batch size (default: 256), this is the total ' 47 | 'batch size of all GPUs on the current node when ' 48 | 'using Data Parallel or Distributed Data Parallel') 49 | parser.add_argument('--lr', '--learning-rate', default=0.001, type=float, 50 | metavar='LR', help='initial learning rate', dest='lr') 51 | parser.add_argument('--momentum', default=0.9, type=float, metavar='M', 52 | help='momentum') 53 | parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, 54 | metavar='W', help='weight decay (default: 1e-4)', 55 | dest='weight_decay') 56 | parser.add_argument('-p', '--print-freq', default=160, type=int, 57 | metavar='N', help='print frequency (default: 10)') 58 | parser.add_argument('--resume', default='', type=str, metavar='PATH', 59 | help='path to latest checkpoint (default: none)') 60 | parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', 61 | help='evaluate model on validation set') 62 | parser.add_argument('--pretrained', dest='pretrained', action='store_true', 63 | help='use pre-trained model') 64 | parser.add_argument('--world-size', default=-1, type=int, 65 | help='number of nodes for distributed training') 66 | parser.add_argument('--rank', default=-1, type=int, 67 | help='node rank for distributed training') 68 | parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, 69 | help='url used to set up distributed training') 70 | parser.add_argument('--dist-backend', default='nccl', type=str, 71 | help='distributed backend') 72 | parser.add_argument('--seed', default=None, type=int, 73 | help='seed for initializing training. ') 74 | parser.add_argument('--gpu', default=None, type=int, 75 | help='GPU id to use.') 76 | parser.add_argument('--multiprocessing-distributed', action='store_true', 77 | help='Use multi-processing distributed training to launch ' 78 | 'N processes per node, which has N GPUs. This is the ' 79 | 'fastest way to use PyTorch for either single node or ' 80 | 'multi node data parallel training') 81 | 82 | parser.add_argument('--seq_len', default=8, type=int) 83 | # parser.add_argument("--split", type=int, default=3, help="Dataset split.") 84 | # parser.add_argument("--scratch", choices=["bc", "bp"], default="bp", help="Computer to run on") 85 | parser.add_argument("--img_size", type=int, default=224, help="Frames per video.") 86 | # parser.add_argument("--way", type=int, default=5, help="Frames per video.") 87 | # parser.add_argument("--shot", type=int, default=5, help="Frames per video.") 88 | # parser.add_argument("--query_per_class", type=int, default=5, help="Frames per video.") 89 | 90 | 91 | parser.add_argument("--dataset", type=str, default="data/ssv2small", help="Path to dataset") 92 | # parser.add_argument("--n_classes", type=int, default=64, help="Frames per video.") 93 | parser.add_argument("out_name", type=str, help="Frames per video.") 94 | 95 | 96 | 97 | 98 | 99 | best_acc1 = 0 100 | 101 | class TSN(nn.Module): 102 | def __init__(self): 103 | super(TSN, self).__init__() 104 | last_layer_idx = -1 105 | self.backbone = models.resnet50(pretrained=True) 106 | #self.resnet = nn.Sequential(*list(self.resnet.children())[:last_layer_idx]) 107 | #self.linear = nn.Linear(2048, 200) 108 | 109 | #self.n_feats = 1000 110 | #self.n_classes = 200 111 | #self.l_norm = torch.nn.LayerNorm(self.n_feats) 112 | #self.classifier = nn.Parameter(torch.zeros(self.n_classes, self.n_feats)) 113 | #self.cos_sim = torch.nn.CosineSimilarity(dim=1) 114 | 115 | 116 | def forward(self, x): 117 | b, s, c, h, w = x.shape 118 | x = x.reshape(b*s, c, h, w) 119 | x = self.backbone(x) 120 | 121 | # use custom linear layer 122 | x = torch.squeeze(x) 123 | #x = self.linear(x) 124 | 125 | ## use cos sim 126 | #x = self.l_norm(x) 127 | #classes = self.l_norm(self.classifier) 128 | #x = self.l_norm(x) 129 | #cos_dist = torch.matmul(x, classes.transpose(-2,-1)) 130 | 131 | 132 | # average after linear layer 133 | x = x.reshape(b,s,-1) 134 | x = torch.mean(x, dim=1) 135 | 136 | return x 137 | 138 | def main(): 139 | args = parser.parse_args() 140 | 141 | if args.seed is not None: 142 | random.seed(args.seed) 143 | torch.manual_seed(args.seed) 144 | cudnn.deterministic = True 145 | warnings.warn('You have chosen to seed training. ' 146 | 'This will turn on the CUDNN deterministic setting, ' 147 | 'which can slow down your training considerably! ' 148 | 'You may see unexpected behavior when restarting ' 149 | 'from checkpoints.') 150 | 151 | if args.gpu is not None: 152 | warnings.warn('You have chosen a specific GPU. This will completely ' 153 | 'disable data parallelism.') 154 | 155 | if args.dist_url == "env://" and args.world_size == -1: 156 | args.world_size = int(os.environ["WORLD_SIZE"]) 157 | 158 | args.distributed = args.world_size > 1 or args.multiprocessing_distributed 159 | 160 | ngpus_per_node = torch.cuda.device_count() 161 | if args.multiprocessing_distributed: 162 | # Since we have ngpus_per_node processes per node, the total world_size 163 | # needs to be adjusted accordingly 164 | args.world_size = ngpus_per_node * args.world_size 165 | # Use torch.multiprocessing.spawn to launch distributed processes: the 166 | # main_worker process function 167 | mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) 168 | else: 169 | # Simply call main_worker function 170 | main_worker(args.gpu, ngpus_per_node, args) 171 | 172 | 173 | def main_worker(gpu, ngpus_per_node, args): 174 | global best_acc1 175 | args.gpu = gpu 176 | 177 | if args.gpu is not None: 178 | print("Use GPU: {} for training".format(args.gpu)) 179 | 180 | if args.distributed: 181 | if args.dist_url == "env://" and args.rank == -1: 182 | args.rank = int(os.environ["RANK"]) 183 | if args.multiprocessing_distributed: 184 | # For multiprocessing distributed training, rank needs to be the 185 | # global rank among all the processes 186 | args.rank = args.rank * ngpus_per_node + gpu 187 | dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, 188 | world_size=args.world_size, rank=args.rank) 189 | # create model 190 | #if args.pretrained: 191 | # print("=> using pre-trained model '{}'".format(args.arch)) 192 | # model = models.__dict__[args.arch](pretrained=True) 193 | #else: 194 | # print("=> creating model '{}'".format(args.arch)) 195 | # model = models.__dict__[args.arch]() 196 | #model = resnet18(pretrained=True, pretrained_model_path="../models/pretrained_resnet.pt.tar") 197 | #model = resnet18(pretrained=False) 198 | model = TSN() 199 | 200 | 201 | 202 | if not torch.cuda.is_available(): 203 | print('using CPU, this will be slow') 204 | elif args.distributed: 205 | # For multiprocessing distributed, DistributedDataParallel constructor 206 | # should always set the single device scope, otherwise, 207 | # DistributedDataParallel will use all available devices. 208 | if args.gpu is not None: 209 | torch.cuda.set_device(args.gpu) 210 | model.cuda(args.gpu) 211 | # When using a single GPU per process and per 212 | # DistributedDataParallel, we need to divide the batch size 213 | # ourselves based on the total number of GPUs we have 214 | args.batch_size = int(args.batch_size / ngpus_per_node) 215 | args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) 216 | model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) 217 | else: 218 | model.cuda() 219 | # DistributedDataParallel will divide and allocate batch_size to all 220 | # available GPUs if device_ids are not set 221 | model = torch.nn.parallel.DistributedDataParallel(model) 222 | elif args.gpu is not None: 223 | torch.cuda.set_device(args.gpu) 224 | model = model.cuda(args.gpu) 225 | else: 226 | # DataParallel will divide and allocate batch_size to all available GPUs 227 | # if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): 228 | # model.features = torch.nn.DataParallel(model.features) 229 | # model.cuda() 230 | # else: 231 | model = torch.nn.DataParallel(model).cuda() 232 | 233 | # define loss function (criterion) and optimizer 234 | criterion = nn.CrossEntropyLoss().cuda(args.gpu) 235 | 236 | optimizer = torch.optim.SGD(model.parameters(), args.lr, 237 | momentum=args.momentum, 238 | weight_decay=args.weight_decay) 239 | 240 | # optionally resume from a checkpoint 241 | if args.resume: 242 | if os.path.isfile(args.resume): 243 | print("=> loading checkpoint '{}'".format(args.resume)) 244 | if args.gpu is None: 245 | checkpoint = torch.load(args.resume) 246 | else: 247 | # Map model to be loaded to specified single gpu. 248 | loc = 'cuda:{}'.format(args.gpu) 249 | checkpoint = torch.load(args.resume, map_location=loc) 250 | args.start_epoch = checkpoint['epoch'] 251 | best_acc1 = checkpoint['best_acc1'] 252 | #if args.gpu is not None: 253 | # best_acc1 may be from a checkpoint from a different GPU 254 | # best_acc1 = best_acc1.to(args.gpu) 255 | model.load_state_dict(checkpoint['state_dict']) 256 | optimizer.load_state_dict(checkpoint['optimizer']) 257 | print("=> loaded checkpoint '{}' (epoch {})" 258 | .format(args.resume, checkpoint['epoch'])) 259 | else: 260 | print("=> no checkpoint found at '{}'".format(args.resume)) 261 | 262 | #cudnn.benchmark = True 263 | 264 | # # Data loading code 265 | # traindir = os.path.join(args.data, 'train') 266 | # valdir = os.path.join(args.data, 'val') 267 | # normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 268 | # std=[0.229, 0.224, 0.225]) 269 | # 270 | # train_dataset = datasets.ImageFolder( 271 | # traindir, 272 | # transforms.Compose([ 273 | # transforms.RandomResizedCrop(224), 274 | # transforms.RandomHorizontalFlip(), 275 | # transforms.ToTensor(), 276 | # normalize, 277 | # ])) 278 | 279 | #train_dataset = ucf_reader.VideoDataset("/mnt/storage/home/tp8961/scratch/video_datasets/data/somethingsomethingv2_128x96q5_class.zip", "/mnt/storage/home/tp8961/scratch/video_datasets/splits/somethingsomethingv2TrainTestlist", 0, train=True, transform=ucf_reader.img_transform, seq_len=1, single_img=True) 280 | 281 | train_dataset = video_reader.VideoDataset(args, meta_batches=False) 282 | if args.distributed: 283 | train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) 284 | else: 285 | train_sampler = None 286 | 287 | train_loader = torch.utils.data.DataLoader( 288 | train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), 289 | num_workers=args.workers, pin_memory=True, sampler=train_sampler) 290 | 291 | # val_loader = torch.utils.data.DataLoader( 292 | # datasets.ImageFolder(valdir, transforms.Compose([ 293 | # transforms.Resize(256), 294 | # transforms.CenterCrop(224), 295 | # transforms.ToTensor(), 296 | # normalize, 297 | # ])), 298 | # batch_size=args.batch_size, shuffle=False, 299 | # num_workers=args.workers, pin_memory=True) 300 | 301 | if args.evaluate: 302 | validate(val_loader, model, criterion, args) 303 | return 304 | 305 | for epoch in range(args.start_epoch, args.epochs): 306 | if args.distributed: 307 | train_sampler.set_epoch(epoch) 308 | adjust_learning_rate(optimizer, epoch, args) 309 | 310 | # train for one epoch 311 | train(train_loader, model, criterion, optimizer, epoch, args) 312 | 313 | # # evaluate on validation set 314 | # acc1 = validate(val_loader, model, criterion, args) 315 | # 316 | # # remember best acc@1 and save checkpoint 317 | # is_best = acc1 > best_acc1 318 | # best_acc1 = max(acc1, best_acc1) 319 | is_best=True 320 | 321 | 322 | if not args.multiprocessing_distributed or (args.multiprocessing_distributed 323 | and args.rank % ngpus_per_node == 0): 324 | torch.save(model.module.backbone.state_dict(), os.path.join(args.out_name, "epoch{}.pth.tar".format(epoch+1))) 325 | 326 | #save_checkpoint({ 327 | # 'epoch': epoch + 1, 328 | # 'arch': args.arch, 329 | # 'state_dict': model.state_dict(), 330 | # 'best_acc1': best_acc1, 331 | # 'optimizer' : optimizer.state_dict(), 332 | #}, is_best, filename=os.path.join(args.scratch, "pretrained_models/{}_epoch{}.pth.tar".format(args.out_name,epoch+1))) 333 | 334 | 335 | def train(train_loader, model, criterion, optimizer, epoch, args): 336 | batch_time = AverageMeter('Time', ':6.3f') 337 | data_time = AverageMeter('Data', ':6.3f') 338 | losses = AverageMeter('Loss', ':.4e') 339 | top1 = AverageMeter('Acc@1', ':6.2f') 340 | top5 = AverageMeter('Acc@5', ':6.2f') 341 | progress = ProgressMeter( 342 | len(train_loader), 343 | [batch_time, data_time, losses, top1, top5], 344 | prefix="Epoch: [{}]".format(epoch)) 345 | 346 | # switch to train mode 347 | model.train() 348 | 349 | end = time.time() 350 | for i, (images, target) in enumerate(train_loader): 351 | # measure data loading time 352 | data_time.update(time.time() - end) 353 | 354 | if args.gpu is not None: 355 | images = images.cuda(args.gpu, non_blocking=True) 356 | if torch.cuda.is_available(): 357 | target = target.cuda(args.gpu, non_blocking=True) 358 | 359 | # compute output 360 | output = model(images) 361 | 362 | loss = criterion(output, target) 363 | 364 | # measure accuracy and record loss 365 | acc1, acc5 = accuracy(output, target, topk=(1, 5)) 366 | losses.update(loss.item(), images.size(0)) 367 | top1.update(acc1[0], images.size(0)) 368 | top5.update(acc5[0], images.size(0)) 369 | 370 | # compute gradient and do SGD step 371 | optimizer.zero_grad() 372 | loss.backward() 373 | optimizer.step() 374 | 375 | # measure elapsed time 376 | batch_time.update(time.time() - end) 377 | end = time.time() 378 | 379 | if i % args.print_freq == 0: 380 | progress.display(i) 381 | 382 | 383 | def validate(val_loader, model, criterion, args): 384 | batch_time = AverageMeter('Time', ':6.3f') 385 | losses = AverageMeter('Loss', ':.4e') 386 | top1 = AverageMeter('Acc@1', ':6.2f') 387 | top5 = AverageMeter('Acc@5', ':6.2f') 388 | progress = ProgressMeter( 389 | len(val_loader), 390 | [batch_time, losses, top1, top5], 391 | prefix='Test: ') 392 | 393 | # switch to evaluate mode 394 | model.eval() 395 | 396 | with torch.no_grad(): 397 | end = time.time() 398 | for i, (images, target) in enumerate(val_loader): 399 | if args.gpu is not None: 400 | images = images.cuda(args.gpu, non_blocking=True) 401 | if torch.cuda.is_available(): 402 | target = target.cuda(args.gpu, non_blocking=True) 403 | 404 | # compute output 405 | output = model(images) 406 | loss = criterion(output, target) 407 | 408 | # measure accuracy and record loss 409 | acc1, acc5 = accuracy(output, target, topk=(1, 5)) 410 | losses.update(loss.item(), images.size(0)) 411 | top1.update(acc1[0], images.size(0)) 412 | top5.update(acc5[0], images.size(0)) 413 | 414 | # measure elapsed time 415 | batch_time.update(time.time() - end) 416 | end = time.time() 417 | 418 | if i % args.print_freq == 0: 419 | progress.display(i) 420 | 421 | # TODO: this should also be done with the ProgressMeter 422 | print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' 423 | .format(top1=top1, top5=top5)) 424 | 425 | return top1.avg 426 | 427 | 428 | def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): 429 | torch.save(state, filename) 430 | if is_best: 431 | shutil.copyfile(filename, 'model_best.pth.tar') 432 | 433 | 434 | class AverageMeter(object): 435 | """Computes and stores the average and current value""" 436 | def __init__(self, name, fmt=':f'): 437 | self.name = name 438 | self.fmt = fmt 439 | self.reset() 440 | 441 | def reset(self): 442 | self.val = 0 443 | self.avg = 0 444 | self.sum = 0 445 | self.count = 0 446 | 447 | def update(self, val, n=1): 448 | self.val = val 449 | self.sum += val * n 450 | self.count += n 451 | self.avg = self.sum / self.count 452 | 453 | def __str__(self): 454 | fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' 455 | return fmtstr.format(**self.__dict__) 456 | 457 | 458 | class ProgressMeter(object): 459 | def __init__(self, num_batches, meters, prefix=""): 460 | self.batch_fmtstr = self._get_batch_fmtstr(num_batches) 461 | self.meters = meters 462 | self.prefix = prefix 463 | 464 | def display(self, batch): 465 | entries = [self.prefix + self.batch_fmtstr.format(batch)] 466 | entries += [str(meter) for meter in self.meters] 467 | print('\t'.join(entries)) 468 | 469 | def _get_batch_fmtstr(self, num_batches): 470 | num_digits = len(str(num_batches // 1)) 471 | fmt = '{:' + str(num_digits) + 'd}' 472 | return '[' + fmt + '/' + fmt.format(num_batches) + ']' 473 | 474 | 475 | def adjust_learning_rate(optimizer, epoch, args): 476 | """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" 477 | lr = args.lr * (0.1 ** (epoch // 30)) 478 | for param_group in optimizer.param_groups: 479 | param_group['lr'] = lr 480 | 481 | 482 | def accuracy(output, target, topk=(1,)): 483 | """Computes the accuracy over the k top predictions for the specified values of k""" 484 | with torch.no_grad(): 485 | maxk = max(topk) 486 | batch_size = target.size(0) 487 | 488 | _, pred = output.topk(maxk, 1, True, True) 489 | pred = pred.t() 490 | correct = pred.eq(target.view(1, -1).expand_as(pred)) 491 | 492 | res = [] 493 | for k in topk: 494 | #correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) 495 | correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) 496 | res.append(correct_k.mul_(100.0 / batch_size)) 497 | return res 498 | 499 | 500 | if __name__ == '__main__': 501 | main() 502 | -------------------------------------------------------------------------------- /Supervised_Learning/main_supervised_hm.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import builtins 3 | import math 4 | import os 5 | import random 6 | import shutil 7 | import time 8 | import warnings 9 | import numpy as np 10 | 11 | import torch 12 | import torch.nn as nn 13 | import torch.nn.parallel 14 | import torch.backends.cudnn as cudnn 15 | import torch.distributed as dist 16 | import torch.optim 17 | import torch.multiprocessing as mp 18 | import torch.utils.data 19 | import torch.utils.data.distributed 20 | 21 | import dataset 22 | import OPA_supervised_hm 23 | import sklearn.metrics 24 | from scipy.stats import mode 25 | import os.path as osp 26 | 27 | 28 | 29 | parser = argparse.ArgumentParser(description='Temporal Alignment Prediction for Supervised Representation Learning and Few-Shot Sequence Classification') 30 | parser.add_argument('-data', metavar='DIR', 31 | help='path to dataset') 32 | parser.add_argument('-savedir', metavar='MIR', 33 | help='path to save models') 34 | parser.add_argument('-j', '--workers', default=2, type=int, metavar='N', 35 | help='number of data loading workers (default: 32)') 36 | parser.add_argument('--epochs', default=101, type=int, metavar='N', 37 | help='number of total epochs to run') 38 | parser.add_argument('--start-epoch', default=0, type=int, metavar='N', 39 | help='manual epoch number (useful on restarts)') 40 | parser.add_argument('-b', '--batch-size', default=128, type=int, 41 | metavar='N', 42 | help='mini-batch size (default: 256), this is the total ' 43 | 'batch size of all GPUs on the current node when ' 44 | 'using Data Parallel or Distributed Data Parallel') 45 | parser.add_argument('--lr', '--learning-rate', default=0.0001, type=float, 46 | metavar='LR', help='initial learning rate', dest='lr') #0.03 47 | parser.add_argument('--schedule', default=[160, 240, 320, 400], nargs='*', type=int, 48 | help='learning rate schedule (when to drop lr by 10x)') #[120, 160, 240, 300] 49 | parser.add_argument('--momentum', default=0.9, type=float, metavar='M', 50 | help='momentum of SGD solver') 51 | parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, 52 | metavar='W', help='weight decay (default: 1e-4)', 53 | dest='weight_decay') 54 | parser.add_argument('-p', '--print-freq', default=10, type=int, 55 | metavar='N', help='print frequency (default: 10)') 56 | parser.add_argument('--resume', default='', type=str, metavar='PATH', 57 | help='path to latest checkpoint (default: none)') 58 | parser.add_argument('--world-size', default=-1, type=int, 59 | help='number of nodes for distributed training') 60 | parser.add_argument('--rank', default=-1, type=int, 61 | help='node rank for distributed training') 62 | parser.add_argument('--dist-url', default='tcp://', type=str, 63 | help='url used to set up distributed training') 64 | parser.add_argument('--dist-backend', default='nccl', type=str, 65 | help='distributed backend') 66 | parser.add_argument('--seed', default=None, type=int, 67 | help='seed for initializing training. ') 68 | parser.add_argument('--gpu', default=0, type=int, 69 | help='GPU id to use.') 70 | parser.add_argument('--multiprocessing-distributed', action='store_true', 71 | help='Use multi-processing distributed training to launch ' 72 | 'N processes per node, which has N GPUs. This is the ' 73 | 'fastest way to use PyTorch for either single node or ' 74 | 'multi node data parallel training') 75 | parser.add_argument('--cos', action='store_true', 76 | help='use cosine lr schedule') 77 | 78 | # TAP specific configs: 79 | parser.add_argument('--inputdim', default=100, type=int, 80 | help='feature dimension (default: 128)') 81 | parser.add_argument('--middledim', default=1024, type=int, 82 | help='hidden nodes in encoder and predictor') 83 | parser.add_argument('--outputdim', default=128, type=float, 84 | help='feature dimension of transformed sequences') 85 | parser.add_argument('--lam', default=1, type=float, 86 | help='weight of relative temporal positions') 87 | parser.add_argument('--lam1', default=1, type=float, 88 | help='weight of dual meta-distance') 89 | parser.add_argument('--lam2', default=0.5, type=float, 90 | help='weight of dual alignment') 91 | parser.add_argument('--normalized', action='store_true', 92 | help='add a normalization layer to encoder and predictor') 93 | 94 | def setup_seed(seed): 95 | torch.manual_seed(seed) 96 | torch.cuda.manual_seed_all(seed) 97 | torch.cuda.manual_seed(seed) 98 | np.random.seed(seed) 99 | random.seed(seed) 100 | cudnn.deterministic = True 101 | #cudnn.benchmark = False 102 | #cudnn.enabled = False 103 | 104 | setup_seed(892101) 105 | 106 | def _init_fn(worker_id): 107 | random.seed(10 + worker_id) 108 | np.random.seed(10 + worker_id) 109 | torch.manual_seed(10 + worker_id) 110 | torch.cuda.manual_seed(10 + worker_id) 111 | torch.cuda.manual_seed_all(10 + worker_id) 112 | 113 | def main(): 114 | args = parser.parse_args() 115 | 116 | if args.seed is not None: 117 | random.seed(args.seed) 118 | torch.manual_seed(args.seed) 119 | cudnn.deterministic = True 120 | warnings.warn('You have chosen to seed training. ' 121 | 'This will turn on the CUDNN deterministic setting, ' 122 | 'which can slow down your training considerably! ' 123 | 'You may see unexpected behavior when restarting ' 124 | 'from checkpoints.') 125 | 126 | if args.gpu is not None: 127 | warnings.warn('You have chosen a specific GPU. This will completely ' 128 | 'disable data parallelism.') 129 | 130 | if args.dist_url == "env://" and args.world_size == -1: 131 | args.world_size = int(os.environ["WORLD_SIZE"]) 132 | 133 | args.distributed = args.world_size > 1 or args.multiprocessing_distributed 134 | 135 | ngpus_per_node = torch.cuda.device_count() 136 | if args.multiprocessing_distributed: 137 | # Since we have ngpus_per_node processes per node, the total world_size 138 | # needs to be adjusted accordingly 139 | args.world_size = ngpus_per_node * args.world_size 140 | # Use torch.multiprocessing.spawn to launch distributed processes: the 141 | # main_worker process function 142 | mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) 143 | else: 144 | # Simply call main_worker function 145 | main_worker(args.gpu, ngpus_per_node, args) 146 | 147 | 148 | def main_worker(gpu, ngpus_per_node, args): 149 | args.gpu = gpu 150 | 151 | # suppress printing if not master 152 | if args.multiprocessing_distributed and args.gpu != 0: 153 | def print_pass(*args): 154 | pass 155 | builtins.print = print_pass 156 | 157 | if args.gpu is not None: 158 | print("Use GPU: {} for training".format(args.gpu)) 159 | 160 | if args.distributed: 161 | if args.dist_url == "env://" and args.rank == -1: 162 | args.rank = int(os.environ["RANK"]) 163 | if args.multiprocessing_distributed: 164 | # For multiprocessing distributed training, rank needs to be the 165 | # global rank among all the processes 166 | args.rank = args.rank * ngpus_per_node + gpu 167 | dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, 168 | world_size=args.world_size, rank=args.rank) 169 | # create model 170 | model = OPA_supervised_hm.BlurContrastiveModelPair(input_dim=args.inputdim) 171 | 172 | if args.distributed: 173 | # For multiprocessing distributed, DistributedDataParallel constructor 174 | # should always set the single device scope, otherwise, 175 | # DistributedDataParallel will use all available devices. 176 | if args.gpu is not None: 177 | torch.cuda.set_device(args.gpu) 178 | model.cuda(args.gpu) 179 | # When using a single GPU per process and per 180 | # DistributedDataParallel, we need to divide the batch size 181 | # ourselves based on the total number of GPUs we have 182 | args.batch_size = int(args.batch_size / ngpus_per_node) 183 | args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) 184 | model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) 185 | else: 186 | model.cuda() 187 | # DistributedDataParallel will divide and allocate batch_size to all 188 | # available GPUs if device_ids are not set 189 | model = torch.nn.parallel.DistributedDataParallel(model) 190 | elif args.gpu is not None: 191 | torch.cuda.set_device(args.gpu) 192 | model = model.cuda(args.gpu) 193 | # comment out the following line for debugging 194 | # raise NotImplementedError("Only DistributedDataParallel is supported.") 195 | else: 196 | # AllGather implementation (batch shuffle, queue update, etc.) in 197 | # this code only supports DistributedDataParallel. 198 | raise NotImplementedError("Only DistributedDataParallel is supported.") 199 | 200 | optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, 201 | weight_decay=args.weight_decay) 202 | 203 | # optionally resume from a checkpoint 204 | if args.resume: 205 | if os.path.isfile(args.resume): 206 | print("=> loading checkpoint '{}'".format(args.resume)) 207 | if args.gpu is None: 208 | checkpoint = torch.load(args.resume) 209 | else: 210 | # Map model to be loaded to specified single gpu. 211 | loc = 'cuda:{}'.format(args.gpu) 212 | checkpoint = torch.load(args.resume, map_location=loc) 213 | args.start_epoch = checkpoint['epoch'] 214 | model.load_state_dict(checkpoint['state_dict']) 215 | optimizer.load_state_dict(checkpoint['optimizer']) 216 | print("=> loaded checkpoint '{}' (epoch {})" 217 | .format(args.resume, checkpoint['epoch'])) 218 | else: 219 | print("=> no checkpoint found at '{}'".format(args.resume)) 220 | 221 | cudnn.benchmark = True 222 | 223 | # Data loading code 224 | #traindir = os.path.join(args.data, 'train') 225 | datasetall = dataset.SequenceDataset(args.data) #, test_dataset 226 | #train_dataset = datasetall.traindata 227 | 228 | if args.distributed: 229 | train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) 230 | else: 231 | train_sampler = None 232 | 233 | train_loader = torch.utils.data.DataLoader( 234 | datasetall.traindata, batch_size=args.batch_size, shuffle=(train_sampler is None), 235 | num_workers=args.workers, pin_memory=True, sampler=train_sampler, drop_last=True, worker_init_fn=_init_fn) 236 | 237 | traintest_loader = torch.utils.data.DataLoader( 238 | datasetall.traindata, batch_size=datasetall.traindata.__len__(), shuffle=False, 239 | num_workers=args.workers, pin_memory=True, worker_init_fn=_init_fn) 240 | 241 | test_loader = torch.utils.data.DataLoader( 242 | datasetall.testdata, batch_size=1, shuffle=False, 243 | num_workers=args.workers, pin_memory=True, worker_init_fn=_init_fn) 244 | 245 | #os.mkdir(args.save_dir) 246 | all_epoch_time = [] 247 | 248 | for epoch in range(args.start_epoch, args.epochs): 249 | if args.distributed: 250 | train_sampler.set_epoch(epoch) 251 | adjust_learning_rate(optimizer, epoch, args) 252 | 253 | epochstart = time.time() 254 | train(train_loader, model, optimizer, epoch, args) 255 | epochend = time.time() 256 | all_epoch_time.append(epochend-epochstart) 257 | 258 | if epoch%10==0: # and epoch>40: # 259 | acc1 = validate(test_loader, traintest_loader, model, args) 260 | 261 | if not args.multiprocessing_distributed or (args.multiprocessing_distributed 262 | and args.rank % ngpus_per_node == 0): 263 | save_checkpoint({ 264 | 'epoch': epoch + 1, 265 | #'arch': args.arch, 266 | 'state_dict': model.state_dict(), 267 | 'optimizer' : optimizer.state_dict(), 268 | }, is_best=False, filename=osp.join(args.savedir, 'checkpoint_{:04d}.pth.tar'.format(epoch))) 269 | #fpath=osp.join(args.save_dir, 'ckp_ep' + str(epoch + 1) + '.pth.tar') 270 | print(all_epoch_time) 271 | print(sum(all_epoch_time)) 272 | 273 | 274 | def train(train_loader, model, optimizer, epoch, args): 275 | batch_time = AverageMeter('Time', ':6.3f') 276 | data_time = AverageMeter('Data', ':6.3f') 277 | losses = AverageMeter('Loss', ':.4e') 278 | progress = ProgressMeter( 279 | len(train_loader), 280 | [batch_time, data_time, losses], 281 | prefix="Epoch: [{}]".format(epoch)) 282 | 283 | # switch to train mode 284 | model.train() 285 | 286 | end = time.time() 287 | for i, (inputs, labels, lens) in enumerate(train_loader): 288 | # measure data loading time 289 | data_time.update(time.time() - end) 290 | 291 | if args.gpu is not None: 292 | inputs = inputs.cuda(args.gpu, non_blocking=True) 293 | labels = labels.cuda(args.gpu, non_blocking=True) 294 | lens = lens.cuda(args.gpu, non_blocking=True) 295 | 296 | # compute output 297 | loss = model(inputs, lens, labels) 298 | losses.update(loss.item(), inputs.size(0)) 299 | 300 | # compute gradient and do SGD step 301 | optimizer.zero_grad() 302 | loss.backward() 303 | optimizer.step() 304 | 305 | # measure elapsed time 306 | batch_time.update(time.time() - end) 307 | end = time.time() 308 | 309 | if i % args.print_freq == 0: 310 | progress.display(i) 311 | 312 | def validate(val_loader, traintest_loader, model, args): 313 | test_batch_time = AverageMeter('Time', ':6.3f') 314 | #losses = AverageMeter('Loss', ':.4e') 315 | map1 = AverageMeter('Map@1', ':6.2f') 316 | top1 = AverageMeter('Acc@1', ':6.2f') 317 | top3 = AverageMeter('Acc@3', ':6.2f') 318 | top5 = AverageMeter('Acc@5', ':6.2f') 319 | top7 = AverageMeter('Acc@7', ':6.2f') 320 | top15 = AverageMeter('Acc@15', ':6.2f') 321 | top30 = AverageMeter('Acc@30', ':6.2f') 322 | progress = ProgressMeter( 323 | len(val_loader), 324 | [test_batch_time, map1, top1, top3, top5, top7, top15, top30], 325 | prefix='Test: ') 326 | 327 | # switch to evaluate mode 328 | model.eval() 329 | 330 | with torch.no_grad(): 331 | end = time.time() 332 | for j, (trinputs, trlabels, trlens) in enumerate(traintest_loader): 333 | #print(j) 334 | if args.gpu is not None: 335 | trinputs = trinputs.cuda(args.gpu, non_blocking=True) 336 | labels2 = trlabels.cuda(args.gpu, non_blocking=True) 337 | lens2 = trlens.cuda(args.gpu, non_blocking=True) 338 | #seq2 = trinputs #model.encoder(trinputs) 339 | seq2 = model.encoder(trinputs) 340 | R2 = model.getlen(seq2,lens2) #.cuda(args.gpu, non_blocking=True) 341 | 342 | trnum = seq2.size(0) 343 | trlabels = trlabels.numpy().reshape(-1) 344 | #print(trlabels) 345 | for i, (inputs, labels, lens) in enumerate(val_loader): 346 | if args.gpu is not None: 347 | inputs = inputs.cuda(args.gpu, non_blocking=True) 348 | labels = labels.cuda(args.gpu, non_blocking=True) 349 | lens = lens.cuda(args.gpu, non_blocking=True) 350 | # compute output 351 | #seq1 = inputs #model.encoder(inputs) 352 | seq1 = model.encoder(inputs) 353 | seq1 = seq1[:,0:lens[0],:] 354 | R1 = model.getlen(seq1,lens) #.cuda(args.gpu, non_blocking=True) 355 | # print(R1.device) 356 | # print(seq1.device) 357 | tenum = seq1.size(0) 358 | assert(tenum==1) 359 | dismat = np.zeros(trnum) 360 | for trcount in range(trnum): 361 | seqtr = seq2[trcount,0:lens2[trcount],:].unsqueeze(0) 362 | Rtr = R2[trcount,0:lens2[trcount]].unsqueeze(0) 363 | # print(Rtr.device) 364 | # print(seqtr.device) 365 | _,dis = model.alignment(seq1,seqtr,R1,Rtr) 366 | #dis = dis.view(-1) 367 | #print(dis) 368 | dismat[trcount] = dis[0].item() 369 | 370 | labels = labels[0].item() 371 | #print(labels) 372 | knn_idx_full = dismat.argsort() 373 | #print(knn_idx_full) 374 | acc = np.zeros(6) 375 | count = 0 376 | for k in [1, 3, 5, 7, 15, 30]: 377 | knn_idx = knn_idx_full[0:k] #[:, :k] 378 | knn_labels = trlabels[knn_idx] 379 | #print(knn_labels) 380 | mode_data = mode(knn_labels) #, axis=1 381 | mode_label = mode_data[0] 382 | #print(mode_label[0]) 383 | #print(mode_data) 384 | if mode_label[0]==labels: 385 | acc[count] = acc[count]+100 386 | count = count + 1 387 | 388 | tru_label = np.where(trlabels==labels,1,0).reshape(-1) 389 | #print(tru_label) 390 | ap = sklearn.metrics.average_precision_score(tru_label,-dismat.reshape(-1)) 391 | 392 | # measure accuracy and record loss 393 | map1.update(ap, inputs.size(0)) 394 | top1.update(acc[0], inputs.size(0)) 395 | top3.update(acc[1], inputs.size(0)) 396 | top5.update(acc[2], inputs.size(0)) 397 | top7.update(acc[3], inputs.size(0)) 398 | top15.update(acc[4], inputs.size(0)) 399 | top30.update(acc[5], inputs.size(0)) 400 | 401 | # measure elapsed time 402 | test_batch_time.update(time.time() - end) 403 | end = time.time() 404 | 405 | if i % args.print_freq == 0: 406 | progress.display(i) 407 | 408 | # TODO: this should also be done with the ProgressMeter 409 | print(' * Map@1 {map1.avg:.3f} Acc@1 {top1.avg:.3f} Acc@3 {top3.avg:.3f} Acc@5 {top5.avg:.3f} Acc@7 {top7.avg:.3f} Acc@15 {top15.avg:.3f} Acc@30 {top30.avg:.3f}' 410 | .format(map1=map1, top1=top1, top3=top3, top5=top5, top7=top7, top15=top15, top30=top30)) 411 | 412 | return top1.avg 413 | 414 | 415 | def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): 416 | torch.save(state, filename) 417 | if is_best: 418 | shutil.copyfile(filename, 'model_best.pth.tar') 419 | 420 | 421 | class AverageMeter(object): 422 | """Computes and stores the average and current value""" 423 | def __init__(self, name, fmt=':f'): 424 | self.name = name 425 | self.fmt = fmt 426 | self.reset() 427 | 428 | def reset(self): 429 | self.val = 0 430 | self.avg = 0 431 | self.sum = 0 432 | self.count = 0 433 | 434 | def update(self, val, n=1): 435 | self.val = val 436 | self.sum += val * n 437 | self.count += n 438 | self.avg = self.sum / self.count 439 | 440 | def __str__(self): 441 | fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' 442 | return fmtstr.format(**self.__dict__) 443 | 444 | 445 | class ProgressMeter(object): 446 | def __init__(self, num_batches, meters, prefix=""): 447 | self.batch_fmtstr = self._get_batch_fmtstr(num_batches) 448 | self.meters = meters 449 | self.prefix = prefix 450 | 451 | def display(self, batch): 452 | entries = [self.prefix + self.batch_fmtstr.format(batch)] 453 | entries += [str(meter) for meter in self.meters] 454 | print('\t'.join(entries)) 455 | 456 | def _get_batch_fmtstr(self, num_batches): 457 | num_digits = len(str(num_batches // 1)) 458 | fmt = '{:' + str(num_digits) + 'd}' 459 | return '[' + fmt + '/' + fmt.format(num_batches) + ']' 460 | 461 | 462 | def adjust_learning_rate(optimizer, epoch, args): 463 | """Decay the learning rate based on schedule""" 464 | lr = args.lr 465 | if args.cos: # cosine lr schedule 466 | lr *= 0.5 * (1. + math.cos(math.pi * epoch / args.epochs)) 467 | else: # stepwise lr schedule 468 | for milestone in args.schedule: 469 | lr *= 0.1 if epoch >= milestone else 1. 470 | for param_group in optimizer.param_groups: 471 | param_group['lr'] = lr 472 | 473 | 474 | def accuracy(output, target, topk=(1,)): 475 | """Computes the accuracy over the k top predictions for the specified values of k""" 476 | with torch.no_grad(): 477 | maxk = max(topk) 478 | batch_size = target.size(0) 479 | 480 | _, pred = output.topk(maxk, 1, True, True) 481 | pred = pred.t() 482 | correct = pred.eq(target.view(1, -1).expand_as(pred)) 483 | 484 | res = [] 485 | for k in topk: 486 | correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) 487 | res.append(correct_k.mul_(100.0 / batch_size)) 488 | return res 489 | 490 | 491 | if __name__ == '__main__': 492 | main() 493 | -------------------------------------------------------------------------------- /Supervised_Learning/main_supervised_binomial.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import builtins 3 | import math 4 | import os 5 | import random 6 | import shutil 7 | import time 8 | import warnings 9 | import numpy as np 10 | 11 | import torch 12 | import torch.nn as nn 13 | import torch.nn.parallel 14 | import torch.backends.cudnn as cudnn 15 | import torch.distributed as dist 16 | import torch.optim 17 | import torch.multiprocessing as mp 18 | import torch.utils.data 19 | import torch.utils.data.distributed 20 | 21 | import dataset 22 | import OPA_supervised_binomial 23 | import sklearn.metrics 24 | from scipy.stats import mode 25 | import os.path as osp 26 | 27 | 28 | 29 | parser = argparse.ArgumentParser(description='Temporal Alignment Prediction for Supervised Representation Learning and Few-Shot Sequence Classification') 30 | parser.add_argument('-data', metavar='DIR', 31 | help='path to dataset') 32 | parser.add_argument('-savedir', metavar='MIR', 33 | help='path to save models') 34 | parser.add_argument('-j', '--workers', default=2, type=int, metavar='N', 35 | help='number of data loading workers (default: 32)') 36 | parser.add_argument('--epochs', default=101, type=int, metavar='N', 37 | help='number of total epochs to run') 38 | parser.add_argument('--start-epoch', default=0, type=int, metavar='N', 39 | help='manual epoch number (useful on restarts)') 40 | parser.add_argument('-b', '--batch-size', default=128, type=int, 41 | metavar='N', 42 | help='mini-batch size (default: 256), this is the total ' 43 | 'batch size of all GPUs on the current node when ' 44 | 'using Data Parallel or Distributed Data Parallel') 45 | parser.add_argument('--lr', '--learning-rate', default=0.0001, type=float, 46 | metavar='LR', help='initial learning rate', dest='lr') #0.03 47 | parser.add_argument('--schedule', default=[160, 240, 320, 400], nargs='*', type=int, 48 | help='learning rate schedule (when to drop lr by 10x)') #[120, 160, 240, 300] 49 | parser.add_argument('--momentum', default=0.9, type=float, metavar='M', 50 | help='momentum of SGD solver') 51 | parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, 52 | metavar='W', help='weight decay (default: 1e-4)', 53 | dest='weight_decay') 54 | parser.add_argument('-p', '--print-freq', default=10, type=int, 55 | metavar='N', help='print frequency (default: 10)') 56 | parser.add_argument('--resume', default='', type=str, metavar='PATH', 57 | help='path to latest checkpoint (default: none)') 58 | parser.add_argument('--world-size', default=-1, type=int, 59 | help='number of nodes for distributed training') 60 | parser.add_argument('--rank', default=-1, type=int, 61 | help='node rank for distributed training') 62 | parser.add_argument('--dist-url', default='tcp://', type=str, 63 | help='url used to set up distributed training') 64 | parser.add_argument('--dist-backend', default='nccl', type=str, 65 | help='distributed backend') 66 | parser.add_argument('--seed', default=None, type=int, 67 | help='seed for initializing training. ') 68 | parser.add_argument('--gpu', default=0, type=int, 69 | help='GPU id to use.') 70 | parser.add_argument('--multiprocessing-distributed', action='store_true', 71 | help='Use multi-processing distributed training to launch ' 72 | 'N processes per node, which has N GPUs. This is the ' 73 | 'fastest way to use PyTorch for either single node or ' 74 | 'multi node data parallel training') 75 | parser.add_argument('--cos', action='store_true', 76 | help='use cosine lr schedule') 77 | 78 | # TAP specific configs: 79 | parser.add_argument('--inputdim', default=100, type=int, 80 | help='feature dimension (default: 128)') 81 | parser.add_argument('--middledim', default=1024, type=int, 82 | help='hidden nodes in encoder and predictor') 83 | parser.add_argument('--outputdim', default=128, type=float, 84 | help='feature dimension of transformed sequences') 85 | parser.add_argument('--lam', default=1, type=float, 86 | help='weight of relative temporal positions') 87 | parser.add_argument('--lam1', default=1, type=float, 88 | help='weight of dual meta-distance') 89 | parser.add_argument('--lam2', default=0.5, type=float, 90 | help='weight of dual alignment') 91 | parser.add_argument('--normalized', action='store_true', 92 | help='add a normalization layer to encoder and predictor') 93 | 94 | def setup_seed(seed): 95 | torch.manual_seed(seed) 96 | torch.cuda.manual_seed_all(seed) 97 | torch.cuda.manual_seed(seed) 98 | np.random.seed(seed) 99 | random.seed(seed) 100 | cudnn.deterministic = True 101 | #cudnn.benchmark = False 102 | #cudnn.enabled = False 103 | 104 | setup_seed(892101) 105 | 106 | def _init_fn(worker_id): 107 | random.seed(10 + worker_id) 108 | np.random.seed(10 + worker_id) 109 | torch.manual_seed(10 + worker_id) 110 | torch.cuda.manual_seed(10 + worker_id) 111 | torch.cuda.manual_seed_all(10 + worker_id) 112 | 113 | def main(): 114 | args = parser.parse_args() 115 | 116 | if args.seed is not None: 117 | random.seed(args.seed) 118 | torch.manual_seed(args.seed) 119 | cudnn.deterministic = True 120 | warnings.warn('You have chosen to seed training. ' 121 | 'This will turn on the CUDNN deterministic setting, ' 122 | 'which can slow down your training considerably! ' 123 | 'You may see unexpected behavior when restarting ' 124 | 'from checkpoints.') 125 | 126 | if args.gpu is not None: 127 | warnings.warn('You have chosen a specific GPU. This will completely ' 128 | 'disable data parallelism.') 129 | 130 | if args.dist_url == "env://" and args.world_size == -1: 131 | args.world_size = int(os.environ["WORLD_SIZE"]) 132 | 133 | args.distributed = args.world_size > 1 or args.multiprocessing_distributed 134 | 135 | ngpus_per_node = torch.cuda.device_count() 136 | if args.multiprocessing_distributed: 137 | # Since we have ngpus_per_node processes per node, the total world_size 138 | # needs to be adjusted accordingly 139 | args.world_size = ngpus_per_node * args.world_size 140 | # Use torch.multiprocessing.spawn to launch distributed processes: the 141 | # main_worker process function 142 | mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) 143 | else: 144 | # Simply call main_worker function 145 | main_worker(args.gpu, ngpus_per_node, args) 146 | 147 | 148 | def main_worker(gpu, ngpus_per_node, args): 149 | args.gpu = gpu 150 | 151 | # suppress printing if not master 152 | if args.multiprocessing_distributed and args.gpu != 0: 153 | def print_pass(*args): 154 | pass 155 | builtins.print = print_pass 156 | 157 | if args.gpu is not None: 158 | print("Use GPU: {} for training".format(args.gpu)) 159 | 160 | if args.distributed: 161 | if args.dist_url == "env://" and args.rank == -1: 162 | args.rank = int(os.environ["RANK"]) 163 | if args.multiprocessing_distributed: 164 | # For multiprocessing distributed training, rank needs to be the 165 | # global rank among all the processes 166 | args.rank = args.rank * ngpus_per_node + gpu 167 | dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, 168 | world_size=args.world_size, rank=args.rank) 169 | # create model 170 | model = OPA_supervised_binomial.BlurContrastiveModelPair(input_dim=args.inputdim) 171 | 172 | if args.distributed: 173 | # For multiprocessing distributed, DistributedDataParallel constructor 174 | # should always set the single device scope, otherwise, 175 | # DistributedDataParallel will use all available devices. 176 | if args.gpu is not None: 177 | torch.cuda.set_device(args.gpu) 178 | model.cuda(args.gpu) 179 | # When using a single GPU per process and per 180 | # DistributedDataParallel, we need to divide the batch size 181 | # ourselves based on the total number of GPUs we have 182 | args.batch_size = int(args.batch_size / ngpus_per_node) 183 | args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) 184 | model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) 185 | else: 186 | model.cuda() 187 | # DistributedDataParallel will divide and allocate batch_size to all 188 | # available GPUs if device_ids are not set 189 | model = torch.nn.parallel.DistributedDataParallel(model) 190 | elif args.gpu is not None: 191 | torch.cuda.set_device(args.gpu) 192 | model = model.cuda(args.gpu) 193 | # comment out the following line for debugging 194 | # raise NotImplementedError("Only DistributedDataParallel is supported.") 195 | else: 196 | # AllGather implementation (batch shuffle, queue update, etc.) in 197 | # this code only supports DistributedDataParallel. 198 | raise NotImplementedError("Only DistributedDataParallel is supported.") 199 | 200 | # define loss function (criterion) and optimizer 201 | #criterion = nn.CrossEntropyLoss().cuda(args.gpu) 202 | # criterion = OPA.BlurContrastiveLossPair() 203 | 204 | # optimizer = torch.optim.SGD(model.parameters(), args.lr, 205 | # momentum=args.momentum, 206 | # weight_decay=args.weight_decay) 207 | optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, 208 | weight_decay=args.weight_decay) 209 | 210 | # optionally resume from a checkpoint 211 | if args.resume: 212 | if os.path.isfile(args.resume): 213 | print("=> loading checkpoint '{}'".format(args.resume)) 214 | if args.gpu is None: 215 | checkpoint = torch.load(args.resume) 216 | else: 217 | # Map model to be loaded to specified single gpu. 218 | loc = 'cuda:{}'.format(args.gpu) 219 | checkpoint = torch.load(args.resume, map_location=loc) 220 | args.start_epoch = checkpoint['epoch'] 221 | model.load_state_dict(checkpoint['state_dict']) 222 | optimizer.load_state_dict(checkpoint['optimizer']) 223 | print("=> loaded checkpoint '{}' (epoch {})" 224 | .format(args.resume, checkpoint['epoch'])) 225 | else: 226 | print("=> no checkpoint found at '{}'".format(args.resume)) 227 | 228 | cudnn.benchmark = True 229 | 230 | # Data loading code 231 | #traindir = os.path.join(args.data, 'train') 232 | datasetall = dataset.SequenceDataset(args.data) #, test_dataset 233 | #train_dataset = datasetall.traindata 234 | 235 | if args.distributed: 236 | train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) 237 | else: 238 | train_sampler = None 239 | 240 | train_loader = torch.utils.data.DataLoader( 241 | datasetall.traindata, batch_size=args.batch_size, shuffle=(train_sampler is None), 242 | num_workers=args.workers, pin_memory=True, sampler=train_sampler, drop_last=True, worker_init_fn=_init_fn) 243 | 244 | traintest_loader = torch.utils.data.DataLoader( 245 | datasetall.traindata, batch_size=datasetall.traindata.__len__(), shuffle=False, 246 | num_workers=args.workers, pin_memory=True, worker_init_fn=_init_fn) 247 | 248 | test_loader = torch.utils.data.DataLoader( 249 | datasetall.testdata, batch_size=1, shuffle=False, 250 | num_workers=args.workers, pin_memory=True, worker_init_fn=_init_fn) 251 | 252 | #os.mkdir(args.save_dir) 253 | all_epoch_time = [] 254 | 255 | for epoch in range(args.start_epoch, args.epochs): 256 | if args.distributed: 257 | train_sampler.set_epoch(epoch) 258 | adjust_learning_rate(optimizer, epoch, args) 259 | 260 | epochstart = time.time() 261 | train(train_loader, model, optimizer, epoch, args) 262 | epochend = time.time() 263 | all_epoch_time.append(epochend-epochstart) 264 | 265 | if epoch%10==0 and epoch>40: # 266 | acc1 = validate(test_loader, traintest_loader, model, args) 267 | 268 | if not args.multiprocessing_distributed or (args.multiprocessing_distributed 269 | and args.rank % ngpus_per_node == 0): 270 | save_checkpoint({ 271 | 'epoch': epoch + 1, 272 | #'arch': args.arch, 273 | 'state_dict': model.state_dict(), 274 | 'optimizer' : optimizer.state_dict(), 275 | }, is_best=False, filename=osp.join(args.savedir, 'checkpoint_{:04d}.pth.tar'.format(epoch))) 276 | #fpath=osp.join(args.save_dir, 'ckp_ep' + str(epoch + 1) + '.pth.tar') 277 | print(all_epoch_time) 278 | print(sum(all_epoch_time)) 279 | 280 | 281 | def train(train_loader, model, optimizer, epoch, args): #, criterion 282 | batch_time = AverageMeter('Time', ':6.3f') 283 | data_time = AverageMeter('Data', ':6.3f') 284 | losses = AverageMeter('Loss', ':.4e') 285 | progress = ProgressMeter( 286 | len(train_loader), 287 | [batch_time, data_time, losses], 288 | prefix="Epoch: [{}]".format(epoch)) 289 | 290 | # switch to train mode 291 | model.train() 292 | 293 | end = time.time() 294 | for i, (inputs, labels, lens) in enumerate(train_loader): 295 | # measure data loading time 296 | data_time.update(time.time() - end) 297 | 298 | if args.gpu is not None: 299 | inputs = inputs.cuda(args.gpu, non_blocking=True) 300 | labels = labels.cuda(args.gpu, non_blocking=True) 301 | lens = lens.cuda(args.gpu, non_blocking=True) 302 | 303 | # compute output 304 | loss = model(inputs, lens, labels) 305 | losses.update(loss.item(), inputs.size(0)) 306 | 307 | # compute gradient and do SGD step 308 | optimizer.zero_grad() 309 | loss.backward() 310 | optimizer.step() 311 | 312 | # measure elapsed time 313 | batch_time.update(time.time() - end) 314 | end = time.time() 315 | 316 | if i % args.print_freq == 0: 317 | progress.display(i) 318 | 319 | def validate(val_loader, traintest_loader, model, args): 320 | test_batch_time = AverageMeter('Time', ':6.3f') 321 | #losses = AverageMeter('Loss', ':.4e') 322 | map1 = AverageMeter('Map@1', ':6.2f') 323 | top1 = AverageMeter('Acc@1', ':6.2f') 324 | top3 = AverageMeter('Acc@3', ':6.2f') 325 | top5 = AverageMeter('Acc@5', ':6.2f') 326 | top7 = AverageMeter('Acc@7', ':6.2f') 327 | top15 = AverageMeter('Acc@15', ':6.2f') 328 | top30 = AverageMeter('Acc@30', ':6.2f') 329 | progress = ProgressMeter( 330 | len(val_loader), 331 | [test_batch_time, map1, top1, top3, top5, top7, top15, top30], 332 | prefix='Test: ') 333 | 334 | # switch to evaluate mode 335 | model.eval() 336 | 337 | with torch.no_grad(): 338 | end = time.time() 339 | for j, (trinputs, trlabels, trlens) in enumerate(traintest_loader): 340 | #print(j) 341 | if args.gpu is not None: 342 | trinputs = trinputs.cuda(args.gpu, non_blocking=True) 343 | labels2 = trlabels.cuda(args.gpu, non_blocking=True) 344 | lens2 = trlens.cuda(args.gpu, non_blocking=True) 345 | #seq2 = trinputs #model.encoder(trinputs) 346 | seq2 = model.encoder(trinputs) 347 | R2 = model.getlen(seq2,lens2) #.cuda(args.gpu, non_blocking=True) 348 | 349 | trnum = seq2.size(0) 350 | trlabels = trlabels.numpy().reshape(-1) 351 | #print(trlabels) 352 | for i, (inputs, labels, lens) in enumerate(val_loader): 353 | if args.gpu is not None: 354 | inputs = inputs.cuda(args.gpu, non_blocking=True) 355 | labels = labels.cuda(args.gpu, non_blocking=True) 356 | lens = lens.cuda(args.gpu, non_blocking=True) 357 | # compute output 358 | #seq1 = inputs #model.encoder(inputs) 359 | seq1 = model.encoder(inputs) 360 | seq1 = seq1[:,0:lens[0],:] 361 | R1 = model.getlen(seq1,lens) #.cuda(args.gpu, non_blocking=True) 362 | # print(R1.device) 363 | # print(seq1.device) 364 | tenum = seq1.size(0) 365 | assert(tenum==1) 366 | dismat = np.zeros(trnum) 367 | for trcount in range(trnum): 368 | seqtr = seq2[trcount,0:lens2[trcount],:].unsqueeze(0) 369 | Rtr = R2[trcount,0:lens2[trcount]].unsqueeze(0) 370 | # print(Rtr.device) 371 | # print(seqtr.device) 372 | _,dis = model.alignment(seq1,seqtr,R1,Rtr) 373 | #dis = dis.view(-1) 374 | #print(dis) 375 | dismat[trcount] = dis[0].item() 376 | 377 | labels = labels[0].item() 378 | #print(labels) 379 | knn_idx_full = dismat.argsort() 380 | #print(knn_idx_full) 381 | acc = np.zeros(6) 382 | count = 0 383 | for k in [1, 3, 5, 7, 15, 30]: 384 | knn_idx = knn_idx_full[0:k] #[:, :k] 385 | knn_labels = trlabels[knn_idx] 386 | #print(knn_labels) 387 | mode_data = mode(knn_labels) #, axis=1 388 | mode_label = mode_data[0] 389 | #print(mode_label[0]) 390 | #print(mode_data) 391 | if mode_label[0]==labels: 392 | acc[count] = acc[count]+100 393 | count = count + 1 394 | 395 | tru_label = np.where(trlabels==labels,1,0).reshape(-1) 396 | #print(tru_label) 397 | ap = sklearn.metrics.average_precision_score(tru_label,-dismat.reshape(-1)) 398 | 399 | # measure accuracy and record loss 400 | map1.update(ap, inputs.size(0)) 401 | top1.update(acc[0], inputs.size(0)) 402 | top3.update(acc[1], inputs.size(0)) 403 | top5.update(acc[2], inputs.size(0)) 404 | top7.update(acc[3], inputs.size(0)) 405 | top15.update(acc[4], inputs.size(0)) 406 | top30.update(acc[5], inputs.size(0)) 407 | 408 | # measure elapsed time 409 | test_batch_time.update(time.time() - end) 410 | end = time.time() 411 | 412 | if i % args.print_freq == 0: 413 | progress.display(i) 414 | 415 | # TODO: this should also be done with the ProgressMeter 416 | print(' * Map@1 {map1.avg:.3f} Acc@1 {top1.avg:.3f} Acc@3 {top3.avg:.3f} Acc@5 {top5.avg:.3f} Acc@7 {top7.avg:.3f} Acc@15 {top15.avg:.3f} Acc@30 {top30.avg:.3f}' 417 | .format(map1=map1, top1=top1, top3=top3, top5=top5, top7=top7, top15=top15, top30=top30)) 418 | 419 | return top1.avg 420 | 421 | 422 | def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): 423 | torch.save(state, filename) 424 | if is_best: 425 | shutil.copyfile(filename, 'model_best.pth.tar') 426 | 427 | 428 | class AverageMeter(object): 429 | """Computes and stores the average and current value""" 430 | def __init__(self, name, fmt=':f'): 431 | self.name = name 432 | self.fmt = fmt 433 | self.reset() 434 | 435 | def reset(self): 436 | self.val = 0 437 | self.avg = 0 438 | self.sum = 0 439 | self.count = 0 440 | 441 | def update(self, val, n=1): 442 | self.val = val 443 | self.sum += val * n 444 | self.count += n 445 | self.avg = self.sum / self.count 446 | 447 | def __str__(self): 448 | fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' 449 | return fmtstr.format(**self.__dict__) 450 | 451 | 452 | class ProgressMeter(object): 453 | def __init__(self, num_batches, meters, prefix=""): 454 | self.batch_fmtstr = self._get_batch_fmtstr(num_batches) 455 | self.meters = meters 456 | self.prefix = prefix 457 | 458 | def display(self, batch): 459 | entries = [self.prefix + self.batch_fmtstr.format(batch)] 460 | entries += [str(meter) for meter in self.meters] 461 | print('\t'.join(entries)) 462 | 463 | def _get_batch_fmtstr(self, num_batches): 464 | num_digits = len(str(num_batches // 1)) 465 | fmt = '{:' + str(num_digits) + 'd}' 466 | return '[' + fmt + '/' + fmt.format(num_batches) + ']' 467 | 468 | 469 | def adjust_learning_rate(optimizer, epoch, args): 470 | """Decay the learning rate based on schedule""" 471 | lr = args.lr 472 | if args.cos: # cosine lr schedule 473 | lr *= 0.5 * (1. + math.cos(math.pi * epoch / args.epochs)) 474 | else: # stepwise lr schedule 475 | for milestone in args.schedule: 476 | lr *= 0.1 if epoch >= milestone else 1. 477 | for param_group in optimizer.param_groups: 478 | param_group['lr'] = lr 479 | 480 | 481 | def accuracy(output, target, topk=(1,)): 482 | """Computes the accuracy over the k top predictions for the specified values of k""" 483 | with torch.no_grad(): 484 | maxk = max(topk) 485 | batch_size = target.size(0) 486 | 487 | _, pred = output.topk(maxk, 1, True, True) 488 | pred = pred.t() 489 | correct = pred.eq(target.view(1, -1).expand_as(pred)) 490 | 491 | res = [] 492 | for k in topk: 493 | correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) 494 | res.append(correct_k.mul_(100.0 / batch_size)) 495 | return res 496 | 497 | 498 | if __name__ == '__main__': 499 | main() 500 | -------------------------------------------------------------------------------- /Supervised_Learning/main_supervised_lifted.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import builtins 3 | import math 4 | import os 5 | import random 6 | import shutil 7 | import time 8 | import warnings 9 | import numpy as np 10 | 11 | import torch 12 | import torch.nn as nn 13 | import torch.nn.parallel 14 | import torch.backends.cudnn as cudnn 15 | import torch.distributed as dist 16 | import torch.optim 17 | import torch.multiprocessing as mp 18 | import torch.utils.data 19 | import torch.utils.data.distributed 20 | 21 | import dataset 22 | import OPA_supervised_lifted 23 | import sklearn.metrics 24 | from scipy.stats import mode 25 | import os.path as osp 26 | 27 | 28 | 29 | parser = argparse.ArgumentParser(description='Temporal Alignment Prediction for Supervised Representation Learning and Few-Shot Sequence Classification') 30 | parser.add_argument('-data', metavar='DIR', 31 | help='path to dataset') 32 | parser.add_argument('-savedir', metavar='MIR', 33 | help='path to save models') 34 | parser.add_argument('-j', '--workers', default=2, type=int, metavar='N', 35 | help='number of data loading workers (default: 32)') 36 | parser.add_argument('--epochs', default=101, type=int, metavar='N', 37 | help='number of total epochs to run') 38 | parser.add_argument('--start-epoch', default=0, type=int, metavar='N', 39 | help='manual epoch number (useful on restarts)') 40 | parser.add_argument('-b', '--batch-size', default=128, type=int, 41 | metavar='N', 42 | help='mini-batch size (default: 256), this is the total ' 43 | 'batch size of all GPUs on the current node when ' 44 | 'using Data Parallel or Distributed Data Parallel') 45 | parser.add_argument('--lr', '--learning-rate', default=0.0001, type=float, 46 | metavar='LR', help='initial learning rate', dest='lr') #0.03 47 | parser.add_argument('--schedule', default=[160, 240, 320, 400], nargs='*', type=int, 48 | help='learning rate schedule (when to drop lr by 10x)') #[120, 160, 240, 300] 49 | parser.add_argument('--momentum', default=0.9, type=float, metavar='M', 50 | help='momentum of SGD solver') 51 | parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, 52 | metavar='W', help='weight decay (default: 1e-4)', 53 | dest='weight_decay') 54 | parser.add_argument('-p', '--print-freq', default=10, type=int, 55 | metavar='N', help='print frequency (default: 10)') 56 | parser.add_argument('--resume', default='', type=str, metavar='PATH', 57 | help='path to latest checkpoint (default: none)') 58 | parser.add_argument('--world-size', default=-1, type=int, 59 | help='number of nodes for distributed training') 60 | parser.add_argument('--rank', default=-1, type=int, 61 | help='node rank for distributed training') 62 | parser.add_argument('--dist-url', default='tcp://', type=str, 63 | help='url used to set up distributed training') 64 | parser.add_argument('--dist-backend', default='nccl', type=str, 65 | help='distributed backend') 66 | parser.add_argument('--seed', default=None, type=int, 67 | help='seed for initializing training. ') 68 | parser.add_argument('--gpu', default=0, type=int, 69 | help='GPU id to use.') 70 | parser.add_argument('--multiprocessing-distributed', action='store_true', 71 | help='Use multi-processing distributed training to launch ' 72 | 'N processes per node, which has N GPUs. This is the ' 73 | 'fastest way to use PyTorch for either single node or ' 74 | 'multi node data parallel training') 75 | parser.add_argument('--cos', action='store_true', 76 | help='use cosine lr schedule') 77 | 78 | # TAP specific configs: 79 | parser.add_argument('--inputdim', default=100, type=int, 80 | help='feature dimension (default: 128)') 81 | parser.add_argument('--middledim', default=1024, type=int, 82 | help='hidden nodes in encoder and predictor') 83 | parser.add_argument('--outputdim', default=128, type=float, 84 | help='feature dimension of transformed sequences') 85 | parser.add_argument('--lam', default=1, type=float, 86 | help='weight of relative temporal positions') 87 | parser.add_argument('--lam1', default=1, type=float, 88 | help='weight of dual meta-distance') 89 | parser.add_argument('--lam2', default=0.5, type=float, 90 | help='weight of dual alignment') 91 | parser.add_argument('--normalized', action='store_true', 92 | help='add a normalization layer to encoder and predictor') 93 | 94 | def setup_seed(seed): 95 | torch.manual_seed(seed) 96 | torch.cuda.manual_seed_all(seed) 97 | torch.cuda.manual_seed(seed) 98 | np.random.seed(seed) 99 | random.seed(seed) 100 | cudnn.deterministic = True 101 | #cudnn.benchmark = False 102 | #cudnn.enabled = False 103 | 104 | setup_seed(892101) 105 | 106 | def _init_fn(worker_id): 107 | random.seed(10 + worker_id) 108 | np.random.seed(10 + worker_id) 109 | torch.manual_seed(10 + worker_id) 110 | torch.cuda.manual_seed(10 + worker_id) 111 | torch.cuda.manual_seed_all(10 + worker_id) 112 | 113 | def main(): 114 | args = parser.parse_args() 115 | 116 | if args.seed is not None: 117 | random.seed(args.seed) 118 | torch.manual_seed(args.seed) 119 | cudnn.deterministic = True 120 | warnings.warn('You have chosen to seed training. ' 121 | 'This will turn on the CUDNN deterministic setting, ' 122 | 'which can slow down your training considerably! ' 123 | 'You may see unexpected behavior when restarting ' 124 | 'from checkpoints.') 125 | 126 | if args.gpu is not None: 127 | warnings.warn('You have chosen a specific GPU. This will completely ' 128 | 'disable data parallelism.') 129 | 130 | if args.dist_url == "env://" and args.world_size == -1: 131 | args.world_size = int(os.environ["WORLD_SIZE"]) 132 | 133 | args.distributed = args.world_size > 1 or args.multiprocessing_distributed 134 | 135 | ngpus_per_node = torch.cuda.device_count() 136 | if args.multiprocessing_distributed: 137 | # Since we have ngpus_per_node processes per node, the total world_size 138 | # needs to be adjusted accordingly 139 | args.world_size = ngpus_per_node * args.world_size 140 | # Use torch.multiprocessing.spawn to launch distributed processes: the 141 | # main_worker process function 142 | mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) 143 | else: 144 | # Simply call main_worker function 145 | main_worker(args.gpu, ngpus_per_node, args) 146 | 147 | 148 | def main_worker(gpu, ngpus_per_node, args): 149 | args.gpu = gpu 150 | 151 | # suppress printing if not master 152 | if args.multiprocessing_distributed and args.gpu != 0: 153 | def print_pass(*args): 154 | pass 155 | builtins.print = print_pass 156 | 157 | if args.gpu is not None: 158 | print("Use GPU: {} for training".format(args.gpu)) 159 | 160 | if args.distributed: 161 | if args.dist_url == "env://" and args.rank == -1: 162 | args.rank = int(os.environ["RANK"]) 163 | if args.multiprocessing_distributed: 164 | # For multiprocessing distributed training, rank needs to be the 165 | # global rank among all the processes 166 | args.rank = args.rank * ngpus_per_node + gpu 167 | dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, 168 | world_size=args.world_size, rank=args.rank) 169 | # create model 170 | model = OPA_supervised_lifted.BlurContrastiveModelPair(input_dim=args.inputdim) 171 | 172 | if args.distributed: 173 | # For multiprocessing distributed, DistributedDataParallel constructor 174 | # should always set the single device scope, otherwise, 175 | # DistributedDataParallel will use all available devices. 176 | if args.gpu is not None: 177 | torch.cuda.set_device(args.gpu) 178 | model.cuda(args.gpu) 179 | # When using a single GPU per process and per 180 | # DistributedDataParallel, we need to divide the batch size 181 | # ourselves based on the total number of GPUs we have 182 | args.batch_size = int(args.batch_size / ngpus_per_node) 183 | args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) 184 | model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) 185 | else: 186 | model.cuda() 187 | # DistributedDataParallel will divide and allocate batch_size to all 188 | # available GPUs if device_ids are not set 189 | model = torch.nn.parallel.DistributedDataParallel(model) 190 | elif args.gpu is not None: 191 | torch.cuda.set_device(args.gpu) 192 | model = model.cuda(args.gpu) 193 | # comment out the following line for debugging 194 | # raise NotImplementedError("Only DistributedDataParallel is supported.") 195 | else: 196 | # AllGather implementation (batch shuffle, queue update, etc.) in 197 | # this code only supports DistributedDataParallel. 198 | raise NotImplementedError("Only DistributedDataParallel is supported.") 199 | 200 | # define loss function (criterion) and optimizer 201 | #criterion = nn.CrossEntropyLoss().cuda(args.gpu) 202 | # criterion = OPA.BlurContrastiveLossPair() 203 | 204 | # optimizer = torch.optim.SGD(model.parameters(), args.lr, 205 | # momentum=args.momentum, 206 | # weight_decay=args.weight_decay) 207 | optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, 208 | weight_decay=args.weight_decay) 209 | 210 | # optionally resume from a checkpoint 211 | if args.resume: 212 | if os.path.isfile(args.resume): 213 | print("=> loading checkpoint '{}'".format(args.resume)) 214 | if args.gpu is None: 215 | checkpoint = torch.load(args.resume) 216 | else: 217 | # Map model to be loaded to specified single gpu. 218 | loc = 'cuda:{}'.format(args.gpu) 219 | checkpoint = torch.load(args.resume, map_location=loc) 220 | args.start_epoch = checkpoint['epoch'] 221 | model.load_state_dict(checkpoint['state_dict']) 222 | optimizer.load_state_dict(checkpoint['optimizer']) 223 | print("=> loaded checkpoint '{}' (epoch {})" 224 | .format(args.resume, checkpoint['epoch'])) 225 | else: 226 | print("=> no checkpoint found at '{}'".format(args.resume)) 227 | 228 | cudnn.benchmark = True 229 | 230 | # Data loading code 231 | #traindir = os.path.join(args.data, 'train') 232 | datasetall = dataset.SequenceDataset(args.data) #, test_dataset 233 | #train_dataset = datasetall.traindata 234 | 235 | if args.distributed: 236 | train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) 237 | else: 238 | train_sampler = None 239 | 240 | train_loader = torch.utils.data.DataLoader( 241 | datasetall.traindata, batch_size=args.batch_size, shuffle=(train_sampler is None), 242 | num_workers=args.workers, pin_memory=True, sampler=train_sampler, drop_last=True, worker_init_fn=_init_fn) 243 | 244 | traintest_loader = torch.utils.data.DataLoader( 245 | datasetall.traindata, batch_size=datasetall.traindata.__len__(), shuffle=False, 246 | num_workers=args.workers, pin_memory=True, worker_init_fn=_init_fn) 247 | 248 | test_loader = torch.utils.data.DataLoader( 249 | datasetall.testdata, batch_size=1, shuffle=False, 250 | num_workers=args.workers, pin_memory=True, worker_init_fn=_init_fn) 251 | 252 | #os.mkdir(args.save_dir) 253 | all_epoch_time = [] 254 | 255 | for epoch in range(args.start_epoch, args.epochs): 256 | if args.distributed: 257 | train_sampler.set_epoch(epoch) 258 | adjust_learning_rate(optimizer, epoch, args) 259 | 260 | # train for one epoch 261 | #with torch.autograd.set_detect_anomaly(True): 262 | # train(train_loader, model, optimizer, epoch, args) #, criterion 263 | epochstart = time.time() 264 | train(train_loader, model, optimizer, epoch, args) #, criterion 265 | epochend = time.time() 266 | all_epoch_time.append(epochend-epochstart) 267 | 268 | if epoch%10==0 and epoch>40: # 269 | acc1 = validate(test_loader, traintest_loader, model, args) 270 | 271 | if not args.multiprocessing_distributed or (args.multiprocessing_distributed 272 | and args.rank % ngpus_per_node == 0): 273 | save_checkpoint({ 274 | 'epoch': epoch + 1, 275 | #'arch': args.arch, 276 | 'state_dict': model.state_dict(), 277 | 'optimizer' : optimizer.state_dict(), 278 | }, is_best=False, filename=osp.join(args.savedir, 'checkpoint_{:04d}.pth.tar'.format(epoch))) 279 | #fpath=osp.join(args.save_dir, 'ckp_ep' + str(epoch + 1) + '.pth.tar') 280 | print(all_epoch_time) 281 | print(sum(all_epoch_time)) 282 | 283 | 284 | def train(train_loader, model, optimizer, epoch, args): #, criterion 285 | batch_time = AverageMeter('Time', ':6.3f') 286 | data_time = AverageMeter('Data', ':6.3f') 287 | losses = AverageMeter('Loss', ':.4e') 288 | # top1 = AverageMeter('Acc@1', ':6.2f') 289 | # top5 = AverageMeter('Acc@5', ':6.2f') 290 | progress = ProgressMeter( 291 | len(train_loader), 292 | [batch_time, data_time, losses], 293 | prefix="Epoch: [{}]".format(epoch)) 294 | 295 | # switch to train mode 296 | model.train() 297 | 298 | end = time.time() 299 | for i, (inputs, labels, lens) in enumerate(train_loader): 300 | # measure data loading time 301 | data_time.update(time.time() - end) 302 | 303 | if args.gpu is not None: 304 | inputs = inputs.cuda(args.gpu, non_blocking=True) 305 | labels = labels.cuda(args.gpu, non_blocking=True) 306 | lens = lens.cuda(args.gpu, non_blocking=True) 307 | 308 | # compute output 309 | loss = model(inputs, lens, labels) 310 | losses.update(loss.item(), inputs.size(0)) 311 | # top1.update(acc1[0], inputs.size(0)) 312 | # top5.update(acc5[0], inputs.size(0)) 313 | 314 | # compute gradient and do SGD step 315 | optimizer.zero_grad() 316 | loss.backward() 317 | optimizer.step() 318 | 319 | # measure elapsed time 320 | batch_time.update(time.time() - end) 321 | end = time.time() 322 | 323 | if i % args.print_freq == 0: 324 | progress.display(i) 325 | 326 | def validate(val_loader, traintest_loader, model, args): 327 | test_batch_time = AverageMeter('Time', ':6.3f') 328 | #losses = AverageMeter('Loss', ':.4e') 329 | map1 = AverageMeter('Map@1', ':6.2f') 330 | top1 = AverageMeter('Acc@1', ':6.2f') 331 | top3 = AverageMeter('Acc@3', ':6.2f') 332 | top5 = AverageMeter('Acc@5', ':6.2f') 333 | top7 = AverageMeter('Acc@7', ':6.2f') 334 | top15 = AverageMeter('Acc@15', ':6.2f') 335 | top30 = AverageMeter('Acc@30', ':6.2f') 336 | progress = ProgressMeter( 337 | len(val_loader), 338 | [test_batch_time, map1, top1, top3, top5, top7, top15, top30], 339 | prefix='Test: ') 340 | 341 | # switch to evaluate mode 342 | model.eval() 343 | 344 | with torch.no_grad(): 345 | end = time.time() 346 | for j, (trinputs, trlabels, trlens) in enumerate(traintest_loader): 347 | #print(j) 348 | if args.gpu is not None: 349 | trinputs = trinputs.cuda(args.gpu, non_blocking=True) 350 | labels2 = trlabels.cuda(args.gpu, non_blocking=True) 351 | lens2 = trlens.cuda(args.gpu, non_blocking=True) 352 | #seq2 = trinputs #model.encoder(trinputs) 353 | seq2 = model.encoder(trinputs) 354 | R2 = model.getlen(seq2,lens2) #.cuda(args.gpu, non_blocking=True) 355 | 356 | trnum = seq2.size(0) 357 | trlabels = trlabels.numpy().reshape(-1) 358 | #print(trlabels) 359 | for i, (inputs, labels, lens) in enumerate(val_loader): 360 | if args.gpu is not None: 361 | inputs = inputs.cuda(args.gpu, non_blocking=True) 362 | labels = labels.cuda(args.gpu, non_blocking=True) 363 | lens = lens.cuda(args.gpu, non_blocking=True) 364 | # compute output 365 | #seq1 = inputs #model.encoder(inputs) 366 | seq1 = model.encoder(inputs) 367 | seq1 = seq1[:,0:lens[0],:] 368 | R1 = model.getlen(seq1,lens) #.cuda(args.gpu, non_blocking=True) 369 | # print(R1.device) 370 | # print(seq1.device) 371 | tenum = seq1.size(0) 372 | assert(tenum==1) 373 | dismat = np.zeros(trnum) 374 | for trcount in range(trnum): 375 | seqtr = seq2[trcount,0:lens2[trcount],:].unsqueeze(0) 376 | Rtr = R2[trcount,0:lens2[trcount]].unsqueeze(0) 377 | # print(Rtr.device) 378 | # print(seqtr.device) 379 | _,dis = model.alignment(seq1,seqtr,R1,Rtr) 380 | #dis = dis.view(-1) 381 | #print(dis) 382 | dismat[trcount] = dis[0].item() 383 | 384 | labels = labels[0].item() 385 | #print(labels) 386 | knn_idx_full = dismat.argsort() 387 | #print(knn_idx_full) 388 | acc = np.zeros(6) 389 | count = 0 390 | for k in [1, 3, 5, 7, 15, 30]: 391 | knn_idx = knn_idx_full[0:k] #[:, :k] 392 | knn_labels = trlabels[knn_idx] 393 | #print(knn_labels) 394 | mode_data = mode(knn_labels) #, axis=1 395 | mode_label = mode_data[0] 396 | #print(mode_label[0]) 397 | #print(mode_data) 398 | if mode_label[0]==labels: 399 | acc[count] = acc[count]+100 400 | count = count + 1 401 | 402 | tru_label = np.where(trlabels==labels,1,0).reshape(-1) 403 | #print(tru_label) 404 | ap = sklearn.metrics.average_precision_score(tru_label,-dismat.reshape(-1)) 405 | 406 | # measure accuracy and record loss 407 | map1.update(ap, inputs.size(0)) 408 | top1.update(acc[0], inputs.size(0)) 409 | top3.update(acc[1], inputs.size(0)) 410 | top5.update(acc[2], inputs.size(0)) 411 | top7.update(acc[3], inputs.size(0)) 412 | top15.update(acc[4], inputs.size(0)) 413 | top30.update(acc[5], inputs.size(0)) 414 | 415 | # measure elapsed time 416 | test_batch_time.update(time.time() - end) 417 | end = time.time() 418 | 419 | if i % args.print_freq == 0: 420 | progress.display(i) 421 | 422 | # TODO: this should also be done with the ProgressMeter 423 | print(' * Map@1 {map1.avg:.3f} Acc@1 {top1.avg:.3f} Acc@3 {top3.avg:.3f} Acc@5 {top5.avg:.3f} Acc@7 {top7.avg:.3f} Acc@15 {top15.avg:.3f} Acc@30 {top30.avg:.3f}' 424 | .format(map1=map1, top1=top1, top3=top3, top5=top5, top7=top7, top15=top15, top30=top30)) 425 | 426 | return top1.avg 427 | 428 | 429 | def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): 430 | torch.save(state, filename) 431 | if is_best: 432 | shutil.copyfile(filename, 'model_best.pth.tar') 433 | 434 | 435 | class AverageMeter(object): 436 | """Computes and stores the average and current value""" 437 | def __init__(self, name, fmt=':f'): 438 | self.name = name 439 | self.fmt = fmt 440 | self.reset() 441 | 442 | def reset(self): 443 | self.val = 0 444 | self.avg = 0 445 | self.sum = 0 446 | self.count = 0 447 | 448 | def update(self, val, n=1): 449 | self.val = val 450 | self.sum += val * n 451 | self.count += n 452 | self.avg = self.sum / self.count 453 | 454 | def __str__(self): 455 | fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' 456 | return fmtstr.format(**self.__dict__) 457 | 458 | 459 | class ProgressMeter(object): 460 | def __init__(self, num_batches, meters, prefix=""): 461 | self.batch_fmtstr = self._get_batch_fmtstr(num_batches) 462 | self.meters = meters 463 | self.prefix = prefix 464 | 465 | def display(self, batch): 466 | entries = [self.prefix + self.batch_fmtstr.format(batch)] 467 | entries += [str(meter) for meter in self.meters] 468 | print('\t'.join(entries)) 469 | 470 | def _get_batch_fmtstr(self, num_batches): 471 | num_digits = len(str(num_batches // 1)) 472 | fmt = '{:' + str(num_digits) + 'd}' 473 | return '[' + fmt + '/' + fmt.format(num_batches) + ']' 474 | 475 | 476 | def adjust_learning_rate(optimizer, epoch, args): 477 | """Decay the learning rate based on schedule""" 478 | lr = args.lr 479 | if args.cos: # cosine lr schedule 480 | lr *= 0.5 * (1. + math.cos(math.pi * epoch / args.epochs)) 481 | else: # stepwise lr schedule 482 | for milestone in args.schedule: 483 | lr *= 0.1 if epoch >= milestone else 1. 484 | for param_group in optimizer.param_groups: 485 | param_group['lr'] = lr 486 | 487 | 488 | def accuracy(output, target, topk=(1,)): 489 | """Computes the accuracy over the k top predictions for the specified values of k""" 490 | with torch.no_grad(): 491 | maxk = max(topk) 492 | batch_size = target.size(0) 493 | 494 | _, pred = output.topk(maxk, 1, True, True) 495 | pred = pred.t() 496 | correct = pred.eq(target.view(1, -1).expand_as(pred)) 497 | 498 | res = [] 499 | for k in topk: 500 | correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) 501 | res.append(correct_k.mul_(100.0 / batch_size)) 502 | return res 503 | 504 | 505 | if __name__ == '__main__': 506 | main() 507 | -------------------------------------------------------------------------------- /Few_Shot_Learning/splits/ssv2_OTAM/val.txt: -------------------------------------------------------------------------------- 1 | val2/188509 2 | val9/154368 3 | val0/136644 4 | val10/84962 5 | val7/67199 6 | val10/18689 7 | val4/212494 8 | val5/193283 9 | val9/84062 10 | val9/210862 11 | val0/187386 12 | val7/216344 13 | val10/91410 14 | val0/153613 15 | val0/218684 16 | val3/45474 17 | val4/38403 18 | val10/120692 19 | val5/194528 20 | val7/217077 21 | val9/29714 22 | val2/96349 23 | val0/155421 24 | val0/97368 25 | val8/65760 26 | val4/204422 27 | val2/92321 28 | val11/199645 29 | val3/24601 30 | val10/29297 31 | val0/205658 32 | val0/156860 33 | val9/115602 34 | val6/79556 35 | val3/38859 36 | val7/144584 37 | val0/51707 38 | val0/71089 39 | val2/44143 40 | val4/199072 41 | val9/162156 42 | val3/39814 43 | val1/106056 44 | val10/198336 45 | val0/179694 46 | val1/181749 47 | val10/80667 48 | val0/110394 49 | val0/76196 50 | val10/38591 51 | val6/42073 52 | val9/55896 53 | val2/171164 54 | val0/115508 55 | val10/23496 56 | val1/159341 57 | val6/94666 58 | val10/203028 59 | val7/205408 60 | val6/18715 61 | val9/180652 62 | val7/156586 63 | val1/93447 64 | val3/61492 65 | val9/4026 66 | val9/193452 67 | val9/200877 68 | val10/175926 69 | val10/214723 70 | val0/112309 71 | val7/155691 72 | val9/190157 73 | val9/101424 74 | val0/218726 75 | val8/102174 76 | val9/64605 77 | val10/209962 78 | val3/76171 79 | val5/196201 80 | val6/97458 81 | val7/193653 82 | val11/62606 83 | val0/38574 84 | val2/23831 85 | val9/121362 86 | val10/151981 87 | val11/81420 88 | val4/115171 89 | val7/183839 90 | val2/220092 91 | val5/30050 92 | val8/141334 93 | val7/28427 94 | val0/196884 95 | val7/148937 96 | val10/127431 97 | val1/31921 98 | val4/159427 99 | val3/127573 100 | val7/76016 101 | val11/80347 102 | val9/112503 103 | val6/141058 104 | val4/42489 105 | val9/28855 106 | val10/6793 107 | val4/58042 108 | val3/52275 109 | val10/108421 110 | val10/185945 111 | val10/165313 112 | val1/140953 113 | val5/2100 114 | val8/8139 115 | val0/164137 116 | val6/101633 117 | val11/165871 118 | val2/68698 119 | val10/138179 120 | val3/3360 121 | val7/165912 122 | val2/71100 123 | val10/113226 124 | val9/68883 125 | val9/107584 126 | val10/59226 127 | val1/170515 128 | val4/193767 129 | val2/56511 130 | val9/218094 131 | val8/18563 132 | val6/75529 133 | val4/69727 134 | val8/178000 135 | val1/121872 136 | val9/172151 137 | val10/182930 138 | val3/146813 139 | val3/138903 140 | val0/12708 141 | val9/165708 142 | val0/188081 143 | val2/186354 144 | val10/28183 145 | val0/143460 146 | val6/193255 147 | val9/118858 148 | val11/29578 149 | val11/190430 150 | val3/73665 151 | val9/59391 152 | val3/74999 153 | val4/145252 154 | val3/137928 155 | val9/157456 156 | val10/195911 157 | val3/101461 158 | val9/73602 159 | val3/182914 160 | val2/117724 161 | val10/197925 162 | val10/65326 163 | val9/69126 164 | val11/124973 165 | val2/11824 166 | val10/158336 167 | val3/128280 168 | val10/104262 169 | val3/149185 170 | val10/52999 171 | val10/121139 172 | val3/160424 173 | val3/146846 174 | val3/121356 175 | val9/87991 176 | val10/215221 177 | val11/180769 178 | val0/143415 179 | val11/198272 180 | val9/81910 181 | val4/197939 182 | val4/137565 183 | val3/90621 184 | val3/208724 185 | val4/13192 186 | val9/141149 187 | val8/118833 188 | val10/24884 189 | val9/36868 190 | val10/124431 191 | val3/8537 192 | val9/145625 193 | val2/149795 194 | val8/3161 195 | val10/35408 196 | val0/189155 197 | val3/131816 198 | val3/210479 199 | val2/120656 200 | val0/22336 201 | val1/134459 202 | val9/160867 203 | val1/100405 204 | val9/100086 205 | val1/62788 206 | val0/120890 207 | val4/173718 208 | val1/63166 209 | val1/123947 210 | val10/151187 211 | val4/193677 212 | val9/52648 213 | val11/166033 214 | val9/69722 215 | val10/43144 216 | val1/177748 217 | val4/85695 218 | val4/64966 219 | val10/154560 220 | val4/40398 221 | val10/5140 222 | val10/64006 223 | val10/179339 224 | val3/95874 225 | val11/65311 226 | val9/197747 227 | val9/141662 228 | val7/104280 229 | val10/154331 230 | val5/9806 231 | val10/148258 232 | val9/46158 233 | val0/164606 234 | val0/74468 235 | val7/60396 236 | val7/152912 237 | val7/133798 238 | val2/92474 239 | val7/184196 240 | val1/27055 241 | val9/41412 242 | val3/198362 243 | val8/102700 244 | val8/47791 245 | val3/23102 246 | val10/208891 247 | val7/114106 248 | val9/79045 249 | val11/74098 250 | val2/113617 251 | val10/173497 252 | val3/194875 253 | val10/206260 254 | val8/77527 255 | val0/166871 256 | val2/120918 257 | val3/32372 258 | val10/126202 259 | val7/130349 260 | val7/146757 261 | val9/46676 262 | val2/111545 263 | val9/173035 264 | val7/202294 265 | val10/117856 266 | val11/19621 267 | val10/162387 268 | val0/197061 269 | val8/204213 270 | val10/132505 271 | val6/160877 272 | val0/195066 273 | val6/46838 274 | val11/124756 275 | val7/142768 276 | val6/66568 277 | val9/3962 278 | val6/127428 279 | val11/19600 280 | val9/176808 281 | val3/102036 282 | val10/741 283 | val0/67804 284 | val11/158708 285 | val6/190411 286 | val2/154849 287 | val11/120781 288 | val2/115383 289 | val6/214534 290 | val6/22430 291 | val3/6102 292 | val3/130289 293 | val9/186395 294 | val9/136201 295 | val3/217957 296 | val7/67855 297 | val0/141208 298 | val10/147079 299 | val7/67854 300 | val10/43276 301 | val0/87807 302 | val2/63580 303 | val1/5538 304 | val11/103151 305 | val3/147065 306 | val0/189590 307 | val3/29699 308 | val10/113152 309 | val11/46143 310 | val10/46963 311 | val0/121267 312 | val3/183926 313 | val10/105671 314 | val1/45350 315 | val10/134267 316 | val6/91894 317 | val10/130213 318 | val10/105091 319 | val10/97082 320 | val11/149542 321 | val9/51761 322 | val10/72552 323 | val9/122777 324 | val3/49280 325 | val1/113364 326 | val3/127980 327 | val9/170528 328 | val7/193671 329 | val3/213350 330 | val1/96842 331 | val9/70536 332 | val10/179512 333 | val2/42640 334 | val9/37666 335 | val1/105208 336 | val10/181858 337 | val3/53974 338 | val2/182974 339 | val10/38922 340 | val9/128832 341 | val7/39512 342 | val10/173811 343 | val10/138482 344 | val0/47387 345 | val10/160765 346 | val5/50150 347 | val11/55818 348 | val5/150419 349 | val0/206004 350 | val9/41495 351 | val0/145765 352 | val3/68352 353 | val10/108505 354 | val11/147885 355 | val1/112889 356 | val0/78265 357 | val0/66983 358 | val10/111502 359 | val10/1241 360 | val2/76073 361 | val3/120887 362 | val10/6973 363 | val10/91802 364 | val10/46005 365 | val10/201616 366 | val0/123668 367 | val1/143272 368 | val10/79097 369 | val3/65485 370 | val10/76825 371 | val10/202581 372 | val4/82068 373 | val0/139227 374 | val0/21812 375 | val0/111358 376 | val10/60583 377 | val0/218316 378 | val8/206521 379 | val9/96234 380 | val10/159438 381 | val9/13147 382 | val10/127704 383 | val8/114755 384 | val0/24643 385 | val3/187593 386 | val3/220500 387 | val2/200343 388 | val1/85196 389 | val4/131883 390 | val8/138724 391 | val6/31455 392 | val9/37195 393 | val2/56463 394 | val9/14204 395 | val5/64291 396 | val10/13558 397 | val3/93464 398 | val4/24807 399 | val0/16041 400 | val10/137184 401 | val1/163458 402 | val3/133940 403 | val10/34926 404 | val11/38045 405 | val0/79888 406 | val5/41326 407 | val0/125693 408 | val11/175704 409 | val10/207394 410 | val10/94899 411 | val2/167504 412 | val10/191567 413 | val9/71345 414 | val10/141321 415 | val6/89713 416 | val0/132724 417 | val9/115917 418 | val10/68993 419 | val4/160141 420 | val3/216535 421 | val10/211062 422 | val11/129899 423 | val6/139264 424 | val9/142546 425 | val9/194068 426 | val3/63861 427 | val10/163545 428 | val9/36281 429 | val4/65851 430 | val7/142814 431 | val2/155545 432 | val9/119718 433 | val2/212796 434 | val3/73021 435 | val1/108727 436 | val7/108484 437 | val9/162835 438 | val4/138381 439 | val10/71016 440 | val6/199375 441 | val7/116063 442 | val9/147091 443 | val10/2784 444 | val9/68397 445 | val6/14361 446 | val9/64279 447 | val11/201433 448 | val0/209695 449 | val10/187937 450 | val2/163636 451 | val6/186256 452 | val7/171965 453 | val1/196683 454 | val1/9108 455 | val10/64907 456 | val11/157575 457 | val0/127077 458 | val3/204534 459 | val10/105680 460 | val0/90208 461 | val9/109321 462 | val9/205989 463 | val10/45417 464 | val2/189395 465 | val0/55078 466 | val0/156289 467 | val0/178430 468 | val3/106562 469 | val9/58692 470 | val7/183205 471 | val1/77758 472 | val9/219388 473 | val3/183188 474 | val10/140211 475 | val6/132312 476 | val9/183368 477 | val7/174342 478 | val2/185853 479 | val10/2791 480 | val7/210640 481 | val1/86186 482 | val3/77290 483 | val8/118632 484 | val6/171590 485 | val10/15944 486 | val10/193353 487 | val2/137779 488 | val7/179122 489 | val9/109666 490 | val10/75918 491 | val11/186553 492 | val5/59005 493 | val10/176204 494 | val5/95180 495 | val9/90483 496 | val1/57802 497 | val10/121410 498 | val10/26785 499 | val9/182749 500 | val10/81790 501 | val0/23188 502 | val0/25484 503 | val10/211101 504 | val10/37072 505 | val7/94872 506 | val8/73965 507 | val0/24378 508 | val7/31431 509 | val2/178388 510 | val4/40514 511 | val0/28970 512 | val0/26287 513 | val2/36373 514 | val10/168726 515 | val11/169827 516 | val3/137165 517 | val8/118939 518 | val7/97768 519 | val8/180612 520 | val10/63348 521 | val7/17700 522 | val3/14538 523 | val10/177582 524 | val4/95527 525 | val3/12636 526 | val11/154287 527 | val7/77429 528 | val9/57791 529 | val6/19995 530 | val10/29666 531 | val10/40294 532 | val10/631 533 | val6/180526 534 | val0/193142 535 | val6/154890 536 | val3/15647 537 | val6/51673 538 | val3/159935 539 | val2/28396 540 | val10/164486 541 | val10/178006 542 | val1/110044 543 | val6/64335 544 | val0/70040 545 | val9/22286 546 | val0/81218 547 | val10/41762 548 | val3/90874 549 | val11/197861 550 | val0/15014 551 | val6/135815 552 | val0/144605 553 | val1/81138 554 | val11/128115 555 | val3/125171 556 | val8/164029 557 | val7/171353 558 | val9/119769 559 | val11/215973 560 | val3/101419 561 | val1/184459 562 | val0/31913 563 | val10/79426 564 | val10/141599 565 | val10/216370 566 | val1/152217 567 | val4/108358 568 | val9/159862 569 | val9/79101 570 | val5/176042 571 | val9/154155 572 | val6/9872 573 | val7/115639 574 | val6/95251 575 | val9/210074 576 | val5/161017 577 | val0/29928 578 | val3/133520 579 | val9/182427 580 | val9/132207 581 | val0/93331 582 | val5/35325 583 | val8/133998 584 | val6/128454 585 | val6/137685 586 | val11/34847 587 | val3/108687 588 | val0/134524 589 | val9/184241 590 | val7/99754 591 | val9/109534 592 | val10/40897 593 | val4/3843 594 | val2/40102 595 | val8/61091 596 | val11/158297 597 | val5/182398 598 | val1/218714 599 | val10/215561 600 | val0/208606 601 | val1/88916 602 | val10/76450 603 | val3/196227 604 | val3/147119 605 | val7/109644 606 | val10/156613 607 | val7/27419 608 | val1/56924 609 | val11/31941 610 | val0/152711 611 | val1/16115 612 | val4/49938 613 | val3/80418 614 | val1/94303 615 | val3/163017 616 | val10/39260 617 | val6/200337 618 | val10/60260 619 | val10/183206 620 | val1/43127 621 | val9/178806 622 | val7/82095 623 | val7/202709 624 | val7/195646 625 | val11/103203 626 | val10/171435 627 | val9/124183 628 | val10/157430 629 | val4/111999 630 | val9/190881 631 | val0/58095 632 | val11/53661 633 | val3/73066 634 | val0/11541 635 | val10/78199 636 | val7/118843 637 | val11/26736 638 | val10/160784 639 | val6/120624 640 | val11/30813 641 | val11/202431 642 | val4/141433 643 | val10/141287 644 | val2/23997 645 | val4/130721 646 | val9/149523 647 | val3/206241 648 | val0/197890 649 | val10/210220 650 | val0/61763 651 | val9/120100 652 | val9/110393 653 | val0/7211 654 | val7/88723 655 | val1/64481 656 | val0/117110 657 | val9/56496 658 | val7/184845 659 | val3/187789 660 | val9/180618 661 | val10/24427 662 | val4/215257 663 | val10/157925 664 | val10/144230 665 | val3/174556 666 | val3/61119 667 | val9/150905 668 | val10/21906 669 | val0/155172 670 | val0/209907 671 | val2/213842 672 | val2/57483 673 | val10/152618 674 | val10/174923 675 | val9/132708 676 | val10/97247 677 | val6/87650 678 | val3/146754 679 | val1/129272 680 | val10/198752 681 | val8/185685 682 | val1/2173 683 | val6/6540 684 | val10/8806 685 | val4/148628 686 | val10/188939 687 | val4/208338 688 | val10/131264 689 | val9/159752 690 | val2/83805 691 | val9/214921 692 | val6/101353 693 | val10/106347 694 | val5/189604 695 | val0/23135 696 | val9/132948 697 | val11/77033 698 | val1/175172 699 | val10/49031 700 | val4/45641 701 | val9/208993 702 | val9/91948 703 | val9/70350 704 | val11/159108 705 | val5/43733 706 | val0/22285 707 | val3/24929 708 | val10/29105 709 | val3/203238 710 | val8/85113 711 | val3/218666 712 | val0/1985 713 | val9/130223 714 | val4/107207 715 | val2/145710 716 | val7/72436 717 | val2/180749 718 | val3/107951 719 | val10/205405 720 | val0/159519 721 | val3/56105 722 | val10/183503 723 | val0/197222 724 | val5/1586 725 | val0/144118 726 | val0/179100 727 | val6/167092 728 | val9/103182 729 | val9/88030 730 | val10/36265 731 | val7/196133 732 | val3/24020 733 | val7/144789 734 | val7/190590 735 | val10/212010 736 | val9/116450 737 | val10/159158 738 | val4/123372 739 | val2/108403 740 | val0/165685 741 | val4/138944 742 | val8/43830 743 | val10/109098 744 | val10/40411 745 | val3/153558 746 | val2/109499 747 | val0/109375 748 | val0/97909 749 | val0/159965 750 | val0/84238 751 | val0/188271 752 | val10/146707 753 | val10/137252 754 | val0/76930 755 | val10/111996 756 | val7/63735 757 | val2/161969 758 | val4/92090 759 | val9/64823 760 | val6/132628 761 | val10/83218 762 | val11/202459 763 | val10/44330 764 | val1/2205 765 | val4/166502 766 | val5/20656 767 | val4/140877 768 | val0/11048 769 | val9/189274 770 | val11/126991 771 | val2/28102 772 | val3/106909 773 | val8/94601 774 | val4/193243 775 | val1/31359 776 | val0/117036 777 | val2/65006 778 | val5/185459 779 | val3/157854 780 | val10/185208 781 | val8/81534 782 | val0/167027 783 | val4/109810 784 | val10/34383 785 | val4/173494 786 | val6/209944 787 | val10/99546 788 | val9/46889 789 | val5/119640 790 | val4/142109 791 | val10/184270 792 | val7/69091 793 | val9/114926 794 | val7/113300 795 | val6/149528 796 | val4/35850 797 | val0/65733 798 | val10/197913 799 | val9/191340 800 | val9/150873 801 | val8/82410 802 | val5/133996 803 | val11/44726 804 | val4/50494 805 | val9/151775 806 | val10/194603 807 | val0/95225 808 | val10/143023 809 | val9/156528 810 | val10/149504 811 | val3/204109 812 | val3/26526 813 | val11/204122 814 | val3/212950 815 | val7/41637 816 | val3/47782 817 | val0/187397 818 | val7/24859 819 | val7/124791 820 | val6/183975 821 | val7/75462 822 | val3/6750 823 | val10/44432 824 | val0/127266 825 | val3/172134 826 | val3/150548 827 | val9/158203 828 | val9/135350 829 | val9/63325 830 | val0/145496 831 | val6/698 832 | val7/115399 833 | val9/213766 834 | val7/67977 835 | val3/181428 836 | val4/162094 837 | val11/85427 838 | val10/152197 839 | val8/123620 840 | val3/200106 841 | val3/214030 842 | val3/152272 843 | val9/16181 844 | val7/87956 845 | val0/118778 846 | val0/97946 847 | val11/94328 848 | val8/211963 849 | val4/70814 850 | val6/100159 851 | val9/187784 852 | val7/115040 853 | val6/155995 854 | val2/99506 855 | val10/149435 856 | val2/91117 857 | val10/195652 858 | val3/174516 859 | val1/10925 860 | val5/167472 861 | val10/65251 862 | val10/75325 863 | val8/55783 864 | val9/11551 865 | val11/94920 866 | val10/72826 867 | val10/154077 868 | val10/31268 869 | val0/117779 870 | val2/199236 871 | val3/92374 872 | val10/89533 873 | val1/51877 874 | val10/217346 875 | val1/129921 876 | val5/619 877 | val3/182027 878 | val3/109252 879 | val6/90398 880 | val10/182059 881 | val5/58475 882 | val10/160050 883 | val7/214642 884 | val10/57055 885 | val9/34256 886 | val1/168047 887 | val11/53422 888 | val3/131185 889 | val1/115929 890 | val2/76055 891 | val3/208801 892 | val9/110857 893 | val4/139092 894 | val6/53740 895 | val7/85067 896 | val3/99735 897 | val0/137870 898 | val0/119063 899 | val1/8659 900 | val1/46173 901 | val5/212458 902 | val6/125075 903 | val10/183991 904 | val1/68350 905 | val9/626 906 | val2/111033 907 | val6/41735 908 | val3/220781 909 | val11/664 910 | val11/119313 911 | val3/86141 912 | val6/91071 913 | val9/11922 914 | val9/65680 915 | val2/90103 916 | val3/218826 917 | val6/143931 918 | val4/7278 919 | val9/138712 920 | val9/90642 921 | val11/1730 922 | val7/105481 923 | val3/85616 924 | val3/54579 925 | val7/156130 926 | val11/29141 927 | val9/206949 928 | val3/84939 929 | val9/42516 930 | val6/38431 931 | val10/30495 932 | val0/45900 933 | val11/4407 934 | val10/157675 935 | val8/85702 936 | val3/77722 937 | val4/180103 938 | val9/115644 939 | val4/177786 940 | val6/23550 941 | val9/102318 942 | val10/105994 943 | val0/169828 944 | val0/151912 945 | val0/54444 946 | val0/4540 947 | val3/141650 948 | val0/38082 949 | val2/119643 950 | val2/179641 951 | val6/53635 952 | val10/208894 953 | val7/205653 954 | val0/51335 955 | val9/193480 956 | val4/219102 957 | val6/34555 958 | val6/37619 959 | val0/69772 960 | val10/24441 961 | val0/11373 962 | val10/6460 963 | val0/105094 964 | val7/210649 965 | val10/62486 966 | val8/36824 967 | val0/177478 968 | val6/95608 969 | val6/52869 970 | val3/62750 971 | val0/202252 972 | val9/59063 973 | val10/45193 974 | val9/78690 975 | val0/100174 976 | val0/70789 977 | val6/46713 978 | val9/76962 979 | val4/104269 980 | val3/193177 981 | val3/86780 982 | val9/48119 983 | val0/8187 984 | val3/195308 985 | val4/89759 986 | val10/123131 987 | val8/11080 988 | val11/107759 989 | val10/156825 990 | val3/76344 991 | val0/195827 992 | val6/66647 993 | val5/116292 994 | val7/59598 995 | val6/55151 996 | val10/179427 997 | val11/127607 998 | val6/157285 999 | val3/470 1000 | val10/15313 1001 | val0/125318 1002 | val6/146405 1003 | val2/169373 1004 | val10/213173 1005 | val9/30152 1006 | val4/28557 1007 | val4/131294 1008 | val5/45054 1009 | val11/122193 1010 | val10/117556 1011 | val4/176232 1012 | val2/15990 1013 | val9/8547 1014 | val3/157613 1015 | val7/25838 1016 | val10/162086 1017 | val9/215361 1018 | val3/145229 1019 | val0/35805 1020 | val3/170900 1021 | val7/105359 1022 | val9/177178 1023 | val3/196818 1024 | val7/214557 1025 | val3/128019 1026 | val3/132932 1027 | val6/27010 1028 | val3/134849 1029 | val8/7414 1030 | val9/196369 1031 | val3/78911 1032 | val2/140578 1033 | val6/14830 1034 | val1/36808 1035 | val10/57930 1036 | val10/129977 1037 | val3/92691 1038 | val2/76998 1039 | val10/83063 1040 | val0/30671 1041 | val11/189110 1042 | val9/168685 1043 | val11/152226 1044 | val4/143551 1045 | val3/90490 1046 | val9/17181 1047 | val8/120079 1048 | val7/129519 1049 | val1/11418 1050 | val3/135641 1051 | val10/65086 1052 | val11/62151 1053 | val6/209553 1054 | val4/55452 1055 | val0/154894 1056 | val11/65822 1057 | val1/16667 1058 | val1/101181 1059 | val10/169292 1060 | val3/125234 1061 | val3/196329 1062 | val7/188913 1063 | val0/89781 1064 | val5/188627 1065 | val7/201889 1066 | val9/83648 1067 | val10/178554 1068 | val0/46186 1069 | val0/124679 1070 | val4/80800 1071 | val0/170139 1072 | val11/30354 1073 | val8/219934 1074 | val7/114783 1075 | val6/17584 1076 | val0/73972 1077 | val0/122630 1078 | val8/13717 1079 | val8/69453 1080 | val6/163839 1081 | val7/204623 1082 | val5/21156 1083 | val10/201811 1084 | val10/217678 1085 | val3/5709 1086 | val0/38741 1087 | val0/215465 1088 | val2/142554 1089 | val9/209204 1090 | val9/151323 1091 | val6/40154 1092 | val10/154658 1093 | val11/83542 1094 | val2/40654 1095 | val9/143711 1096 | val11/62881 1097 | val10/52494 1098 | val9/181088 1099 | val3/27728 1100 | val9/200689 1101 | val10/164792 1102 | val11/83682 1103 | val1/114335 1104 | val0/66194 1105 | val10/12895 1106 | val3/59950 1107 | val0/98524 1108 | val3/157374 1109 | val4/199306 1110 | val10/201087 1111 | val4/5633 1112 | val3/40871 1113 | val9/204006 1114 | val6/198072 1115 | val0/88321 1116 | val9/200602 1117 | val0/120878 1118 | val10/153589 1119 | val1/23505 1120 | val10/197920 1121 | val2/170379 1122 | val11/1580 1123 | val7/81136 1124 | val9/95730 1125 | val2/62770 1126 | val10/41675 1127 | val9/195337 1128 | val2/194336 1129 | val0/204032 1130 | val0/189783 1131 | val9/190407 1132 | val10/52805 1133 | val10/62553 1134 | val3/82165 1135 | val0/123033 1136 | val1/58412 1137 | val3/199151 1138 | val0/196256 1139 | val11/58840 1140 | val7/12315 1141 | val9/98791 1142 | val10/27494 1143 | val6/197831 1144 | val0/126557 1145 | val7/167347 1146 | val3/152269 1147 | val9/195270 1148 | val9/108798 1149 | val10/102693 1150 | val4/97173 1151 | val9/3053 1152 | val9/85124 1153 | val11/73350 1154 | val9/56806 1155 | val2/84598 1156 | val6/211247 1157 | val11/75814 1158 | val3/156677 1159 | val9/157156 1160 | val2/124050 1161 | val0/25379 1162 | val3/3082 1163 | val3/85066 1164 | val3/66545 1165 | val3/115061 1166 | val7/72315 1167 | val2/49256 1168 | val3/191966 1169 | val0/134682 1170 | val9/18152 1171 | val7/65007 1172 | val7/154344 1173 | val1/179394 1174 | val9/120493 1175 | val0/62147 1176 | val3/15490 1177 | val9/91033 1178 | val3/69683 1179 | val0/210152 1180 | val2/107921 1181 | val10/41116 1182 | val5/11652 1183 | val3/161435 1184 | val5/53153 1185 | val3/124448 1186 | val7/57197 1187 | val3/119546 1188 | val9/152112 1189 | val6/123982 1190 | val3/178084 1191 | val10/52130 1192 | val6/83183 1193 | val3/11687 1194 | val0/4093 1195 | val11/84998 1196 | val10/204017 1197 | val0/141169 1198 | val4/55417 1199 | val6/34517 1200 | val11/214178 1201 | val9/21158 1202 | val4/34993 1203 | val10/144126 1204 | val1/140945 1205 | val10/64506 1206 | val1/36328 1207 | val10/217306 1208 | val1/202442 1209 | val8/114986 1210 | val3/62930 1211 | val10/218360 1212 | val3/95927 1213 | val6/96158 1214 | val11/38164 1215 | val3/44055 1216 | val3/55458 1217 | val10/62458 1218 | val10/186424 1219 | val9/88017 1220 | val8/116520 1221 | val9/110043 1222 | val2/30791 1223 | val0/107888 1224 | val6/91344 1225 | val7/191184 1226 | val3/174376 1227 | val10/32811 1228 | val9/68196 1229 | val3/144130 1230 | val10/217331 1231 | val0/30348 1232 | val8/100400 1233 | val6/140978 1234 | val9/23207 1235 | val7/14278 1236 | val10/72554 1237 | val9/112098 1238 | val1/92712 1239 | val10/122023 1240 | val1/19489 1241 | val4/43082 1242 | val3/38673 1243 | val10/199203 1244 | val10/195366 1245 | val0/72310 1246 | val2/12340 1247 | val10/176333 1248 | val10/189780 1249 | val11/78060 1250 | val1/200482 1251 | val9/142184 1252 | val3/107257 1253 | val9/38937 1254 | val10/158046 1255 | val0/116005 1256 | val9/99674 1257 | val0/150162 1258 | val4/1451 1259 | val0/195268 1260 | val6/91140 1261 | val0/208553 1262 | val4/37333 1263 | val11/74856 1264 | val9/106879 1265 | val8/47482 1266 | val2/194162 1267 | val10/17384 1268 | val10/199115 1269 | val9/32563 1270 | val10/100820 1271 | val0/31749 1272 | val2/8221 1273 | val10/43560 1274 | val6/210724 1275 | val10/220247 1276 | val5/162390 1277 | val10/182237 1278 | val4/83377 1279 | val7/1488 1280 | val0/29924 1281 | val0/219653 1282 | val2/86581 1283 | val3/86353 1284 | val4/178287 1285 | val4/36879 1286 | val9/119652 1287 | val10/64089 1288 | val0/64482 1289 | val3/72154 1290 | val5/88324 1291 | val3/49120 1292 | val11/49832 1293 | val9/199060 1294 | val10/7440 1295 | val9/211423 1296 | val10/28464 1297 | val9/11177 1298 | val10/86510 1299 | val1/220385 1300 | val11/72702 1301 | val10/75216 1302 | val2/79690 1303 | val0/190464 1304 | val9/79717 1305 | val1/88607 1306 | val3/109237 1307 | val1/123900 1308 | val0/66247 1309 | val10/182928 1310 | val1/51921 1311 | val2/30072 1312 | val4/152154 1313 | val7/171081 1314 | val0/81730 1315 | val10/113329 1316 | val0/113626 1317 | val0/93699 1318 | val7/100579 1319 | val7/64685 1320 | val10/28815 1321 | val3/146799 1322 | val0/117177 1323 | val0/98091 1324 | val9/126191 1325 | val0/6695 1326 | val7/79932 1327 | val0/164064 1328 | val4/113012 1329 | val3/58097 1330 | val10/73758 1331 | val3/178399 1332 | val7/75967 1333 | val3/187835 1334 | val9/83464 1335 | val8/32578 1336 | val9/26814 1337 | val4/199243 1338 | val1/30095 1339 | val10/167326 1340 | val3/75252 1341 | val4/34155 1342 | val9/65876 1343 | val11/214480 1344 | val9/175315 1345 | val7/4386 1346 | val3/8361 1347 | val2/143913 1348 | val4/218922 1349 | val4/40480 1350 | val3/105308 1351 | val7/176966 1352 | val0/198766 1353 | val9/173351 1354 | val9/127533 1355 | val8/11386 1356 | val3/100240 1357 | val1/196028 1358 | val9/64104 1359 | val10/218735 1360 | val2/110948 1361 | val11/102548 1362 | val3/206488 1363 | val9/131302 1364 | val3/126945 1365 | val9/79993 1366 | val1/64635 1367 | val9/195895 1368 | val2/2346 1369 | val11/122642 1370 | val7/23005 1371 | val0/167629 1372 | val6/140309 1373 | val3/120436 1374 | val1/122041 1375 | val2/108070 1376 | val10/30636 1377 | val3/162969 1378 | val0/149360 1379 | val9/159713 1380 | val0/99896 1381 | val1/61092 1382 | val10/114865 1383 | val9/76138 1384 | val4/186569 1385 | val9/26309 1386 | val10/48691 1387 | val10/46385 1388 | val7/20889 1389 | val1/114145 1390 | val9/56419 1391 | val11/20048 1392 | val10/209009 1393 | val0/200108 1394 | val3/77203 1395 | val0/175722 1396 | val10/180205 1397 | val5/144928 1398 | val10/156929 1399 | val3/213228 1400 | val10/36529 1401 | val6/160457 1402 | val10/134687 1403 | val7/216143 1404 | val10/197143 1405 | val0/122253 1406 | val10/174759 1407 | val9/162351 1408 | val0/121681 1409 | val7/172666 1410 | val0/172769 1411 | val1/205120 1412 | val10/177671 1413 | val10/37029 1414 | val3/8713 1415 | val10/122534 1416 | val2/191438 1417 | val0/216777 1418 | val0/70655 1419 | val4/7244 1420 | val3/27664 1421 | val3/16293 1422 | val10/75821 1423 | val10/188086 1424 | val10/101018 1425 | val9/69266 1426 | val1/96964 1427 | val10/200057 1428 | val0/147427 1429 | val10/15451 1430 | val10/218479 1431 | val10/184846 1432 | val6/152716 1433 | val4/17246 1434 | val7/9679 1435 | val3/220641 1436 | val0/78863 1437 | val10/8283 1438 | val3/50750 1439 | val10/69679 1440 | val10/73507 1441 | val4/76281 1442 | val1/48460 1443 | val6/36752 1444 | val0/10161 1445 | val2/24994 1446 | val6/162226 1447 | val11/49990 1448 | val5/137384 1449 | val8/115277 1450 | val10/4288 1451 | val10/137064 1452 | val10/44451 1453 | val1/205724 1454 | val9/71742 1455 | val9/28351 1456 | val3/137966 1457 | val11/80697 1458 | val3/27776 1459 | val10/154500 1460 | val0/127658 1461 | val4/183215 1462 | val0/40907 1463 | val9/34848 1464 | val0/2399 1465 | val7/1484 1466 | val0/107346 1467 | val3/34745 1468 | val5/8683 1469 | val3/54227 1470 | val3/42610 1471 | val10/60164 1472 | val6/74614 1473 | val5/106291 1474 | val9/31231 1475 | val11/194501 1476 | val10/114314 1477 | val7/94371 1478 | val2/11210 1479 | val11/51535 1480 | val0/39750 1481 | val4/5154 1482 | val11/166720 1483 | val9/145958 1484 | val0/207869 1485 | val0/38718 1486 | val0/172414 1487 | val3/6597 1488 | val3/60451 1489 | val4/66273 1490 | val0/108632 1491 | val6/163180 1492 | val11/44147 1493 | val8/208059 1494 | val6/125648 1495 | val10/126188 1496 | val0/96956 1497 | val2/88569 1498 | val9/205766 1499 | val1/125877 1500 | val8/18048 1501 | val8/150353 1502 | val4/180881 1503 | val2/177779 1504 | val9/117071 1505 | val7/119103 1506 | val0/182201 1507 | val2/141215 1508 | val9/79403 1509 | val7/161610 1510 | val0/72176 1511 | val10/145109 1512 | val11/109906 1513 | val9/163277 1514 | val10/77214 1515 | val10/111958 1516 | val4/170266 1517 | val10/60410 1518 | val9/207643 1519 | val1/215193 1520 | val8/140784 1521 | val6/158334 1522 | val10/205071 1523 | val0/122718 1524 | val7/132832 1525 | val9/210998 1526 | val7/45963 1527 | val9/105246 1528 | val11/61314 1529 | val0/148335 1530 | val0/144017 1531 | val10/33082 1532 | val1/154488 1533 | val0/37727 1534 | val11/27864 1535 | val10/114296 1536 | val0/220592 1537 | val9/141241 1538 | val10/126129 1539 | val9/84125 1540 | val9/168741 1541 | val9/94521 1542 | val0/157531 1543 | val9/1006 1544 | val0/160674 1545 | val8/7703 1546 | val2/84260 1547 | val0/205016 1548 | val10/100751 1549 | val10/128081 1550 | val3/139906 1551 | val3/4971 1552 | val3/130330 1553 | val10/70471 1554 | val3/51547 1555 | val3/169808 1556 | val3/136669 1557 | val10/56558 1558 | val7/165723 1559 | val7/34319 1560 | val10/30303 1561 | val4/86248 1562 | val10/50831 1563 | val2/84619 1564 | val8/24798 1565 | val0/118286 1566 | val0/132285 1567 | val2/140947 1568 | val8/18227 1569 | val1/17063 1570 | val11/175179 1571 | val7/67128 1572 | val7/82217 1573 | val10/139566 1574 | val6/90007 1575 | val9/1271 1576 | val4/132130 1577 | val3/215316 1578 | val9/44844 1579 | val10/15390 1580 | val3/183412 1581 | val10/118107 1582 | val1/153214 1583 | val7/68381 1584 | val4/27853 1585 | val9/147259 1586 | val1/189425 1587 | val2/95374 1588 | val8/31481 1589 | val10/43140 1590 | val3/158153 1591 | val2/80128 1592 | val4/95906 1593 | val3/177267 1594 | val10/53565 1595 | val7/177838 1596 | val7/115216 1597 | val4/204775 1598 | val10/85070 1599 | val2/164119 1600 | val10/219213 1601 | val0/105939 1602 | val9/76235 1603 | val10/108061 1604 | val6/27794 1605 | val10/39568 1606 | val9/39505 1607 | val5/143497 1608 | val3/56980 1609 | val5/164514 1610 | val3/182007 1611 | val10/9028 1612 | val5/127879 1613 | val0/133172 1614 | val10/24130 1615 | val6/33211 1616 | val3/98332 1617 | val10/168601 1618 | val7/118498 1619 | val0/22279 1620 | val0/160387 1621 | val10/172786 1622 | val0/68298 1623 | val10/165984 1624 | val3/14551 1625 | val11/19698 1626 | val5/93514 1627 | val9/114795 1628 | val0/48917 1629 | val10/72232 1630 | val6/26864 1631 | val4/154913 1632 | val3/82024 1633 | val3/65224 1634 | val10/210275 1635 | val10/199597 1636 | val0/194264 1637 | val0/50195 1638 | val11/197905 1639 | val10/108927 1640 | val10/129665 1641 | val10/119678 1642 | val10/113076 1643 | val3/140125 1644 | val7/147010 1645 | val3/112372 1646 | val8/11239 1647 | val10/76379 1648 | val9/176872 1649 | val0/52031 1650 | val0/97449 1651 | val7/199768 1652 | val11/123928 1653 | val11/30666 1654 | val3/143743 1655 | val6/121711 1656 | val3/85262 1657 | val10/54968 1658 | val9/47092 1659 | val2/216605 1660 | val7/51893 1661 | val2/7272 1662 | val0/102389 1663 | val10/189909 1664 | val10/55877 1665 | val4/185235 1666 | val1/125822 1667 | val1/49789 1668 | val0/65651 1669 | val3/113381 1670 | val2/71488 1671 | val0/109734 1672 | val0/40431 1673 | val6/178800 1674 | val2/101927 1675 | val8/62681 1676 | val2/1722 1677 | val9/60108 1678 | val9/135300 1679 | val0/187864 1680 | val1/33576 1681 | val9/28718 1682 | val10/182111 1683 | val3/200865 1684 | val1/131764 1685 | val10/43853 1686 | val2/29334 1687 | val1/93029 1688 | val6/107958 1689 | val9/108013 1690 | val5/148416 1691 | val10/195135 1692 | val6/13161 1693 | val10/214590 1694 | val10/86492 1695 | val10/200151 1696 | val9/197712 1697 | val5/167506 1698 | val3/142943 1699 | val1/72472 1700 | val6/65707 1701 | val10/126106 1702 | val1/135971 1703 | val7/133808 1704 | val9/34363 1705 | val11/219710 1706 | val4/29943 1707 | val5/63097 1708 | val10/122116 1709 | val3/25707 1710 | val7/128840 1711 | val7/188676 1712 | val3/151836 1713 | val10/115374 1714 | val1/34550 1715 | val10/163752 1716 | val0/141390 1717 | val10/73616 1718 | val9/73494 1719 | val3/35227 1720 | val9/194240 1721 | val0/135139 1722 | val0/184073 1723 | val10/219464 1724 | val6/209628 1725 | val9/131253 1726 | val4/187925 1727 | val10/64592 1728 | val1/188499 1729 | val2/28039 1730 | val2/120743 1731 | val0/60452 1732 | val4/32433 1733 | val10/72009 1734 | val0/44356 1735 | val3/50067 1736 | val8/15326 1737 | val9/22311 1738 | val9/4133 1739 | val9/91838 1740 | val6/161990 1741 | val5/218607 1742 | val8/45713 1743 | val9/124688 1744 | val9/74950 1745 | val7/41267 1746 | val0/93742 1747 | val2/206542 1748 | val6/148210 1749 | val5/18186 1750 | val7/47349 1751 | val10/21313 1752 | val0/195209 1753 | val6/18088 1754 | val4/182518 1755 | val1/85349 1756 | val11/118329 1757 | val3/52242 1758 | val6/108096 1759 | val3/142889 1760 | val1/82303 1761 | val11/166977 1762 | val0/30474 1763 | val6/67326 1764 | val0/99324 1765 | val6/152489 1766 | val2/66573 1767 | val3/88989 1768 | val9/141533 1769 | val4/182508 1770 | val0/71948 1771 | val0/105319 1772 | val10/17410 1773 | val9/56403 1774 | val4/183089 1775 | val9/190040 1776 | val6/57664 1777 | val9/25058 1778 | val5/135503 1779 | val11/16285 1780 | val0/188869 1781 | val3/47486 1782 | val11/95742 1783 | val3/32510 1784 | val7/51775 1785 | val9/114976 1786 | val10/105836 1787 | val10/68678 1788 | val1/130952 1789 | val0/115017 1790 | val8/50874 1791 | val5/38795 1792 | val0/101094 1793 | val9/20162 1794 | val3/154829 1795 | val9/51717 1796 | val10/142245 1797 | val9/118648 1798 | val2/54939 1799 | val10/182132 1800 | val7/108246 1801 | val0/207856 1802 | val7/88041 1803 | val10/37854 1804 | val9/54905 1805 | val9/16474 1806 | val6/100499 1807 | val6/32256 1808 | val8/42650 1809 | val3/69566 1810 | val7/195109 1811 | val10/57009 1812 | val10/158134 1813 | val10/150634 1814 | val2/166397 1815 | val0/152709 1816 | val7/35391 1817 | val10/29050 1818 | val8/24550 1819 | val9/212965 1820 | val6/101639 1821 | val3/33495 1822 | val9/55257 1823 | val0/44211 1824 | val0/209264 1825 | val6/83132 1826 | val3/22712 1827 | val9/77895 1828 | val11/105986 1829 | val2/117231 1830 | val3/50413 1831 | val5/216846 1832 | val11/132916 1833 | val0/196788 1834 | val1/163960 1835 | val8/162700 1836 | val2/193199 1837 | val6/116734 1838 | val3/1255 1839 | val9/192236 1840 | val0/98345 1841 | val3/99268 1842 | val0/220041 1843 | val3/7068 1844 | val3/52685 1845 | val10/151021 1846 | val1/55921 1847 | val0/66863 1848 | val0/36349 1849 | val4/103924 1850 | val3/128710 1851 | val5/149113 1852 | val9/133156 1853 | val0/189650 1854 | val0/181279 1855 | val10/5107 1856 | val2/71985 1857 | val1/41892 1858 | val6/58926 1859 | val5/120187 1860 | val7/95402 1861 | val0/15017 1862 | val9/63242 1863 | val3/18865 1864 | val1/97161 1865 | val9/23819 1866 | val0/15863 1867 | val10/109836 1868 | val2/8159 1869 | val11/78015 1870 | val7/181328 1871 | val4/185081 1872 | val9/60459 1873 | val10/166887 1874 | val0/135492 1875 | val0/47742 1876 | val0/106035 1877 | val8/188449 1878 | val11/144274 1879 | val5/88549 1880 | val1/47180 1881 | val7/217807 1882 | val7/59710 1883 | val1/172617 1884 | val6/99948 1885 | val8/203266 1886 | val4/156459 1887 | val1/211041 1888 | val10/15242 1889 | val3/15512 1890 | val6/80787 1891 | val9/13597 1892 | val3/174626 1893 | val9/10082 1894 | val6/114037 1895 | val5/172589 1896 | val10/21830 1897 | val10/76136 1898 | val6/39117 1899 | val1/164644 1900 | val1/62317 1901 | val8/181154 1902 | val1/120828 1903 | val3/52026 1904 | val7/136563 1905 | val3/106098 1906 | val6/129609 1907 | val0/85483 1908 | val10/87145 1909 | val0/41462 1910 | val9/433 1911 | val6/162877 1912 | val2/28911 1913 | val10/139001 1914 | val3/196305 1915 | val11/91770 1916 | val10/5589 1917 | val11/153678 1918 | val4/62940 1919 | val10/139649 1920 | val5/69252 1921 | val10/152835 1922 | val2/116511 1923 | val10/86588 1924 | val10/163685 1925 | val11/88665 1926 | val9/137805 1927 | --------------------------------------------------------------------------------