├── 151+16_cls_from_1K_ft ├── 151_cls_from_1K_ft ├── DA_LOGS ├── base1 ├── base2 ├── base3 ├── base4 ├── base5 ├── mmd_0.01_1 ├── mmd_0.01_2 ├── mmd_0.01_3 ├── mmd_0.01_4 ├── mmd_0.01_5 ├── pmd_0.005_1 ├── pmd_0.005_2 ├── pmd_0.005_3 ├── pmd_0.005_4 ├── pmd_0.005_5 ├── sgmd_0.005_1 ├── sgmd_0.005_2 ├── sgmd_0.005_3 ├── sgmd_0.005_4 ├── sgmd_0.005_5 ├── sgmd_0.01_1 ├── sgmd_0.01_2 ├── sgmd_0.01_3 ├── sgmd_0.01_4 └── sgmd_0.01_5 ├── DA_mmd.py ├── DA_pmd.py ├── DA_sgmd.py ├── GCN ├── __pycache__ │ └── utils.cpython-36.pyc ├── awa_50_cls_basic ├── extract_50_cls_from_all_graph_for_awa.py ├── materials │ └── AWA2 │ │ ├── 151+16_cls_from_1K_ft │ │ ├── 151_cls_from_1K │ │ ├── 151_cls_from_1K_ft │ │ ├── README.md │ │ ├── __pycache__ │ │ └── glove.cpython-36.pyc │ │ ├── animals_graph_all.json │ │ ├── animals_graph_dense.json │ │ ├── animals_graph_grouped.json │ │ ├── animals_graph_step1.json │ │ ├── awa2-split.json │ │ ├── extract_127+24_from_1K.py │ │ ├── fc_weights_pretrained_on_I2AwA2_source_only.pkl │ │ ├── glove.py │ │ ├── imagenet-split.json │ │ ├── make_dense_animal.py │ │ ├── make_dense_graph.py │ │ ├── make_dense_grouped_graph_animal.py │ │ ├── make_graph_animal_step1.py │ │ ├── make_graph_animal_step2.py │ │ └── web_wnids.json ├── models │ ├── __init__.py │ ├── __init__.pyc │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── gcn.cpython-36.pyc │ │ ├── gcn.cpython-37.pyc │ │ ├── gcn_dense.cpython-36.pyc │ │ ├── gcn_dense.cpython-37.pyc │ │ ├── gcn_dense_att.cpython-36.pyc │ │ ├── gcn_dense_att.cpython-37.pyc │ │ └── resnet.cpython-37.pyc │ ├── gcn.py │ ├── gcn.pyc │ ├── gcn_dense.py │ ├── gcn_dense.pyc │ ├── gcn_dense_att.py │ ├── resnet.py │ └── resnet.pyc ├── train_gcn_att_ezhuo_2019.py ├── train_gcn_basic_awa_ezhuo_2019.py ├── train_gcn_dense_awa_ezhuo_2019.py └── utils.py ├── README.md ├── __pycache__ ├── data_list.cpython-36.pyc ├── loss.cpython-36.pyc ├── lr_schedule.cpython-36.pyc ├── network.cpython-36.pyc ├── pre_process.cpython-36.pyc └── utils.cpython-36.pyc ├── baseline_pmd+bgcn.py ├── data ├── I2AWA2.txt ├── I2AWA2_40.txt ├── Matching │ ├── data_list.py │ ├── extract_feature.py │ ├── match_5_split.py │ ├── pre_process.py │ └── resnet.py ├── WEB_3D3_2.txt └── new_AwA2.txt ├── data_list.py ├── framework5.png ├── gcn_lib ├── __pycache__ │ └── gcn.cpython-36.pyc └── gcn.py ├── loss.py ├── lr_schedule.py ├── network.py ├── pre_process.py ├── train_40_cls_with_help_of_1K.py ├── train_UODTN.py └── utils.py /151+16_cls_from_1K_ft: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/151+16_cls_from_1K_ft -------------------------------------------------------------------------------- /151_cls_from_1K_ft: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/151_cls_from_1K_ft -------------------------------------------------------------------------------- /DA_mmd.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import os.path as osp 4 | 5 | import numpy as np 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | import torch.optim as optim 10 | import torch.utils.data as util_data 11 | from torch.autograd import Variable 12 | 13 | import time 14 | import json 15 | import random 16 | 17 | from data_list import ImageList 18 | import network 19 | import loss 20 | import pre_process as prep 21 | import lr_schedule 22 | from gcn.gcn import GCN 23 | 24 | optim_dict = {"SGD": optim.SGD} 25 | 26 | def image_classification_test(iter_test,len_now, base, class1,bottelneck, gpu=True): 27 | start_test = True 28 | Bd = 29410 29 | COR = 0. 30 | Total = 0. 31 | print('Testing ...') 32 | for i in range(len_now): 33 | data = iter_test.next() 34 | inputs = data[0] 35 | labels = data[1] 36 | if gpu: 37 | inputs = Variable(inputs.cuda()) 38 | labels = Variable(labels.cuda()) 39 | else: 40 | inputs = Variable(inputs) 41 | labels = Variable(labels) 42 | output = base(inputs) 43 | # output = bottelneck(output) 44 | outputs = class1(output) 45 | if start_test: 46 | all_output = outputs.data.float() 47 | all_label = labels.data.float() 48 | _, predict = torch.max(all_output, 1) 49 | COR = COR + torch.sum(torch.squeeze(predict).float() == all_label) 50 | Total = Total + all_label.size()[0] 51 | accuracy = float(COR)/float(Total) 52 | return accuracy 53 | 54 | def train_classification(config): 55 | ## set pre-process 56 | prep_train = prep.image_train(resize_size=256, crop_size=224) 57 | prep_test = prep.image_test(resize_size=256, crop_size=224) 58 | 59 | ## set loss 60 | class_criterion = nn.CrossEntropyLoss() 61 | transfer_criterion = loss.loss_dict["MMD"] 62 | 63 | ## prepare data 64 | TRAIN_LIST = 'data/WEB_3D3_2.txt' 65 | TEST_LIST = 'data/new_AwA2_common.txt' 66 | BSZ = args.batch_size 67 | 68 | dsets_train1 = ImageList(open(TRAIN_LIST).readlines(), shape = (args.img_size,args.img_size), transform=prep_train, train=False) 69 | loaders_train1 = util_data.DataLoader(dsets_train1, batch_size=BSZ, shuffle=True, num_workers=6, pin_memory=True) 70 | 71 | dsets_test = ImageList(open(TEST_LIST).readlines(), shape = (args.img_size,args.img_size),transform=prep_test, train=False) 72 | loaders_test = util_data.DataLoader(dsets_test, batch_size=BSZ, shuffle=True, num_workers=4, pin_memory=True) 73 | ## set base network 74 | net_config = config["network"] 75 | base_network = network.network_dict[net_config["name"]]() 76 | # classifier_layer1 = nn.Linear(256, class_num) 77 | classifier_layer1 = nn.Linear(base_network.output_num(), class_num) 78 | ## initialization 79 | for param in base_network.parameters(): 80 | param.requires_grad = False 81 | for param in base_network.layer4.parameters(): 82 | param.requires_grad = True 83 | for param in base_network.layer3.parameters(): 84 | param.requires_grad = True 85 | 86 | use_gpu = torch.cuda.is_available() 87 | if use_gpu: 88 | classifier_layer1 = classifier_layer1.cuda() 89 | base_network = base_network.cuda() 90 | 91 | ## collect parameters 92 | parameter_list = [{"params":classifier_layer1.parameters(), "lr":10}, 93 | {"params": base_network.layer3.parameters(), "lr":1}, 94 | {"params": base_network.layer4.parameters(), "lr":5}] 95 | 96 | 97 | ## set optimizer 98 | optimizer_config = config["optimizer"] 99 | optimizer = optim_dict[optimizer_config["type"]](parameter_list, **(optimizer_config["optim_params"])) 100 | param_lr = [] 101 | for param_group in optimizer.param_groups: 102 | param_lr.append(param_group["lr"]) 103 | schedule_param = optimizer_config["lr_param"] 104 | lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] 105 | 106 | 107 | len_train_source = len(loaders_train1) - 1 108 | len_test_source = len(loaders_test) - 1 109 | optimizer.zero_grad() 110 | for i in range(config["num_iterations"]): 111 | if (i + 1) % config["test_interval"] == 0: 112 | base_network.train(False) 113 | classifier_layer1.train(False) 114 | print(str(i)+' ACC:') 115 | iter_target = iter(loaders_test) 116 | print(image_classification_test(iter_target,len_test_source, base_network, classifier_layer1, bottelneck, gpu=use_gpu)) 117 | iter_target = iter(loaders_test) 118 | if not osp.exists(osp.join('model',args.save_name)): 119 | os.mkdir(osp.join('model',args.save_name)) 120 | torch.save(base_network.state_dict(),osp.join('model',args.save_name,'base_net%d.pkl'%(i+1))) 121 | torch.save(classifier_layer1.state_dict(),osp.join('model',args.save_name,'class%d.pkl'%(i+1))) 122 | 123 | classifier_layer1.train(True) 124 | base_network.train(True) 125 | 126 | optimizer = lr_scheduler(param_lr, optimizer, i, **schedule_param) 127 | 128 | if i % (len_train_source-1) == 0: 129 | iter_source1 = iter(loaders_train1) 130 | if i % (len_test_source ) == 0: 131 | iter_target = iter(loaders_test) 132 | 133 | inputs_source, labels_source = iter_source1.next() 134 | inputs_target, _ = iter_source2.next() 135 | 136 | if use_gpu: 137 | inputs_source, labels_source, inputs_target = Variable(inputs_source).cuda(), Variable(labels_source).cuda(), Variable(inputs_target).cuda() 138 | else: 139 | inputs_source, labels_source, inputs_target = Variable(inputs_source), Variable(labels_source),Variable(inputs_target) 140 | 141 | features_source = base_network(inputs_source) 142 | features_target = base_network(inputs_target) 143 | 144 | outputs_source1 = classifier_layer1(features_source) 145 | outputs_target1 = classifier_layer1(features_target) 146 | 147 | 148 | cls_loss = class_criterion(outputs_source1, labels_source) 149 | 150 | transfer_loss = transfer_criterion(features_source, features_target) 151 | 152 | 153 | 154 | total_loss = cls_loss + transfer_loss * args.w_align 155 | print("Step "+str(i)+": cls_loss: "+str(cls_loss.cpu().data.numpy())+ 156 | " transfer_loss: "+str(transfer_loss.cpu().data.numpy())) 157 | 158 | total_loss.backward(retain_graph=True) 159 | if (i+1)% config["opt_num"] ==0: 160 | optimizer.step() 161 | optimizer.zero_grad() 162 | 163 | 164 | if __name__ == "__main__": 165 | parser = argparse.ArgumentParser(description='Transfer Learning') 166 | parser.add_argument('--gpu_id', type=str, nargs='?', default='0', help="device id to run") 167 | parser.add_argument('--batch_size', type=int, nargs='?', default=128, help="batch size") 168 | parser.add_argument('--img_size', type=int, nargs='?', default=256, help="image size") 169 | parser.add_argument('--save_name', type=str, nargs='?', default='SOURCE_ONLY', help="loss name") 170 | parser.add_argument('--w_align', type=float, nargs='?', default=0.1, help="percent of unseen data") 171 | args = parser.parse_args() 172 | os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id 173 | 174 | config = {} 175 | config["num_iterations"] = 3000 176 | config["test_interval"] = 200 177 | config["save_num"] = 200 178 | config["opt_num"] = 1 179 | config["network"] = {"name":"ResNet50"} 180 | config["optimizer"] = {"type":"SGD", "optim_params":{"lr":1.0, "momentum":0.9, "weight_decay":0.0001, "nesterov":True}, "lr_type":"inv", "lr_param":{"init_lr":0.001, "gamma":0.001, "power":0.75} } 181 | print(config) 182 | print(args) 183 | train_classification(config) 184 | -------------------------------------------------------------------------------- /DA_pmd.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import os.path as osp 4 | 5 | import numpy as np 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | import torch.optim as optim 10 | import torch.utils.data as util_data 11 | from torch.autograd import Variable 12 | 13 | import time 14 | import json 15 | import random 16 | 17 | from data_list import ImageList 18 | import network 19 | import loss 20 | import pre_process as prep 21 | import lr_schedule 22 | from gcn.gcn import GCN 23 | 24 | optim_dict = {"SGD": optim.SGD} 25 | 26 | 27 | def image_classification_test(iter_test,len_now, base, class1,bottelneck, gpu=True): 28 | start_test = True 29 | COR = 0. 30 | Total = 0. 31 | print('Testing ...') 32 | for i in range(len_now): 33 | data = iter_test.next() 34 | inputs = data[0] 35 | labels = data[1] 36 | if gpu: 37 | inputs = Variable(inputs.cuda()) 38 | labels = Variable(labels.cuda()) 39 | else: 40 | inputs = Variable(inputs) 41 | labels = Variable(labels) 42 | output = base(inputs) 43 | outputs = class1(output) 44 | if start_test: 45 | all_output = outputs.data.float() 46 | all_label = labels.data.float() 47 | _, predict = torch.max(all_output, 1) 48 | COR = COR + torch.sum(torch.squeeze(predict).float() == all_label) 49 | Total = Total + all_label.size()[0] 50 | accuracy = float(COR)/float(Total) 51 | return accuracy 52 | 53 | def train_classification(config): 54 | ## set pre-process 55 | prep_train = prep.image_train(resize_size=256, crop_size=224) 56 | prep_test = prep.image_test(resize_size=256, crop_size=224) 57 | 58 | ## set loss 59 | class_criterion = nn.CrossEntropyLoss() 60 | transfer_criterion = loss.loss_dict["LP"] 61 | 62 | ## prepare data 63 | TEST_LIST = 'data/new_AwA2_common.txt'#AWA_T.txt'#'data/WEB_72.txt' 64 | TRAIN_LIST = 'data/I2AWA2_40.txt'#'AWA_SS.txt#'data/new_AwA2_common.txt' 65 | BSZ = args.batch_size 66 | 67 | dsets_train1 = ImageList(open(TRAIN_LIST).readlines(), shape = (args.img_size,args.img_size), transform=prep_train, train=True) 68 | loaders_train1 = util_data.DataLoader(dsets_train1, batch_size=BSZ, shuffle=True, num_workers=8, pin_memory=True) 69 | 70 | dsets_test = ImageList(open(TEST_LIST).readlines(), shape = (args.img_size,args.img_size),transform=prep_test, train=False) 71 | loaders_test = util_data.DataLoader(dsets_test, batch_size=BSZ, shuffle=True, num_workers=4, pin_memory=True) 72 | net_config = config["network"] 73 | base_network = network.network_dict[net_config["name"]]() 74 | classifier_layer1 = nn.Linear(base_network.output_num(), class_num) 75 | ## initialization 76 | for param in base_network.parameters(): 77 | param.requires_grad = False 78 | for param in base_network.layer4.parameters(): 79 | param.requires_grad = True 80 | for param in base_network.layer3.parameters(): 81 | param.requires_grad = True 82 | 83 | use_gpu = torch.cuda.is_available() 84 | if use_gpu: 85 | classifier_layer1 = classifier_layer1.cuda() 86 | base_network = base_network.cuda() 87 | 88 | ## collect parameters 89 | parameter_list = [{"params":classifier_layer1.parameters(), "lr":10}, 90 | {"params": base_network.layer3.parameters(), "lr":1}, 91 | {"params": base_network.layer4.parameters(), "lr":5}] 92 | 93 | 94 | ## set optimizer 95 | optimizer_config = config["optimizer"] 96 | optimizer = optim_dict[optimizer_config["type"]](parameter_list, **(optimizer_config["optim_params"])) 97 | param_lr = [] 98 | for param_group in optimizer.param_groups: 99 | param_lr.append(param_group["lr"]) 100 | schedule_param = optimizer_config["lr_param"] 101 | lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] 102 | 103 | 104 | len_train_source = len(loaders_train1) - 1 105 | len_test_source = len(loaders_test) - 1 106 | optimizer.zero_grad() 107 | for i in range(config["num_iterations"]): 108 | if (i + 1) % config["test_interval"] == 0: 109 | base_network.train(False) 110 | classifier_layer1.train(False) 111 | print(str(i)+' ACC:') 112 | iter_target = iter(loaders_test) 113 | print(image_classification_test(iter_target,len_test_source, base_network, classifier_layer1, bottelneck, gpu=use_gpu)) 114 | iter_target = iter(loaders_test) 115 | if not osp.exists(osp.join('model',args.save_name)): 116 | os.mkdir(osp.join('model',args.save_name)) 117 | torch.save(base_network.state_dict(),osp.join('model',args.save_name,'base_net%d.pkl'%(i+1))) 118 | torch.save(classifier_layer1.state_dict(),osp.join('model',args.save_name,'class%d.pkl'%(i+1))) 119 | 120 | classifier_layer1.train(True) 121 | base_network.train(True) 122 | 123 | optimizer = lr_scheduler(param_lr, optimizer, i, **schedule_param) 124 | 125 | if i % (len_train_source-1) == 0: 126 | iter_source = iter(loaders_train1) 127 | if i % (len_test_source ) == 0: 128 | iter_target = iter(loaders_test) 129 | 130 | inputs_source, labels_source, labels_source_father, inputs_target = iter_source.next() 131 | 132 | if use_gpu: 133 | inputs_source, labels_source, inputs_target = Variable(inputs_source).cuda(), Variable(labels_source).cuda(), Variable(inputs_target).cuda() 134 | else: 135 | inputs_source, labels_source, inputs_target = Variable(inputs_source), Variable(labels_source),Variable(inputs_target) 136 | 137 | features_source = base_network(inputs_source) 138 | features_target = base_network(inputs_target) 139 | 140 | outputs_source1 = classifier_layer1(features_source) 141 | outputs_target1 = classifier_layer1(features_target) 142 | 143 | 144 | cls_loss = class_criterion(outputs_source1, labels_source) 145 | 146 | transfer_loss = transfer_criterion(features_source, features_target) 147 | 148 | 149 | 150 | total_loss = cls_loss + transfer_loss * args.w_align 151 | print("Step "+str(i)+": cls_loss: "+str(cls_loss.cpu().data.numpy())+ 152 | " transfer_loss: "+str(transfer_loss.cpu().data.numpy())) 153 | 154 | total_loss.backward(retain_graph=True) 155 | if (i+1)% config["opt_num"] ==0: 156 | optimizer.step() 157 | optimizer.zero_grad() 158 | 159 | 160 | if __name__ == "__main__": 161 | parser = argparse.ArgumentParser(description='Transfer Learning') 162 | parser.add_argument('--gpu_id', type=str, nargs='?', default='0', help="device id to run") 163 | parser.add_argument('--batch_size', type=int, nargs='?', default=32, help="batch size") 164 | parser.add_argument('--img_size', type=int, nargs='?', default=256, help="image size") 165 | parser.add_argument('--save_name', type=str, nargs='?', default='base', help="loss name") 166 | args = parser.parse_args() 167 | os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id 168 | 169 | config = {} 170 | config["num_iterations"] = 3000 171 | config["test_interval"] = 200 172 | config["save_num"] = 200 173 | config["opt_num"] = 1 174 | config["network"] = {"name":"ResNet50"} 175 | config["optimizer"] = {"type":"SGD", "optim_params":{"lr":1.0, "momentum":0.9, "weight_decay":0.0001, "nesterov":True}, "lr_type":"inv", "lr_param":{"init_lr":0.001, "gamma":0.001, "power":0.75} } 176 | print(config) 177 | print(args) 178 | train_classification(config) 179 | -------------------------------------------------------------------------------- /DA_sgmd.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import os.path as osp 4 | 5 | import numpy as np 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | import torch.optim as optim 10 | import torch.utils.data as util_data 11 | from torch.autograd import Variable 12 | 13 | import time 14 | import json 15 | import random 16 | 17 | from data_list import ImageList 18 | import network 19 | import loss 20 | import pre_process as prep 21 | import lr_schedule 22 | from gcn.gcn import GCN 23 | 24 | optim_dict = {"SGD": optim.SGD} 25 | 26 | 27 | def image_classification_test(iter_test,len_now, base, class1,bottelneck, gpu=True): 28 | start_test = True 29 | COR = 0. 30 | Total = 0. 31 | print('Testing ...') 32 | for i in range(len_now): 33 | data = iter_test.next() 34 | inputs = data[0] 35 | labels = data[1] 36 | if gpu: 37 | inputs = Variable(inputs.cuda()) 38 | labels = Variable(labels.cuda()) 39 | else: 40 | inputs = Variable(inputs) 41 | labels = Variable(labels) 42 | output = base(inputs) 43 | outputs = class1(output) 44 | if start_test: 45 | all_output = outputs.data.float() 46 | all_label = labels.data.float() 47 | _, predict = torch.max(all_output, 1) 48 | COR = COR + torch.sum(torch.squeeze(predict).float() == all_label) 49 | Total = Total + all_label.size()[0] 50 | accuracy = float(COR)/float(Total) 51 | return accuracy 52 | 53 | def train_classification(config): 54 | ## set pre-process 55 | prep_train = prep.image_train(resize_size=256, crop_size=224) 56 | prep_test = prep.image_test(resize_size=256, crop_size=224) 57 | 58 | ## set loss 59 | class_criterion = nn.CrossEntropyLoss() 60 | transfer_criterion = loss.loss_dict["LP"] 61 | 62 | ## prepare data 63 | TEST_LIST = 'data/new_AwA2_common.txt'#AWA_T.txt'#'data/WEB_72.txt' 64 | TRAIN_LIST = 'data/I2AWA2_40.txt'#'AWA_SS.txt#'data/new_AwA2_common.txt' 65 | BSZ = args.batch_size 66 | 67 | dsets_train1 = ImageList(open(TRAIN_LIST).readlines(), shape = (args.img_size,args.img_size), transform=prep_train, train=True) 68 | loaders_train1 = util_data.DataLoader(dsets_train1, batch_size=BSZ, shuffle=True, num_workers=8, pin_memory=True) 69 | 70 | dsets_test = ImageList(open(TEST_LIST).readlines(), shape = (args.img_size,args.img_size),transform=prep_test, train=False) 71 | loaders_test = util_data.DataLoader(dsets_test, batch_size=BSZ, shuffle=True, num_workers=4, pin_memory=True) 72 | ## set base network 73 | net_config = config["network"] 74 | base_network = network.network_dict[net_config["name"]]() 75 | classifier_layer1 = nn.Linear(base_network.output_num(), class_num) 76 | ## initialization 77 | for param in base_network.parameters(): 78 | param.requires_grad = False 79 | for param in base_network.layer4.parameters(): 80 | param.requires_grad = True 81 | for param in base_network.layer3.parameters(): 82 | param.requires_grad = True 83 | 84 | use_gpu = torch.cuda.is_available() 85 | if use_gpu: 86 | classifier_layer1 = classifier_layer1.cuda() 87 | base_network = base_network.cuda() 88 | 89 | ## collect parameters 90 | parameter_list = [{"params":classifier_layer1.parameters(), "lr":10}, 91 | {"params": base_network.layer3.parameters(), "lr":1}, 92 | {"params": base_network.layer4.parameters(), "lr":5}] 93 | 94 | 95 | ## set optimizer 96 | optimizer_config = config["optimizer"] 97 | optimizer = optim_dict[optimizer_config["type"]](parameter_list, **(optimizer_config["optim_params"])) 98 | param_lr = [] 99 | for param_group in optimizer.param_groups: 100 | param_lr.append(param_group["lr"]) 101 | schedule_param = optimizer_config["lr_param"] 102 | lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] 103 | 104 | 105 | len_train_source = len(loaders_train1) - 1 106 | len_test_source = len(loaders_test) - 1 107 | optimizer.zero_grad() 108 | for i in range(config["num_iterations"]): 109 | if (i + 1) % config["test_interval"] == 0: 110 | base_network.train(False) 111 | classifier_layer1.train(False) 112 | bottelneck.train(False) 113 | print(str(i)+' ACC:') 114 | iter_target = iter(loaders_test) 115 | print(image_classification_test(iter_target,len_test_source, base_network, classifier_layer1, bottelneck, gpu=use_gpu)) 116 | iter_target = iter(loaders_test) 117 | if not osp.exists(osp.join('model',args.save_name)): 118 | os.mkdir(osp.join('model',args.save_name)) 119 | torch.save(base_network.state_dict(),osp.join('model',args.save_name,'base_net%d.pkl'%(i+1))) 120 | torch.save(classifier_layer1.state_dict(),osp.join('model',args.save_name,'class%d.pkl'%(i+1))) 121 | 122 | classifier_layer1.train(True) 123 | base_network.train(True) 124 | bottelneck.train(True) 125 | 126 | optimizer = lr_scheduler(param_lr, optimizer, i, **schedule_param) 127 | 128 | if i % (len_train_source-1) == 0: 129 | iter_source = iter(loaders_train1) 130 | if i % (len_test_source ) == 0: 131 | iter_target = iter(loaders_test) 132 | 133 | inputs_source, labels_source, labels_source_father, inputs_target = iter_source.next() 134 | 135 | if use_gpu: 136 | inputs_source, labels_source, inputs_target = Variable(inputs_source).cuda(), Variable(labels_source).cuda(), Variable(inputs_target).cuda() 137 | else: 138 | inputs_source, labels_source, inputs_target = Variable(inputs_source), Variable(labels_source),Variable(inputs_target) 139 | 140 | features_source = base_network(inputs_source) 141 | features_target = base_network(inputs_target) 142 | 143 | outputs_source1 = classifier_layer1(features_source) 144 | outputs_target1 = classifier_layer1(features_target) 145 | 146 | 147 | cls_loss = class_criterion(outputs_source1, labels_source) 148 | 149 | outputs_softmax = F.softmax(outputs_target1, dim=1) 150 | WEIGHT = torch.sum(torch.softmax(outputs_source1, dim=1) * outputs_softmax, 1) 151 | 152 | transfer_loss = transfer_criterion(features_source, features_target, WEIGHT.gt(0.6).float(), True) 153 | 154 | 155 | 156 | total_loss = cls_loss + transfer_loss * args.w_align 157 | print("Step "+str(i)+": cls_loss: "+str(cls_loss.cpu().data.numpy())+ 158 | " transfer_loss: "+str(transfer_loss.cpu().data.numpy())) 159 | 160 | total_loss.backward(retain_graph=True) 161 | if (i+1)% config["opt_num"] ==0: 162 | optimizer.step() 163 | optimizer.zero_grad() 164 | 165 | 166 | if __name__ == "__main__": 167 | parser = argparse.ArgumentParser(description='Transfer Learning') 168 | parser.add_argument('--gpu_id', type=str, nargs='?', default='0', help="device id to run") 169 | parser.add_argument('--batch_size', type=int, nargs='?', default=32, help="batch size") 170 | parser.add_argument('--img_size', type=int, nargs='?', default=256, help="image size") 171 | parser.add_argument('--save_name', type=str, nargs='?', default='base', help="loss name") 172 | parser.add_argument('--w_align', type=float, nargs='?', default=0.1, help="percent of unseen data") 173 | args = parser.parse_args() 174 | os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id 175 | 176 | config = {} 177 | config["num_iterations"] = 3000 178 | config["test_interval"] = 200 179 | config["save_num"] = 200 180 | config["opt_num"] = 1 181 | config["network"] = {"name":"ResNet50"} 182 | config["optimizer"] = {"type":"SGD", "optim_params":{"lr":1.0, "momentum":0.9, "weight_decay":0.0001, "nesterov":True}, "lr_type":"inv", "lr_param":{"init_lr":0.001, "gamma":0.001, "power":0.75} } 183 | print(config) 184 | print(args) 185 | train_classification(config) 186 | -------------------------------------------------------------------------------- /GCN/__pycache__/utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/GCN/__pycache__/utils.cpython-36.pyc -------------------------------------------------------------------------------- /GCN/awa_50_cls_basic: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/GCN/awa_50_cls_basic -------------------------------------------------------------------------------- /GCN/extract_50_cls_from_all_graph_for_awa.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import json 3 | import numpy as np 4 | # from IPython import embed;embed();exit(); 5 | all_good = torch.load('RESULTS_MODELS/awa-att/epoch-3000.pred') 6 | # all_good = torch.load('RESULTS_MODELS/awa-dense/epoch-3000.pred') 7 | # all_good = torch.load('RESULTS_MODELS/awa-basic3/epoch-3000.pred') 8 | all_id = all_good['wnids'] 9 | all_matric = all_good['pred'] 10 | test_sets = json.load(open('materials/AWA2/animals_graph_grouped.json')) 11 | # test_sets = json.load(open('materials/AWA2/animals_graph_dense.json')) 12 | # test_sets = json.load(open('materials/AWA2/animals_graph_all.json')) 13 | #test_sets = json.load(open('../ADM/AWA2/awa2-ezhuo-graph.json', 'r')) 14 | good_id = test_sets['wnids'][127:177] 15 | 16 | mytensor = torch.zeros(len(good_id),all_matric.shape[1]) 17 | # mytensor = torch.zeros(len(good_id),all_matric[0].shape[1]) 18 | for j in range(len(good_id)): 19 | find=0 20 | for i in range(len(all_id)): 21 | if all_id[i] ==good_id[j]: 22 | # mytensor[j]=all_matric[0][i] 23 | mytensor[j]=all_matric[i] 24 | find=1 25 | break 26 | if find ==0: 27 | print(good_id[0]) 28 | torch.save({'fc50':mytensor},'awa_50_cls_att') 29 | 30 | -------------------------------------------------------------------------------- /GCN/materials/AWA2/151+16_cls_from_1K_ft: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/GCN/materials/AWA2/151+16_cls_from_1K_ft -------------------------------------------------------------------------------- /GCN/materials/AWA2/151_cls_from_1K: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/GCN/materials/AWA2/151_cls_from_1K -------------------------------------------------------------------------------- /GCN/materials/AWA2/151_cls_from_1K_ft: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/GCN/materials/AWA2/151_cls_from_1K_ft -------------------------------------------------------------------------------- /GCN/materials/AWA2/README.md: -------------------------------------------------------------------------------- 1 | run python make_graph_animal_step1.py 2 | run python make_graph_animal_step2.py 3 | python make_dense_graph_animal.py 4 | python make_dense_grouped_graph_animal.py 5 | 6 | pretrained model 84.1% on AwA2 (40) 7 | -------------------------------------------------------------------------------- /GCN/materials/AWA2/__pycache__/glove.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/GCN/materials/AWA2/__pycache__/glove.cpython-36.pyc -------------------------------------------------------------------------------- /GCN/materials/AWA2/awa2-split.json: -------------------------------------------------------------------------------- 1 | {"train": ["n02071294", "n02363005", "n02110341", "n02123394", "n02106662", "n02123597", "n02445715", "n01889520", "n02129604", "n02398521", "n02128385", "n02493793", "n02503517", "n02480855", "n02403003", "n02481823", "n02342885", "n02118333", "n02355227", "n02324045", "n02114100", "n02085620", "n02441942", "n02444819", "n02410702", "n02391049", "n02510455", "n02395406", "n02129165", "n02134084", "n02106030", "n02403454", "n02430045", "n02330245", "n02065726", "n02419796", "n02132580", "n02391994", "n02508021", "n02432983"], "test": ["n02411705", "n02068974", "n02139199", "n02076196", "n02064816", "n02331046", "n02374451", "n02081571", "n02439033", "n02127482"], "train_names": ["killer+whale", "beaver", "dalmatian", "persian+cat", "german+shepherd", "siamese+cat", "skunk", "mole", "tiger", "hippopotamus", "leopard", "spider+monkey", "elephant", "gorilla", "ox", "chimpanzee", "hamster", "fox", "squirrel", "rabbit", "wolf", "chihuahua", "weasel", "otter", "buffalo", "zebra", "giant+panda", "pig", "lion", "polar+bear", "collie", "cow", "deer", "mouse", "humpback+whale", "antelope", "grizzly+bear", "rhinoceros", "raccoon", "moose"], "test_names": ["sheep", "dolphin", "bat", "seal", "blue+whale", "rat", "horse", "walrus", "giraffe", "bobcat"]} -------------------------------------------------------------------------------- /GCN/materials/AWA2/extract_127+24_from_1K.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import json 3 | import numpy as np 4 | import torch 5 | import torch.nn.functional as F 6 | p = torch.load('resnet50-raw.pth') 7 | w = p['fc.weight'].data 8 | b = p['fc.bias'].data 9 | v = torch.cat([w, b.unsqueeze(1)], dim=1).tolist() 10 | 11 | wnids = json.load(open('imagenet-split.json', 'r'))['train'] 12 | wnids = sorted(wnids) 13 | 14 | 15 | 16 | test_sets = json.load(open('animals_graph_all.json')) 17 | good_id = test_sets['wnids'][:127+24] 18 | 19 | mytensor = torch.zeros(len(good_id),2049) 20 | # from IPython import embed;embed();exit(); 21 | for j in range(len(good_id)): 22 | find=0 23 | for i in range(len(wnids)): 24 | if wnids[i] ==good_id[j]: 25 | mytensor[j]=torch.FloatTensor(v[i]) 26 | find=1 27 | break 28 | if find ==0: 29 | print(good_id[0]) 30 | torch.save({'fc151':mytensor},'151_cls_from_1K') 31 | 32 | -------------------------------------------------------------------------------- /GCN/materials/AWA2/fc_weights_pretrained_on_I2AwA2_source_only.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/GCN/materials/AWA2/fc_weights_pretrained_on_I2AwA2_source_only.pkl -------------------------------------------------------------------------------- /GCN/materials/AWA2/glove.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class GloVe(): 5 | 6 | def __init__(self, file_path): 7 | self.dimension = None 8 | self.embedding = dict() 9 | with open(file_path, 'r') as f: 10 | for line in f.readlines(): 11 | strs = line.strip().split() 12 | word = strs[0] 13 | vector = torch.FloatTensor(list(map(float, strs[1:]))) 14 | self.embedding[word] = vector 15 | if self.dimension is None: 16 | self.dimension = len(vector) 17 | 18 | def _fix_word(self, word): 19 | terms = word.replace('_', ' ').split(' ') 20 | ret = self.zeros() 21 | cnt = 0 22 | for term in terms: 23 | v = self.embedding.get(term) 24 | if v is None: 25 | subterms = term.split('-') 26 | subterm_sum = self.zeros() 27 | subterm_cnt = 0 28 | for subterm in subterms: 29 | subv = self.embedding.get(subterm) 30 | if subv is not None: 31 | subterm_sum += subv 32 | subterm_cnt += 1 33 | if subterm_cnt > 0: 34 | v = subterm_sum / subterm_cnt 35 | if v is not None: 36 | ret += v 37 | cnt += 1 38 | return ret / cnt if cnt > 0 else None 39 | 40 | def __getitem__(self, words): 41 | if type(words) is str: 42 | words = [words] 43 | ret = self.zeros() 44 | cnt = 0 45 | for word in words: 46 | v = self.embedding.get(word) 47 | if v is None: 48 | v = self._fix_word(word) 49 | if v is not None: 50 | ret += v 51 | cnt += 1 52 | if cnt > 0: 53 | return ret / cnt 54 | else: 55 | return self.zeros() 56 | 57 | def zeros(self): 58 | return torch.zeros(self.dimension) 59 | 60 | -------------------------------------------------------------------------------- /GCN/materials/AWA2/make_dense_animal.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | 4 | parser = argparse.ArgumentParser() 5 | parser.add_argument('--input', default='animals_graph_all.json') 6 | parser.add_argument('--output', default='animals_graph_dense.json') 7 | args = parser.parse_args() 8 | 9 | js = json.load(open(args.input, 'r')) 10 | wnids = js['wnids'] 11 | vectors = js['vectors'] 12 | edges = js['edges'] 13 | 14 | n = len(wnids) 15 | adjs = {} 16 | for i in range(n): 17 | adjs[i] = [] 18 | for u, v in edges: 19 | adjs[u].append(v) 20 | 21 | new_edges = [] 22 | 23 | for u, wnid in enumerate(wnids): 24 | q = [u] 25 | l = 0 26 | d = {} 27 | d[u] = 0 28 | while l < len(q): 29 | x = q[l] 30 | l += 1 31 | for y in adjs[x]: 32 | if d.get(y) is None: 33 | d[y] = d[x] + 1 34 | q.append(y) 35 | for x, dis in d.items(): 36 | new_edges.append((u, x)) 37 | 38 | json.dump({'wnids': wnids, 'vectors': vectors, 'edges': new_edges}, 39 | open(args.output, 'w')) 40 | 41 | -------------------------------------------------------------------------------- /GCN/materials/AWA2/make_dense_graph.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | 4 | parser = argparse.ArgumentParser() 5 | parser.add_argument('--input', default='imagenet-induced-graph.json') 6 | parser.add_argument('--output', default='imagenet-dense-graph.json') 7 | args = parser.parse_args() 8 | 9 | js = json.load(open(args.input, 'r')) 10 | wnids = js['wnids'] 11 | vectors = js['vectors'] 12 | edges = js['edges'] 13 | 14 | n = len(wnids) 15 | adjs = {} 16 | for i in range(n): 17 | adjs[i] = [] 18 | for u, v in edges: 19 | adjs[u].append(v) 20 | 21 | new_edges = [] 22 | 23 | for u, wnid in enumerate(wnids): 24 | q = [u] 25 | l = 0 26 | d = {} 27 | d[u] = 0 28 | while l < len(q): 29 | x = q[l] 30 | l += 1 31 | for y in adjs[x]: 32 | if d.get(y) is None: 33 | d[y] = d[x] + 1 34 | q.append(y) 35 | for x, dis in d.items(): 36 | new_edges.append((u, x)) 37 | 38 | json.dump({'wnids': wnids, 'vectors': vectors, 'edges': new_edges}, 39 | open(args.output, 'w')) 40 | 41 | -------------------------------------------------------------------------------- /GCN/materials/AWA2/make_dense_grouped_graph_animal.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | 4 | parser = argparse.ArgumentParser() 5 | parser.add_argument('--input', default='animals_graph_all.json') 6 | parser.add_argument('--output', default='animals_graph_grouped.json') 7 | args = parser.parse_args() 8 | 9 | js = json.load(open(args.input, 'r')) 10 | wnids = js['wnids'] 11 | vectors = js['vectors'] 12 | edges = js['edges'] 13 | 14 | n = len(wnids) 15 | adjs = {} 16 | for i in range(n): 17 | adjs[i] = [] 18 | for u, v in edges: 19 | adjs[u].append(v) 20 | 21 | new_edges = [[] for i in range(99)] 22 | 23 | for u, wnid in enumerate(wnids): 24 | q = [u] 25 | l = 0 26 | d = {} 27 | d[u] = 0 28 | while l < len(q): 29 | x = q[l] 30 | l += 1 31 | for y in adjs[x]: 32 | if d.get(y) is None: 33 | d[y] = d[x] + 1 34 | q.append(y) 35 | for x, dis in d.items(): 36 | new_edges[dis].append((u, x)) 37 | 38 | while new_edges[-1] == []: 39 | new_edges.pop() 40 | 41 | json.dump({'wnids': wnids, 'vectors': vectors, 'edges_set': new_edges}, 42 | open(args.output, 'w')) 43 | 44 | -------------------------------------------------------------------------------- /GCN/materials/AWA2/make_graph_animal_step1.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | 4 | from nltk.corpus import wordnet as wn 5 | import torch 6 | 7 | from glove import GloVe 8 | 9 | 10 | def getnode(x): 11 | return wn.synset_from_pos_and_offset('n', int(x[1:])) 12 | 13 | 14 | def getwnid(u): 15 | s = str(u.offset()) 16 | return 'n' + (8 - len(s)) * '0' + s 17 | 18 | 19 | def getedges(s): 20 | dic = {x: i for i, x in enumerate(s)} 21 | edges = [] 22 | for i, u in enumerate(s): 23 | for v in u.hypernyms(): 24 | j = dic.get(v) 25 | if j is not None: 26 | edges.append((i, j)) 27 | return edges 28 | 29 | 30 | def induce_parents(s, stop_set): 31 | q = s 32 | vis = set(s) 33 | l = 0 34 | while l < len(q): 35 | u = q[l] 36 | l += 1 37 | #if len(u.hypernyms())>1: 38 | # from IPython import embed;embed();exit(); 39 | for p in u.hypernyms(): 40 | if p not in vis: 41 | vis.add(p) 42 | q.append(p) 43 | add = stop_set 44 | #from IPython import embed;embed();exit(); 45 | l = 0 46 | while l1: 38 | # from IPython import embed;embed();exit(); 39 | for p in u.hypernyms(): 40 | if p not in vis: 41 | vis.add(p) 42 | q.append(p) 43 | '''add = stop_set 44 | #from IPython import embed;embed();exit(); 45 | l = 0 46 | while l 0: 120 | val_loss = mask_l2_loss(output_vectors, fc_vectors, tlist[n_train:]).item() 121 | loss = val_loss 122 | else: 123 | val_loss = 0 124 | loss = train_loss 125 | print('epoch {}, train_loss={:.4f}, val_loss={:.4f}' 126 | .format(epoch, train_loss, val_loss)) 127 | 128 | trlog['train_loss'].append(train_loss) 129 | trlog['val_loss'].append(val_loss) 130 | trlog['min_loss'] = min_loss 131 | torch.save(trlog, osp.join(save_path, 'trlog')) 132 | 133 | if (epoch % args.save_epoch == 0): 134 | if args.no_pred: 135 | pred_obj = None 136 | else: 137 | pred_obj = { 138 | 'wnids': wnids, 139 | 'pred': output_vectors 140 | } 141 | 142 | if epoch % args.save_epoch == 0: 143 | save_checkpoint('epoch-{}'.format(epoch)) 144 | 145 | pred_obj = None 146 | 147 | -------------------------------------------------------------------------------- /GCN/train_gcn_basic_awa_ezhuo_2019.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import random 4 | import os.path as osp 5 | 6 | import torch 7 | import torch.nn.functional as F 8 | 9 | from utils import ensure_path, set_gpu, l2_loss 10 | from models.gcn import GCN 11 | 12 | 13 | def save_checkpoint(name): 14 | torch.save(gcn.state_dict(), osp.join(save_path, name + '.pth')) 15 | torch.save(pred_obj, osp.join(save_path, name + '.pred')) 16 | 17 | 18 | def mask_l2_loss(a, b, mask): 19 | #good = [127]*len(mask) 20 | # print([i for i in range(127,127+40)]) 21 | # from IPython import embed;embed();exit(); 22 | return l2_loss(a[0][:127+40][mask], b[mask]) 23 | # return l2_loss(a[mask], b[mask]) 24 | #return l2_loss(a[[(x+127) for x in mask]], b[mask]) 25 | 26 | 27 | if __name__ == '__main__': 28 | parser = argparse.ArgumentParser() 29 | 30 | parser.add_argument('--max-epoch', type=int, default=3000) 31 | parser.add_argument('--trainval', default='10,0') 32 | parser.add_argument('--lr', type=float, default=0.001) 33 | parser.add_argument('--weight-decay', type=float, default=0.0005) 34 | parser.add_argument('--save-epoch', type=int, default=300) 35 | parser.add_argument('--save-path', default='RESULTS_MODELS/awa-basic3') 36 | 37 | parser.add_argument('--gpu', default='0') 38 | 39 | parser.add_argument('--no-pred', action='store_true') 40 | args = parser.parse_args() 41 | 42 | set_gpu(args.gpu) 43 | 44 | save_path = args.save_path 45 | ensure_path(save_path) 46 | # graph =json.load(open('materials/AWA2/animals_graph_dense.json')) 47 | graph =json.load(open('materials/AWA2/animals_graph_all.json')) 48 | #graph = json.load(open('../ADM/AWA2/awa2-ezhuo-graph.json', 'r')) 49 | wnids = graph['wnids'] 50 | n = len(wnids) 51 | edges = graph['edges'] 52 | 53 | edges = edges + [(v, u) for (u, v) in edges] 54 | edges = edges + [(u, u) for u in range(n)] 55 | 56 | word_vectors = torch.tensor(graph['vectors']).cuda() 57 | word_vectors = F.normalize(word_vectors) 58 | #fc_vectors = torch.load('../ADM/AWA2/class54000.pkl') 59 | #fcfile = json.load(open('materials/fc-weights.json', 'r')) 60 | #train_wnids = [x[0] for x in fcfile] 61 | #fc_vectors = [x[1] for x in fcfile] 62 | #assert train_wnids == wnids[:len(train_wnids)] 63 | # fc_vectors_=torch.load('materials/AWA2/fc_weights_pretrained_on_I2AwA2_source_only.pkl')#['fc151'] 64 | fc_vectors =torch.load('materials/AWA2/151+16_cls_from_1K_ft')['fc151+16_ft'] 65 | #fc_vectors = torch.cat((fc_vectors_['weight'].cpu(), fc_vectors_['bias'].cpu().view(-1,1)),dim=1) 66 | #fc_vectors=F.normalize(torch.load('materials/fc151')['fc151']) 67 | fc_vectors = torch.tensor(fc_vectors).cuda() 68 | #fc_vectors =torch.cat((fc_vectors, torch.load('../ADM/AWA2/gcn/class16_fix5000.pkl')),dim=0) 69 | #laji = torch.load('../ADM/AWA2/gcn/class24_fix5000.pkl') 70 | #assert laji ==fc_vectors[127:151] 71 | #fc_vectors = torch.cat((fc_vectors,torch.load('class248000.pkl')[8:]),dim=0) 72 | #fc_vectors = torch.cat((fc_vectors,torch.load('class168000.pkl')),dim=0) 73 | # fc_vectors = F.normalize(fc_vectors) 74 | 75 | hidden_layers = 'd2048,d' 76 | gcn = GCN(n, edges, word_vectors.shape[1], fc_vectors.shape[1], hidden_layers).cuda() 77 | 78 | print('{} nodes, {} edges'.format(n, len(edges))) 79 | print('word vectors:', word_vectors.shape) 80 | print('fc vectors:', fc_vectors.shape) 81 | print('hidden layers:', hidden_layers) 82 | 83 | optimizer = torch.optim.Adam(gcn.parameters(), lr=args.lr, weight_decay=args.weight_decay) 84 | 85 | v_train, v_val = map(float, args.trainval.split(',')) 86 | n_trainval = len(fc_vectors) 87 | n_train = round(n_trainval * (v_train / (v_train + v_val))) 88 | print('num train: {}, num val: {}'.format(n_train, n_trainval - n_train)) 89 | tlist = list(range(len(fc_vectors))) 90 | random.shuffle(tlist) 91 | 92 | min_loss = 1e18 93 | 94 | trlog = {} 95 | trlog['train_loss'] = [] 96 | trlog['val_loss'] = [] 97 | trlog['min_loss'] = 0 98 | 99 | for epoch in range(1, args.max_epoch + 1): 100 | gcn.train() 101 | output_vectors = gcn(word_vectors) 102 | # simi = torch.matmul(output_vectors[:127], output_vectors[:127].transpose(0,1)) 103 | # simi_loss = torch.mean(torch.abs(simi - torch.diag(torch.diag(simi)))) 104 | loss = mask_l2_loss(output_vectors, fc_vectors, tlist[:151+16]) #+ 0.15 * simi_loss 105 | optimizer.zero_grad() 106 | #loss.backward() 107 | loss.backward(retain_graph=True) 108 | optimizer.step() 109 | 110 | gcn.eval() 111 | output_vectors = gcn(word_vectors) 112 | train_loss = mask_l2_loss(output_vectors, fc_vectors, tlist[:151+16]).item() 113 | if v_val > 0: 114 | val_loss = mask_l2_loss(output_vectors, fc_vectors, tlist[n_train:]).item() 115 | loss = val_loss 116 | else: 117 | val_loss = 0 118 | loss = train_loss 119 | print('epoch {}, train_loss={:.4f}, val_loss={:.4f}' 120 | .format(epoch, train_loss, val_loss)) 121 | 122 | trlog['train_loss'].append(train_loss) 123 | trlog['val_loss'].append(val_loss) 124 | trlog['min_loss'] = min_loss 125 | torch.save(trlog, osp.join(save_path, 'trlog')) 126 | 127 | if (epoch % args.save_epoch == 0): 128 | if args.no_pred: 129 | pred_obj = None 130 | else: 131 | pred_obj = { 132 | 'wnids': wnids, 133 | 'pred': output_vectors 134 | } 135 | 136 | if epoch % args.save_epoch == 0: 137 | save_checkpoint('epoch-{}'.format(epoch)) 138 | 139 | pred_obj = None 140 | 141 | -------------------------------------------------------------------------------- /GCN/train_gcn_dense_awa_ezhuo_2019.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import random 4 | import os.path as osp 5 | 6 | import torch 7 | import torch.nn.functional as F 8 | 9 | from utils import ensure_path, set_gpu, l2_loss 10 | from models.gcn_dense import GCN_Dense as GCN 11 | 12 | 13 | def save_checkpoint(name): 14 | torch.save(gcn.state_dict(), osp.join(save_path, name + '.pth')) 15 | torch.save(pred_obj, osp.join(save_path, name + '.pred')) 16 | 17 | 18 | def mask_l2_loss(a, b, mask): 19 | #good = [127]*len(mask) 20 | # print([i for i in range(127,127+40)]) 21 | # from IPython import embed;embed();exit(); 22 | return l2_loss(a[:127+40][mask], b[mask]) 23 | # return l2_loss(a[mask], b[mask]) 24 | #return l2_loss(a[[(x+127) for x in mask]], b[mask]) 25 | 26 | 27 | if __name__ == '__main__': 28 | parser = argparse.ArgumentParser() 29 | 30 | parser.add_argument('--max-epoch', type=int, default=3000) 31 | parser.add_argument('--trainval', default='10,0') 32 | parser.add_argument('--lr', type=float, default=0.001) 33 | parser.add_argument('--weight-decay', type=float, default=0.0005) 34 | parser.add_argument('--save-epoch', type=int, default=300) 35 | parser.add_argument('--save-path', default='RESULTS_MODELS/awa-dense') 36 | 37 | parser.add_argument('--gpu', default='0') 38 | 39 | parser.add_argument('--no-pred', action='store_true') 40 | args = parser.parse_args() 41 | 42 | set_gpu(args.gpu) 43 | 44 | save_path = args.save_path 45 | ensure_path(save_path) 46 | graph =json.load(open('materials/AWA2/animals_graph_dense.json')) 47 | # graph =json.load(open('materials/AWA2/animals_graph_all.json')) 48 | #graph = json.load(open('../ADM/AWA2/awa2-ezhuo-graph.json', 'r')) 49 | wnids = graph['wnids'] 50 | n = len(wnids) 51 | edges = graph['edges'] 52 | 53 | # edges = edges + [(v, u) for (u, v) in edges] 54 | # edges = edges + [(u, u) for u in range(n)] 55 | 56 | word_vectors = torch.tensor(graph['vectors']).cuda() 57 | word_vectors = F.normalize(word_vectors) 58 | #fc_vectors = torch.load('../ADM/AWA2/class54000.pkl') 59 | #fcfile = json.load(open('materials/fc-weights.json', 'r')) 60 | #train_wnids = [x[0] for x in fcfile] 61 | #fc_vectors = [x[1] for x in fcfile] 62 | #assert train_wnids == wnids[:len(train_wnids)] 63 | # fc_vectors_=torch.load('materials/AWA2/fc_weights_pretrained_on_I2AwA2_source_only.pkl')#['fc151'] 64 | fc_vectors =torch.load('materials/AWA2/151+16_cls_from_1K_ft')['fc151+16_ft'] 65 | #fc_vectors = torch.cat((fc_vectors_['weight'].cpu(), fc_vectors_['bias'].cpu().view(-1,1)),dim=1) 66 | #fc_vectors=F.normalize(torch.load('materials/fc151')['fc151']) 67 | fc_vectors = torch.tensor(fc_vectors).cuda() 68 | #fc_vectors =torch.cat((fc_vectors, torch.load('../ADM/AWA2/gcn/class16_fix5000.pkl')),dim=0) 69 | #laji = torch.load('../ADM/AWA2/gcn/class24_fix5000.pkl') 70 | #assert laji ==fc_vectors[127:151] 71 | #fc_vectors = torch.cat((fc_vectors,torch.load('class248000.pkl')[8:]),dim=0) 72 | #fc_vectors = torch.cat((fc_vectors,torch.load('class168000.pkl')),dim=0) 73 | # fc_vectors = F.normalize(fc_vectors) 74 | 75 | hidden_layers = 'd2048,d' 76 | gcn = GCN(n, edges, word_vectors.shape[1], fc_vectors.shape[1], hidden_layers).cuda() 77 | 78 | print('{} nodes, {} edges'.format(n, len(edges))) 79 | print('word vectors:', word_vectors.shape) 80 | print('fc vectors:', fc_vectors.shape) 81 | print('hidden layers:', hidden_layers) 82 | 83 | optimizer = torch.optim.Adam(gcn.parameters(), lr=args.lr, weight_decay=args.weight_decay) 84 | 85 | v_train, v_val = map(float, args.trainval.split(',')) 86 | n_trainval = len(fc_vectors) 87 | n_train = round(n_trainval * (v_train / (v_train + v_val))) 88 | print('num train: {}, num val: {}'.format(n_train, n_trainval - n_train)) 89 | tlist = list(range(len(fc_vectors))) 90 | random.shuffle(tlist) 91 | 92 | min_loss = 1e18 93 | 94 | trlog = {} 95 | trlog['train_loss'] = [] 96 | trlog['val_loss'] = [] 97 | trlog['min_loss'] = 0 98 | 99 | for epoch in range(1, args.max_epoch + 1): 100 | gcn.train() 101 | output_vectors = gcn(word_vectors) 102 | # simi = torch.matmul(output_vectors[:127], output_vectors[:127].transpose(0,1)) 103 | # simi_loss = torch.mean(torch.abs(simi - torch.diag(torch.diag(simi)))) 104 | loss = mask_l2_loss(output_vectors, fc_vectors, tlist[:151+16]) #+ 0.15 * simi_loss 105 | optimizer.zero_grad() 106 | #loss.backward() 107 | loss.backward(retain_graph=True) 108 | optimizer.step() 109 | 110 | gcn.eval() 111 | output_vectors = gcn(word_vectors) 112 | train_loss = mask_l2_loss(output_vectors, fc_vectors, tlist[:151+16]).item() 113 | if v_val > 0: 114 | val_loss = mask_l2_loss(output_vectors, fc_vectors, tlist[n_train:]).item() 115 | loss = val_loss 116 | else: 117 | val_loss = 0 118 | loss = train_loss 119 | print('epoch {}, train_loss={:.4f}, val_loss={:.4f}' 120 | .format(epoch, train_loss, val_loss)) 121 | 122 | trlog['train_loss'].append(train_loss) 123 | trlog['val_loss'].append(val_loss) 124 | trlog['min_loss'] = min_loss 125 | torch.save(trlog, osp.join(save_path, 'trlog')) 126 | 127 | if (epoch % args.save_epoch == 0): 128 | if args.no_pred: 129 | pred_obj = None 130 | else: 131 | pred_obj = { 132 | 'wnids': wnids, 133 | 'pred': output_vectors 134 | } 135 | 136 | if epoch % args.save_epoch == 0: 137 | save_checkpoint('epoch-{}'.format(epoch)) 138 | 139 | pred_obj = None 140 | 141 | -------------------------------------------------------------------------------- /GCN/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import shutil 4 | 5 | import numpy as np 6 | import scipy.sparse as sp 7 | import torch 8 | 9 | 10 | def ensure_path(path): 11 | if osp.exists(path): 12 | if input('{} exists, remove? ([y]/n)'.format(path)) != 'n': 13 | shutil.rmtree(path) 14 | os.mkdir(path) 15 | else: 16 | os.mkdir(path) 17 | 18 | 19 | def set_gpu(gpu): 20 | os.environ['CUDA_VISIBLE_DEVICES'] = gpu 21 | print('using gpu {}'.format(gpu)) 22 | 23 | 24 | def pick_vectors(dic, wnids, is_tensor=False): 25 | o = next(iter(dic.values())) 26 | dim = len(o) 27 | ret = [] 28 | for wnid in wnids: 29 | v = dic.get(wnid) 30 | if v is None: 31 | if not is_tensor: 32 | v = [0] * dim 33 | else: 34 | v = torch.zeros(dim) 35 | ret.append(v) 36 | if not is_tensor: 37 | return torch.FloatTensor(ret) 38 | else: 39 | return torch.stack(ret) 40 | 41 | 42 | def l2_loss(a, b): 43 | return ((a - b)**2).sum() / (len(a) * 2) 44 | 45 | 46 | def normt_spm(mx, method='in'): 47 | if method == 'in': 48 | mx = mx.transpose() 49 | rowsum = np.array(mx.sum(1)) 50 | r_inv = np.power(rowsum, -1).flatten() 51 | r_inv[np.isinf(r_inv)] = 0. 52 | r_mat_inv = sp.diags(r_inv) 53 | mx = r_mat_inv.dot(mx) 54 | return mx 55 | 56 | if method == 'sym': 57 | rowsum = np.array(mx.sum(1)) 58 | r_inv = np.power(rowsum, -0.5).flatten() 59 | r_inv[np.isinf(r_inv)] = 0. 60 | r_mat_inv = sp.diags(r_inv) 61 | mx = mx.dot(r_mat_inv).transpose().dot(r_mat_inv) 62 | return mx 63 | 64 | 65 | def spm_to_tensor(sparse_mx): 66 | sparse_mx = sparse_mx.tocoo().astype(np.float32) 67 | indices = torch.from_numpy(np.vstack( 68 | (sparse_mx.row, sparse_mx.col))).long() 69 | values = torch.from_numpy(sparse_mx.data) 70 | shape = torch.Size(sparse_mx.shape) 71 | return torch.sparse.FloatTensor(indices, values, shape) 72 | 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # UODTN 2 | Code release for ["Unsupervised Open Domain Recognition by Semantic Discrepancy Minimization"](http://openaccess.thecvf.com/content_CVPR_2019/papers/Zhuo_Unsupervised_Open_Domain_Recognition_by_Semantic_Discrepancy_Minimization_CVPR_2019_paper.pdf) (CVPR 2019) 3 | 4 | We implement our experiments with python 3.6, numpy 1.15 and PyTorch 1.0. 5 | ![alt text](https://raw.githubusercontent.com/junbaoZHUO/UODTN/master/framework5.png) 6 | ## Step 1 Generate graph for AwA 7 | ``` 8 | cd GCN/materials/AWA2 9 | ``` 10 | 11 | Download: http://nlp.stanford.edu/data/glove.6B.zip 12 | Unzip it, find and put glove.6B.300d.txt here. 13 | 14 | ### Construct graph: 15 | ``` 16 | python make_graph_animal_step1.py 17 | python make_graph_animal_step2.py 18 | ``` 19 | Result in animals_graph_all.json, the graph for AwA2 which contains 255 nodes. 20 | 21 | ### Extract clasifiers from Resnet-50 pretrained on ImageNet for categories that shared by the construted graph and ImageNet-1K. 22 | Download: https://download.pytorch.org/models/resnet50-19c8e357.pth 23 | Rename and put it as materials/resnet50-raw.pth 24 | ``` 25 | python extract_127+24_from_1K.py 26 | ``` 27 | 28 | Result in 151_cls_from_1K. 29 | 30 | ## Step 2 Prepare data 31 | 32 | We upload the source domain of I2AwA : 33 | https://drive.google.com/file/d/1GdDZ1SvEqGin_zeCAGaJn0821vC_PJmc/view?usp=sharing 34 | Or 35 | https://pan.baidu.com/s/122-cvnjhYb0mB1zf4nShdA with password: ibee 36 | 37 | For the target domain, one can download from http://cvml.ist.ac.at/AwA2/. The link is as follow: 38 | 39 | http://cvml.ist.ac.at/AwA2/AwA2-data.zip 40 | 41 | NOTE THAT WE DO NOT HAVE THE RIGHTS FOR THE SHARED IMAGES. PLEASE DO NOT USE OUR DATASET FOR COMMERCIAL PURPOSE. 42 | 43 | We recommend that I2AwA can be used for traditional domain adaptation. 44 | 45 | ## Step 3 Train classifier on source domain 46 | 47 | We find that directly finetune classifiers on source domain and then transfer 40 known classifiers to unknown categories via GCN resulting very poor results. The reason may be in that using only 40 known classifiers as supervision information to train GCN is insufficient. Therefore, we utilize 127+24 classifiers pretrained on ImageNet-1K to train GCN. Specifically, 24 indicates that the classifiers are in source domain while the other 127 classifiers are not in target domain but are included in the constructed graph for target domain. We need to train the missing 16 classifiers for source domain. However, the classifiers pretrained on ImageNet-1K is discrepant from source domain. Thus, we fix the 24 classifiers and train 16 classifiers on source domain. Then we use these 127+24+16 classifiers to train GCN and obtain the initail classifiers for unknown categories. 48 | 49 | ``` 50 | cd UODTN 51 | python train_40_cls_with_help_of_1K.py 52 | ``` 53 | 54 | Result in 151+16_cls_from_1K_ft and base_net_pretrained_on_I2AwA2_source_only. 151+16_cls_from_1K_ft includes the original 127+24 classifiers and the additional 16 classifiers finetuned from the source domain. ["base_net_pretrained_on_I2AwA2_source_only"](https://drive.google.com/file/d/1FiHB8HV8U2Isfx0A6ipWEIaE4q-sekoO/view?usp=sharing) is a trained feature extractor for I2AwA. 55 | 56 | ## Step 4 Train UODTN 57 | 58 | Train a GCN: 59 | 60 | ``` 61 | cd GCN 62 | python train_gcn_basic_awa_ezhuo_2019.py 63 | python extract_50_cls_from_all_graph_for_awa.py 64 | ``` 65 | 66 | Result in ["awa_50_cls_basic"](https://drive.google.com/file/d/1DLeCpM7-k1xBianFEmc3L6c9526WEha4/view?usp=sharing), which contains 50 initial classifiers for AwA2. 67 | 68 | ### Prepare matching: 69 | 70 | ``` 71 | cd data/Matching 72 | python extract_feature.py 73 | python match_5_split.py 74 | ``` 75 | 76 | 77 | ### Train UODTN: 78 | 79 | ``` 80 | python train_UODTN.py 81 | ``` 82 | ### Citation 83 | Please cite our paper: 84 | @inproceedings{zhuo2019unsupervised, 85 | title={Unsupervised Open Domain Recognition by Semantic Discrepancy Minimization}, 86 | author={Zhuo, Junbao and Wang, Shuhui and Cui, Shuhao and Huang, Qingming}, 87 | booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition}, 88 | pages={750--759}, 89 | year={2019} 90 | } 91 | 92 | Acknowledgements: Our codes are mainly based on https://github.com/cyvius96/adgpm and https://github.com/thuml/Xlearn/tree/master/pytorch 93 | -------------------------------------------------------------------------------- /__pycache__/data_list.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/__pycache__/data_list.cpython-36.pyc -------------------------------------------------------------------------------- /__pycache__/loss.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/__pycache__/loss.cpython-36.pyc -------------------------------------------------------------------------------- /__pycache__/lr_schedule.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/__pycache__/lr_schedule.cpython-36.pyc -------------------------------------------------------------------------------- /__pycache__/network.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/__pycache__/network.cpython-36.pyc -------------------------------------------------------------------------------- /__pycache__/pre_process.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/__pycache__/pre_process.cpython-36.pyc -------------------------------------------------------------------------------- /__pycache__/utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/__pycache__/utils.cpython-36.pyc -------------------------------------------------------------------------------- /baseline_pmd+bgcn.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import os.path as osp 4 | 5 | import numpy as np 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | import torch.optim as optim 10 | import torch.utils.data as util_data 11 | from torch.autograd import Variable 12 | 13 | import time 14 | import json 15 | import random 16 | 17 | from data_list import ImageList 18 | import network 19 | import loss 20 | import pre_process as prep 21 | import lr_schedule 22 | from gcn_lib.gcn import GCN 23 | 24 | optim_dict = {"SGD": optim.SGD} 25 | 26 | 27 | 28 | def image_classification_test(iter_test,len_now, base, class1, class2, gpu=True): 29 | start_test = True 30 | Bd = 29410 31 | Total_1k = 0. 32 | Total_4k = 0. 33 | COR_1k = 0. 34 | COR_4k = 0. 35 | COR = 0. 36 | Total = 0. 37 | print('Testing ...') 38 | for i in range(len_now): 39 | data = iter_test.next() 40 | inputs = data[0] 41 | labels = data[1] 42 | if gpu: 43 | inputs = Variable(inputs.cuda()) 44 | labels = Variable(labels.cuda()) 45 | else: 46 | inputs = Variable(inputs) 47 | labels = Variable(labels) 48 | output = base(inputs) 49 | out1 = class1(output) 50 | out2 = class2(output) 51 | outputs = torch.cat((out1,out2),dim=1) 52 | if start_test: 53 | all_output = outputs.data.float() 54 | all_label = labels.data.float() 55 | _, predict = torch.max(all_output, 1) 56 | ind_1K = all_label.gt(39) 57 | ind_4K = 1-all_label.gt(39) 58 | COR = COR + torch.sum(torch.squeeze(predict).float() == all_label) 59 | Total = Total + all_label.size()[0] 60 | COR_1k = COR_1k + torch.sum(torch.squeeze(predict).float()[ind_1K] == all_label[ind_1K]) 61 | Total_1k = Total_1k + torch.sum(ind_1K) 62 | COR_4k = COR_4k + torch.sum(torch.squeeze(predict).float()[ind_4K] == all_label[ind_4K]) 63 | Total_4k = Total_4k + torch.sum(ind_4K) 64 | print('Unkown_acc: '+ str(float(COR_1k)/float(Total_1k))) 65 | print('Known_acc: '+ str(float(COR_4k)/float(Total_4k))) 66 | accuracy = float(COR)/float(Total) 67 | return accuracy 68 | 69 | def train_classification(config): 70 | ## set pre-process 71 | prep_train = prep.image_train(resize_size=256, crop_size=224) 72 | prep_test = prep.image_test(resize_size=256, crop_size=224) 73 | 74 | ## set loss 75 | class_criterion = nn.CrossEntropyLoss() 76 | transfer_criterion = loss.loss_dict["LP"] 77 | 78 | ## prepare data 79 | TRAIN_LIST = 'data/I2AWA2_40.txt' 80 | TEST_LIST = 'data/new_AwA2.txt' 81 | BSZ = args.batch_size 82 | 83 | dsets_train = ImageList(open(TRAIN_LIST).readlines(), shape = (args.img_size,args.img_size), transform=prep_train) 84 | loaders_train = util_data.DataLoader(dsets_train, batch_size=BSZ, shuffle=True, num_workers=8, pin_memory=True) 85 | 86 | dsets_test = ImageList(open(TEST_LIST).readlines(), shape = (args.img_size,args.img_size),transform=prep_test, train=False) 87 | loaders_test = util_data.DataLoader(dsets_test, batch_size=BSZ, shuffle=True, num_workers=4, pin_memory=True) 88 | begin_num = 127 89 | class_num = 40 90 | all_num = 50 91 | ## set base network 92 | net_config = config["network"] 93 | base_network = network.network_dict[net_config["name"]]() 94 | base_network.load_state_dict(torch.load('GCN/materials/AWA2/base_net_pretrained_on_I2AwA2_source_only.pkl')) 95 | classifier_layer1 = nn.Linear(base_network.output_num(), class_num) 96 | classifier_layer2 = nn.Linear(base_network.output_num(), all_num-class_num) 97 | for param in base_network.parameters(): 98 | param.requires_grad = False 99 | for param in base_network.layer4.parameters(): 100 | param.requires_grad = True 101 | for param in base_network.layer3.parameters(): 102 | param.requires_grad = True 103 | ## initialization 104 | 105 | weight_bias=torch.load('GCN/awa_50_cls_basic')['fc50'] 106 | classifier_layer1.weight.data = weight_bias[:class_num,:2048] 107 | classifier_layer2.weight.data = weight_bias[class_num:,:2048] 108 | classifier_layer1.bias.data = weight_bias[:class_num,-1] 109 | classifier_layer2.bias.data = weight_bias[class_num:,-1] 110 | 111 | graph = json.load(open('GCN/materials/AWA2/animals_graph_all.json','r')) 112 | word_vectors = torch.tensor(graph['vectors']) 113 | wnids = graph['wnids'] 114 | n = len(wnids) 115 | use_att =False 116 | if use_att: 117 | edges_set = graph['edges_set'] 118 | print('edges_set', [len(l) for l in edges_set]) 119 | lim = 4 120 | for i in range(lim + 1, len(edges_set)): 121 | edges_set[lim].extend(edges_set[i]) 122 | edges_set = edges_set[:lim + 1] 123 | print('edges_set', [len(l) for l in edges_set]) 124 | edges = edges_set 125 | else: 126 | edges = graph['edges'] 127 | edges = edges + [(v, u) for (u, v) in edges] 128 | edges = edges + [(u, u) for u in range(n)] 129 | word_vectors = F.normalize(word_vectors).cuda() 130 | hidden_layers = 'd2048,d' 131 | gcn = GCN(n, edges, word_vectors.shape[1], 2049, hidden_layers) 132 | gcn.load_state_dict(torch.load('GCN/RESULTS_MODELS/awa-basic/epoch-3000.pth')) 133 | gcn.train() 134 | use_gpu = torch.cuda.is_available() 135 | if use_gpu: 136 | classifier_layer1 = classifier_layer1.cuda() 137 | classifier_layer2 = classifier_layer2.cuda() 138 | base_network = base_network.cuda() 139 | gcn =gcn.cuda() 140 | 141 | 142 | ## collect parameters 143 | parameter_list = [{"params":classifier_layer2.parameters(), "lr":2}, 144 | {"params":classifier_layer1.parameters(), "lr":5}, 145 | {"params": base_network.layer3.parameters(), "lr":1}, 146 | {"params": base_network.layer4.parameters(), "lr":2}] 147 | 148 | 149 | ## set optimizer 150 | optimizer_config = config["optimizer"] 151 | optimizer = optim_dict[optimizer_config["type"]](parameter_list, **(optimizer_config["optim_params"])) 152 | param_lr = [] 153 | for param_group in optimizer.param_groups: 154 | param_lr.append(param_group["lr"]) 155 | schedule_param = optimizer_config["lr_param"] 156 | lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] 157 | 158 | 159 | len_train_source = len(loaders_train) - 1 160 | len_test_source = len(loaders_test) - 1 161 | optimizer.zero_grad() 162 | for i in range(config["num_iterations"]): 163 | if ((i + 0) % config["test_interval"] == 0 and i > 100) or i== config["num_iterations"]-1 : 164 | base_network.layer3.train(False) 165 | base_network.layer4.train(False) 166 | classifier_layer1.train(False) 167 | classifier_layer2.train(False) 168 | print(str(i)+' ACC:') 169 | iter_target = iter(loaders_test) 170 | print(image_classification_test(iter_target,len_test_source, base_network, classifier_layer1,classifier_layer2, gpu=use_gpu)) 171 | iter_target = iter(loaders_test) 172 | 173 | classifier_layer1.train(True) 174 | classifier_layer2.train(True) 175 | base_network.layer3.train(True) 176 | base_network.layer4.train(True) 177 | 178 | optimizer = lr_scheduler(param_lr, optimizer, i, **schedule_param) 179 | 180 | if i % len_train_source == 0: 181 | iter_source = iter(loaders_train) 182 | if i % (len_test_source ) == 0: 183 | iter_target = iter(loaders_test) 184 | 185 | inputs_source, labels_source, labels_source_father, inputs_target = iter_source.next() 186 | 187 | if use_gpu: 188 | inputs_source, labels_source, inputs_target = Variable(inputs_source).cuda(), Variable(labels_source).cuda(), Variable(inputs_target).cuda() 189 | else: 190 | inputs_source, labels_source, inputs_target = Variable(inputs_source), Variable(labels_source),Variable(inputs_target) 191 | 192 | features_source = base_network(inputs_source) 193 | features_target = base_network(inputs_target) 194 | 195 | outputs_source1 = classifier_layer1(features_source) 196 | outputs_source2 = classifier_layer2(features_source) 197 | outputs_target1 = classifier_layer1(features_target) 198 | outputs_target2 = classifier_layer2(features_target) 199 | 200 | outputs_source = torch.cat((outputs_source1,outputs_source2),dim=1) 201 | outputs_target = torch.cat((outputs_target1,outputs_target2),dim=1) 202 | output_vectors = gcn(word_vectors) 203 | 204 | cls_loss = class_criterion(outputs_source, labels_source) 205 | 206 | outputs_softmax = F.softmax(outputs_target, dim=1) 207 | 208 | transfer_loss = transfer_criterion(features_source, features_target) 209 | entropy_loss = -torch.mean(torch.sum(outputs_softmax[:,class_num:],1)) 210 | 211 | class1_weightbias = torch.cat((classifier_layer1.weight,classifier_layer1.bias.view(-1, 1)),dim=1) 212 | class2_weightbias = torch.cat((classifier_layer2.weight,classifier_layer2.bias.view(-1, 1)),dim=1) 213 | classifier_weight_bias = torch.cat((class1_weightbias,class2_weightbias), dim=0) 214 | 215 | total_loss = cls_loss + args.w_entropy * entropy_loss + transfer_loss * args.w_align 216 | print("Step "+str(i)+": cls_loss: "+str(cls_loss.cpu().data.numpy())+ 217 | " entropy_loss: "+str(entropy_loss.cpu().data.numpy())+ 218 | " transfer_loss: "+str(transfer_loss.cpu().data.numpy())) 219 | 220 | if ( i + 0 ) % config["save_num"] == 0: 221 | if not osp.exists(osp.join('save',args.save_name)): 222 | os.mkdir(osp.join('save',args.save_name)) 223 | torch.save(base_network.state_dict(),osp.join('save',args.save_name,'base_net%d.pkl'%(i+1))) 224 | torch.save(classifier_weight_bias,osp.join('save',args.save_name,'class%d.pkl'%(i+1))) 225 | torch.save(gcn.state_dict(),osp.join('save',args.save_name,'gcn_net%d.pkl'%(i+1))) 226 | 227 | total_loss.backward(retain_graph=True) 228 | if (i+1)% config["opt_num"] ==0: 229 | optimizer.step() 230 | optimizer.zero_grad() 231 | 232 | 233 | if __name__ == "__main__": 234 | parser = argparse.ArgumentParser(description='Transfer Learning') 235 | parser.add_argument('--gpu_id', type=str, nargs='?', default='0', help="device id to run") 236 | parser.add_argument('--batch_size', type=int, nargs='?', default=90, help="batch size") 237 | parser.add_argument('--img_size', type=int, nargs='?', default=256, help="image size") 238 | parser.add_argument('--save_name', type=str, nargs='?', default='base', help="loss name") 239 | parser.add_argument('--w_entropy', type=float, nargs='?', default=0.4, help="weight of entropy for target domain") 240 | parser.add_argument('--w_align', type=float, nargs='?', default=0.1, help="percent of unseen data") # Set w_align=0 to get the result of baseline: bGCN 241 | args = parser.parse_args() 242 | os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id 243 | 244 | config = {} 245 | config["num_iterations"] = 1200 246 | config["test_interval"] = 200 247 | config["save_num"] = 200 248 | config["opt_num"] = 1 249 | config["network"] = {"name":"ResNet50"} 250 | config["optimizer"] = {"type":"SGD", "optim_params":{"lr":1.0, "momentum":0.9, "weight_decay":0.0001, "nesterov":True}, "lr_type":"inv", "lr_param":{"init_lr":0.0001, "gamma":0.001, "power":0.75} } 251 | print(config) 252 | print(args) 253 | train_classification(config) 254 | -------------------------------------------------------------------------------- /data/Matching/data_list.py: -------------------------------------------------------------------------------- 1 | #from __future__ import print_function, division 2 | 3 | import torch 4 | import numpy as np 5 | from sklearn.preprocessing import StandardScaler 6 | import random 7 | from PIL import Image 8 | from PIL import ImageFile 9 | ImageFile.LOAD_TRUNCATED_IMAGES = True 10 | import torch.utils.data as data 11 | import os 12 | import os.path 13 | import time 14 | 15 | def make_dataset(image_list, labels): 16 | if len(image_list[0].split()) > 2: 17 | images = [(val.split()[0], int(val.split()[1]), int(val.split()[2]), val.split()[3]) for val in image_list] 18 | else: 19 | images = [(val.split()[0], int(val.split()[1])) for val in image_list] 20 | return images 21 | 22 | 23 | def pil_loader(path): 24 | # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835) 25 | with open(path, 'rb') as f: 26 | with Image.open(f) as img: 27 | return img.convert('RGB') 28 | 29 | 30 | def accimage_loader(path): 31 | import accimage 32 | try: 33 | return accimage.Image(path) 34 | except IOError: 35 | # Potentially a decoding problem, fall back to PIL.Image 36 | return pil_loader(path) 37 | 38 | 39 | def default_loader(path): 40 | #from torchvision import get_image_backend 41 | #if get_image_backend() == 'accimage': 42 | # return accimage_loader(path) 43 | #else: 44 | return pil_loader(path) 45 | 46 | 47 | class ImageList(object): 48 | """A generic data loader where the images are arranged in this way: :: 49 | root/dog/xxx.png 50 | root/dog/xxy.png 51 | root/dog/xxz.png 52 | root/cat/123.png 53 | root/cat/nsdf3.png 54 | root/cat/asd932_.png 55 | Args: 56 | root (string): Root directory path. 57 | transform (callable, optional): A function/transform that takes in an PIL image 58 | and returns a transformed version. E.g, ``transforms.RandomCrop`` 59 | target_transform (callable, optional): A function/transform that takes in the 60 | target and transforms it. 61 | loader (callable, optional): A function to load an image given its path. 62 | Attributes: 63 | classes (list): List of the class names. 64 | class_to_idx (dict): Dict with items (class_name, class_index). 65 | imgs (list): List of (image path, class_index) tuples 66 | """ 67 | 68 | def __init__(self, image_list, shape=None,labels=None, transform=None, target_transform=None, 69 | loader=default_loader, train=True): 70 | imgs = make_dataset(image_list, labels) 71 | if len(imgs) == 0: 72 | raise(RuntimeError("Found 0 images in subfolders of: " + root + "\n" 73 | "Supported image extensions are: " + ",".join(IMG_EXTENSIONS))) 74 | 75 | self.imgs = imgs 76 | self.transform = transform 77 | self.target_transform = target_transform 78 | self.loader = loader 79 | self.train = train 80 | self.shape = shape#hassassin 81 | def __getitem__(self, index): 82 | """ 83 | Args: 84 | index (int): Index 85 | Returns: 86 | tuple: (image, target) where target is class_index of the target class. 87 | """ 88 | #print(index,'image time',time.time()) 89 | if self.train: 90 | path, target, target2, path2 = self.imgs[index] 91 | else: 92 | path, target = self.imgs[index] 93 | img = self.loader(path) 94 | if self.train: 95 | img2 = self.loader(path2) 96 | if self.shape: 97 | img = img.resize(self.shape) 98 | if self.train: 99 | img2= img2.resize(self.shape) 100 | if self.transform is not None: 101 | img = self.transform(img) 102 | if self.train: 103 | img2 = self.transform(img2) 104 | #print(index,'deal time',time.time()) 105 | if self.train: 106 | return img, target, target2, img2 107 | else: 108 | return img, target 109 | 110 | def __len__(self): 111 | return len(self.imgs) 112 | 113 | def ClassSamplingImageList(image_list, transform, return_keys=False): 114 | data = open(image_list).readlines() 115 | label_dict = {} 116 | for line in data: 117 | label_dict[int(line.split()[1])] = [] 118 | for line in data: 119 | label_dict[int(line.split()[1])].append(line) 120 | all_image_list = {} 121 | for i in label_dict.keys(): 122 | all_image_list[i] = ImageList(label_dict[i], transform=transform) 123 | if return_keys: 124 | return all_image_list, label_dict.keys() 125 | else: 126 | return all_image_list 127 | -------------------------------------------------------------------------------- /data/Matching/extract_feature.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os.path 3 | import numpy as np 4 | import torch 5 | import torch.nn.functional as F 6 | import torch.utils.data as util_data 7 | 8 | 9 | from data_list import ImageList 10 | import pre_process as prep 11 | from resnet import make_resnet50_base 12 | 13 | 14 | model_path ='resnet50-base.pth' 15 | cnn = make_resnet50_base() 16 | cnn.load_state_dict(torch.load(model_path)) 17 | for param in cnn.parameters(): 18 | param.requires_grad = False 19 | cnn.cuda() 20 | cnn.eval() 21 | infile = open('S_features.csv','a') 22 | TEST_LIST = 'WEB_3D3_2.txt' 23 | prep_test = prep.image_test(resize_size=256, crop_size=224) 24 | dsets_test = ImageList(open(TEST_LIST).readlines(), shape = (256,256),transform=prep_test, train=False) 25 | loaders_test = util_data.DataLoader(dsets_test, batch_size=512, shuffle=False, num_workers=16, pin_memory=True) 26 | for batch_id ,batch in enumerate(loaders_test,1): 27 | print(str(batch_id)) 28 | data,label = batch 29 | data = data.cuda() 30 | features = cnn(data) 31 | now = features.cpu().data.numpy() 32 | np.savetxt(infile, now, delimiter = ',') 33 | torch.cuda.empty_cache() 34 | 35 | infile = open('T_features.csv','a') 36 | TEST_LIST = 'new_AwA2.txt' 37 | prep_test = prep.image_test(resize_size=256, crop_size=224) 38 | dsets_test = ImageList(open(TEST_LIST).readlines(), shape = (256,256),transform=prep_test, train=False) 39 | loaders_test = util_data.DataLoader(dsets_test, batch_size=512, shuffle=False, num_workers=16, pin_memory=True) 40 | for batch_id ,batch in enumerate(loaders_test,1): 41 | print(str(batch_id)) 42 | data,label = batch 43 | data = data.cuda() 44 | features = cnn(data) 45 | now = features.cpu().data.numpy() 46 | np.savetxt(infile, now, delimiter = ',') 47 | torch.cuda.empty_cache() 48 | -------------------------------------------------------------------------------- /data/Matching/match_5_split.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os, sys 3 | import json 4 | import random 5 | import hungarian 6 | 7 | import os 8 | 9 | os.system('cat ../WEB_3D3_2.txt ../WEB_3D3_2.txt ../WEB_3D3_2.txt ../WEB_3D3_2.txt ../WEB_3D3_2.txt ../WEB_3D3_2.txt ../WEB_3D3_2.txt ../WEB_3D3_2.txt > WEB_3D3_.txt') 10 | print('Loading S') 11 | imgs = np.loadtxt(open('S_features.csv','rb'),delimiter=',',skiprows=0) 12 | imgs = np.concatenate((imgs, imgs), axis=0) 13 | imgs = np.concatenate((imgs, imgs), axis=0) 14 | imgs = np.concatenate((imgs, imgs), axis=0) 15 | paths = open('./WEB_3D3_.txt','r').readlines() 16 | print('Loading T') 17 | imgs_web = np.loadtxt(open('T_features.csv','rb'),delimiter=',',skiprows=0) 18 | paths_web = open('./new_AwA2.txt','r').readlines() 19 | print('Loaded !') 20 | 21 | L = imgs.shape[0] 22 | print(L) 23 | print(imgs.shape[0]) 24 | print(imgs.shape[1]) 25 | shuffle = [i for i in range(L)] 26 | random.shuffle(shuffle) 27 | Tail = imgs_web.shape[0] - imgs.shape[0] 28 | imgs = np.concatenate((imgs, imgs[shuffle[:Tail]]), axis=0) 29 | print(len(paths)) 30 | paths_tail = [paths[i] for i in shuffle[:Tail]] 31 | paths = paths + paths_tail 32 | 33 | L = imgs_web.shape[0] 34 | shuffle = [i for i in range(L)] 35 | random.shuffle(shuffle) 36 | 37 | imgs_web = imgs_web[shuffle] 38 | paths_web = [paths_web[i] for i in shuffle] 39 | 40 | random.shuffle(shuffle) 41 | imgs = imgs[shuffle] 42 | paths = [paths[i] for i in shuffle] 43 | 44 | 45 | 46 | d_h1 = imgs[:8500].astype(np.float32) 47 | d_h2 = imgs_web[:8500].astype(np.float32) 48 | L1_ = np.zeros((d_h1.shape[0], d_h2.shape[0])) 49 | ws = [] 50 | BSZ = 50 51 | for i in range(int(d_h1.shape[0]/BSZ)+1): 52 | print(i) 53 | if i!=int(d_h1.shape[0]/BSZ): 54 | d_h1_ = np.repeat(d_h1[int(i*BSZ):int((i+1)*BSZ)], d_h2.shape[0], axis=0) 55 | d_h2_ = np.tile(d_h2, [BSZ,1]) 56 | L1 = np.sum(np.abs(d_h1_ - d_h2_), 1) 57 | L1_[int(i*BSZ):int((i+1)*BSZ),:] = np.reshape(L1, (BSZ, d_h2.shape[0])) 58 | else: 59 | d_h1_ = np.repeat(d_h1[int(i*BSZ):], d_h2.shape[0], axis=0) 60 | d_h2_ = np.tile(d_h2, [int(d_h1.shape[0]%BSZ),1]) 61 | L1 = np.sum(np.abs(d_h1_ - d_h2_), 1) 62 | L1_[int(i*BSZ):,:] = np.reshape(L1, (int(d_h1.shape[0]%BSZ), d_h2.shape[0])) 63 | 64 | 65 | obj_matrix = L1_ 66 | f=open('COR1.txt','w') 67 | print('Begin') 68 | rs, cs = hungarian.lap(obj_matrix) 69 | print('Write result') 70 | for i in range(8500): 71 | f.write(paths[i][:-1]+' '+ paths_web[rs[i]]) 72 | f.close() 73 | f=open('COR2.txt','w') 74 | d_h1 = imgs[8500:17000].astype(np.float32) 75 | d_h2 = imgs_web[8500:17000].astype(np.float32) 76 | L1_ = np.zeros((d_h1.shape[0], d_h2.shape[0])) 77 | ws = [] 78 | BSZ = 50 79 | for i in range(int(d_h1.shape[0]/BSZ)+1): 80 | print(i) 81 | if i!=int(d_h1.shape[0]/BSZ): 82 | d_h1_ = np.repeat(d_h1[int(i*BSZ):int((i+1)*BSZ)], d_h2.shape[0], axis=0) 83 | d_h2_ = np.tile(d_h2, [BSZ,1]) 84 | L1 = np.sum(np.abs(d_h1_ - d_h2_), 1) 85 | L1_[int(i*BSZ):int((i+1)*BSZ),:] = np.reshape(L1, (BSZ, d_h2.shape[0])) 86 | else: 87 | d_h1_ = np.repeat(d_h1[int(i*BSZ):], d_h2.shape[0], axis=0) 88 | d_h2_ = np.tile(d_h2, [int(d_h1.shape[0]%BSZ),1]) 89 | L1 = np.sum(np.abs(d_h1_ - d_h2_), 1) 90 | L1_[int(i*BSZ):,:] = np.reshape(L1, (int(d_h1.shape[0]%BSZ), d_h2.shape[0])) 91 | 92 | 93 | obj_matrix = L1_ 94 | print('Begin') 95 | rs, cs = hungarian.lap(obj_matrix) 96 | print('Write result') 97 | for i in range(8500): 98 | f.write(paths[i+8500][:-1]+' '+ paths_web[rs[i]+8500]) 99 | f.close() 100 | f=open('COR3.txt','w') 101 | 102 | d_h1 = imgs[17000:25500].astype(np.float32) 103 | d_h2 = imgs_web[17000:25500].astype(np.float32) 104 | L1_ = np.zeros((d_h1.shape[0], d_h2.shape[0])) 105 | ws = [] 106 | BSZ = 50 107 | for i in range(int(d_h1.shape[0]/BSZ)+1): 108 | print(i) 109 | if i!=int(d_h1.shape[0]/BSZ): 110 | d_h1_ = np.repeat(d_h1[int(i*BSZ):int((i+1)*BSZ)], d_h2.shape[0], axis=0) 111 | d_h2_ = np.tile(d_h2, [BSZ,1]) 112 | L1 = np.sum(np.abs(d_h1_ - d_h2_), 1) 113 | L1_[int(i*BSZ):int((i+1)*BSZ),:] = np.reshape(L1, (BSZ, d_h2.shape[0])) 114 | else: 115 | d_h1_ = np.repeat(d_h1[int(i*BSZ):], d_h2.shape[0], axis=0) 116 | d_h2_ = np.tile(d_h2, [int(d_h1.shape[0]%BSZ),1]) 117 | L1 = np.sum(np.abs(d_h1_ - d_h2_), 1) 118 | L1_[int(i*BSZ):,:] = np.reshape(L1, (int(d_h1.shape[0]%BSZ), d_h2.shape[0])) 119 | 120 | 121 | obj_matrix = L1_ 122 | print('Begin') 123 | rs, cs = hungarian.lap(obj_matrix) 124 | print('Write result') 125 | for i in range(8500): 126 | f.write(paths[i+17000][:-1]+' '+ paths_web[rs[i]+17000]) 127 | 128 | f.close() 129 | f=open('COR4.txt','w') 130 | d_h1 = imgs[25500:34000].astype(np.float32) 131 | d_h2 = imgs_web[25500:34000].astype(np.float32) 132 | L1_ = np.zeros((d_h1.shape[0], d_h2.shape[0])) 133 | ws = [] 134 | BSZ = 50 135 | for i in range(int(d_h1.shape[0]/BSZ)+1): 136 | print(i) 137 | if i!=int(d_h1.shape[0]/BSZ): 138 | d_h1_ = np.repeat(d_h1[int(i*BSZ):int((i+1)*BSZ)], d_h2.shape[0], axis=0) 139 | d_h2_ = np.tile(d_h2, [BSZ,1]) 140 | L1 = np.sum(np.abs(d_h1_ - d_h2_), 1) 141 | L1_[int(i*BSZ):int((i+1)*BSZ),:] = np.reshape(L1, (BSZ, d_h2.shape[0])) 142 | else: 143 | d_h1_ = np.repeat(d_h1[int(i*BSZ):], d_h2.shape[0], axis=0) 144 | d_h2_ = np.tile(d_h2, [int(d_h1.shape[0]%BSZ),1]) 145 | L1 = np.sum(np.abs(d_h1_ - d_h2_), 1) 146 | L1_[int(i*BSZ):,:] = np.reshape(L1, (int(d_h1.shape[0]%BSZ), d_h2.shape[0])) 147 | 148 | 149 | obj_matrix = L1_ 150 | print('Begin') 151 | rs, cs = hungarian.lap(obj_matrix) 152 | print('Write result') 153 | for i in range(min(8500,8268)): 154 | f.write(paths[i+25500][:-1]+' '+ paths_web[rs[i]+25500]) 155 | f.close() 156 | f=open('COR5.txt','w') 157 | 158 | d_h1 = imgs[34000:].astype(np.float32) 159 | d_h2 = imgs_web[34000:].astype(np.float32) 160 | L1_ = np.zeros((d_h1.shape[0], d_h2.shape[0])) 161 | ws = [] 162 | BSZ = 50 163 | for i in range(int(d_h1.shape[0]/BSZ)+1): 164 | print(i) 165 | if i!=int(d_h1.shape[0]/BSZ): 166 | d_h1_ = np.repeat(d_h1[int(i*BSZ):int((i+1)*BSZ)], d_h2.shape[0], axis=0) 167 | d_h2_ = np.tile(d_h2, [BSZ,1]) 168 | L1 = np.sum(np.abs(d_h1_ - d_h2_), 1) 169 | L1_[int(i*BSZ):int((i+1)*BSZ),:] = np.reshape(L1, (BSZ, d_h2.shape[0])) 170 | else: 171 | d_h1_ = np.repeat(d_h1[int(i*BSZ):], d_h2.shape[0], axis=0) 172 | d_h2_ = np.tile(d_h2, [int(d_h1.shape[0]%BSZ),1]) 173 | L1 = np.sum(np.abs(d_h1_ - d_h2_), 1) 174 | L1_[int(i*BSZ):,:] = np.reshape(L1, (int(d_h1.shape[0]%BSZ), d_h2.shape[0])) 175 | 176 | 177 | obj_matrix = L1_ 178 | print('Begin') 179 | rs, cs = hungarian.lap(obj_matrix) 180 | print('Write result') 181 | for i in range(len(imgs)-34000): 182 | f.write(paths[i+34000][:-1]+' '+ paths_web[rs[i]+34000]) 183 | 184 | f.close() 185 | 186 | os.system('cat COR1.txt COR2.txt COR3.txt COR4.txt COR5.txt > Matched_pairs.txt') 187 | -------------------------------------------------------------------------------- /data/Matching/pre_process.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from torchvision import transforms 3 | import os 4 | from PIL import Image, ImageOps 5 | import numbers 6 | import torch 7 | 8 | class ResizeImage(): 9 | def __init__(self, size): 10 | if isinstance(size, int): 11 | self.size = (int(size), int(size)) 12 | else: 13 | self.size = size 14 | def __call__(self, img): 15 | th, tw = self.size 16 | return img.resize((th, tw)) 17 | 18 | 19 | class PlaceCrop(object): 20 | """Crops the given PIL.Image at the particular index. 21 | Args: 22 | size (sequence or int): Desired output size of the crop. If size is an 23 | int instead of sequence like (w, h), a square crop (size, size) is 24 | made. 25 | """ 26 | 27 | def __init__(self, size, start_x, start_y): 28 | if isinstance(size, int): 29 | self.size = (int(size), int(size)) 30 | else: 31 | self.size = size 32 | self.start_x = start_x 33 | self.start_y = start_y 34 | 35 | def __call__(self, img): 36 | """ 37 | Args: 38 | img (PIL.Image): Image to be cropped. 39 | Returns: 40 | PIL.Image: Cropped image. 41 | """ 42 | th, tw = self.size 43 | return img.crop((self.start_x, self.start_y, self.start_x + tw, self.start_y + th)) 44 | 45 | 46 | class ForceFlip(object): 47 | """Horizontally flip the given PIL.Image randomly with a probability of 0.5.""" 48 | 49 | def __call__(self, img): 50 | """ 51 | Args: 52 | img (PIL.Image): Image to be flipped. 53 | Returns: 54 | PIL.Image: Randomly flipped image. 55 | """ 56 | return img.transpose(Image.FLIP_LEFT_RIGHT) 57 | 58 | def image_train(resize_size=256, crop_size=224): 59 | normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 60 | std=[0.229, 0.224, 0.225]) 61 | return transforms.Compose([ 62 | #ResizeImage(resize_size), 63 | # transforms.RandomResizedCrop(crop_size), 64 | transforms.RandomCrop(crop_size), 65 | transforms.RandomHorizontalFlip(), 66 | transforms.ToTensor(), 67 | normalize 68 | ]) 69 | 70 | def image_test(resize_size=256, crop_size=224): 71 | normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 72 | std=[0.229, 0.224, 0.225]) 73 | #ten crops for image when validation, input the data_transforms dictionary 74 | start_first = 0 75 | start_center = (resize_size - crop_size - 1) / 2 76 | start_last = resize_size - crop_size - 1 77 | 78 | return transforms.Compose([ 79 | transforms.Resize((resize_size,resize_size)), 80 | # ResizeImage(resize_size), NEW_JUNBAO 81 | PlaceCrop(crop_size, start_center, start_center), 82 | transforms.ToTensor(), 83 | normalize 84 | ]) 85 | 86 | # def image_test_10crop(resize_size=256, crop_size=224): 87 | # normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 88 | # std=[0.229, 0.224, 0.225]) 89 | # #ten crops for image when validation, input the data_transforms dictionary 90 | # start_first = 0 91 | # start_center = (resize_size - crop_size - 1) / 2 92 | # start_last = resize_size - crop_size - 1 93 | # data_transforms = {} 94 | # data_transforms['val0'] = transforms.Compose([ 95 | # ResizeImage(resize_size),ForceFlip(), 96 | # PlaceCrop(crop_size, start_first, start_first), 97 | # transforms.ToTensor(), 98 | # normalize 99 | # ]) 100 | # data_transforms['val1'] = transforms.Compose([ 101 | # ResizeImage(resize_size),ForceFlip(), 102 | # PlaceCrop(crop_size, start_last, start_last), 103 | # transforms.ToTensor(), 104 | # normalize 105 | # ]) 106 | # data_transforms['val2'] = transforms.Compose([ 107 | # ResizeImage(resize_size),ForceFlip(), 108 | # PlaceCrop(crop_size, start_last, start_first), 109 | # transforms.ToTensor(), 110 | # normalize 111 | # ]) 112 | # data_transforms['val3'] = transforms.Compose([ 113 | # ResizeImage(resize_size),ForceFlip(), 114 | # PlaceCrop(crop_size, start_first, start_last), 115 | # transforms.ToTensor(), 116 | # normalize 117 | # ]) 118 | # data_transforms['val4'] = transforms.Compose([ 119 | # ResizeImage(resize_size),ForceFlip(), 120 | # PlaceCrop(crop_size, start_center, start_center), 121 | # transforms.ToTensor(), 122 | # normalize 123 | # ]) 124 | # data_transforms['val5'] = transforms.Compose([ 125 | # ResizeImage(resize_size), 126 | # PlaceCrop(crop_size, start_first, start_first), 127 | # transforms.ToTensor(), 128 | # normalize 129 | # ]) 130 | # data_transforms['val6'] = transforms.Compose([ 131 | # ResizeImage(resize_size), 132 | # PlaceCrop(crop_size, start_last, start_last), 133 | # transforms.ToTensor(), 134 | # normalize 135 | # ]) 136 | # data_transforms['val7'] = transforms.Compose([ 137 | # ResizeImage(resize_size), 138 | # PlaceCrop(crop_size, start_last, start_first), 139 | # transforms.ToTensor(), 140 | # normalize 141 | # ]) 142 | # data_transforms['val8'] = transforms.Compose([ 143 | # ResizeImage(resize_size), 144 | # PlaceCrop(crop_size, start_first, start_last), 145 | # transforms.ToTensor(), 146 | # normalize 147 | # ]) 148 | # data_transforms['val9'] = transforms.Compose([ 149 | # ResizeImage(resize_size), 150 | # PlaceCrop(crop_size, start_center, start_center), 151 | # transforms.ToTensor(), 152 | # normalize 153 | # ]) 154 | # return data_transforms 155 | # 156 | -------------------------------------------------------------------------------- /data/Matching/resnet.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | 7 | __all__ = ['ResNetBase', 'make_resnet_base', 'ResNet'] 8 | 9 | 10 | ''' 11 | model_urls = { 12 | 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', 13 | 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', 14 | 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', 15 | 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', 16 | 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', 17 | } 18 | ''' 19 | 20 | 21 | def conv3x3(in_planes, out_planes, stride=1): 22 | """3x3 convolution with padding""" 23 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 24 | padding=1, bias=False) 25 | 26 | 27 | class BasicBlock(nn.Module): 28 | expansion = 1 29 | 30 | def __init__(self, inplanes, planes, stride=1, downsample=None): 31 | super(BasicBlock, self).__init__() 32 | self.conv1 = conv3x3(inplanes, planes, stride) 33 | self.bn1 = nn.BatchNorm2d(planes) 34 | self.relu = nn.ReLU(inplace=True) 35 | self.conv2 = conv3x3(planes, planes) 36 | self.bn2 = nn.BatchNorm2d(planes) 37 | self.downsample = downsample 38 | self.stride = stride 39 | 40 | def forward(self, x): 41 | residual = x 42 | 43 | out = self.conv1(x) 44 | out = self.bn1(out) 45 | out = self.relu(out) 46 | 47 | out = self.conv2(out) 48 | out = self.bn2(out) 49 | 50 | if self.downsample is not None: 51 | residual = self.downsample(x) 52 | 53 | out += residual 54 | out = self.relu(out) 55 | 56 | return out 57 | 58 | 59 | class Bottleneck(nn.Module): 60 | expansion = 4 61 | 62 | def __init__(self, inplanes, planes, stride=1, downsample=None): 63 | super(Bottleneck, self).__init__() 64 | self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) 65 | self.bn1 = nn.BatchNorm2d(planes) 66 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, 67 | padding=1, bias=False) 68 | self.bn2 = nn.BatchNorm2d(planes) 69 | self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) 70 | self.bn3 = nn.BatchNorm2d(planes * 4) 71 | self.relu = nn.ReLU(inplace=True) 72 | self.downsample = downsample 73 | self.stride = stride 74 | 75 | def forward(self, x): 76 | residual = x 77 | 78 | out = self.conv1(x) 79 | out = self.bn1(out) 80 | out = self.relu(out) 81 | 82 | out = self.conv2(out) 83 | out = self.bn2(out) 84 | out = self.relu(out) 85 | 86 | out = self.conv3(out) 87 | out = self.bn3(out) 88 | 89 | if self.downsample is not None: 90 | residual = self.downsample(x) 91 | 92 | out += residual 93 | out = self.relu(out) 94 | return out 95 | 96 | 97 | class ResNetBase(nn.Module): 98 | 99 | def __init__(self, block, layers): 100 | self.inplanes = 64 101 | super(ResNetBase, self).__init__() 102 | self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, 103 | bias=False) 104 | self.bn1 = nn.BatchNorm2d(64) 105 | self.relu = nn.ReLU(inplace=True) 106 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) 107 | self.layer1 = self._make_layer(block, 64, layers[0]) 108 | self.layer2 = self._make_layer(block, 128, layers[1], stride=2) 109 | self.layer3 = self._make_layer(block, 256, layers[2], stride=2) 110 | self.layer4 = self._make_layer(block, 512, layers[3], stride=2) 111 | 112 | self.out_channels = 512 * block.expansion 113 | 114 | for m in self.modules(): 115 | if isinstance(m, nn.Conv2d): 116 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 117 | m.weight.data.normal_(0, math.sqrt(2. / n)) 118 | elif isinstance(m, nn.BatchNorm2d): 119 | m.weight.data.fill_(1) 120 | m.bias.data.zero_() 121 | 122 | def _make_layer(self, block, planes, blocks, stride=1): 123 | downsample = None 124 | if stride != 1 or self.inplanes != planes * block.expansion: 125 | downsample = nn.Sequential( 126 | nn.Conv2d(self.inplanes, planes * block.expansion, 127 | kernel_size=1, stride=stride, bias=False), 128 | nn.BatchNorm2d(planes * block.expansion), 129 | ) 130 | 131 | layers = [] 132 | layers.append(block(self.inplanes, planes, stride, downsample)) 133 | self.inplanes = planes * block.expansion 134 | for i in range(1, blocks): 135 | layers.append(block(self.inplanes, planes)) 136 | 137 | return nn.Sequential(*layers) 138 | 139 | def forward(self, x): 140 | x = self.conv1(x) 141 | x = self.bn1(x) 142 | x = self.relu(x) 143 | x = self.maxpool(x) 144 | 145 | x = self.layer1(x) 146 | x = self.layer2(x) 147 | x = self.layer3(x) 148 | x = self.layer4(x) 149 | 150 | x = x.view(x.shape[0], x.shape[1], -1).mean(dim=2) 151 | return x 152 | 153 | 154 | def make_resnet18_base(**kwargs): 155 | """Constructs a ResNet-18 model. 156 | """ 157 | model = ResNetBase(BasicBlock, [2, 2, 2, 2], **kwargs) 158 | return model 159 | 160 | 161 | def make_resnet34_base(**kwargs): 162 | """Constructs a ResNet-34 model. 163 | """ 164 | model = ResNetBase(BasicBlock, [3, 4, 6, 3], **kwargs) 165 | return model 166 | 167 | 168 | def make_resnet50_base(**kwargs): 169 | """Constructs a ResNet-50 model. 170 | """ 171 | model = ResNetBase(Bottleneck, [3, 4, 6, 3], **kwargs) 172 | return model 173 | 174 | 175 | def make_resnet101_base(**kwargs): 176 | """Constructs a ResNet-101 model. 177 | """ 178 | model = ResNetBase(Bottleneck, [3, 4, 23, 3], **kwargs) 179 | return model 180 | 181 | 182 | def make_resnet152_base(**kwargs): 183 | """Constructs a ResNet-152 model. 184 | """ 185 | model = ResNetBase(Bottleneck, [3, 8, 36, 3], **kwargs) 186 | return model 187 | 188 | 189 | def make_resnet_base(version, pretrained=None): 190 | maker = { 191 | 'resnet18': make_resnet18_base, 192 | 'resnet34': make_resnet34_base, 193 | 'resnet50': make_resnet50_base, 194 | 'resnet101': make_resnet101_base, 195 | 'resnet152': make_resnet152_base 196 | } 197 | resnet = maker[version]() 198 | if pretrained is not None: 199 | sd = torch.load(pretrained) 200 | sd.pop('fc.weight') 201 | sd.pop('fc.bias') 202 | resnet.load_state_dict(sd) 203 | return resnet 204 | 205 | 206 | class ResNet(nn.Module): 207 | 208 | def __init__(self, version, num_classes, pretrained=None): 209 | super().__init__() 210 | self.resnet_base = make_resnet_base(version, pretrained=pretrained) 211 | self.fc = nn.Linear(self.resnet_base.out_channels, num_classes) 212 | 213 | def forward(self, x, need_features=False): 214 | x = self.resnet_base(x) 215 | feat = x 216 | x = self.fc(x) 217 | if need_features: 218 | return x, feat 219 | else: 220 | return x 221 | 222 | -------------------------------------------------------------------------------- /data_list.py: -------------------------------------------------------------------------------- 1 | #from __future__ import print_function, division 2 | 3 | import torch 4 | import numpy as np 5 | from sklearn.preprocessing import StandardScaler 6 | import random 7 | from PIL import Image 8 | from PIL import ImageFile 9 | ImageFile.LOAD_TRUNCATED_IMAGES = True 10 | import torch.utils.data as data 11 | import os 12 | import os.path 13 | import time 14 | 15 | def make_dataset(image_list, labels): 16 | if len(image_list[0].split()) > 2: 17 | images = [(val.split()[0], int(val.split()[1]), val.split()[2], int(val.split()[3])) for val in image_list] 18 | else: 19 | images = [(val.split()[0], int(val.split()[1])) for val in image_list] 20 | return images 21 | 22 | 23 | def pil_loader(path): 24 | # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835) 25 | with open(path, 'rb') as f: 26 | with Image.open(f) as img: 27 | return img.convert('RGB') 28 | 29 | 30 | def accimage_loader(path): 31 | import accimage 32 | try: 33 | return accimage.Image(path) 34 | except IOError: 35 | # Potentially a decoding problem, fall back to PIL.Image 36 | return pil_loader(path) 37 | 38 | 39 | def default_loader(path): 40 | #from torchvision import get_image_backend 41 | #if get_image_backend() == 'accimage': 42 | # return accimage_loader(path) 43 | #else: 44 | return pil_loader(path) 45 | 46 | 47 | class ImageList(object): 48 | 49 | def __init__(self, image_list, shape=None,labels=None, transform=None, target_transform=None, 50 | loader=default_loader, train=True): 51 | imgs = make_dataset(image_list, labels) 52 | if len(imgs) == 0: 53 | raise(RuntimeError("Found 0 images in subfolders of: " + root + "\n" 54 | "Supported image extensions are: " + ",".join(IMG_EXTENSIONS))) 55 | 56 | self.imgs = imgs 57 | self.transform = transform 58 | self.target_transform = target_transform 59 | self.loader = loader 60 | self.train = train 61 | self.shape = shape#hassassin 62 | def __getitem__(self, index): 63 | """ 64 | Args: 65 | index (int): Index 66 | Returns: 67 | tuple: (image, target) where target is class_index of the target class. 68 | """ 69 | if self.train: 70 | path, target, path2, target2 = self.imgs[index] 71 | else: 72 | path, target = self.imgs[index] 73 | img = self.loader(path) 74 | if self.train: 75 | img2 = self.loader(path2) 76 | if self.transform is not None: 77 | img = self.transform(img) 78 | if self.train: 79 | img2 = self.transform(img2) 80 | if self.train: 81 | return img, target, target2, img2 82 | else: 83 | return img, target 84 | 85 | def __len__(self): 86 | return len(self.imgs) 87 | 88 | def ClassSamplingImageList(image_list, transform, return_keys=False): 89 | data = open(image_list).readlines() 90 | label_dict = {} 91 | for line in data: 92 | label_dict[int(line.split()[1])] = [] 93 | for line in data: 94 | label_dict[int(line.split()[1])].append(line) 95 | all_image_list = {} 96 | for i in label_dict.keys(): 97 | all_image_list[i] = ImageList(label_dict[i], transform=transform) 98 | if return_keys: 99 | return all_image_list, label_dict.keys() 100 | else: 101 | return all_image_list 102 | -------------------------------------------------------------------------------- /framework5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/framework5.png -------------------------------------------------------------------------------- /gcn_lib/__pycache__/gcn.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/junbaoZHUO/UODTN/cdfdfa2dc711ed49f1dab9184f017d39b47ce6f3/gcn_lib/__pycache__/gcn.cpython-36.pyc -------------------------------------------------------------------------------- /gcn_lib/gcn.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.sparse as sp 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | # from torch.nn.init import xavier_uniform 8 | from torch.nn.init import xavier_uniform_ 9 | 10 | from utils import normt_spm, spm_to_tensor 11 | 12 | 13 | class GraphConv(nn.Module): 14 | 15 | def __init__(self, in_channels, out_channels, dropout=False, relu=True): 16 | super(GraphConv,self).__init__() 17 | 18 | if dropout: 19 | self.dropout = nn.Dropout(p=0.5) 20 | else: 21 | self.dropout = None 22 | 23 | self.w = nn.Parameter(torch.empty(in_channels, out_channels)) 24 | self.b = nn.Parameter(torch.zeros(out_channels)) 25 | xavier_uniform_(self.w) 26 | 27 | if relu: 28 | self.relu = nn.LeakyReLU(negative_slope=0.2) 29 | else: 30 | self.relu = None 31 | 32 | def forward(self, inputs, adj): 33 | if self.dropout is not None: 34 | inputs = self.dropout(inputs) 35 | 36 | outputs = torch.mm(adj, torch.mm(inputs, self.w)) + self.b 37 | 38 | if self.relu is not None: 39 | outputs = self.relu(outputs) 40 | return outputs 41 | 42 | 43 | class GCN(nn.Module): 44 | 45 | def __init__(self, n, edges, in_channels, out_channels, hidden_layers): 46 | super(GCN,self).__init__() 47 | 48 | edges = np.array(edges) 49 | adj = sp.coo_matrix((np.ones(len(edges)), (edges[:, 0], edges[:, 1])), 50 | shape=(n, n), dtype='float32') 51 | adj = normt_spm(adj, method='in') 52 | adj = spm_to_tensor(adj) 53 | self.adj = adj.cuda() 54 | 55 | hl = hidden_layers.split(',') 56 | if hl[-1] == 'd': 57 | dropout_last = True 58 | hl = hl[:-1] 59 | else: 60 | dropout_last = False 61 | 62 | i = 0 63 | layers = [] 64 | last_c = in_channels 65 | for c in hl: 66 | if c[0] == 'd': 67 | dropout = True 68 | c = c[1:] 69 | else: 70 | dropout = False 71 | c = int(c) 72 | 73 | i += 1 74 | conv = GraphConv(last_c, c, dropout=dropout) 75 | self.add_module('conv{}'.format(i), conv) 76 | layers.append(conv) 77 | 78 | last_c = c 79 | 80 | conv = GraphConv(last_c, out_channels, relu=False, dropout=dropout_last) 81 | self.add_module('conv-last', conv) 82 | layers.append(conv) 83 | 84 | self.layers = layers 85 | 86 | def forward(self, x): 87 | for conv in self.layers: 88 | x = conv(x, self.adj) 89 | return x 90 | # return F.normalize(x) 91 | 92 | -------------------------------------------------------------------------------- /loss.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | from torch.autograd import Variable 5 | 6 | def EntropyLoss(input_): 7 | mask = input_.ge(0.000001) 8 | mask_out = torch.masked_select(input_, mask) 9 | entropy = -(torch.sum(mask_out * torch.log(mask_out))) 10 | return entropy / float(input_.size(0)) 11 | 12 | def guassian_kernel(source, target, kernel_mul=2.0, kernel_num=5, fix_sigma=None): 13 | n_samples = int(source.size()[0])+int(target.size()[0]) 14 | total = torch.cat([source, target], dim=0) 15 | total0 = total.unsqueeze(0).expand(int(total.size(0)), int(total.size(0)), int(total.size(1))) 16 | total1 = total.unsqueeze(1).expand(int(total.size(0)), int(total.size(0)), int(total.size(1))) 17 | L2_distance = ((total0-total1)**2).sum(2) 18 | if fix_sigma: 19 | bandwidth = fix_sigma 20 | else: 21 | bandwidth = torch.sum(L2_distance.data) / (n_samples**2-n_samples) 22 | bandwidth /= kernel_mul ** (kernel_num // 2) 23 | bandwidth_list = [bandwidth * (kernel_mul**i) for i in range(kernel_num)] 24 | kernel_val = [torch.exp(-L2_distance / bandwidth_temp) for bandwidth_temp in bandwidth_list] 25 | return sum(kernel_val) 26 | 27 | 28 | def MK_MMD(source, target, kernel_mul=2.0, kernel_num=8, fix_sigma=None): 29 | batch_size = int(source.size()[0]) 30 | kernels = guassian_kernel(source, target, 31 | kernel_mul=kernel_mul, kernel_num=kernel_num, fix_sigma=fix_sigma) 32 | loss = 0 33 | for i in range(batch_size): 34 | s1, s2 = i, (i+1)%batch_size 35 | t1, t2 = s1+batch_size, s2+batch_size 36 | loss += kernels[s1, s2] + kernels[t1, t2] 37 | loss -= kernels[s1, t2] + kernels[s2, t1] 38 | return loss / float(batch_size) 39 | 40 | 41 | def JAN(source_list, target_list, kernel_muls=[2.0, 2.0], kernel_nums=[5, 1], fix_sigma_list=[None, 1.68]): 42 | batch_size = int(source_list[0].size()[0]) 43 | layer_num = len(source_list) 44 | joint_kernels = None 45 | for i in range(layer_num): 46 | source = source_list[i] 47 | target = target_list[i] 48 | kernel_mul = kernel_muls[i] 49 | kernel_num = kernel_nums[i] 50 | fix_sigma = fix_sigma_list[i] 51 | kernels = guassian_kernel(source, target, 52 | kernel_mul=kernel_mul, kernel_num=kernel_num, fix_sigma=fix_sigma) 53 | if joint_kernels is not None: 54 | joint_kernels = joint_kernels * kernels 55 | else: 56 | joint_kernels = kernels 57 | 58 | loss = 0 59 | for i in range(batch_size): 60 | s1, s2 = i, (i+1)%batch_size 61 | t1, t2 = s1+batch_size, s2+batch_size 62 | loss += joint_kernels[s1, s2] + joint_kernels[t1, t2] 63 | loss -= joint_kernels[s1, t2] + joint_kernels[s2, t1] 64 | return loss / float(batch_size) 65 | 66 | 67 | def L1N(source, target, WEIGHT=None, if_weight=False): 68 | if not if_weight: 69 | L1 = torch.mean(torch.sum(torch.abs(source - target), dim=1)) 70 | L_s = torch.mean(torch.sum(torch.abs(source), dim=1)) 71 | L_t = torch.mean(torch.sum(torch.abs(target), dim=1)) 72 | else: 73 | L1 = torch.mean(torch.sum(torch.abs(source - target), dim=1) * WEIGHT.view(-1, 1)) 74 | L_s = torch.mean(torch.sum(torch.abs(source), dim=1)* WEIGHT.view(-1, 1)) 75 | L_t = torch.mean(torch.sum(torch.abs(target), dim=1)* WEIGHT.view(-1, 1)) 76 | return L1/(L_s + L_t) 77 | 78 | def LP(source, target, WEIGHT=None, if_weight=False, P=2): 79 | if not if_weight: 80 | return torch.mean(torch.nn.PairwiseDistance(p=P)(source, target)) 81 | else: 82 | return torch.sum(torch.nn.PairwiseDistance(p=P)(source, target) * WEIGHT.view(-1))/(torch.sum(WEIGHT) + 0.0001) 83 | 84 | def Matching_MMD(source, target, WEIGHT, if_weight=False): 85 | Mask = WEIGHT.view(-1, 1).repeat(1, 2048).byte() 86 | source_matched = torch.masked_select(source, Mask).view(-1, 2048) 87 | target_matched = torch.masked_select(target, Mask).view(-1, 2048) 88 | return MK_MMD(source_matched, target_matched) 89 | 90 | loss_dict = {"MMD":MK_MMD, "JAN":JAN, "L1N":L1N, "LP":LP, "Matching-MMD":Matching_MMD} 91 | -------------------------------------------------------------------------------- /lr_schedule.py: -------------------------------------------------------------------------------- 1 | def inv_lr_scheduler(param_lr, optimizer, iter_num, gamma, power, init_lr=0.001): 2 | """Decay learning rate by a factor of 0.1 every lr_decay_epoch epochs.""" 3 | lr = init_lr * (1 + gamma * iter_num) ** (-power) 4 | 5 | i=0 6 | for param_group in optimizer.param_groups: 7 | param_group['lr'] = lr * param_lr[i] 8 | i+=1 9 | 10 | return optimizer 11 | 12 | 13 | schedule_dict = {"inv":inv_lr_scheduler} 14 | -------------------------------------------------------------------------------- /network.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | import torchvision 5 | from torchvision import models 6 | from torch.autograd import Variable 7 | 8 | class SilenceLayer(torch.autograd.Function): 9 | def __init__(self): 10 | pass 11 | def forward(self, input): 12 | return input * 1.0 13 | 14 | def backward(self, gradOutput): 15 | return 0 * gradOutput 16 | 17 | 18 | # convnet without the last layer 19 | class AlexNetFc(nn.Module): 20 | def __init__(self): 21 | super(AlexNetFc, self).__init__() 22 | model_alexnet = models.alexnet(pretrained=True) 23 | self.features = model_alexnet.features 24 | self.classifier = nn.Sequential() 25 | for i in xrange(6): 26 | self.classifier.add_module("classifier"+str(i), model_alexnet.classifier[i]) 27 | self.__in_features = model_alexnet.classifier[6].in_features 28 | 29 | def forward(self, x): 30 | x = self.features(x) 31 | x = x.view(x.size(0), 256*6*6) 32 | x = self.classifier(x) 33 | return x 34 | 35 | def output_num(self): 36 | return self.__in_features 37 | 38 | class ResNet18Fc(nn.Module): 39 | def __init__(self): 40 | super(ResNet18Fc, self).__init__() 41 | model_resnet18 = models.resnet18(pretrained=True) 42 | self.conv1 = model_resnet18.conv1 43 | self.bn1 = model_resnet18.bn1 44 | self.relu = model_resnet18.relu 45 | self.maxpool = model_resnet18.maxpool 46 | self.layer1 = model_resnet18.layer1 47 | self.layer2 = model_resnet18.layer2 48 | self.layer3 = model_resnet18.layer3 49 | self.layer4 = model_resnet18.layer4 50 | self.avgpool = model_resnet18.avgpool 51 | self.__in_features = model_resnet18.fc.in_features 52 | 53 | def forward(self, x): 54 | x = self.conv1(x) 55 | x = self.bn1(x) 56 | x = self.relu(x) 57 | x = self.maxpool(x) 58 | x = self.layer1(x) 59 | x = self.layer2(x) 60 | x = self.layer3(x) 61 | x = self.layer4(x) 62 | x = self.avgpool(x) 63 | x = x.view(x.size(0), -1) 64 | return x 65 | 66 | def output_num(self): 67 | return self.__in_features 68 | 69 | class ResNet34Fc(nn.Module): 70 | def __init__(self): 71 | super(ResNet34Fc, self).__init__() 72 | model_resnet34 = models.resnet34(pretrained=True) 73 | self.conv1 = model_resnet34.conv1 74 | self.bn1 = model_resnet34.bn1 75 | self.relu = model_resnet34.relu 76 | self.maxpool = model_resnet34.maxpool 77 | self.layer1 = model_resnet34.layer1 78 | self.layer2 = model_resnet34.layer2 79 | self.layer3 = model_resnet34.layer3 80 | self.layer4 = model_resnet34.layer4 81 | self.avgpool = model_resnet34.avgpool 82 | self.__in_features = model_resnet34.fc.in_features 83 | 84 | def forward(self, x): 85 | x = self.conv1(x) 86 | x = self.bn1(x) 87 | x = self.relu(x) 88 | x = self.maxpool(x) 89 | x = self.layer1(x) 90 | x = self.layer2(x) 91 | x = self.layer3(x) 92 | x = self.layer4(x) 93 | x = self.avgpool(x) 94 | x = x.view(x.size(0), -1) 95 | return x 96 | 97 | def output_num(self): 98 | return self.__in_features 99 | 100 | class ResNet50Fc(nn.Module): 101 | def __init__(self): 102 | super(ResNet50Fc, self).__init__() 103 | model_resnet50 = models.resnet50(pretrained=True) 104 | self.conv1 = model_resnet50.conv1 105 | self.bn1 = model_resnet50.bn1 106 | self.relu = model_resnet50.relu 107 | self.maxpool = model_resnet50.maxpool 108 | self.layer1 = model_resnet50.layer1 109 | self.layer2 = model_resnet50.layer2 110 | self.layer3 = model_resnet50.layer3 111 | self.layer4 = model_resnet50.layer4 112 | self.avgpool = model_resnet50.avgpool 113 | self.__in_features = model_resnet50.fc.in_features 114 | 115 | def forward(self, x): 116 | x = self.conv1(x) 117 | x = self.bn1(x) 118 | x = self.relu(x) 119 | x = self.maxpool(x) 120 | x = self.layer1(x) 121 | x = self.layer2(x) 122 | x = self.layer3(x) 123 | x = self.layer4(x) 124 | x = self.avgpool(x) 125 | x = x.view(x.size(0), -1) 126 | return x 127 | 128 | def output_num(self): 129 | return self.__in_features 130 | 131 | class ResNet101Fc(nn.Module): 132 | def __init__(self): 133 | super(ResNet101Fc, self).__init__() 134 | model_resnet101 = models.resnet101(pretrained=True) 135 | self.conv1 = model_resnet101.conv1 136 | self.bn1 = model_resnet101.bn1 137 | self.relu = model_resnet101.relu 138 | self.maxpool = model_resnet101.maxpool 139 | self.layer1 = model_resnet101.layer1 140 | self.layer2 = model_resnet101.layer2 141 | self.layer3 = model_resnet101.layer3 142 | self.layer4 = model_resnet101.layer4 143 | self.avgpool = model_resnet101.avgpool 144 | self.__in_features = model_resnet101.fc.in_features 145 | 146 | def forward(self, x): 147 | x = self.conv1(x) 148 | x = self.bn1(x) 149 | x = self.relu(x) 150 | x = self.maxpool(x) 151 | x = self.layer1(x) 152 | x = self.layer2(x) 153 | x = self.layer3(x) 154 | x = self.layer4(x) 155 | x = self.avgpool(x) 156 | x = x.view(x.size(0), -1) 157 | return x 158 | 159 | def output_num(self): 160 | return self.__in_features 161 | 162 | 163 | class ResNet152Fc(nn.Module): 164 | def __init__(self): 165 | super(ResNet152Fc, self).__init__() 166 | model_resnet152 = models.resnet152(pretrained=True) 167 | self.conv1 = model_resnet152.conv1 168 | self.bn1 = model_resnet152.bn1 169 | self.relu = model_resnet152.relu 170 | self.maxpool = model_resnet152.maxpool 171 | self.layer1 = model_resnet152.layer1 172 | self.layer2 = model_resnet152.layer2 173 | self.layer3 = model_resnet152.layer3 174 | self.layer4 = model_resnet152.layer4 175 | self.avgpool = model_resnet152.avgpool 176 | self.__in_features = model_resnet152.fc.in_features 177 | 178 | def forward(self, x): 179 | x = self.conv1(x) 180 | x = self.bn1(x) 181 | x = self.relu(x) 182 | x = self.maxpool(x) 183 | x = self.layer1(x) 184 | x = self.layer2(x) 185 | x = self.layer3(x) 186 | x = self.layer4(x) 187 | x = self.avgpool(x) 188 | x = x.view(x.size(0), -1) 189 | return x 190 | 191 | def output_num(self): 192 | return self.__in_features 193 | 194 | network_dict = {"AlexNet":AlexNetFc, "ResNet18":ResNet18Fc, "ResNet34":ResNet34Fc, "ResNet50":ResNet50Fc, "ResNet101":ResNet101Fc, "ResNet152":ResNet152Fc} 195 | -------------------------------------------------------------------------------- /pre_process.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from torchvision import transforms 3 | import os 4 | from PIL import Image, ImageOps 5 | import numbers 6 | import torch 7 | 8 | class ResizeImage(): 9 | def __init__(self, size): 10 | if isinstance(size, int): 11 | self.size = (int(size), int(size)) 12 | else: 13 | self.size = size 14 | def __call__(self, img): 15 | th, tw = self.size 16 | return img.resize((th, tw)) 17 | 18 | 19 | class PlaceCrop(object): 20 | """Crops the given PIL.Image at the particular index. 21 | Args: 22 | size (sequence or int): Desired output size of the crop. If size is an 23 | int instead of sequence like (w, h), a square crop (size, size) is 24 | made. 25 | """ 26 | 27 | def __init__(self, size, start_x, start_y): 28 | if isinstance(size, int): 29 | self.size = (int(size), int(size)) 30 | else: 31 | self.size = size 32 | self.start_x = start_x 33 | self.start_y = start_y 34 | 35 | def __call__(self, img): 36 | """ 37 | Args: 38 | img (PIL.Image): Image to be cropped. 39 | Returns: 40 | PIL.Image: Cropped image. 41 | """ 42 | th, tw = self.size 43 | return img.crop((self.start_x, self.start_y, self.start_x + tw, self.start_y + th)) 44 | 45 | 46 | class ForceFlip(object): 47 | """Horizontally flip the given PIL.Image randomly with a probability of 0.5.""" 48 | 49 | def __call__(self, img): 50 | """ 51 | Args: 52 | img (PIL.Image): Image to be flipped. 53 | Returns: 54 | PIL.Image: Randomly flipped image. 55 | """ 56 | return img.transpose(Image.FLIP_LEFT_RIGHT) 57 | 58 | def image_train(resize_size=256, crop_size=224): 59 | normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 60 | std=[0.229, 0.224, 0.225]) 61 | return transforms.Compose([ 62 | transforms.Resize((resize_size,resize_size)), 63 | transforms.RandomCrop(crop_size), 64 | transforms.RandomHorizontalFlip(), 65 | transforms.ToTensor(), 66 | normalize 67 | ]) 68 | 69 | def image_test(resize_size=256, crop_size=224): 70 | normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 71 | std=[0.229, 0.224, 0.225]) 72 | #ten crops for image when validation, input the data_transforms dictionary 73 | start_first = 0 74 | start_center = (resize_size - crop_size - 1) / 2 75 | start_last = resize_size - crop_size - 1 76 | 77 | return transforms.Compose([ 78 | transforms.Resize((resize_size,resize_size)), 79 | PlaceCrop(crop_size, start_center, start_center), 80 | transforms.ToTensor(), 81 | normalize 82 | ]) 83 | -------------------------------------------------------------------------------- /train_40_cls_with_help_of_1K.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import os.path as osp 4 | 5 | import numpy as np 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | import torch.optim as optim 10 | import torch.utils.data as util_data 11 | from torch.autograd import Variable 12 | 13 | import time 14 | import json 15 | import random 16 | 17 | from data_list import ImageList 18 | import network 19 | import loss 20 | import pre_process as prep 21 | import lr_schedule 22 | 23 | optim_dict = {"SGD": optim.SGD} 24 | 25 | 26 | 27 | 28 | def train_classification(config): 29 | ## set pre-process 30 | prep_train = prep.image_train(resize_size=256, crop_size=224) 31 | 32 | ## set loss 33 | class_criterion = nn.CrossEntropyLoss() 34 | 35 | ## prepare data 36 | TRAIN_LIST = 'data/WEB_3D3_2.txt' 37 | BSZ = args.batch_size 38 | 39 | dsets_train1 = ImageList(open(TRAIN_LIST).readlines(), shape = (args.img_size,args.img_size), transform=prep_train, train=False) 40 | loaders_train1 = util_data.DataLoader(dsets_train1, batch_size=BSZ, shuffle=True, num_workers=6, pin_memory=True) 41 | 42 | begin_num = 127 43 | class_num = 40 44 | all_num = 50 45 | ## set base network 46 | net_config = config["network"] 47 | base_network = network.network_dict[net_config["name"]]() 48 | classifier_layer1 = nn.Linear(base_network.output_num(), 24) 49 | classifier_layer2 = nn.Linear(base_network.output_num(), 16) 50 | weight_bias=torch.load('GCN/materials/AWA2/151_cls_from_1K')['fc151'] 51 | classifier_layer1.weight.data = weight_bias[127:127+24,:2048] 52 | classifier_layer1.bias.data = weight_bias[127:127+24,-1] 53 | 54 | ## initialization 55 | for param in base_network.parameters(): 56 | param.requires_grad = False 57 | for param in base_network.layer4.parameters(): 58 | param.requires_grad = True 59 | for param in base_network.layer3.parameters(): 60 | param.requires_grad = True 61 | for param in classifier_layer1.parameters(): 62 | param.requires_grad = False 63 | 64 | use_gpu = torch.cuda.is_available() 65 | if use_gpu: 66 | classifier_layer1 = classifier_layer1.cuda() 67 | classifier_layer2 = classifier_layer2.cuda() 68 | base_network = base_network.cuda() 69 | 70 | ## collect parameters 71 | parameter_list = [{"params":classifier_layer2.parameters(), "lr":10}, 72 | {"params": base_network.layer3.parameters(), "lr":1}, 73 | {"params": base_network.layer4.parameters(), "lr":5}] 74 | 75 | 76 | ## set optimizer 77 | optimizer_config = config["optimizer"] 78 | optimizer = optim_dict[optimizer_config["type"]](parameter_list, **(optimizer_config["optim_params"])) 79 | param_lr = [] 80 | for param_group in optimizer.param_groups: 81 | param_lr.append(param_group["lr"]) 82 | schedule_param = optimizer_config["lr_param"] 83 | lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] 84 | 85 | 86 | len_train_source = len(loaders_train1) - 1 87 | optimizer.zero_grad() 88 | for i in range(config["num_iterations"]): 89 | if (i + 1) % config["test_interval"] == 0: 90 | if not osp.exists(osp.join('save',args.save_name)): 91 | os.mkdir(osp.join('save',args.save_name)) 92 | weight_bias[127:127+24,:2048] = classifier_layer1.weight.data 93 | weight_bias[127:127+24,-1] = classifier_layer1.bias.data 94 | weight_bias2 = torch.cat((classifier_layer2.weight.data,classifier_layer2.bias.data.unsqueeze(1)),dim=1) 95 | torch.save(base_network.state_dict(),osp.join('save',args.save_name,'base_net%d.pkl'%(i+1))) 96 | torch.save({'fc151+16_ft':torch.cat((weight_bias, weight_bias2.cpu()),dim=0)},'151+16_cls_from_1K_ft') 97 | 98 | classifier_layer1.train(True) 99 | classifier_layer2.train(True) 100 | base_network.train(True) 101 | 102 | optimizer = lr_scheduler(param_lr, optimizer, i, **schedule_param) 103 | 104 | if i % (len_train_source-1) == 0: 105 | iter_source1 = iter(loaders_train1) 106 | 107 | inputs_source, labels_source = iter_source1.next() 108 | 109 | if use_gpu: 110 | inputs_source, labels_source = Variable(inputs_source).cuda(), Variable(labels_source).cuda() 111 | else: 112 | inputs_source, labels_source = Variable(inputs_source), Variable(labels_source) 113 | 114 | features_source = base_network(inputs_source) 115 | 116 | outputs_source1 = classifier_layer1(features_source) 117 | outputs_source2 = classifier_layer2(features_source) 118 | cls_loss = class_criterion(torch.cat((outputs_source1, outputs_source2), dim=1), labels_source) 119 | 120 | total_loss = cls_loss 121 | print("Step "+str(i)+": cls_loss: "+str(cls_loss.cpu().data.numpy())) 122 | 123 | total_loss.backward(retain_graph=True) 124 | if (i+1)% config["opt_num"] ==0: 125 | optimizer.step() 126 | optimizer.zero_grad() 127 | 128 | 129 | if __name__ == "__main__": 130 | parser = argparse.ArgumentParser(description='Transfer Learning') 131 | parser.add_argument('--gpu_id', type=str, nargs='?', default='0', help="device id to run") 132 | parser.add_argument('--batch_size', type=int, nargs='?', default=64, help="batch size") 133 | parser.add_argument('--img_size', type=int, nargs='?', default=256, help="image size") 134 | parser.add_argument('--save_name', type=str, nargs='?', default='SOURCE_ONLY', help="loss name") 135 | args = parser.parse_args() 136 | os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id 137 | 138 | config = {} 139 | config["num_iterations"] = 2000 140 | config["test_interval"] = 500 141 | config["save_num"] = 500 142 | config["opt_num"] = 1 143 | config["network"] = {"name":"ResNet50"} 144 | config["optimizer"] = {"type":"SGD", "optim_params":{"lr":1.0, "momentum":0.9, "weight_decay":0.0001, "nesterov":True}, "lr_type":"inv", "lr_param":{"init_lr":0.001, "gamma":0.001, "power":0.75} } 145 | print(config) 146 | print(args) 147 | train_classification(config) 148 | -------------------------------------------------------------------------------- /train_UODTN.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import os.path as osp 4 | 5 | import numpy as np 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | import torch.optim as optim 10 | import torch.utils.data as util_data 11 | from torch.autograd import Variable 12 | 13 | import time 14 | import json 15 | import random 16 | 17 | from data_list import ImageList 18 | import network 19 | import loss 20 | import pre_process as prep 21 | import lr_schedule 22 | from gcn_lib.gcn import GCN 23 | 24 | optim_dict = {"SGD": optim.SGD} 25 | 26 | 27 | def Entropy(input_): 28 | bs = input_.size(0) 29 | epsilon = 1e-7 30 | entropy = -input_ * torch.log(input_ + epsilon) 31 | entropy = torch.sum(entropy, dim=1) 32 | return entropy 33 | 34 | def grl_hook(coeff): 35 | def fun1(grad): 36 | return -coeff*grad.clone() 37 | return fun1 38 | 39 | def gfl_hook(coeff): 40 | def fun1(grad): 41 | return coeff*grad.clone() 42 | return fun1 43 | 44 | def calc_coeff(iter_num, high=1.0, low=0.0, alpha=10.0, max_iter=1200.0): 45 | return np.float(2.0 * (high - low) / (1.0 + np.exp(-alpha*iter_num / max_iter)) - (high - low) + low) 46 | 47 | 48 | class GradReverse(torch.autograd.Function): 49 | def forward(self, x): 50 | return x.view_as(x) 51 | 52 | def backward(self, grad_output): 53 | return (grad_output * -1) 54 | 55 | def grad_reverse(x): 56 | return GradReverse()(x) 57 | 58 | def init_weights(m): 59 | classname = m.__class__.__name__ 60 | if classname.find('Conv2d') != -1 or classname.find('ConvTranspose2d') != -1: 61 | nn.init.kaiming_uniform_(m.weight) 62 | nn.init.zeros_(m.bias) 63 | elif classname.find('BatchNorm') != -1: 64 | nn.init.normal_(m.weight, 1.0, 0.02) 65 | nn.init.zeros_(m.bias) 66 | elif classname.find('Linear') != -1: 67 | nn.init.xavier_normal_(m.weight) 68 | nn.init.zeros_(m.bias) 69 | 70 | class AdversarialNetwork(nn.Module): 71 | def __init__(self, in_feature, hidden_size): 72 | super(AdversarialNetwork, self).__init__() 73 | self.ad_layer1 = nn.Linear(in_feature, hidden_size) 74 | self.ad_layer2 = nn.Linear(hidden_size, hidden_size) 75 | self.ad_layer3 = nn.Linear(hidden_size, 1) 76 | self.relu1 = nn.ReLU() 77 | self.relu2 = nn.ReLU() 78 | self.dropout1 = nn.Dropout(0.5) 79 | self.dropout2 = nn.Dropout(0.5) 80 | self.sigmoid = nn.Sigmoid() 81 | self.apply(init_weights) 82 | self.iter_num = 0 83 | self.alpha = 10 84 | self.low = 0.0 85 | self.high = 1.0 86 | self.max_iter = 1200.0 87 | 88 | def forward(self, x): 89 | if self.training: 90 | self.iter_num += 1 91 | coeff = calc_coeff(self.iter_num, self.high, self.low, self.alpha, self.max_iter) 92 | x = x * 1.0 93 | x.register_hook(grl_hook(coeff)) 94 | x = self.ad_layer1(x) 95 | x = self.relu1(x) 96 | x = self.dropout1(x) 97 | x = self.ad_layer2(x) 98 | x = self.relu2(x) 99 | x = self.dropout2(x) 100 | y = self.ad_layer3(x) 101 | y = self.sigmoid(y) 102 | return y 103 | 104 | def output_num(self): 105 | return 1 106 | def get_parameters(self): 107 | return [{"params":self.parameters(), "lr_mult":10, 'decay_mult':2}] 108 | def DANN(features, ad_net,size1,size2, entropy=None): 109 | ad_out = ad_net(features) 110 | batch_size = ad_out.size(0) // 2 111 | dc_target = torch.from_numpy(np.array([[1]] * size1 + [[0]] * size2)).float().cuda() 112 | if entropy is not None: 113 | return torch.mean(entropy.view(-1 ,1)*nn.BCELoss(reduction='none')(ad_out, dc_target)) 114 | else: 115 | return nn.BCELoss()(ad_out, dc_target) 116 | 117 | 118 | 119 | def my_l2_loss(a, b): 120 | return ((a - b)**2).sum() / (len(a) * 2) 121 | 122 | def image_classification_test(iter_test,len_now, base, class1, class2, gpu=True): 123 | start_test = True 124 | Total_1k = 0. 125 | Total_4k = 0. 126 | COR_1k = 0. 127 | COR_4k = 0. 128 | COR = 0. 129 | Total = 0. 130 | print('Testing ...') 131 | for i in range(len_now): 132 | data = iter_test.next() 133 | inputs = data[0] 134 | labels = data[1] 135 | if gpu: 136 | inputs = Variable(inputs.cuda()) 137 | labels = Variable(labels.cuda()) 138 | else: 139 | inputs = Variable(inputs) 140 | labels = Variable(labels) 141 | output = base(inputs) 142 | out1 = class1(output) 143 | out2 = class2(output) 144 | outputs = torch.cat((out1,out2),dim=1) 145 | if start_test: 146 | all_output = outputs.data.float() 147 | all_label = labels.data.float() 148 | _, predict = torch.max(all_output, 1) 149 | ind_1K = all_label.gt(39) 150 | ind_4K = 1-all_label.gt(39) 151 | COR = COR + torch.sum(torch.squeeze(predict).float() == all_label) 152 | Total = Total + all_label.size()[0] 153 | COR_1k = COR_1k + torch.sum(torch.squeeze(predict).float()[ind_1K] == all_label[ind_1K]) 154 | Total_1k = Total_1k + torch.sum(ind_1K) 155 | COR_4k = COR_4k + torch.sum(torch.squeeze(predict).float()[ind_4K] == all_label[ind_4K]) 156 | Total_4k = Total_4k + torch.sum(ind_4K) 157 | print('Unkown_acc: '+ str(float(COR_1k)/float(Total_1k))) 158 | print('Known_acc: '+ str(float(COR_4k)/float(Total_4k))) 159 | accuracy = float(COR)/float(Total) 160 | return accuracy 161 | 162 | def train_classification(config): 163 | ## set pre-process 164 | prep_train = prep.image_train(resize_size=256, crop_size=224) 165 | prep_test = prep.image_test(resize_size=256, crop_size=224) 166 | 167 | ## set loss 168 | class_criterion = nn.CrossEntropyLoss() 169 | 170 | ## prepare data 171 | TRAIN_LIST = 'data/I2AWA2.txt'#'AWA_SS.txt#'data/new_AwA2_common.txt' 172 | TEST_LIST = 'data/new_AwA2.txt' 173 | BSZ = args.batch_size 174 | 175 | dsets_train = ImageList(open(TRAIN_LIST).readlines(), shape = (args.img_size,args.img_size), transform=prep_train) 176 | loaders_train = util_data.DataLoader(dsets_train, batch_size=BSZ, shuffle=True, num_workers=8, pin_memory=True) 177 | 178 | dsets_test = ImageList(open(TEST_LIST).readlines(), shape = (args.img_size,args.img_size),transform=prep_test, train=False) 179 | loaders_test = util_data.DataLoader(dsets_test, batch_size=BSZ, shuffle=True, num_workers=4, pin_memory=True) 180 | begin_num = 127 181 | class_num = 40 182 | all_num = 50 183 | ## set base network 184 | net_config = config["network"] 185 | base_network = network.network_dict[net_config["name"]]() 186 | base_network.load_state_dict(torch.load('GCN/materials/AWA2/base_net_pretrained_on_I2AwA2_source_only.pkl')) 187 | classifier_layer1 = nn.Linear(base_network.output_num(), class_num) 188 | classifier_layer2 = nn.Linear(base_network.output_num(), all_num-class_num) 189 | for param in base_network.parameters(): 190 | param.requires_grad = False 191 | for param in base_network.layer4.parameters(): 192 | param.requires_grad = True 193 | for param in base_network.layer3.parameters(): 194 | param.requires_grad = True 195 | ## initialization 196 | 197 | weight_bias=torch.load('GCN/awa_50_cls_basic')['fc50'] 198 | weight_bias_127=torch.load('GCN/materials/AWA2/151_cls_from_1K')['fc151'] 199 | classifier_layer1.weight.data = weight_bias[:class_num,:2048] 200 | classifier_layer2.weight.data = weight_bias[class_num:,:2048] 201 | classifier_layer1.bias.data = weight_bias[:class_num,-1] 202 | classifier_layer2.bias.data = weight_bias[class_num:,-1] 203 | ad_net = AdversarialNetwork(2048, 1024) 204 | 205 | graph = json.load(open('GCN/materials/AWA2/animals_graph_all.json','r')) 206 | word_vectors = torch.tensor(graph['vectors']) 207 | wnids = graph['wnids'] 208 | n = len(wnids) 209 | use_att =False 210 | if use_att: 211 | edges_set = graph['edges_set'] 212 | print('edges_set', [len(l) for l in edges_set]) 213 | lim = 4 214 | for i in range(lim + 1, len(edges_set)): 215 | edges_set[lim].extend(edges_set[i]) 216 | edges_set = edges_set[:lim + 1] 217 | print('edges_set', [len(l) for l in edges_set]) 218 | edges = edges_set 219 | else: 220 | edges = graph['edges'] 221 | edges = edges + [(v, u) for (u, v) in edges] 222 | edges = edges + [(u, u) for u in range(n)] 223 | word_vectors = F.normalize(word_vectors).cuda() 224 | hidden_layers = 'd2048,d' 225 | gcn = GCN(n, edges, word_vectors.shape[1], 2049, hidden_layers) 226 | gcn.load_state_dict(torch.load('GCN/RESULTS_MODELS/awa-basic/epoch-3000.pth')) 227 | gcn.train() 228 | use_gpu = torch.cuda.is_available() 229 | if use_gpu: 230 | classifier_layer1 = classifier_layer1.cuda() 231 | classifier_layer2 = classifier_layer2.cuda() 232 | base_network = base_network.cuda() 233 | gcn =gcn.cuda() 234 | ad_net = ad_net.cuda() 235 | 236 | 237 | ## collect parameters 238 | parameter_list = [{"params": classifier_layer2.parameters(), "lr":2}, 239 | {"params": classifier_layer1.parameters(), "lr":5}, 240 | {"params": ad_net.parameters(), "lr":5}, 241 | {"params": base_network.layer3.parameters(), "lr":1}, 242 | {"params": base_network.layer4.parameters(), "lr":2}, 243 | {"params": gcn.parameters(), "lr":5}] 244 | 245 | 246 | ## set optimizer 247 | optimizer_config = config["optimizer"] 248 | optimizer = optim_dict[optimizer_config["type"]](parameter_list, **(optimizer_config["optim_params"])) 249 | param_lr = [] 250 | for param_group in optimizer.param_groups: 251 | param_lr.append(param_group["lr"]) 252 | schedule_param = optimizer_config["lr_param"] 253 | lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] 254 | 255 | 256 | len_train_source = len(loaders_train) - 1 257 | len_test_source = len(loaders_test) - 1 258 | optimizer.zero_grad() 259 | for i in range(config["num_iterations"]): 260 | if ((i + 0) % config["test_interval"] == 0 and i > 100) or i== config["num_iterations"]-1 : 261 | base_network.layer3.train(False) 262 | base_network.layer4.train(False) 263 | classifier_layer1.train(False) 264 | classifier_layer2.train(False) 265 | print(str(i)+' ACC:') 266 | iter_target = iter(loaders_test) 267 | print(image_classification_test(iter_target,len_test_source, base_network, classifier_layer1,classifier_layer2, gpu=use_gpu)) 268 | iter_target = iter(loaders_test) 269 | 270 | classifier_layer1.train(True) 271 | classifier_layer2.train(True) 272 | base_network.layer3.train(True) 273 | base_network.layer4.train(True) 274 | ad_net.train(True) 275 | 276 | optimizer = lr_scheduler(param_lr, optimizer, i, **schedule_param) 277 | 278 | if i % len_train_source == 0: 279 | iter_source = iter(loaders_train) 280 | if i % (len_test_source ) == 0: 281 | iter_target = iter(loaders_test) 282 | 283 | inputs_source, labels_source, labels_source_father, inputs_target = iter_source.next() 284 | 285 | if use_gpu: 286 | inputs_source, labels_source, inputs_target = Variable(inputs_source).cuda(), Variable(labels_source).cuda(), Variable(inputs_target).cuda() 287 | else: 288 | inputs_source, labels_source, inputs_target = Variable(inputs_source), Variable(labels_source),Variable(inputs_target) 289 | 290 | features_source = base_network(inputs_source) 291 | features_target = base_network(inputs_target) 292 | 293 | outputs_source1 = classifier_layer1(features_source) 294 | outputs_source2 = classifier_layer2(features_source) 295 | outputs_target1 = classifier_layer1(features_target) 296 | outputs_target2 = classifier_layer2(features_target) 297 | 298 | outputs_source = torch.cat((outputs_source1,outputs_source2),dim=1) 299 | outputs_target = torch.cat((outputs_target1,outputs_target2),dim=1) 300 | output_vectors = gcn(word_vectors) 301 | 302 | cls_loss = class_criterion(outputs_source, labels_source) 303 | 304 | outputs_softmax = F.softmax(outputs_target, dim=1) 305 | 306 | WEIGHT = torch.sum(torch.softmax(outputs_source, dim=1)[:,:40] * outputs_softmax[:,:40], 1)# - 0.2 307 | max_s = torch.max(outputs_softmax[:,:40], 1)[0]# - 0.2 308 | coeff = calc_coeff(i) 309 | entropy = Entropy(outputs_softmax) 310 | entropy.register_hook(grl_hook(coeff)) 311 | entropy = 1.0+torch.exp(-entropy) 312 | entropy_s = Entropy(F.softmax(outputs_source, dim=1)) 313 | entropy_s.register_hook(grl_hook(coeff)) 314 | entropy_s = 1.0+torch.exp(-entropy_s) 315 | Mask_ = (max_s.gt(0.4).float()*WEIGHT.gt(0.5).float()).byte() 316 | Mask = (max_s.gt(0.4).float()*WEIGHT.gt(0.5).float()).view(-1, 1).repeat(1, 2048).byte() 317 | source_matched = torch.masked_select(features_source, Mask).view(-1, 2048) 318 | target_matched = torch.masked_select(features_target, Mask).view(-1, 2048) 319 | transfer_loss = DANN(torch.cat((features_source, target_matched), dim=0), ad_net, BSZ, target_matched.size()[0], torch.cat((entropy_s, torch.masked_select(entropy, Mask_)), dim=0)) 320 | entropy_loss = torch.sum(torch.sum(outputs_softmax[:,class_num:],1) * (1.0 - max_s.gt(0.5).float()*WEIGHT.gt(0.6).float()))/torch.sum(1.0 - max_s.gt(0.5).float()*WEIGHT.gt(0.6).float()) 321 | 322 | class1_weightbias = torch.cat((classifier_layer1.weight,classifier_layer1.bias.view(-1, 1)),dim=1) 323 | class2_weightbias = torch.cat((classifier_layer2.weight,classifier_layer2.bias.view(-1, 1)),dim=1) 324 | classifier_weight_bias = torch.cat((class1_weightbias,class2_weightbias), dim=0) 325 | 326 | gcn_loss_1k = my_l2_loss(output_vectors[begin_num:(begin_num+class_num)], class1_weightbias) 327 | gcn_loss_4k = my_l2_loss(output_vectors[(begin_num+class_num):(begin_num+all_num)], class2_weightbias) 328 | gcn_loss_127 = my_l2_loss(output_vectors[:begin_num], weight_bias_127[:127].cuda()) 329 | 330 | 331 | total_loss = cls_loss + args.w_gcn * (gcn_loss_1k + gcn_loss_4k + gcn_loss_127 ) + args.w_entropy * (entropy_loss + args.w_data * args.w_data / entropy_loss) + transfer_loss * args.w_align 332 | print("Step "+str(i)+": cls_loss: "+str(cls_loss.cpu().data.numpy())+ 333 | " entropy_loss: "+str(entropy_loss.cpu().data.numpy())+ 334 | " transfer_loss: "+str(transfer_loss.cpu().data.numpy())+ 335 | " gcn_1k_loss: "+str(gcn_loss_1k.cpu().data.numpy())+ 336 | " gcn_4k_loss: "+str(gcn_loss_4k.cpu().data.numpy())+ 337 | " gcn_127_loss: "+str(gcn_loss_4k.cpu().data.numpy())) 338 | 339 | if ( i + 0 ) % config["save_num"] == 0: 340 | if not osp.exists(osp.join('save',args.save_name)): 341 | os.mkdir(osp.join('save',args.save_name)) 342 | torch.save(base_network.state_dict(),osp.join('save',args.save_name,'base_net%d.pkl'%(i+1))) 343 | torch.save(classifier_weight_bias,osp.join('save',args.save_name,'class%d.pkl'%(i+1))) 344 | torch.save(gcn.state_dict(),osp.join('save',args.save_name,'gcn_net%d.pkl'%(i+1))) 345 | 346 | total_loss.backward(retain_graph=True) 347 | if (i+1)% config["opt_num"] ==0: 348 | optimizer.step() 349 | optimizer.zero_grad() 350 | 351 | 352 | if __name__ == "__main__": 353 | parser = argparse.ArgumentParser(description='Transfer Learning') 354 | parser.add_argument('--gpu_id', type=str, nargs='?', default='0', help="device id to run") 355 | parser.add_argument('--batch_size', type=int, nargs='?', default=90, help="batch size") 356 | parser.add_argument('--img_size', type=int, nargs='?', default=256, help="image size") 357 | parser.add_argument('--save_name', type=str, nargs='?', default='UODTN', help="loss name") 358 | parser.add_argument('--w_entropy', type=float, nargs='?', default=3, help="weight of entropy for target domain") 359 | parser.add_argument('--w_gcn', type=float, nargs='?', default=1.5, help="weight of gcn") 360 | parser.add_argument('--w_data', type=float, nargs='?', default=0.09, help="percent of unseen data") 361 | parser.add_argument('--w_align', type=float, nargs='?', default=0.7, help="percent of unseen data") 362 | args = parser.parse_args() 363 | os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id 364 | 365 | config = {} 366 | config["num_iterations"] = 1200 367 | config["test_interval"] = 200 368 | config["save_num"] = 200 369 | config["opt_num"] = 1 370 | config["network"] = {"name":"ResNet50"} 371 | config["optimizer"] = {"type":"SGD", "optim_params":{"lr":1.0, "momentum":0.9, "weight_decay":0.0001, "nesterov":True}, "lr_type":"inv", "lr_param":{"init_lr":0.0001, "gamma":0.001, "power":0.75} } 372 | print(config) 373 | print(args) 374 | train_classification(config) 375 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import shutil 4 | 5 | import numpy as np 6 | import scipy.sparse as sp 7 | import torch 8 | 9 | 10 | def ensure_path(path): 11 | if osp.exists(path): 12 | if input('{} exists, remove? ([y]/n)'.format(path)) != 'n': 13 | shutil.rmtree(path) 14 | os.mkdir(path) 15 | else: 16 | os.mkdir(path) 17 | 18 | 19 | def set_gpu(gpu): 20 | os.environ['CUDA_VISIBLE_DEVICES'] = gpu 21 | print('using gpu {}'.format(gpu)) 22 | 23 | 24 | def pick_vectors(dic, wnids, is_tensor=False): 25 | o = next(iter(dic.values())) 26 | dim = len(o) 27 | ret = [] 28 | for wnid in wnids: 29 | v = dic.get(wnid) 30 | if v is None: 31 | if not is_tensor: 32 | v = [0] * dim 33 | else: 34 | v = torch.zeros(dim) 35 | ret.append(v) 36 | if not is_tensor: 37 | return torch.FloatTensor(ret) 38 | else: 39 | return torch.stack(ret) 40 | 41 | 42 | def l2_loss(a, b): 43 | return ((a - b)**2).sum() / (len(a) * 2) 44 | 45 | 46 | def normt_spm(mx, method='in'): 47 | if method == 'in': 48 | mx = mx.transpose() 49 | rowsum = np.array(mx.sum(1)) 50 | r_inv = np.power(rowsum, -1).flatten() 51 | r_inv[np.isinf(r_inv)] = 0. 52 | r_mat_inv = sp.diags(r_inv) 53 | mx = r_mat_inv.dot(mx) 54 | return mx 55 | 56 | if method == 'sym': 57 | rowsum = np.array(mx.sum(1)) 58 | r_inv = np.power(rowsum, -0.5).flatten() 59 | r_inv[np.isinf(r_inv)] = 0. 60 | r_mat_inv = sp.diags(r_inv) 61 | mx = mx.dot(r_mat_inv).transpose().dot(r_mat_inv) 62 | return mx 63 | 64 | 65 | def spm_to_tensor(sparse_mx): 66 | sparse_mx = sparse_mx.tocoo().astype(np.float32) 67 | indices = torch.from_numpy(np.vstack( 68 | (sparse_mx.row, sparse_mx.col))).long().cuda() 69 | values = torch.from_numpy(sparse_mx.data).cuda() 70 | shape = torch.Size(sparse_mx.shape) 71 | return torch.sparse.FloatTensor(indices, values, shape) 72 | 73 | --------------------------------------------------------------------------------