├── README.md ├── extrasensory ├── example.csv ├── models │ ├── AE.py │ ├── LocalModel.py │ └── ServerModel.py ├── script_extrasensory.sh ├── sensor_ae_train.py ├── sensor_infer_attack.py ├── sensor_normal_train.py ├── sensor_server_ae_train.py └── utils │ ├── robustae.py │ └── sensor_util.py ├── nus-wide ├── NUS_WIDE │ └── README.md ├── models │ ├── AE.py │ ├── LocalModel.py │ └── ServerModel.py ├── nus_ae_train.py ├── nus_infer_attack.py ├── nus_results │ └── normal_2i_2t_try1 │ │ ├── acc_test.txt │ │ ├── acc_train.txt │ │ ├── ae_ckpt.tf.data-00000-of-00001 │ │ ├── ae_ckpt.tf.index │ │ ├── checkpoint │ │ ├── final_server_AETrue │ │ ├── best_checkpoints.data-00000-of-00001 │ │ ├── best_checkpoints.index │ │ ├── checkpoint │ │ ├── checkpoints.data-00000-of-00001 │ │ └── checkpoints.index │ │ ├── img_0_360 │ │ ├── best_checkpoints.data-00000-of-00001 │ │ ├── best_checkpoints.index │ │ ├── checkpoint │ │ ├── epoch30_checkpoints.data-00000-of-00001 │ │ ├── epoch30_checkpoints.index │ │ ├── epoch40_checkpoints.data-00000-of-00001 │ │ ├── epoch40_checkpoints.index │ │ ├── epoch50_checkpoints.data-00000-of-00001 │ │ └── epoch50_checkpoints.index │ │ ├── img_360_634 │ │ ├── best_checkpoints.data-00000-of-00001 │ │ ├── best_checkpoints.index │ │ ├── checkpoint │ │ ├── epoch30_checkpoints.data-00000-of-00001 │ │ ├── epoch30_checkpoints.index │ │ ├── epoch40_checkpoints.data-00000-of-00001 │ │ ├── epoch40_checkpoints.index │ │ ├── epoch50_checkpoints.data-00000-of-00001 │ │ └── epoch50_checkpoints.index │ │ ├── server │ │ ├── best_checkpoints.data-00000-of-00001 │ │ ├── best_checkpoints.index │ │ ├── checkpoint │ │ ├── epoch30_checkpoints.data-00000-of-00001 │ │ ├── epoch30_checkpoints.index │ │ ├── epoch40_checkpoints.data-00000-of-00001 │ │ ├── epoch40_checkpoints.index │ │ ├── epoch50_checkpoints.data-00000-of-00001 │ │ └── epoch50_checkpoints.index │ │ ├── text_0_500 │ │ ├── best_checkpoints.data-00000-of-00001 │ │ ├── best_checkpoints.index │ │ ├── checkpoint │ │ ├── epoch30_checkpoints.data-00000-of-00001 │ │ ├── epoch30_checkpoints.index │ │ ├── epoch40_checkpoints.data-00000-of-00001 │ │ ├── epoch40_checkpoints.index │ │ ├── epoch50_checkpoints.data-00000-of-00001 │ │ └── epoch50_checkpoints.index │ │ └── text_500_1000 │ │ ├── best_checkpoints.data-00000-of-00001 │ │ ├── best_checkpoints.index │ │ ├── checkpoint │ │ ├── epoch30_checkpoints.data-00000-of-00001 │ │ ├── epoch30_checkpoints.index │ │ ├── epoch40_checkpoints.data-00000-of-00001 │ │ ├── epoch40_checkpoints.index │ │ ├── epoch50_checkpoints.data-00000-of-00001 │ │ └── epoch50_checkpoints.index ├── nus_server_ae_train.py ├── script_nus.sh └── utils │ ├── nus_utils.py │ ├── nus_wide_data_util.py │ └── robustae.py └── requirement.txt /README.md: -------------------------------------------------------------------------------- 1 | # CoPur: Certifiably Robust Collaborative Inference via Feature Purification 2 | 3 | This repository is the official implementation of "[CoPur: Certifiably Robust Collaborative Inference via Feature Purification](https://openreview.net/forum?id=r5rzV51GZx)". 4 | 5 | # Download and Installation 6 | The required packages can be installed by: 7 | 8 | ``` 9 | pip install -r requirement.txt 10 | ``` 11 | 12 | For datasets: 13 | - Download [NUS_WIDE dataset](https://lms.comp.nus.edu.sg/wp-content/uploads/2019/research/nuswide/NUS-WIDE.html) to folder `nus-wide/NUS_WIDE` 14 | - The Extrasensory dataset is available in `extrasensory/example.csv` 15 | 16 | # Usage 17 | 18 | 19 | 20 | ## Training and testing on NUS-WIDE dataset: 21 | 0. Enter the folder 22 | 23 | ``` 24 | cd nus-wide 25 | ``` 26 | 1. The pre-trained Local feature extractors can be found in the folder ``nus_results`` 27 | 2. Feature subspace learning 28 | 29 | ``` 30 | python nus_ae_train.py --ae_epochs 400 --ae_lr 0.001 --img_feature_div 0 360 634 --text_feature_div 0 500 1000 31 | ``` 32 | 33 | 3. Fusion center training 34 | 35 | ``` 36 | python nus_server_ae_train.py --use_ae --lr 0.001 --epochs 200 --img_feature_div 0 360 634 --text_feature_div 0 500 1000 37 | ``` 38 | 39 | 4. Inference under untargeted attack 40 | 41 | ``` 42 | python nus_infer_attack.py --use_ae --L_lr 0.001 --img_feature_div 0 360 634 --text_feature_div 0 500 1000 --corruption_amp 10 43 | ``` 44 | 45 | 5. Inference under targeted attack 46 | 47 | ``` 48 | python nus_infer_attack.py --use_ae --L_lr 0.001 --img_feature_div 0 360 634 --text_feature_div 0 500 1000 --num_test_samples 1000 --targeted 1 49 | ``` 50 | 51 | ## Training and testing on Extrasensory dataset: 52 | 53 | 0. Enter the folder 54 | 55 | ``` 56 | cd extrasensory 57 | ``` 58 | 1. Local feature extractor training (trained models are saved in `nus_results' folder) 59 | ``` 60 | python sensor_normal_train.py --epochs 200 --emb_dim 32 --num_class 1 --lr 0.001 --text_feature_div 0 26 52 83 129 138 155 183 209 213 221 61 | ``` 62 | 2. Feature subspace learning 63 | ``` 64 | python sensor_ae_train.py --ae_epochs 400 --ae_lr 0.001 --text_feature_div 0 26 52 83 129 138 155 183 209 213 221 65 | ``` 66 | 3. Fusion center training 67 | ``` 68 | python sensor_server_ae_train.py --use_ae --epochs 200 --lr 0.001 --text_feature_div 0 26 52 83 129 138 155 183 209 213 221 69 | ``` 70 | 4. Inference under untargeted attack 71 | ``` 72 | python sensor_infer_attack.py --use_ae --L_lr 0.001 --text_feature_div 0 26 52 83 129 138 155 183 209 213 221 --corruption_amp 10 73 | ``` 74 | 5. Inference under targeted attack 75 | ``` 76 | python sensor_infer_attack.py --use_ae --L_lr 0.001 --text_feature_div 0 26 52 83 129 138 155 183 209 213 221 --targeted 1 77 | ``` 78 | 79 | 80 | ## Citation 81 | If you find our work useful in your research, please consider citing: 82 | ``` 83 | @inproceedings{ 84 | liu2022copur, 85 | title={CoPur: Certifiably Robust Collaborative Inference via Feature Purification}, 86 | author={Jing Liu and Chulin Xie and Oluwasanmi O Koyejo and Bo Li}, 87 | booktitle={Advances in Neural Information Processing Systems}, 88 | editor={Alice H. Oh and Alekh Agarwal and Danielle Belgrave and Kyunghyun Cho}, 89 | year={2022}, 90 | url={https://openreview.net/forum?id=r5rzV51GZx} 91 | } 92 | ``` 93 | -------------------------------------------------------------------------------- /extrasensory/models/AE.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.keras.layers import Dense, Flatten, LayerNormalization 3 | from tensorflow.keras import Model 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | import copy 7 | def my_leaky_relu(x): 8 | return tf.nn.leaky_relu(x, 0.8) 9 | 10 | 11 | class AE_NUS(Model): 12 | def __init__(self,out_dim = 160 , hidden_dim=200 , encode_dim= 150): 13 | super(AE_NUS, self).__init__() 14 | self.d1 = Dense(hidden_dim, name="dense1", activation=my_leaky_relu) 15 | self.d2 = Dense(encode_dim, name="dense2", activation=None) 16 | self.d3 = Dense(hidden_dim, name="dense3", activation=my_leaky_relu) 17 | self.d4 = Dense(out_dim, name="dense4", activation=None) 18 | 19 | def call(self, x): 20 | x = self.d1(x) 21 | x = self.d2(x) 22 | x2 =x 23 | x = self.d3(x2) 24 | x = self.d4(x) 25 | return x,x2 26 | 27 | class AE(Model): 28 | def __init__(self,out_dim = 160, encode_dim= 100): 29 | super(AE, self).__init__() 30 | self.d1 = Dense(encode_dim, name="dense1", activation=None) 31 | self.d2 = Dense(out_dim, name="dense2", activation=None) 32 | 33 | def call(self, x): 34 | x = self.d1(x) 35 | x2 =x 36 | x = self.d2(x2) 37 | return x,x2 -------------------------------------------------------------------------------- /extrasensory/models/LocalModel.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.keras.layers import Dense, Flatten, LayerNormalization 3 | from tensorflow.keras import Model, datasets, layers, models 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | import copy 7 | 8 | 9 | class VFLPassiveModel(Model): 10 | def __init__(self,emb_dim=32): 11 | super(VFLPassiveModel, self).__init__() 12 | self.flatten = Flatten() 13 | self.d1 = Dense(64, name="dense1", activation='relu') 14 | self.d2 = Dense(emb_dim, name="dense2", activation=None)#'relu' 15 | 16 | 17 | def call(self, x): 18 | x = self.flatten(x) 19 | #x= self.d1(x) 20 | x= self.d2(x) 21 | return x 22 | 23 | 24 | class VFLPassiveModelMNIST(Model): 25 | def __init__(self,emb_dim=32): 26 | super(VFLPassiveModelMNIST, self).__init__() 27 | self.flatten = Flatten() 28 | self.d1 = Dense(128, name="dense1", activation='relu') 29 | self.d2 = Dense(64, name="dense2", activation='relu') 30 | self.d3 = Dense(emb_dim, name="dense3", activation='relu') 31 | 32 | def call(self, x): 33 | x = self.flatten(x) 34 | x= self.d1(x) 35 | x= self.d2(x) 36 | x= self.d3(x) 37 | return x 38 | 39 | 40 | 41 | class VFLPassiveModelCIFAR(Model): 42 | def __init__(self,emb_dim=32): 43 | super(VFLPassiveModelCIFAR, self).__init__() 44 | self.d0 = layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)) 45 | self.d1 = layers.MaxPooling2D((2, 2)) 46 | self.d2 = layers.Conv2D(64, (3, 3), activation='relu') 47 | self.d3 = layers.MaxPooling2D((2,2)) 48 | self.d4 = layers.Conv2D(64, (3, 3), activation='relu') 49 | self.d5 = layers.Flatten() 50 | self.d6= layers.Dense(emb_dim, activation='relu') 51 | # self.d7= layers.Dense(10) 52 | 53 | 54 | def call(self, x): 55 | x=self.d0(x) 56 | x=self.d1(x) 57 | x=self.d2(x) 58 | x=self.d3(x) 59 | x=self.d4(x) 60 | x=self.d5(x) 61 | x=self.d6(x) 62 | # x=self.d7(x) 63 | return x -------------------------------------------------------------------------------- /extrasensory/models/ServerModel.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.keras.layers import Dense, Flatten, LayerNormalization 3 | from tensorflow.keras import Model 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | import copy 7 | 8 | def my_leaky_relu(x): 9 | return tf.nn.leaky_relu(x, 0.9) 10 | class QuanVFLActiveModelWithOneLayer(Model): 11 | def __init__(self, emb_dim=16,class_num=1,nonlinear1 = my_leaky_relu): 12 | super(QuanVFLActiveModelWithOneLayer, self).__init__() 13 | 14 | self.d1 = Dense(emb_dim, name="dense1", activation=nonlinear1)#'relu' 15 | self.out = Dense(class_num, name="out", activation=None)#'softmax' 16 | 17 | def call(self, x): 18 | x = self.d1(x) 19 | return self.out(x) 20 | 21 | 22 | class VFLActiveModelWithOneLayer(Model): 23 | def __init__(self,emb_dim=16,class_num=1,nonlinear1 = my_leaky_relu): 24 | super(VFLActiveModelWithOneLayer, self).__init__() 25 | self.concatenated = tf.keras.layers.Concatenate() 26 | self.d1 = Dense(emb_dim, name="dense1", activation=nonlinear1)# 27 | self.out = Dense(class_num, name="out", activation=None)#'softmax' 28 | 29 | def call(self, x): 30 | x = self.concatenated(x) 31 | x = self.d1(x) 32 | return self.out(x) 33 | -------------------------------------------------------------------------------- /extrasensory/script_extrasensory.sh: -------------------------------------------------------------------------------- 1 | # Local feature extractor training (trained models are saved in `sensor_results' folder) 2 | python sensor_normal_train.py --epochs 200 --emb_dim 32 --num_class 1 --lr 0.001 --text_feature_div 0 26 52 83 129 138 155 183 209 213 221 3 | # Feature subspace learning 4 | python sensor_ae_train.py --ae_epochs 400 --ae_lr 0.001 --text_feature_div 0 26 52 83 129 138 155 183 209 213 221 5 | # Fusion center training 6 | python sensor_server_ae_train.py --use_ae --epochs 200 --lr 0.001 --text_feature_div 0 26 52 83 129 138 155 183 209 213 221 7 | # inference under untargeted attack 8 | python sensor_infer_attack.py --use_ae --L_lr 0.001 --text_feature_div 0 26 52 83 129 138 155 183 209 213 221 --corruption_amp 10 9 | # inference under targeted attack 10 | python sensor_infer_attack.py --use_ae --L_lr 0.001 --text_feature_div 0 26 52 83 129 138 155 183 209 213 221 --targeted 1 11 | -------------------------------------------------------------------------------- /extrasensory/sensor_ae_train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import time 4 | from utils.sensor_util import read_user_data, load_sensor_data, create_models, get_local_outputs 5 | import tensorflow as tf 6 | from models.AE import AE_NUS as AutoEncoder 7 | 8 | def parse_path(args): 9 | SavedPaths=[] 10 | 11 | for i in range(args.num_img_clients): 12 | _dir= os.path.join(args.path_predix, 'img_{}_{}'.format(args.img_feature_div[i], args.img_feature_div[i+1])) 13 | SavedPaths.append(_dir) 14 | for i in range(args.num_text_clients): 15 | _dir= os.path.join(args.path_predix, 'sensor_{}_{}'.format(args.text_feature_div[i], args.text_feature_div[i+1] )) 16 | SavedPaths.append(_dir) 17 | 18 | print(SavedPaths) 19 | return SavedPaths 20 | 21 | def create_models(args): 22 | from models.LocalModel import VFLPassiveModel 23 | 24 | LocalModels= [] 25 | for i in range(args.num_clients): 26 | local_model = VFLPassiveModel(emb_dim=args.emb_dim) 27 | local_model.built = True 28 | 29 | local_model.load_weights(os.path.join(args.local_paths[i],'best_checkpoints')) 30 | local_model.trainable = False 31 | LocalModels.append(local_model) 32 | 33 | 34 | return LocalModels 35 | 36 | def parse_command(): 37 | parser = argparse.ArgumentParser() 38 | 39 | parser.add_argument('--emb_dim', type=int, default=32) 40 | parser.add_argument('--batch_size', type=int, default=256) 41 | parser.add_argument('--ae_epochs', type=int, default=10) 42 | parser.add_argument('--ae_lr', type=float, default=0.001) 43 | parser.add_argument('--num_class', type=int, default=1) 44 | parser.add_argument('--seed', type=int, default=0) 45 | 46 | parser.add_argument('--num_test_samples', type=int, default=10000) 47 | parser.add_argument('--num_train_samples', type=int, default=60000) 48 | 49 | parser.add_argument('--path_predix', type=str, default='sensor_results') 50 | parser.add_argument('--mode', type=str, default='normal', 51 | choices=[ 52 | 'normal', 53 | ]) 54 | parser.add_argument('--img_feature_div', nargs='+', type=int, default=[0, 360, 634]) 55 | parser.add_argument('--text_feature_div', nargs='+', type=int, default=[0,500, 1000]) 56 | 57 | parser.add_argument('--vis', action='store_true') 58 | 59 | 60 | args = parser.parse_args() 61 | if args.seed is not None: 62 | import random 63 | random.seed(args.seed) 64 | tf.random.set_seed(args.seed) 65 | args.num_img_clients=0# len(args.img_feature_div)-1 66 | args.num_text_clients= len(args.text_feature_div)-1 67 | args.num_clients = args.num_img_clients+ args.num_text_clients 68 | 69 | args.path_predix_load = args.path_predix 70 | savefolder= '{}_{}i_{}t_try{}'.format(args.mode, args.num_img_clients, args.num_text_clients, args.seed) 71 | # savefolder= 'AE_{}_{}i_{}t_try{}'.format(args.mode, args.num_img_clients,args.num_text_clients, args.seed) 72 | args.path_predix=os.path.join(args.path_predix, savefolder) 73 | local_paths = parse_path(args) 74 | args.local_paths= local_paths 75 | print("save to", args.path_predix) 76 | args.top_k= ['buildings', 'grass', 'animal', 'water', 'person'] 77 | 78 | return args 79 | 80 | 81 | 82 | args = parse_command() 83 | if args.vis: 84 | from tensorboardX import SummaryWriter 85 | import datetime 86 | current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") 87 | log_dir = os.path.join(args.path_predix,current_time) 88 | args.train_writer = tf.summary.create_file_writer(log_dir) 89 | 90 | (X1,Y1,M1,timestamps1,feature_names,feat_sensor_names,label_names) = read_user_data('example.csv') 91 | # print(M1.shape) 92 | print('111111111111') 93 | # print(timestamps1.shape) 94 | sensors_to_use = ['Acc','Gyro','Magnet','WAcc','Compass','Loc','Aud','PS','LF','TS'] 95 | # 96 | target_label ='SITTING'# 'FIX_walking'; # this is just code name for the cleaned version of the label 'Walking' 97 | x_train, y_train= load_sensor_data(X1[0:2500,:],Y1[0:2500],M1[0:2500,:],feat_sensor_names,label_names,sensors_to_use,target_label,0) 98 | x_test, y_test= load_sensor_data(X1[2500:-1,:],Y1[2500:-1],M1[2500:-1,:],feat_sensor_names,label_names,sensors_to_use,target_label,0) 99 | 100 | 101 | LocalModels = create_models(args) 102 | autoencoder_model=AutoEncoder(out_dim= args.num_clients * args.emb_dim) 103 | #(x_train_images, x_train_texts) = x_train 104 | LocalOutputs = get_local_outputs(args.num_text_clients, x_train, args.text_feature_div, LocalModels) 105 | H_input=tf.concat(LocalOutputs,1) 106 | rae_loss_object=tf.keras.losses.MeanSquaredError() 107 | rae_optimizer = tf.keras.optimizers.SGD(learning_rate=args.ae_lr) 108 | rae_train_loss= tf.keras.metrics.Mean(name='rae_train_loss') 109 | 110 | def train_ae(epoch): 111 | 112 | train_ds = tf.data.Dataset.from_tensor_slices( 113 | (H_input)).batch(args.batch_size) 114 | 115 | ae_step= len(train_ds) *epoch 116 | for h_input in train_ds: 117 | ae_step+=1 118 | with tf.GradientTape() as active_tape: 119 | rae_output, _ = autoencoder_model(h_input) 120 | loss= rae_loss_object(rae_output, h_input) 121 | 122 | rae_gradients = active_tape.gradient(loss, autoencoder_model.trainable_variables) 123 | rae_optimizer.apply_gradients(zip(rae_gradients, autoencoder_model.trainable_variables)) 124 | 125 | rae_train_loss(loss) 126 | if args.vis: 127 | with args.train_writer.as_default(): 128 | tf.summary.scalar('train_AE_loss', loss, step=ae_step) 129 | 130 | 131 | 132 | for epoch in range(args.ae_epochs): 133 | start= time.time() 134 | train_ae(epoch) 135 | end= time.time() 136 | print("AE training epoch: ", epoch, "loss", rae_train_loss.result(), "time: ",time.time()- start) 137 | rae_train_loss.reset_states() 138 | autoencoder_model.save_weights(os.path.join(args.path_predix, 'ae_ckpt.tf')) 139 | 140 | 141 | 142 | -------------------------------------------------------------------------------- /extrasensory/sensor_infer_attack.py: -------------------------------------------------------------------------------- 1 | 2 | import argparse 3 | import os 4 | import time 5 | import numpy as np 6 | from utils.sensor_util import read_user_data, load_sensor_data, load_models, get_local_outputs 7 | import tensorflow as tf 8 | from math import ceil 9 | 10 | from models.AE import AE_NUS as AutoEncoder 11 | from utils.robustae import inference_purify_miss 12 | 13 | 14 | def parse_path(args): 15 | SavedPaths=[] 16 | for i in range(args.num_img_clients): 17 | _dir= os.path.join(args.path_predix, 'img_{}_{}'.format(args.img_feature_div[i], args.img_feature_div[i+1])) 18 | 19 | SavedPaths.append(_dir) 20 | for i in range(args.num_text_clients): 21 | _dir= os.path.join(args.path_predix, 'sensor_{}_{}'.format(args.text_feature_div[i], args.text_feature_div[i+1])) 22 | 23 | SavedPaths.append(_dir) 24 | if args.server_model_path == '': 25 | _dir= os.path.join(args.path_predix,'final_server_AE{}'.format( args.use_ae)) 26 | else: 27 | _dir = os.path.join(args.path_predix, args.server_model_path) 28 | 29 | server_savedpath=_dir 30 | 31 | return SavedPaths, server_savedpath 32 | 33 | 34 | def parse_command(): 35 | parser = argparse.ArgumentParser() 36 | 37 | parser.add_argument('--emb_dim', type=int, default=32) 38 | parser.add_argument('--batch_size', type=int, default=1) 39 | 40 | parser.add_argument('--num_class', type=int, default=1) 41 | parser.add_argument('--seed', type=int, default=0) 42 | 43 | parser.add_argument('--num_test_samples', type=int, default=10000) 44 | parser.add_argument('--num_train_samples', type=int, default=60000) 45 | parser.add_argument('--path_predix', type=str, default='sensor_results') 46 | parser.add_argument('--ckpt_name', type=str, default='best_checkpoints') 47 | parser.add_argument('--mode', type=str, default='normal', 48 | choices=[ 49 | 'normal', 50 | ]) 51 | parser.add_argument('--img_feature_div', nargs='+', type=int, default=[0, 360, 634]) 52 | parser.add_argument('--text_feature_div', nargs='+', type=int, default=[0, 500, 1000]) 53 | parser.add_argument('--use_ae', action='store_true') 54 | parser.add_argument('--purify_epochs', type=int, default=10) 55 | parser.add_argument('--initial_epochs', type=int, default=100) 56 | parser.add_argument('--L_lr', type=float, default=0.001) 57 | parser.add_argument('--L0_lr', type=float, default=0.1) 58 | parser.add_argument('--attack', action='store_true') 59 | parser.add_argument('--tau',type=float, default=1000) 60 | parser.add_argument('--sigma',type=float, default=0.5) 61 | parser.add_argument('--corruption_amp',type=float, default=10) 62 | 63 | parser.add_argument('--server_model_path',type=str, default='') 64 | parser.add_argument('--num_noise', type=int, default=100) 65 | parser.add_argument('--attackers_index', nargs='+', type=int, default=[9])######## 66 | parser.add_argument('--miss_index', nargs='+', type=int, default=[]) ######## 67 | parser.add_argument('--PGDeps',type=float, default=0.5) 68 | parser.add_argument('--PGDiters', type=int, default=30) 69 | parser.add_argument('--targeted', type=int, default=0) 70 | 71 | args = parser.parse_args() 72 | if args.seed is not None: 73 | import random 74 | random.seed(args.seed) 75 | tf.random.set_seed(args.seed) 76 | 77 | args.num_img_clients=0 78 | args.num_text_clients= len(args.text_feature_div)-1 79 | args.num_clients = args.num_img_clients+ args.num_text_clients 80 | args.num_attacker = len(args.attackers_index)############## 81 | print('attacker index', args.attackers_index) 82 | args.num_miss = len(args.miss_index) ############## 83 | print('miss index', args.miss_index) 84 | 85 | args.observe_list=list(set(list(range(0,args.num_text_clients))) - set(args.miss_index)) 86 | print('observed',args.observe_list) 87 | args.path_predix_load = args.path_predix 88 | savefolder= '{}_{}i_{}t_try{}'.format(args.mode, args.num_img_clients, args.num_text_clients, args.seed) 89 | args.path_predix=os.path.join(args.path_predix, savefolder) 90 | 91 | 92 | args.top_k= ['buildings', 'grass', 'animal', 'water', 'person'] 93 | 94 | return args 95 | 96 | args = parse_command() 97 | 98 | (X1,Y1,M1,timestamps1,feature_names,feat_sensor_names,label_names) = read_user_data('example.csv') 99 | 100 | sensors_to_use = ['Acc','Gyro','Magnet','WAcc','Compass','Loc','Aud','PS','LF','TS'] 101 | # 102 | target_label ='SITTING'# 'FIX_walking'; # this is just code name for the cleaned version of the label 'Walking' 103 | x_train, y_train= load_sensor_data(X1[0:2500,:],Y1[0:2500],M1[0:2500,:],feat_sensor_names,label_names,sensors_to_use,target_label,0) 104 | x_test, y_test= load_sensor_data(X1[2500:-1,:],Y1[2500:-1],M1[2500:-1,:],feat_sensor_names,label_names,sensors_to_use,target_label,0) 105 | 106 | L_optimizer = tf.keras.optimizers.SGD(learning_rate=args.L0_lr) 107 | L_optimizer2 = tf.keras.optimizers.SGD(learning_rate=args.L_lr) 108 | rae_loss_object=tf.keras.losses.MeanSquaredError() 109 | 110 | 111 | loss_object = tf.keras.losses.CategoricalCrossentropy() 112 | 113 | test_loss = tf.keras.metrics.Mean(name='test_loss') 114 | test_accuracy = tf.keras.metrics.BinaryAccuracy(name='test_accuracy') 115 | 116 | test_loss1 = tf.keras.metrics.Mean(name='test_loss1') 117 | test_accuracy1 = tf.keras.metrics.BinaryAccuracy(name='test_accuracy1') 118 | 119 | test_loss2 = tf.keras.metrics.Mean(name='test_loss2') 120 | test_accuracy2 = tf.keras.metrics.BinaryAccuracy(name='test_accuracy2') 121 | 122 | test_loss3 = tf.keras.metrics.Mean(name='test_loss3') 123 | test_accuracy3 = tf.keras.metrics.BinaryAccuracy(name='test_accuracy3') 124 | 125 | if args.use_ae: 126 | print("use AE") 127 | autoencoder_model = AutoEncoder(out_dim= args.num_clients * args.emb_dim) 128 | autoencoder_model.built = True 129 | autoencoder_model.load_weights(os.path.join(args.path_predix,'ae_ckpt.tf')) 130 | 131 | 132 | 133 | def RandomizedSmooth(args, LocalOutputs, active_model,batch_size=100): 134 | counts = 0 135 | x=np.concatenate(tuple(LocalOutputs), axis=1) 136 | num=args.num_noise 137 | for _ in range(ceil(num / batch_size)): 138 | this_batch_size = min(batch_size, num) 139 | num -= this_batch_size 140 | batch = x 141 | batch = np.repeat(batch, this_batch_size, axis=0) 142 | noise = np.random.randn(*batch.shape) * args.sigma 143 | batch +=noise 144 | predictions = active_model(tf.split(batch , args.num_clients, 1 )) 145 | predictions1=np.sign(predictions) 146 | counts += sum(predictions1) 147 | 148 | return counts 149 | 150 | 151 | def RandomizedSmooth_block(args, LocalOutputs, active_model, batch_size=100): 152 | counts = 0#np.zeros(args.num_class, dtype=int) 153 | x = np.concatenate(tuple(LocalOutputs), axis=1) 154 | num = args.num_noise 155 | for _ in range(ceil(num / batch_size)): 156 | this_batch_size = min(batch_size, num) 157 | num -= this_batch_size 158 | 159 | batch = x 160 | batch = np.repeat(batch, this_batch_size, axis=0) 161 | noise = np.random.randn(batch.shape[0], args.emb_dim) * args.sigma 162 | 163 | for j in range(args.num_attacker): 164 | atk_index = args.attackers_index[j] 165 | batch[:, atk_index * args.emb_dim:(atk_index+1)* args.emb_dim] += noise 166 | for j in range(args.num_miss): 167 | missing_index = args.miss_index[j] 168 | batch[:, missing_index * args.emb_dim:(missing_index+1)* args.emb_dim] += noise 169 | 170 | 171 | predictions = active_model(tf.split(batch, args.num_clients, 1)) 172 | 173 | predictions1=np.sign(predictions) 174 | counts += sum(predictions1) 175 | 176 | 177 | return counts 178 | 179 | def count_arr(arr: np.ndarray, length: int) -> np.ndarray: 180 | counts = np.zeros(length, dtype=int) 181 | for idx in arr: 182 | counts[idx] += 1 183 | return counts 184 | 185 | 186 | def create_adversarial_pattern(_LocalOutputs,args, labels, server_model,PGD=True): 187 | 188 | loss_object = tf.keras.losses.BinaryCrossentropy(from_logits=True) 189 | 190 | if PGD == True: 191 | LocalOutputs = _LocalOutputs 192 | 193 | hadv = [_LocalOutputs[index] for index in args.attackers_index] 194 | h0=tf.convert_to_tensor(np.concatenate(tuple(hadv), axis=1),dtype=tf.float32) 195 | 196 | for i in range(args.PGDiters): 197 | 198 | with tf.GradientTape() as tape: 199 | tape.watch(h0) # the adversarial's feature 200 | 201 | ADV_Outputs=tf.split(h0, args.num_attacker, 1) 202 | for j in range(args.num_attacker): 203 | atk_index = args.attackers_index[j] 204 | LocalOutputs[atk_index] = ADV_Outputs[j] 205 | 206 | output = server_model(LocalOutputs) 207 | loss = loss_object(1-labels, output) 208 | 209 | gradient = tape.gradient(loss, h0) 210 | 211 | h0 -= args.PGDeps * gradient 212 | 213 | return LocalOutputs 214 | 215 | 216 | 217 | def test(_LocalModels,_active_model ): 218 | 219 | test_ds = tf.data.Dataset.from_tensor_slices( 220 | (x_test, y_test)).batch(args.batch_size) 221 | for texts, labels in test_ds: 222 | 223 | _LocalOutputs= get_local_outputs(args.num_text_clients, texts, args.text_feature_div, _LocalModels) 224 | 225 | 226 | for j in range(args.num_miss): 227 | missing_index = args.miss_index[j] 228 | _LocalOutputs[missing_index] = 0 * _LocalOutputs[missing_index] 229 | if args.targeted==1: 230 | _LocalOutputs=create_adversarial_pattern(_LocalOutputs, args, labels, _active_model, PGD=True) 231 | else: 232 | for j in range(args.num_attacker): 233 | atk_index = args.attackers_index[j] 234 | _LocalOutputs[atk_index] = -args.corruption_amp * _LocalOutputs[atk_index] 235 | print('corrupted -',args.corruption_amp) 236 | 237 | temp=np.concatenate(tuple(_LocalOutputs), axis=1) 238 | 239 | rae_output,layer_output=autoencoder_model(temp) 240 | 241 | if args.use_ae: 242 | LocalOutputs = inference_purify_miss(args, _LocalOutputs,autoencoder_model,L_optimizer,L_optimizer2,rae_loss_object, vis=False) 243 | else: 244 | LocalOutputs = _LocalOutputs 245 | 246 | 247 | test_output = _active_model(LocalOutputs) 248 | 249 | 250 | test_output1 = _active_model(_LocalOutputs)#no AE, just use corrupted h 251 | test_output2 = _active_model(tf.split(rae_output, args.num_clients, 1 ))# use DE(h) 252 | 253 | test_loss(loss_object(labels, test_output)) 254 | test_accuracy(labels, test_output) 255 | 256 | 257 | test_loss1(loss_object(labels, test_output1)) 258 | test_accuracy1(labels, test_output1) 259 | 260 | test_loss2(loss_object(labels, test_output2)) 261 | test_accuracy2(labels, test_output2) 262 | label_RS=RandomizedSmooth(args, LocalOutputs, _active_model,100) 263 | 264 | 265 | test_accuracy3(labels,label_RS) 266 | 267 | 268 | local_paths, sever_path= parse_path(args) 269 | args.local_paths= local_paths 270 | args.sever_path= sever_path 271 | LocalModels, active_model= load_models(args) 272 | start= time.time() 273 | test(LocalModels, active_model) 274 | 275 | end= time.time() 276 | template = '{} Test Loss: {}, Test Accuracy: {}, Test Accuracy3: {}, Time: {}' 277 | print(template.format(args.path_predix, 278 | test_loss.result(), 279 | test_accuracy.result()*100, 280 | test_accuracy3.result()*100, 281 | end-start 282 | )) 283 | 284 | test_loss.reset_states() 285 | test_accuracy.reset_states() 286 | -------------------------------------------------------------------------------- /extrasensory/sensor_normal_train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import time 4 | import numpy as np 5 | from utils.sensor_util import read_user_data, load_sensor_data, create_folder, create_models, get_local_outputs 6 | import tensorflow as tf 7 | 8 | def parse_command(): 9 | parser = argparse.ArgumentParser() 10 | 11 | parser.add_argument('--emb_dim', type=int, default=60) 12 | parser.add_argument('--batch_size', type=int, default=32) 13 | parser.add_argument('--epochs', type=int, default=80) 14 | parser.add_argument('--save_epochs', nargs='+', type=int, default=[30,40,50]) 15 | parser.add_argument('--num_class', type=int, default=1) 16 | parser.add_argument('--seed', type=int, default=0) 17 | 18 | parser.add_argument('--num_test_samples', type=int, default=10000) 19 | parser.add_argument('--num_train_samples', type=int, default=60000) 20 | 21 | parser.add_argument('--path_predix', type=str, default='sensor_results') 22 | parser.add_argument('--mode', type=str, default='normal', 23 | ) 24 | parser.add_argument('--img_feature_div', nargs='+', type=int, default=[0, 360, 634]) 25 | parser.add_argument('--text_feature_div', nargs='+', type=int, default=[0, 500, 1000]) 26 | parser.add_argument('--lr', type=float, default=0.005) 27 | 28 | 29 | args = parser.parse_args() 30 | if args.seed is not None: 31 | import random 32 | random.seed(args.seed) 33 | tf.random.set_seed(args.seed) 34 | args.num_img_clients= 0 35 | args.num_text_clients= len(args.text_feature_div)-1 36 | args.num_clients = args.num_img_clients+ args.num_text_clients 37 | 38 | savefolder= '{}_{}i_{}t_try{}'.format(args.mode, args.num_img_clients,args.num_text_clients , args.seed) 39 | args.path_predix=os.path.join(args.path_predix, savefolder) 40 | local_paths, sever_path= create_folder(args) 41 | args.local_paths= local_paths 42 | args.sever_path= sever_path 43 | args.top_k= ['buildings', 'grass', 'animal', 'water', 'person'] 44 | print("will save in epochs:", args.save_epochs) 45 | 46 | 47 | return args 48 | 49 | 50 | args = parse_command() 51 | 52 | 53 | (X1,Y1,M1,timestamps1,feature_names,feat_sensor_names,label_names) = read_user_data('example.csv') 54 | 55 | sensors_to_use = ['Acc','Gyro','Magnet','WAcc','Compass','Loc','Aud','PS','LF','TS'] 56 | # 57 | target_label ='SITTING'# 'FIX_walking'; # this is just code name for the cleaned version of the label 'Walking' 58 | x_train, y_train= load_sensor_data(X1[0:2500,:],Y1[0:2500],M1[0:2500,:],feat_sensor_names,label_names,sensors_to_use,target_label,0) 59 | 60 | print(x_train.shape) 61 | print(sum(y_train)) 62 | 63 | x_test, y_test= load_sensor_data(X1[2500:-1,:],Y1[2500:-1],M1[2500:-1,:],feat_sensor_names,label_names,sensors_to_use,target_label,0) 64 | print(x_test.shape) 65 | 66 | LocalModels, GradientsRes, active_model= create_models(args) 67 | lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay( 68 | initial_learning_rate=args.lr, 69 | decay_steps=10000, 70 | decay_rate=0.99) 71 | 72 | optimizer = tf.keras.optimizers.SGD(learning_rate=lr_schedule) 73 | loss_object = tf.keras.losses.BinaryCrossentropy() 74 | 75 | train_loss = tf.keras.metrics.Mean(name='train_loss') 76 | train_accuracy = tf.keras.metrics.BinaryAccuracy(name='train_accuracy') 77 | test_loss = tf.keras.metrics.Mean(name='test_loss') 78 | test_accuracy = tf.keras.metrics.BinaryAccuracy(name='test_accuracy') 79 | test_label_accuracy = tf.keras.metrics.BinaryAccuracy(name='test_label_accuracy') 80 | acc_train = [] 81 | acc_test = [] 82 | loss_train = [] 83 | loss_test = [] 84 | best_acc=0 85 | 86 | 87 | def train(epoch): 88 | train_ds = tf.data.Dataset.from_tensor_slices( 89 | (x_train, y_train)).shuffle(y_train.shape[0]).batch(args.batch_size) 90 | current_step = 0 91 | for texts, labels in train_ds: 92 | current_step += 1 93 | LocalOutputs = [] 94 | LocalEmbeds = [] 95 | 96 | with tf.GradientTape() as passive_tape: 97 | 98 | for i in range(args.num_text_clients): 99 | texts_div = texts[:, args.text_feature_div[i]:args.text_feature_div[i + 1]] 100 | local_output = local_embed = LocalModels[i + args.num_img_clients](texts_div) 101 | LocalOutputs.append(local_output) 102 | LocalEmbeds.append(local_embed) 103 | 104 | with tf.GradientTape() as active_tape: 105 | for i in range(args.num_clients): 106 | active_tape.watch(LocalOutputs[i]) 107 | 108 | output = active_model(LocalOutputs) 109 | loss = loss_object(labels, output) 110 | 111 | trainable_variables = [] 112 | for i in range(args.num_clients): 113 | trainable_variables.append(LocalOutputs[i]) 114 | trainable_variables.append(active_model.trainable_variables) 115 | 116 | gradients = active_tape.gradient(loss, trainable_variables) 117 | EmbedGradients = gradients[:-1] 118 | server_model_gradients = gradients[-1] 119 | optimizer.apply_gradients( 120 | zip(server_model_gradients, active_model.trainable_variables)) # update server model 121 | 122 | EmbLoss = [] 123 | local_trainable_varaibles = [] 124 | for i in range(args.num_clients): 125 | EmbLoss.append(tf.multiply(LocalEmbeds[i], EmbedGradients[i].numpy())) 126 | local_trainable_varaibles.append(LocalModels[i].trainable_variables) 127 | LocalGradients = passive_tape.gradient(EmbLoss, local_trainable_varaibles) 128 | for i in range(args.num_clients): 129 | optimizer.apply_gradients(zip(LocalGradients[i], local_trainable_varaibles[i])) 130 | 131 | train_loss(loss) 132 | train_accuracy(labels, output) 133 | 134 | 135 | 136 | def test(epoch): 137 | LocalOutputs = get_local_outputs(args.num_text_clients, x_test, args.text_feature_div, LocalModels) 138 | test_output = active_model(LocalOutputs) 139 | print(test_output.shape) 140 | print( np.mean(test_output == y_test)) 141 | test_accuracy(y_test, test_output) 142 | 143 | 144 | for epoch in range(args.epochs): 145 | start = time.time() 146 | train(epoch) 147 | test(epoch) 148 | 149 | end = time.time() 150 | template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}, Time: {}' 151 | print(template.format(epoch + 1, 152 | train_loss.result(), 153 | train_accuracy.result() * 100, 154 | test_loss.result(), 155 | test_accuracy.result() * 100, 156 | end - start 157 | )) 158 | 159 | acc_train.append(train_accuracy.result()) 160 | acc_test.append(test_accuracy.result()) 161 | loss_train.append(train_loss.result()) 162 | loss_test.append(test_loss.result()) 163 | if best_acc <= test_accuracy.result(): 164 | best_acc = test_accuracy.result() 165 | # Save the weights 166 | for i in range(args.num_clients): 167 | LocalModels[i].save_weights(os.path.join(args.local_paths[i], 'best_checkpoints')) 168 | active_model.save_weights(os.path.join(args.sever_path, 'best_checkpoints')) 169 | train_loss.reset_states() 170 | train_accuracy.reset_states() 171 | test_loss.reset_states() 172 | test_accuracy.reset_states() 173 | if epoch + 1 in args.save_epochs: 174 | epoch_filename = "epoch{}_checkpoints".format(epoch + 1) 175 | for i in range(args.num_clients): 176 | LocalModels[i].save_weights(os.path.join(args.local_paths[i], epoch_filename)) 177 | active_model.save_weights(os.path.join(args.sever_path, epoch_filename)) 178 | 179 | with open(os.path.join(args.path_predix, 'acc_test.txt'), "w") as outfile: 180 | outfile.write("\n".join("{:.4f}".format(item * 100) for item in acc_test)) 181 | 182 | with open(os.path.join(args.path_predix, 'acc_train.txt'), "w") as outfile: 183 | outfile.write("\n".join("{:.4f}".format(item * 100) for item in acc_train)) -------------------------------------------------------------------------------- /extrasensory/sensor_server_ae_train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import time 4 | import numpy as np 5 | import copy 6 | from utils.sensor_util import read_user_data, load_sensor_data, get_local_outputs 7 | import tensorflow as tf 8 | from models.AE import AE_NUS as AutoEncoder 9 | 10 | 11 | 12 | def parse_path(args): 13 | SavedPaths=[] 14 | 15 | for i in range(args.num_img_clients): 16 | _dir= os.path.join(args.path_predix, 'img_{}_{}'.format(args.img_feature_div[i], args.img_feature_div[i+1])) 17 | SavedPaths.append(_dir) 18 | for i in range(args.num_text_clients): 19 | _dir= os.path.join(args.path_predix, 'sensor_{}_{}'.format(args.text_feature_div[i], args.text_feature_div[i+1] )) 20 | SavedPaths.append(_dir) 21 | 22 | _dir= os.path.join(args.path_predix,'final_server_AE{}'.format( args.use_ae)) 23 | if not os.path.exists(_dir): 24 | os.makedirs(_dir) 25 | server_savedpath=_dir 26 | print(SavedPaths) 27 | print("server_savedpath", server_savedpath) 28 | return SavedPaths, server_savedpath 29 | 30 | def create_models(args): 31 | from models.LocalModel import VFLPassiveModel 32 | from models.ServerModel import VFLActiveModelWithOneLayer 33 | 34 | LocalModels= [] 35 | for i in range(args.num_clients): 36 | local_model = VFLPassiveModel(emb_dim=args.emb_dim) 37 | local_model.built = True 38 | 39 | local_model.load_weights(os.path.join(args.local_paths[i],'best_checkpoints')) 40 | local_model.trainable = False 41 | LocalModels.append(local_model) 42 | 43 | active_model = VFLActiveModelWithOneLayer(emb_dim=args.emb_dim,class_num= args.num_class) 44 | 45 | return LocalModels, active_model 46 | 47 | def parse_command(): 48 | parser = argparse.ArgumentParser() 49 | 50 | parser.add_argument('--emb_dim', type=int, default=32) 51 | parser.add_argument('--batch_size', type=int, default=256) 52 | parser.add_argument('--epochs', type=int, default=10) 53 | parser.add_argument('--num_class', type=int, default=1) 54 | parser.add_argument('--seed', type=int, default=0) 55 | 56 | parser.add_argument('--num_test_samples', type=int, default=10000) 57 | parser.add_argument('--num_train_samples', type=int, default=60000) 58 | 59 | parser.add_argument('--path_predix', type=str, default='sensor_results') 60 | parser.add_argument('--mode', type=str, default='normal', 61 | choices=[ 62 | 'normal', 63 | 64 | ]) 65 | parser.add_argument('--img_feature_div', nargs='+', type=int, default=[0 , 360, 634]) 66 | parser.add_argument('--text_feature_div', nargs='+', type=int, default=[0, 500, 1000]) 67 | 68 | 69 | parser.add_argument('--vis', action='store_true') 70 | parser.add_argument('--use_ae', action='store_true') 71 | 72 | parser.add_argument('--lr', type=float, default=0.001) 73 | 74 | 75 | args = parser.parse_args() 76 | if args.seed is not None: 77 | import random 78 | random.seed(args.seed) 79 | tf.random.set_seed(args.seed) 80 | args.num_img_clients=0# len(args.img_feature_div)-1 81 | args.num_text_clients= len(args.text_feature_div)-1 82 | args.num_clients = args.num_img_clients+ args.num_text_clients 83 | 84 | args.path_predix_load = args.path_predix 85 | 86 | savefolder= '{}_{}i_{}t_try{}'.format(args.mode, args.num_img_clients, args.num_text_clients, args.seed) 87 | 88 | args.path_predix=os.path.join(args.path_predix, savefolder) 89 | local_paths, sever_path= parse_path(args) 90 | args.local_paths= local_paths 91 | args.sever_path= sever_path 92 | args.top_k= ['buildings', 'grass', 'animal', 'water', 'person'] 93 | print("save to", args.path_predix) 94 | 95 | 96 | return args 97 | 98 | 99 | args = parse_command() 100 | if args.vis: 101 | from tensorboardX import SummaryWriter 102 | import datetime 103 | current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") 104 | log_dir = os.path.join(args.path_predix,current_time) 105 | args.train_writer = tf.summary.create_file_writer(log_dir) 106 | else: 107 | args.train_writer=None 108 | 109 | (X1,Y1,M1,timestamps1,feature_names,feat_sensor_names,label_names) = read_user_data('example.csv') 110 | sensors_to_use = ['Acc','Gyro','Magnet','WAcc','Compass','Loc','Aud','PS','LF','TS'] 111 | # 112 | target_label ='SITTING'# 'FIX_walking'; # this is just code name for the cleaned version of the label 'Walking' 113 | x_train, y_train= load_sensor_data(X1[0:2500,:],Y1[0:2500],M1[0:2500,:],feat_sensor_names,label_names,sensors_to_use,target_label,0) 114 | x_test, y_test= load_sensor_data(X1[2500:-1,:],Y1[2500:-1],M1[2500:-1,:],feat_sensor_names,label_names,sensors_to_use,target_label,0) 115 | LocalModels, active_model= create_models(args) 116 | 117 | if args.use_ae: 118 | autoencoder_model = AutoEncoder(out_dim= args.num_clients * args.emb_dim) 119 | autoencoder_model.built = True 120 | autoencoder_model.load_weights(os.path.join(args.path_predix,'ae_ckpt.tf')) 121 | 122 | 123 | optimizer_server = tf.keras.optimizers.SGD(learning_rate=args.lr) 124 | 125 | 126 | loss_object = tf.keras.losses.CategoricalCrossentropy() 127 | train_loss = tf.keras.metrics.Mean(name='train_loss') 128 | train_accuracy = tf.keras.metrics.BinaryAccuracy(name='train_accuracy') 129 | test_loss = tf.keras.metrics.Mean(name='test_loss') 130 | test_accuracy = tf.keras.metrics.BinaryAccuracy(name='test_accuracy') 131 | acc_train = [] 132 | acc_test = [] 133 | loss_train = [] 134 | loss_test = [] 135 | best_acc=0 136 | 137 | def train(epoch): 138 | 139 | train_ds = tf.data.Dataset.from_tensor_slices( 140 | (x_train, y_train)).shuffle(y_train.shape[0]).batch(args.batch_size) 141 | server_step=0 142 | for texts, labels in train_ds: 143 | server_step+=1 144 | _LocalOutputs= get_local_outputs(args.num_text_clients, texts, args.text_feature_div, LocalModels) 145 | 146 | if args.use_ae: 147 | rae_output,layer_output=autoencoder_model(tf.concat(_LocalOutputs,1)) 148 | LocalOutputs = tf.split(rae_output, args.num_clients, 1 ) 149 | else: 150 | LocalOutputs = _LocalOutputs 151 | with tf.GradientTape() as active_tape: 152 | active_output = active_model(LocalOutputs) 153 | loss = loss_object(labels, active_output) 154 | 155 | [active_model_gradients] = active_tape.gradient(loss, [active_model.trainable_variables]) 156 | optimizer_server.apply_gradients(zip(active_model_gradients, active_model.trainable_variables)) 157 | 158 | train_loss(loss) 159 | train_accuracy(labels, active_output) 160 | if args.vis: 161 | with args.train_writer.as_default(): 162 | tf.summary.scalar('Server_train_loss', loss, step=server_step+epoch*len(train_ds)) 163 | 164 | def test(epoch): 165 | 166 | _LocalOutputs = get_local_outputs(args.num_text_clients, x_test, args.text_feature_div, LocalModels) 167 | if args.use_ae: 168 | rae_output,layer_output=autoencoder_model(tf.concat(_LocalOutputs,1)) 169 | LocalOutputs = tf.split(rae_output, args.num_clients, 1 ) 170 | else: 171 | LocalOutputs = _LocalOutputs 172 | test_output = active_model(LocalOutputs) 173 | test_accuracy(y_test, test_output) 174 | if args.vis: 175 | with args.train_writer.as_default(): 176 | tf.summary.scalar('Server_test_loss', test_loss.result(), step=epoch ) 177 | tf.summary.scalar('Server_test_acc', test_accuracy.result(), step=epoch ) 178 | 179 | 180 | for epoch in range(args.epochs): 181 | start= time.time() 182 | train(epoch) 183 | test(epoch) 184 | 185 | end= time.time() 186 | template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}, Time: {}' 187 | print(template.format(epoch+1, 188 | train_loss.result(), 189 | train_accuracy.result()*100, 190 | test_loss.result(), 191 | test_accuracy.result()*100, 192 | end-start 193 | )) 194 | 195 | acc_train.append(train_accuracy.result()) 196 | acc_test.append(test_accuracy.result()) 197 | loss_train.append(train_loss.result()) 198 | loss_test.append(test_loss.result()) 199 | if best_acc <= test_accuracy.result(): 200 | best_acc= test_accuracy.result() 201 | active_model.save_weights(os.path.join(args.sever_path, 'best_checkpoints')) 202 | train_loss.reset_states() 203 | train_accuracy.reset_states() 204 | test_loss.reset_states() 205 | test_accuracy.reset_states() 206 | active_model.save_weights(os.path.join(args.sever_path, 'checkpoints')) 207 | with open(os.path.join(args.path_predix, 'acc_test.txt'), "w") as outfile: 208 | outfile.write("\n".join("{:.4f}".format(item*100) for item in acc_test)) 209 | 210 | with open(os.path.join(args.path_predix, 'acc_train.txt'), "w") as outfile: 211 | outfile.write("\n".join("{:.4f}".format(item*100) for item in acc_train)) 212 | 213 | -------------------------------------------------------------------------------- /extrasensory/utils/robustae.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | def l21_rownorm(X): 4 | """ 5 | This function calculates the l21 norm of a matrix X, i.e., \sum ||X[i,:]||_2 6 | Input: 7 | ----- 8 | X: {numpy array} 9 | Output: 10 | ------ 11 | l21_norm: {float} 12 | """ 13 | return (np.sqrt(np.multiply(X, X).sum(1))).sum() 14 | 15 | def l21_colnorm(X): 16 | """ 17 | This function calculates the l21 norm of a matrix X, i.e., \sum ||X[:,j]||_2 18 | Input: 19 | ----- 20 | X: {numpy array} 21 | Output: 22 | ------ 23 | l21_norm: {float} 24 | """ 25 | return (np.sqrt(np.multiply(X, X).sum(0))).sum() 26 | 27 | 28 | def inference_purify_miss(args, LocalOutputs,ae_model,L_optimizer,L_optimizer2, rae_loss_object, vis=False): 29 | h=np.concatenate(tuple(LocalOutputs), axis=1) 30 | rae_output,layer_output=ae_model(h) 31 | L=tf.Variable(rae_output,trainable=True) 32 | purify_epochs= args.purify_epochs 33 | for epoch in range(args.initial_epochs): 34 | with tf.GradientTape() as passive_tape: 35 | rae_output,layer_output=ae_model(L) 36 | rae_output_split= tf.split(rae_output, args.num_clients, 1 ) 37 | 38 | loss=0 39 | for i in range(args.num_clients): 40 | 41 | loss += tf.sqrt(rae_loss_object(rae_output_split[i], LocalOutputs[i])) 42 | L_gradients = passive_tape.gradient(loss,[L]) 43 | if tf.norm(L_gradients)<0.1: 44 | break 45 | L_optimizer.apply_gradients(zip(L_gradients, [L])) 46 | L2 = tf.Variable(rae_output, trainable=True) 47 | for epoch in range(purify_epochs): 48 | with tf.GradientTape() as passive_tape: 49 | rae_output,layer_output=ae_model(L2) 50 | rae_output_split= tf.split(rae_output, args.num_clients, 1 ) 51 | L_split2 = tf.split(L2, args.num_clients, 1 ) 52 | loss2=0 53 | for i in range(args.num_clients): 54 | loss2 += args.tau* tf.sqrt(rae_loss_object(rae_output_split[i], L_split2[i])) 55 | 56 | for i in range(len(args.observe_list)): 57 | loss2 += tf.sqrt(rae_loss_object( LocalOutputs[args.observe_list[i]],L_split2[args.observe_list[i]])) 58 | L_gradients = passive_tape.gradient(loss2,[L2]) 59 | 60 | if tf.norm(L_gradients)<0.1: 61 | break 62 | 63 | L_optimizer2.apply_gradients(zip(L_gradients, [L2])) 64 | if vis: 65 | print('Purify epoch {}, purify L2 Loss2: {}'.format(epoch+1,loss2.numpy())) 66 | 67 | return tf.split(rae_output, args.num_clients, 1 ) 68 | 69 | -------------------------------------------------------------------------------- /extrasensory/utils/sensor_util.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | 4 | from io import StringIO 5 | import os.path 6 | 7 | import numpy as np 8 | from models.LocalModel import VFLPassiveModel 9 | from models.ServerModel import VFLActiveModelWithOneLayer,QuanVFLActiveModelWithOneLayer 10 | 11 | 12 | #from data_util.data_loader import TwoPartyDataLoader 13 | def parse_header_of_csv(headline): 14 | # Isolate the headline columns: 15 | # headline = csv_str[:csv_str.index('\n')] 16 | columns = headline.split(',') 17 | 18 | # The first column should be timestamp: 19 | assert columns[0] == 'timestamp' 20 | # The last column should be label_source: 21 | assert columns[-1] == 'label_source' 22 | 23 | # Search for the column of the first label: 24 | for (ci, col) in enumerate(columns): 25 | if col.startswith('label:'): 26 | first_label_ind = ci 27 | break 28 | pass 29 | 30 | # Feature columns come after timestamp and before the labels: 31 | feature_names = columns[1:first_label_ind] 32 | # Then come the labels, till the one-before-last column: 33 | label_names = columns[first_label_ind:-1] 34 | for (li, label) in enumerate(label_names): 35 | # In the CSV the label names appear with prefix 'label:', but we don't need it after reading the data: 36 | assert label.startswith('label:') 37 | label_names[li] = label.replace('label:', '') 38 | pass 39 | 40 | return (feature_names, label_names) 41 | 42 | 43 | def parse_body_of_csv(csv_str, n_features): 44 | # Read the entire CSV body into a single numeric matrix: 45 | full_table = np.loadtxt(StringIO(csv_str), delimiter=',') # ,skiprows=1) 46 | 47 | # Timestamp is the primary key for the records (examples): 48 | timestamps = full_table[:, 0].astype(int) 49 | 50 | # Read the sensor features: 51 | X = full_table[:, 1:(n_features + 1)] 52 | 53 | # Read the binary label values, and the 'missing label' indicators: 54 | trinary_labels_mat = full_table[:, (n_features + 1):-1] # This should have values of either 0., 1. or NaN 55 | M = np.isnan(trinary_labels_mat) # M is the missing label matrix 56 | Y = np.where(M, 0, trinary_labels_mat) > 0. # Y is the label matrix 57 | 58 | return (X, Y, M, timestamps) 59 | 60 | 61 | ''' 62 | Interpret the feature names to figure out for each feature what is the sensor it was extracted from. 63 | ''' 64 | 65 | 66 | def get_sensor_names_from_features(feature_names): 67 | feat_sensor_names = np.array([None for feat in feature_names]) 68 | for (fi, feat) in enumerate(feature_names): 69 | if feat.startswith('raw_acc'): 70 | feat_sensor_names[fi] = 'Acc' 71 | pass 72 | elif feat.startswith('proc_gyro'): 73 | feat_sensor_names[fi] = 'Gyro' 74 | pass 75 | elif feat.startswith('raw_magnet'): 76 | feat_sensor_names[fi] = 'Magnet' 77 | pass 78 | elif feat.startswith('watch_acceleration'): 79 | feat_sensor_names[fi] = 'WAcc' 80 | pass 81 | elif feat.startswith('watch_heading'): 82 | feat_sensor_names[fi] = 'Compass' 83 | pass 84 | elif feat.startswith('location'): 85 | feat_sensor_names[fi] = 'Loc' 86 | pass 87 | elif feat.startswith('location_quick_features'): 88 | feat_sensor_names[fi] = 'Loc' 89 | pass 90 | elif feat.startswith('audio_naive'): 91 | feat_sensor_names[fi] = 'Aud' 92 | pass 93 | elif feat.startswith('audio_properties'): 94 | feat_sensor_names[fi] = 'Aud'#AP 95 | pass 96 | elif feat.startswith('discrete'): 97 | feat_sensor_names[fi] = 'PS' 98 | pass 99 | elif feat.startswith('Tdiscrete'): 100 | feat_sensor_names[fi] = 'TS' 101 | pass 102 | elif feat.startswith('lf_measurements'): 103 | feat_sensor_names[fi] = 'LF' 104 | pass 105 | else: 106 | raise ValueError("!!! Unsupported feature name: %s" % feat) 107 | 108 | pass 109 | 110 | return feat_sensor_names 111 | 112 | 113 | ''' 114 | Read the data (precomputed sensor-features and labels) for a user. 115 | This function assumes the user's data file is present. 116 | ''' 117 | 118 | 119 | def read_user_data(uuid): 120 | user_data_file = os.path.join('', '%s' % uuid) 121 | 122 | # Read the entire csv file of the user: 123 | with open(user_data_file, 'r') as fid: 124 | csv_headline = fid.readline().strip() 125 | csv_body = fid.read() 126 | pass 127 | 128 | (feature_names, label_names) = parse_header_of_csv(csv_headline) 129 | print(len(feature_names)) 130 | feat_sensor_names = get_sensor_names_from_features(feature_names) 131 | print(feat_sensor_names) 132 | n_features = len(feature_names) 133 | (X, Y, M, timestamps) = parse_body_of_csv(csv_body, n_features) 134 | 135 | return (X, Y, M, timestamps, feature_names, feat_sensor_names, label_names) 136 | 137 | 138 | def validate_column_names_are_consistent(old_column_names, new_column_names): 139 | if len(old_column_names) != len(new_column_names): 140 | raise ValueError("!!! Inconsistent number of columns.") 141 | 142 | for ci in range(len(old_column_names)): 143 | if old_column_names[ci] != new_column_names[ci]: 144 | raise ValueError("!!! Inconsistent column %d) %s != %s" % (ci, old_column_names[ci], new_column_names[ci])) 145 | pass 146 | return 147 | 148 | 149 | def read_multiple_users_data(uuids): 150 | feature_names = None 151 | feat_sensor_names = None 152 | label_names = None 153 | X_parts = [] 154 | Y_parts = [] 155 | M_parts = [] 156 | timestamps_parts = [] 157 | uuid_inds_parts = [] 158 | for (ui, uuid) in enumerate(uuids): 159 | (X_i, Y_i, M_i, timestamps_i, feature_names_i, feat_sensor_names_i, label_names_i) 160 | # Make sure the feature names are consistent among all users: 161 | if feature_names is None: 162 | feature_names = feature_names_i 163 | feat_sensor_names = feat_sensor_names_i 164 | pass 165 | else: 166 | validate_column_names_are_consistent(feature_names, feature_names_i) 167 | pass 168 | # Make sure the label names are consistent among all users: 169 | if label_names is None: 170 | label_names = label_names_i 171 | pass 172 | else: 173 | validate_column_names_are_consistent(label_names, label_names_i) 174 | pass 175 | # Accumulate this user's data: 176 | X_parts.append(X_i) 177 | Y_parts.append(Y_i) 178 | M_parts.append(M_i) 179 | timestamps_parts.append(timestamps_i) 180 | uuid_inds_parts.append(ui * np.ones(len(timestamps_i))) 181 | pass 182 | 183 | # Combine all the users' data: 184 | X = np.concatenate(tuple(X_parts), axis=0) 185 | Y = np.concatenate(tuple(Y_parts), axis=0) 186 | M = np.concatenate(tuple(M_parts), axis=0) 187 | timestamps = np.concatenate(tuple(timestamps_parts), axis=0) 188 | uuid_inds = np.concatenate(tuple(uuid_inds_parts), axis=0) 189 | 190 | return (X, Y, M, uuid_inds, timestamps, feature_names, feat_sensor_names, label_names) 191 | 192 | 193 | def project_features_to_selected_sensors(X, feat_sensor_names, sensors_to_use): 194 | use_feature = np.zeros(len(feat_sensor_names), dtype=bool) 195 | for sensor in sensors_to_use: 196 | is_from_sensor = (feat_sensor_names == sensor) 197 | 198 | use_feature = np.logical_or(use_feature, is_from_sensor) 199 | pass 200 | X = X[:, use_feature] 201 | #print(use_feature) 202 | return X 203 | 204 | 205 | def estimate_standardization_params(X_train): 206 | mean_vec = np.nanmean(X_train, axis=0) 207 | std_vec = np.nanstd(X_train, axis=0) 208 | return (mean_vec, std_vec) 209 | 210 | 211 | def standardize_features(X, mean_vec, std_vec): 212 | # Subtract the mean, to centralize all features around zero: 213 | X_centralized = X - mean_vec.reshape((1, -1)) 214 | # Divide by the standard deviation, to get unit-variance for all features: 215 | # * Avoid dividing by zero, in case some feature had estimate of zero variance 216 | normalizers = np.where(std_vec > 0., std_vec, 1.).reshape((1, -1)) 217 | X_standard = X_centralized / normalizers 218 | return X_standard 219 | 220 | 221 | def load_sensor_data(X_train, Y_train, M_train, feat_sensor_names, label_names, sensors_to_use, target_label,delete_nan): 222 | # Project the feature matrix to the features from the desired sensors: 223 | X_train = project_features_to_selected_sensors(X_train, feat_sensor_names, sensors_to_use) 224 | print(X_train.shape) 225 | print("== Projected the features to %d features from the sensors: %s" % ( 226 | X_train.shape[1], ', '.join(sensors_to_use))) 227 | 228 | # It is recommended to standardize the features (subtract mean and divide by standard deviation), 229 | # so that all their values will be roughly in the same range: 230 | (mean_vec, std_vec) = estimate_standardization_params(X_train) 231 | X_train = standardize_features(X_train, mean_vec, std_vec) 232 | 233 | # The single target label: 234 | label_ind = label_names.index(target_label) 235 | y = Y_train[:, label_ind] 236 | missing_label = M_train[:, label_ind] 237 | existing_label = np.logical_not(missing_label) 238 | 239 | # Select only the examples that are not missing the target label: 240 | X_train = X_train[existing_label, :] 241 | y = y[existing_label] 242 | 243 | # Also, there may be missing sensor-features (represented in the data as NaN). 244 | # You can handle those by imputing a value of zero (since we standardized, this is equivalent to assuming average value). 245 | # You can also further select examples - only those that have values for all the features. 246 | # For this tutorial, let's use the simple heuristic of zero-imputation: 247 | if delete_nan: 248 | index_good = ~np.isnan(X_train).any(axis=1) 249 | # a[index_good, :] 250 | y = y[index_good] 251 | X_train = X_train[index_good, :] 252 | else: 253 | X_train[np.isnan(X_train)] = 0. 254 | print("== Training with %d examples. For label '%s' we have %d positive and %d negative examples." % \ 255 | (len(y), target_label, sum(y), sum(np.logical_not(y)))) 256 | 257 | yy=y.astype(float) 258 | return X_train, yy 259 | 260 | 261 | def test_model(X_test, Y_test, M_test, timestamps, feat_sensor_names, label_names, model): 262 | # Project the feature matrix to the features from the sensors that the classifier is based on: 263 | X_test = project_features_to_selected_sensors(X_test, feat_sensor_names, model['sensors_to_use']) 264 | print("== Projected the features to %d features from the sensors: %s" % ( 265 | X_test.shape[1], ', '.join(model['sensors_to_use']))) 266 | 267 | # We should standardize the features the same way the train data was standardized: 268 | X_test = standardize_features(X_test, model['mean_vec'], model['std_vec']) 269 | 270 | # The single target label: 271 | label_ind = label_names.index(model['target_label']) 272 | y = Y_test[:, label_ind] 273 | missing_label = M_test[:, label_ind] 274 | existing_label = np.logical_not(missing_label) 275 | 276 | # Select only the examples that are not missing the target label: 277 | X_test = X_test[existing_label, :] 278 | y = y[existing_label] 279 | timestamps = timestamps[existing_label] 280 | 281 | # Do the same treatment for missing features as done to the training data: 282 | X_test[np.isnan(X_test)] = 0. 283 | 284 | print("== Testing with %d examples. For label '%s' we have %d positive and %d negative examples." % \ 285 | (len(y), model['target_label'], sum(y), sum(np.logical_not(y)))) 286 | 287 | # Preform the prediction: 288 | 289 | y_pred = model['mlp'].predict(X_test) 290 | # Naive accuracy (correct classification rate): 291 | accuracy = np.mean(y_pred == y) 292 | 293 | # Count occorrences of true-positive, true-negative, false-positive, and false-negative: 294 | tp = np.sum(np.logical_and(y_pred, y)) 295 | tn = np.sum(np.logical_and(np.logical_not(y_pred), np.logical_not(y))) 296 | fp = np.sum(np.logical_and(y_pred, np.logical_not(y))) 297 | fn = np.sum(np.logical_and(np.logical_not(y_pred), y)) 298 | 299 | # Sensitivity (=recall=true positive rate) and Specificity (=true negative rate): 300 | sensitivity = float(tp) / (tp + fn) 301 | specificity = float(tn) / (tn + fp) 302 | 303 | # Balanced accuracy is a more fair replacement for the naive accuracy: 304 | balanced_accuracy = (sensitivity + specificity) / 2. 305 | 306 | # Precision: 307 | # Beware from this metric, since it may be too sensitive to rare labels. 308 | # In the ExtraSensory Dataset, there is large skew among the positive and negative classes, 309 | # and for each label the pos/neg ratio is different. 310 | # This can cause undesirable and misleading results when averaging precision across different labels. 311 | precision = float(tp) / (tp + fp) 312 | 313 | print("-" * 10) 314 | print('Accuracy*: %.2f' % accuracy) 315 | print('Sensitivity (TPR): %.2f' % sensitivity) 316 | print('Specificity (TNR): %.2f' % specificity) 317 | print('Balanced accuracy: %.2f' % balanced_accuracy) 318 | print('Precision**: %.2f' % precision) 319 | print("-" * 10) 320 | 321 | print( 322 | '* The accuracy metric is misleading - it is dominated by the negative examples (typically there are many more negatives).') 323 | print( 324 | '** Precision is very sensitive to rare labels. It can cause misleading results when averaging precision over different labels.') 325 | 326 | 327 | return 328 | 329 | 330 | def create_folder(args): 331 | if not os.path.exists(args.path_predix): 332 | os.makedirs(args.path_predix) 333 | print("save to", args.path_predix) 334 | 335 | SavedPaths = [] 336 | 337 | for i in range(args.num_text_clients): 338 | _dir = os.path.join(args.path_predix, 339 | 'sensor_{}_{}'.format(args.text_feature_div[i], args.text_feature_div[i + 1])) 340 | if not os.path.exists(_dir): 341 | os.makedirs(_dir) 342 | SavedPaths.append(_dir) 343 | 344 | _dir = os.path.join(args.path_predix, 'server') 345 | if not os.path.exists(_dir): 346 | os.makedirs(_dir) 347 | server_savedpath = _dir 348 | 349 | return SavedPaths, server_savedpath 350 | 351 | 352 | def create_models(args, quan_server=False): 353 | if quan_server == False: 354 | LocalModels = [] 355 | GradientsRes = [] 356 | for i in range(args.num_clients): 357 | local_model = VFLPassiveModel(emb_dim=args.emb_dim) 358 | LocalModels.append(local_model) 359 | GradientsRes.append(None) 360 | active_model = VFLActiveModelWithOneLayer(emb_dim=args.emb_dim, class_num=args.num_class) 361 | return LocalModels, GradientsRes, active_model 362 | else: 363 | local_model = VFLPassiveModel(emb_dim=args.emb_dim) 364 | active_model = QuanVFLActiveModelWithOneLayer(emb_dim=args.emb_dim, class_num=args.num_class) 365 | return local_model, active_model 366 | 367 | 368 | def load_models(args, quan_server=False): 369 | if quan_server == False: 370 | LocalModels = [] 371 | for i in range(args.num_clients): 372 | local_model = VFLPassiveModel(emb_dim=args.emb_dim) 373 | local_model.built = True 374 | local_model.load_weights(os.path.join(args.local_paths[i], args.ckpt_name)) 375 | LocalModels.append(local_model) 376 | 377 | active_model = VFLActiveModelWithOneLayer(emb_dim=args.emb_dim, class_num=args.num_class) 378 | active_model.built = True 379 | active_model.load_weights(os.path.join(args.sever_path, args.ckpt_name)) 380 | return LocalModels, active_model 381 | else: 382 | local_model = VFLPassiveModel(emb_dim=args.emb_dim) 383 | local_model.built = True 384 | local_model.load_weights(os.path.join(args.local_paths[0], args.ckpt_name)) 385 | active_model = QuanVFLActiveModelWithOneLayer(emb_dim=args.emb_dim, class_num=args.num_class) 386 | active_model.built = True 387 | active_model.load_weights(os.path.join(args.sever_path, args.ckpt_name)) 388 | return local_model, active_model 389 | 390 | def get_local_outputs(num_text_clients, texts, text_feature_div, LocalModels): 391 | LocalOutputs =[] 392 | for i in range(num_text_clients): 393 | text_div = texts[:,text_feature_div[i]:text_feature_div[i+1]] 394 | LocalOutputs.append(LocalModels[i](text_div)) 395 | 396 | return LocalOutputs 397 | -------------------------------------------------------------------------------- /nus-wide/NUS_WIDE/README.md: -------------------------------------------------------------------------------- 1 | Please download NUS_WIDE dataset (https://lms.comp.nus.edu.sg/wp-content/uploads/2019/research/nuswide/NUS-WIDE.html) and unzip to this folder -------------------------------------------------------------------------------- /nus-wide/models/AE.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.keras.layers import Dense, Flatten, LayerNormalization 3 | from tensorflow.keras import Model 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | import copy 7 | def my_leaky_relu(x): 8 | return tf.nn.leaky_relu(x, 0.8) 9 | 10 | 11 | class AE_NUS(Model): 12 | def __init__(self,out_dim = 160 , hidden_dim=200 , encode_dim= 150): 13 | super(AE_NUS, self).__init__() 14 | self.d1 = Dense(hidden_dim, name="dense1", activation=my_leaky_relu) 15 | self.d2 = Dense(encode_dim, name="dense2", activation=None) 16 | self.d3 = Dense(hidden_dim, name="dense3", activation=my_leaky_relu) 17 | self.d4 = Dense(out_dim, name="dense4", activation=None) 18 | 19 | def call(self, x): 20 | # print(x.shape) 21 | x = self.d1(x) 22 | x = self.d2(x) 23 | # x2 = LayerNormalization(axis=-1 , center=False , scale=True)(x) 24 | x2 =x 25 | x = self.d3(x2) 26 | x = self.d4(x) 27 | return x,x2 28 | 29 | class AE(Model): 30 | def __init__(self,out_dim = 160, encode_dim= 100): 31 | super(AE, self).__init__() 32 | self.d1 = Dense(encode_dim, name="dense1", activation=None) 33 | self.d2 = Dense(out_dim, name="dense2", activation=None) 34 | 35 | def call(self, x): 36 | x = self.d1(x) 37 | x2 =x 38 | x = self.d2(x2) 39 | return x,x2 -------------------------------------------------------------------------------- /nus-wide/models/LocalModel.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.keras.layers import Dense, Flatten, LayerNormalization 3 | from tensorflow.keras import Model, datasets, layers, models 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | import copy 7 | 8 | 9 | class VFLPassiveModel(Model): 10 | def __init__(self,emb_dim=32): 11 | super(VFLPassiveModel, self).__init__() 12 | self.flatten = Flatten() 13 | self.d1 = Dense(128, name="dense1", activation='relu') 14 | self.d2 = Dense(emb_dim, name="dense2", activation='relu') 15 | 16 | 17 | def call(self, x): 18 | x = self.flatten(x) 19 | x= self.d1(x) 20 | x= self.d2(x) 21 | return x 22 | 23 | 24 | class VFLPassiveModelMNIST(Model): 25 | def __init__(self,emb_dim=32): 26 | super(VFLPassiveModelMNIST, self).__init__() 27 | self.flatten = Flatten() 28 | self.d1 = Dense(128, name="dense1", activation='relu') 29 | self.d2 = Dense(64, name="dense2", activation='relu') 30 | self.d3 = Dense(emb_dim, name="dense3", activation='relu') 31 | 32 | def call(self, x): 33 | x = self.flatten(x) 34 | x= self.d1(x) 35 | x= self.d2(x) 36 | x= self.d3(x) 37 | return x 38 | 39 | 40 | class VFLPassiveModelCIFAR(Model): 41 | def __init__(self,emb_dim=32): 42 | super(VFLPassiveModelCIFAR, self).__init__() 43 | self.d0 = layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)) 44 | self.d1 = layers.MaxPooling2D((2, 2)) 45 | self.d2 = layers.Conv2D(64, (3, 3), activation='relu') 46 | self.d3 = layers.MaxPooling2D((2,2)) 47 | self.d4 = layers.Conv2D(64, (3, 3), activation='relu') 48 | self.d5 = layers.Flatten() 49 | self.d6= layers.Dense(emb_dim, activation='relu') 50 | 51 | 52 | 53 | def call(self, x): 54 | x=self.d0(x) 55 | x=self.d1(x) 56 | x=self.d2(x) 57 | x=self.d3(x) 58 | x=self.d4(x) 59 | x=self.d5(x) 60 | x=self.d6(x) 61 | return x -------------------------------------------------------------------------------- /nus-wide/models/ServerModel.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.keras.layers import Dense, Flatten, LayerNormalization 3 | from tensorflow.keras import Model 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | import copy 7 | 8 | class QuanVFLActiveModelWithOneLayer(Model): 9 | def __init__(self, emb_dim=16,class_num=5): 10 | super(QuanVFLActiveModelWithOneLayer, self).__init__() 11 | 12 | self.d1 = Dense(emb_dim, name="dense1", activation='relu') 13 | self.out = Dense(class_num, name="out", activation='softmax') 14 | 15 | def call(self, x): 16 | x = self.d1(x) 17 | return self.out(x) 18 | 19 | 20 | class VFLActiveModelWithOneLayer(Model): 21 | def __init__(self,emb_dim=16,class_num=5): 22 | super(VFLActiveModelWithOneLayer, self).__init__() 23 | self.concatenated = tf.keras.layers.Concatenate() 24 | self.d1 = Dense(emb_dim, name="dense1", activation='relu') 25 | self.out = Dense(class_num, name="out", activation='softmax') 26 | 27 | def call(self, x): 28 | x = self.concatenated(x) 29 | x = self.d1(x) 30 | return self.out(x) 31 | -------------------------------------------------------------------------------- /nus-wide/nus_ae_train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import time 4 | import numpy as np 5 | import copy 6 | from utils.nus_utils import get_local_outputs,load_data 7 | import tensorflow as tf 8 | from models.AE import AE_NUS as AutoEncoder 9 | 10 | def parse_path(args): 11 | SavedPaths=[] 12 | # _mode = 'normal' 13 | # savefolder= '{}_{}i_{}t_try{}'.format(_mode, args.num_img_clients, args.num_text_clients, args.seed) 14 | for i in range(args.num_img_clients): 15 | _dir= os.path.join(args.path_predix, 'img_{}_{}'.format(args.img_feature_div[i], args.img_feature_div[i+1])) 16 | SavedPaths.append(_dir) 17 | for i in range(args.num_text_clients): 18 | _dir= os.path.join(args.path_predix, 'text_{}_{}'.format(args.text_feature_div[i], args.text_feature_div[i+1] )) 19 | SavedPaths.append(_dir) 20 | 21 | print(SavedPaths) 22 | return SavedPaths 23 | 24 | def create_models(args): 25 | from models.LocalModel import VFLPassiveModel 26 | 27 | LocalModels= [] 28 | for i in range(args.num_clients): 29 | local_model = VFLPassiveModel(emb_dim=args.emb_dim) 30 | local_model.built = True 31 | 32 | local_model.load_weights(os.path.join(args.local_paths[i],'best_checkpoints')) 33 | local_model.trainable = False 34 | LocalModels.append(local_model) 35 | 36 | 37 | return LocalModels 38 | 39 | def parse_command(): 40 | parser = argparse.ArgumentParser() 41 | 42 | parser.add_argument('--emb_dim', type=int, default=60) 43 | parser.add_argument('--batch_size', type=int, default=256) 44 | parser.add_argument('--ae_epochs', type=int, default=10) 45 | parser.add_argument('--ae_lr', type=float, default=0.001) 46 | parser.add_argument('--num_class', type=int, default=5) 47 | parser.add_argument('--seed', type=int, default=1) 48 | 49 | parser.add_argument('--num_test_samples', type=int, default=10000) 50 | parser.add_argument('--num_train_samples', type=int, default=60000) 51 | 52 | parser.add_argument('--path_predix', type=str, default='nus_results') 53 | parser.add_argument('--mode', type=str, default='normal', 54 | choices=[ 55 | 'normal', 56 | ]) 57 | parser.add_argument('--img_feature_div', nargs='+', type=int, default=[0, 360, 634]) 58 | parser.add_argument('--text_feature_div', nargs='+', type=int, default=[0,500, 1000]) 59 | 60 | parser.add_argument('--vis', action='store_true') 61 | 62 | 63 | args = parser.parse_args() 64 | if args.seed is not None: 65 | import random 66 | random.seed(args.seed) 67 | tf.random.set_seed(args.seed) 68 | args.num_img_clients= len(args.img_feature_div)-1 69 | args.num_text_clients= len(args.text_feature_div)-1 70 | args.num_clients = args.num_img_clients+ args.num_text_clients 71 | 72 | args.path_predix_load = args.path_predix 73 | savefolder= '{}_{}i_{}t_try{}'.format(args.mode, args.num_img_clients, args.num_text_clients, args.seed) 74 | args.path_predix=os.path.join(args.path_predix, savefolder) 75 | local_paths = parse_path(args) 76 | args.local_paths= local_paths 77 | print("save to", args.path_predix) 78 | args.top_k= ['buildings', 'grass', 'animal', 'water', 'person'] 79 | 80 | return args 81 | 82 | 83 | 84 | args = parse_command() 85 | if args.vis: 86 | from tensorboardX import SummaryWriter 87 | import datetime 88 | current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") 89 | log_dir = os.path.join(args.path_predix,current_time) 90 | args.train_writer = tf.summary.create_file_writer(log_dir) 91 | 92 | 93 | x_train, (test_images, test_texts), y_train, y_test = load_data(args) 94 | LocalModels = create_models(args) 95 | autoencoder_model=AutoEncoder(out_dim= args.num_clients * args.emb_dim) 96 | (x_train_images, x_train_texts) = x_train 97 | LocalOutputs= get_local_outputs(args.num_img_clients, x_train_images, args.img_feature_div, 98 | args.num_text_clients, x_train_texts, args.text_feature_div, LocalModels) 99 | H_input=tf.concat(LocalOutputs,1) 100 | rae_loss_object=tf.keras.losses.MeanSquaredError() 101 | rae_optimizer = tf.keras.optimizers.SGD(learning_rate=args.ae_lr) 102 | rae_train_loss= tf.keras.metrics.Mean(name='rae_train_loss') 103 | 104 | def train_ae(epoch): 105 | 106 | train_ds = tf.data.Dataset.from_tensor_slices( 107 | (H_input)).batch(args.batch_size) 108 | 109 | ae_step= len(train_ds) *epoch 110 | for h_input in train_ds: 111 | ae_step+=1 112 | with tf.GradientTape() as active_tape: 113 | rae_output, _ = autoencoder_model(h_input) 114 | loss= rae_loss_object(rae_output, h_input) 115 | 116 | rae_gradients = active_tape.gradient(loss, autoencoder_model.trainable_variables) 117 | rae_optimizer.apply_gradients(zip(rae_gradients, autoencoder_model.trainable_variables)) 118 | 119 | rae_train_loss(loss) 120 | if args.vis: 121 | with args.train_writer.as_default(): 122 | tf.summary.scalar('train_AE_loss', loss, step=ae_step) 123 | 124 | 125 | 126 | for epoch in range(args.ae_epochs): 127 | start= time.time() 128 | train_ae(epoch) 129 | end= time.time() 130 | print("AE training epoch: ", epoch, "loss", rae_train_loss.result(), "time: ",time.time()- start) 131 | rae_train_loss.reset_states() 132 | autoencoder_model.save_weights(os.path.join(args.path_predix, 'ae_ckpt.tf')) 133 | 134 | 135 | 136 | -------------------------------------------------------------------------------- /nus-wide/nus_infer_attack.py: -------------------------------------------------------------------------------- 1 | 2 | import argparse 3 | import os 4 | import time 5 | import numpy as np 6 | import copy 7 | from utils.nus_utils import get_local_outputs,load_data, load_models 8 | import tensorflow as tf 9 | from math import ceil 10 | 11 | from models.AE import AE_NUS as AutoEncoder 12 | from utils.robustae import inference_purify_miss 13 | 14 | 15 | def parse_path(args): 16 | SavedPaths=[] 17 | for i in range(args.num_img_clients): 18 | _dir= os.path.join(args.path_predix, 'img_{}_{}'.format(args.img_feature_div[i], args.img_feature_div[i+1])) 19 | 20 | SavedPaths.append(_dir) 21 | for i in range(args.num_text_clients): 22 | _dir= os.path.join(args.path_predix, 'text_{}_{}'.format(args.text_feature_div[i], args.text_feature_div[i+1])) 23 | 24 | SavedPaths.append(_dir) 25 | if args.server_model_path == '': 26 | _dir= os.path.join(args.path_predix,'final_server_AE{}'.format( args.use_ae)) 27 | else: 28 | _dir = os.path.join(args.path_predix, args.server_model_path) 29 | 30 | server_savedpath=_dir 31 | 32 | return SavedPaths, server_savedpath 33 | 34 | 35 | def parse_command(): 36 | parser = argparse.ArgumentParser() 37 | 38 | parser.add_argument('--emb_dim', type=int, default=60) 39 | parser.add_argument('--batch_size', type=int, default=1) 40 | 41 | parser.add_argument('--num_class', type=int, default=5) 42 | parser.add_argument('--seed', type=int, default=1) 43 | 44 | parser.add_argument('--num_test_samples', type=int, default=10000) 45 | parser.add_argument('--num_train_samples', type=int, default=60000) 46 | parser.add_argument('--path_predix', type=str, default='nus_results') 47 | parser.add_argument('--ckpt_name', type=str, default='best_checkpoints') 48 | parser.add_argument('--mode', type=str, default='normal', 49 | choices=[ 50 | 'normal', 51 | 52 | ]) 53 | 54 | parser.add_argument('--img_feature_div', nargs='+', type=int, default=[0, 360, 634]) 55 | parser.add_argument('--text_feature_div', nargs='+', type=int, default=[0, 500, 1000]) 56 | parser.add_argument('--attackers_index', nargs='+', type=int, default=[3]) 57 | parser.add_argument('--use_ae', action='store_true') 58 | parser.add_argument('--purify_epochs', type=int, default=3) 59 | parser.add_argument('--initial_epochs', type=int, default=100) 60 | parser.add_argument('--L_lr', type=float, default=0.001) 61 | parser.add_argument('--L0_lr', type=float, default=0.1) 62 | parser.add_argument('--attack', action='store_true') 63 | parser.add_argument('--tau',type=float, default=1000) 64 | parser.add_argument('--sigma',type=float, default=0.01) 65 | parser.add_argument('--corruption_amp',type=float, default=10) 66 | 67 | parser.add_argument('--server_model_path',type=str, default='') 68 | parser.add_argument('--num_noise', type=int, default=100) 69 | parser.add_argument('--miss_index', nargs='+', type=int, default=[]) ######## 70 | parser.add_argument('--PGDeps', type=float, default=0.1) 71 | parser.add_argument('--PGDiters', type=int, default=50) 72 | parser.add_argument('--targeted', type=int, default=0) 73 | 74 | args = parser.parse_args() 75 | if args.seed is not None: 76 | import random 77 | random.seed(args.seed) 78 | tf.random.set_seed(args.seed) 79 | 80 | args.num_img_clients= len(args.img_feature_div)-1 81 | args.num_text_clients= len(args.text_feature_div)-1 82 | args.num_clients = args.num_img_clients + args.num_text_clients 83 | args.num_attacker = len(args.attackers_index) ############## 84 | print('attacker index', args.attackers_index) 85 | args.num_miss = len(args.miss_index) ############## 86 | print('miss index', args.miss_index) 87 | ######args.num_text_clients 88 | args.observe_list = list(set(list(range(0, args.num_clients))) - set(args.miss_index)) 89 | print('observed', args.observe_list) 90 | 91 | args.path_predix_load = args.path_predix 92 | savefolder= '{}_{}i_{}t_try{}'.format(args.mode, args.num_img_clients, args.num_text_clients, args.seed) 93 | args.path_predix=os.path.join(args.path_predix, savefolder) 94 | args.num_attacker= len(args.attackers_index) 95 | 96 | args.top_k= ['buildings', 'grass', 'animal', 'water', 'person'] 97 | 98 | return args 99 | 100 | args = parse_command() 101 | 102 | (test_images, test_texts), y_test = load_data(args, test=True) 103 | #print(y_test[0:100,]) 104 | print(np.sum(y_test, axis=0)) 105 | L_optimizer = tf.keras.optimizers.SGD(learning_rate=args.L0_lr) 106 | L_optimizer2 = tf.keras.optimizers.SGD(learning_rate=args.L_lr) 107 | rae_loss_object=tf.keras.losses.MeanSquaredError() 108 | 109 | 110 | loss_object = tf.keras.losses.CategoricalCrossentropy() 111 | 112 | test_loss = tf.keras.metrics.Mean(name='test_loss') 113 | test_accuracy = tf.keras.metrics.CategoricalAccuracy(name='test_accuracy') 114 | 115 | test_loss1 = tf.keras.metrics.Mean(name='test_loss1') 116 | test_accuracy1 = tf.keras.metrics.CategoricalAccuracy(name='test_accuracy1') 117 | 118 | test_loss2 = tf.keras.metrics.Mean(name='test_loss2') 119 | test_accuracy2 = tf.keras.metrics.CategoricalAccuracy(name='test_accuracy2') 120 | 121 | test_loss3 = tf.keras.metrics.Mean(name='test_loss3') 122 | test_accuracy3 = tf.keras.metrics.CategoricalAccuracy(name='test_accuracy3') 123 | 124 | if args.use_ae: 125 | print("use AE") 126 | autoencoder_model = AutoEncoder(out_dim= args.num_clients * args.emb_dim) 127 | autoencoder_model.built = True 128 | autoencoder_model.load_weights(os.path.join(args.path_predix,'ae_ckpt.tf')) 129 | 130 | 131 | def RandomizedSmooth(args, LocalOutputs, active_model,batch_size=100): 132 | counts = np.zeros(args.num_class, dtype=int) 133 | x=np.concatenate(tuple(LocalOutputs), axis=1) 134 | num=args.num_noise 135 | for _ in range(ceil(num / batch_size)): 136 | this_batch_size = min(batch_size, num) 137 | num -= this_batch_size 138 | 139 | batch = x#np.expand_dims(x, 0) 140 | batch = np.repeat(batch, this_batch_size, axis=0) 141 | noise = np.random.randn(*batch.shape) * args.sigma 142 | 143 | batch +=noise 144 | 145 | predictions = active_model(tf.split(batch , args.num_clients, 1 )) 146 | predictions1=tf.Variable(predictions).numpy() 147 | predictions1=predictions1.argmax(axis=1) 148 | counts += count_arr(predictions1, args.num_class) 149 | 150 | 151 | 152 | top2 = counts.argsort()[::-1][:2] 153 | 154 | return top2[0] 155 | 156 | 157 | def count_arr(arr: np.ndarray, length: int) -> np.ndarray: 158 | counts = np.zeros(length, dtype=int) 159 | for idx in arr: 160 | counts[idx] += 1 161 | return counts 162 | 163 | def create_adversarial_pattern(_LocalOutputs, args, labels, target_labels, server_model, PGD=True): 164 | loss_object = tf.keras.losses.CategoricalCrossentropy() 165 | if PGD == True: 166 | LocalOutputs = _LocalOutputs 167 | hadv = [_LocalOutputs[index] for index in args.attackers_index] 168 | h0 = tf.convert_to_tensor(np.concatenate(tuple(hadv), axis=1), dtype=tf.float32) 169 | 170 | 171 | for i in range(args.PGDiters): 172 | 173 | with tf.GradientTape() as tape: 174 | tape.watch(h0) # the adversarial's feature 175 | ADV_Outputs = tf.split(h0, args.num_attacker, 1) 176 | for j in range(args.num_attacker): 177 | atk_index = args.attackers_index[j] 178 | LocalOutputs[atk_index] = ADV_Outputs[j] 179 | 180 | 181 | output = server_model(LocalOutputs) 182 | 183 | if args.targeted == 1: 184 | loss = loss_object(target_labels, output) # targeted attack 185 | else: 186 | loss = loss_object(labels, output) # untargeted attack 187 | 188 | 189 | gradient = tape.gradient(loss, h0) 190 | 191 | if args.targeted == 1: 192 | h0 -= args.PGDeps * gradient 193 | else: 194 | h0 += args.PGDeps * gradient 195 | 196 | return LocalOutputs 197 | 198 | 199 | 200 | def test(_LocalModels,_active_model ): 201 | 202 | test_ds = tf.data.Dataset.from_tensor_slices( 203 | ((test_images, test_texts), y_test)).batch(args.batch_size) 204 | count_index = 0 205 | for (images, texts), labels in test_ds: 206 | count_index += 1 207 | print('Sample index:', count_index) 208 | 209 | _LocalOutputs= get_local_outputs(args.num_img_clients, images, args.img_feature_div, 210 | args.num_text_clients, texts, args.text_feature_div, _LocalModels) 211 | 212 | for j in range(args.num_miss): 213 | missing_index = args.miss_index[j] 214 | _LocalOutputs[missing_index] = 0 * _LocalOutputs[missing_index] 215 | 216 | 217 | if args.targeted==1: 218 | _LocalOutputs = create_adversarial_pattern(_LocalOutputs, args, labels, target_labels, _active_model, PGD=True) 219 | else: 220 | for j in range(args.num_attacker): 221 | atk_index = args.attackers_index[j] 222 | _LocalOutputs[atk_index] = -args.corruption_amp * _LocalOutputs[atk_index] 223 | 224 | 225 | temp=np.concatenate(tuple(_LocalOutputs), axis=1) 226 | 227 | rae_output,layer_output=autoencoder_model(temp) 228 | 229 | if args.use_ae: 230 | LocalOutputs = inference_purify_miss(args, _LocalOutputs,autoencoder_model,L_optimizer,L_optimizer2,rae_loss_object, vis=False)#LocalOutputs = inference_purify(args, _LocalOutputs,autoencoder_model,L_optimizer,L_optimizer2,rae_loss_object, vis=True) 231 | else: 232 | LocalOutputs = _LocalOutputs 233 | 234 | test_output = _active_model(LocalOutputs) 235 | 236 | test_output1 = _active_model(_LocalOutputs)#no AE, just use corrupted h 237 | test_output2 = _active_model(tf.split(rae_output, args.num_clients, 1 ))# use DE(h) 238 | 239 | test_loss(loss_object(labels, test_output)) 240 | test_accuracy(labels, test_output) 241 | 242 | test_loss1(loss_object(labels, test_output1)) 243 | test_accuracy1(labels, test_output1) 244 | 245 | test_loss2(loss_object(labels, test_output2)) 246 | test_accuracy2(labels, test_output2) 247 | 248 | test_accuracy3(labels,tf.one_hot(RandomizedSmooth(args, LocalOutputs, _active_model,100),5)) 249 | 250 | 251 | 252 | target_labels = tf.constant([0, 1, 0, 0, 0], dtype=tf.float32, shape=(1, 5)) 253 | local_paths, sever_path= parse_path(args) 254 | args.local_paths= local_paths 255 | args.sever_path= sever_path 256 | LocalModels, active_model= load_models(args) 257 | start= time.time() 258 | test(LocalModels, active_model) 259 | 260 | end= time.time() 261 | template = '{} Test Loss: {}, Test Accuracy: {}, Test Accuracy3: {}, Time: {}' 262 | print(template.format(args.path_predix, 263 | test_loss.result(), 264 | test_accuracy.result()*100, 265 | test_accuracy3.result()*100, 266 | end-start 267 | )) 268 | 269 | test_loss.reset_states() 270 | test_accuracy.reset_states() 271 | -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/acc_test.txt: -------------------------------------------------------------------------------- 1 | 39.8400 2 | 42.2400 3 | 47.0900 4 | 49.9900 5 | 52.6500 6 | 55.1900 7 | 57.5600 8 | 59.3100 9 | 61.2500 10 | 62.7200 11 | 64.0600 12 | 65.1300 13 | 66.1800 14 | 66.9600 15 | 67.9100 16 | 68.5300 17 | 69.1900 18 | 69.9100 19 | 70.4600 20 | 71.2100 21 | 71.7800 22 | 72.3600 23 | 72.9000 24 | 73.4400 25 | 74.0400 26 | 74.4600 27 | 74.7900 28 | 75.4200 29 | 75.7200 30 | 76.1300 31 | 76.5200 32 | 76.9800 33 | 77.2900 34 | 77.5800 35 | 77.7300 36 | 77.8400 37 | 78.1900 38 | 78.5300 39 | 78.8200 40 | 78.8800 41 | 79.2000 42 | 79.4300 43 | 79.5600 44 | 79.7400 45 | 79.8500 46 | 79.9700 47 | 80.1000 48 | 80.1200 49 | 80.2000 50 | 80.3000 51 | 80.3200 52 | 80.5100 53 | 80.5900 54 | 80.6100 55 | 80.8100 56 | 80.8400 57 | 80.8700 58 | 80.9100 59 | 80.9600 60 | 81.0200 61 | 81.0200 62 | 81.0500 63 | 81.0900 64 | 81.1600 65 | 81.2600 66 | 81.3100 67 | 81.3500 68 | 81.4200 69 | 81.4600 70 | 81.4700 71 | 81.5000 72 | 81.5400 73 | 81.5900 74 | 81.5700 75 | 81.6200 76 | 81.6800 77 | 81.7500 78 | 81.7500 79 | 81.7500 80 | 81.8000 81 | 81.9100 82 | 81.9200 83 | 81.8600 84 | 81.9100 85 | 81.8900 86 | 81.9200 87 | 81.8700 88 | 81.9300 89 | 81.9400 90 | 81.9600 91 | 81.9800 92 | 82.0300 93 | 82.0200 94 | 82.0800 95 | 82.0700 96 | 82.1200 97 | 82.1200 98 | 82.1500 99 | 82.1700 100 | 82.1200 101 | 82.1400 102 | 82.1500 103 | 82.1400 104 | 82.1500 105 | 82.2000 106 | 82.2200 107 | 82.1900 108 | 82.2300 109 | 82.2500 110 | 82.2900 111 | 82.2600 112 | 82.2600 113 | 82.2900 114 | 82.3500 115 | 82.3500 116 | 82.3700 117 | 82.3800 118 | 82.4200 119 | 82.4300 120 | 82.4600 121 | 82.4900 122 | 82.5000 123 | 82.5500 124 | 82.5400 125 | 82.5800 126 | 82.5900 127 | 82.6200 128 | 82.6500 129 | 82.6200 130 | 82.6500 131 | 82.7000 132 | 82.7200 133 | 82.7500 134 | 82.7500 135 | 82.7800 136 | 82.8000 137 | 82.8400 138 | 82.8200 139 | 82.8800 140 | 82.8900 141 | 82.8700 142 | 82.9200 143 | 82.9000 144 | 82.9500 145 | 83.0200 146 | 83.0000 147 | 82.9900 148 | 82.9800 149 | 83.0200 150 | 82.9800 151 | 82.9700 152 | 82.9800 153 | 82.9900 154 | 83.0100 155 | 83.0300 156 | 83.0200 157 | 83.0200 158 | 83.0100 159 | 83.0400 160 | 83.0300 161 | 83.0400 162 | 83.0900 163 | 83.0900 164 | 83.0600 165 | 83.1200 166 | 83.1400 167 | 83.1600 168 | 83.1300 169 | 83.1400 170 | 83.1200 171 | 83.1900 172 | 83.2100 173 | 83.2000 174 | 83.2400 175 | 83.2000 176 | 83.2500 177 | 83.2400 178 | 83.2400 179 | 83.2800 180 | 83.2500 181 | 83.2700 182 | 83.3000 183 | 83.2800 184 | 83.3200 185 | 83.3300 186 | 83.3600 187 | 83.3800 188 | 83.4000 189 | 83.3800 190 | 83.3900 191 | 83.4300 192 | 83.4200 193 | 83.4000 194 | 83.4800 195 | 83.4900 196 | 83.4700 197 | 83.4800 198 | 83.5200 199 | 83.5400 200 | 83.5200 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/acc_train.txt: -------------------------------------------------------------------------------- 1 | 31.9433 2 | 40.8967 3 | 44.9383 4 | 49.3650 5 | 52.2267 6 | 54.8017 7 | 57.1483 8 | 59.3417 9 | 61.0933 10 | 62.6850 11 | 63.9467 12 | 65.0867 13 | 66.0500 14 | 67.0783 15 | 67.8400 16 | 68.7417 17 | 69.4667 18 | 70.1983 19 | 70.9433 20 | 71.6283 21 | 72.5000 22 | 73.1183 23 | 73.7583 24 | 74.3983 25 | 74.9000 26 | 75.4717 27 | 76.0317 28 | 76.3683 29 | 76.9250 30 | 77.2517 31 | 77.6317 32 | 77.9250 33 | 78.2317 34 | 78.5183 35 | 78.8033 36 | 79.0383 37 | 79.2533 38 | 79.4483 39 | 79.7650 40 | 79.9600 41 | 80.0867 42 | 80.2633 43 | 80.4717 44 | 80.5567 45 | 80.7367 46 | 80.8433 47 | 80.9700 48 | 81.1417 49 | 81.2233 50 | 81.3500 51 | 81.4533 52 | 81.5350 53 | 81.6350 54 | 81.7850 55 | 81.8150 56 | 81.9617 57 | 82.0517 58 | 82.0917 59 | 82.1717 60 | 82.2333 61 | 82.3100 62 | 82.3317 63 | 82.4317 64 | 82.4717 65 | 82.5583 66 | 82.6083 67 | 82.6367 68 | 82.6850 69 | 82.7317 70 | 82.7850 71 | 82.8267 72 | 82.8633 73 | 82.9083 74 | 82.9650 75 | 82.9817 76 | 82.9950 77 | 83.0517 78 | 83.0800 79 | 83.0933 80 | 83.1417 81 | 83.1933 82 | 83.1850 83 | 83.2167 84 | 83.2400 85 | 83.2600 86 | 83.3050 87 | 83.3167 88 | 83.3400 89 | 83.3767 90 | 83.3967 91 | 83.4167 92 | 83.4333 93 | 83.4867 94 | 83.5233 95 | 83.5583 96 | 83.5783 97 | 83.6083 98 | 83.6267 99 | 83.6417 100 | 83.6467 101 | 83.6650 102 | 83.6983 103 | 83.7233 104 | 83.7467 105 | 83.7583 106 | 83.7950 107 | 83.8017 108 | 83.8250 109 | 83.8500 110 | 83.8817 111 | 83.8783 112 | 83.8950 113 | 83.9133 114 | 83.9267 115 | 83.9283 116 | 83.9683 117 | 83.9550 118 | 83.9767 119 | 84.0117 120 | 84.0083 121 | 84.0117 122 | 84.0367 123 | 84.0667 124 | 84.0883 125 | 84.1000 126 | 84.1083 127 | 84.1400 128 | 84.1467 129 | 84.1350 130 | 84.1567 131 | 84.1800 132 | 84.1767 133 | 84.2050 134 | 84.2150 135 | 84.2367 136 | 84.2267 137 | 84.2550 138 | 84.2650 139 | 84.2700 140 | 84.3050 141 | 84.2950 142 | 84.3017 143 | 84.3350 144 | 84.2983 145 | 84.3367 146 | 84.3417 147 | 84.3617 148 | 84.3633 149 | 84.3650 150 | 84.3733 151 | 84.3633 152 | 84.3817 153 | 84.3933 154 | 84.4083 155 | 84.4000 156 | 84.4200 157 | 84.4250 158 | 84.4317 159 | 84.4517 160 | 84.4450 161 | 84.4700 162 | 84.4600 163 | 84.4883 164 | 84.5000 165 | 84.5067 166 | 84.4900 167 | 84.5067 168 | 84.5400 169 | 84.5350 170 | 84.5633 171 | 84.5517 172 | 84.5567 173 | 84.6033 174 | 84.6100 175 | 84.6000 176 | 84.6433 177 | 84.6350 178 | 84.6550 179 | 84.6767 180 | 84.6667 181 | 84.6767 182 | 84.6867 183 | 84.6917 184 | 84.7250 185 | 84.7217 186 | 84.7167 187 | 84.7367 188 | 84.7583 189 | 84.7500 190 | 84.7800 191 | 84.7800 192 | 84.7850 193 | 84.7783 194 | 84.7850 195 | 84.8200 196 | 84.8050 197 | 84.8050 198 | 84.8183 199 | 84.8200 200 | 84.8333 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/ae_ckpt.tf.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/ae_ckpt.tf.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/ae_ckpt.tf.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/ae_ckpt.tf.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "ae_ckpt.tf" 2 | all_model_checkpoint_paths: "ae_ckpt.tf" 3 | -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/final_server_AETrue/best_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/final_server_AETrue/best_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/final_server_AETrue/best_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/final_server_AETrue/best_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/final_server_AETrue/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "checkpoints" 2 | all_model_checkpoint_paths: "checkpoints" 3 | -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/final_server_AETrue/checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/final_server_AETrue/checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/final_server_AETrue/checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/final_server_AETrue/checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_0_360/best_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_0_360/best_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_0_360/best_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_0_360/best_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_0_360/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "best_checkpoints" 2 | all_model_checkpoint_paths: "best_checkpoints" 3 | -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_0_360/epoch30_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_0_360/epoch30_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_0_360/epoch30_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_0_360/epoch30_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_0_360/epoch40_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_0_360/epoch40_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_0_360/epoch40_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_0_360/epoch40_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_0_360/epoch50_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_0_360/epoch50_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_0_360/epoch50_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_0_360/epoch50_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_360_634/best_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_360_634/best_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_360_634/best_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_360_634/best_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_360_634/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "best_checkpoints" 2 | all_model_checkpoint_paths: "best_checkpoints" 3 | -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_360_634/epoch30_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_360_634/epoch30_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_360_634/epoch30_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_360_634/epoch30_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_360_634/epoch40_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_360_634/epoch40_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_360_634/epoch40_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_360_634/epoch40_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_360_634/epoch50_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_360_634/epoch50_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/img_360_634/epoch50_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/img_360_634/epoch50_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/server/best_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/server/best_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/server/best_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/server/best_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/server/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "best_checkpoints" 2 | all_model_checkpoint_paths: "best_checkpoints" 3 | -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/server/epoch30_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/server/epoch30_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/server/epoch30_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/server/epoch30_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/server/epoch40_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/server/epoch40_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/server/epoch40_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/server/epoch40_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/server/epoch50_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/server/epoch50_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/server/epoch50_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/server/epoch50_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_0_500/best_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_0_500/best_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_0_500/best_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_0_500/best_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_0_500/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "best_checkpoints" 2 | all_model_checkpoint_paths: "best_checkpoints" 3 | -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_0_500/epoch30_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_0_500/epoch30_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_0_500/epoch30_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_0_500/epoch30_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_0_500/epoch40_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_0_500/epoch40_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_0_500/epoch40_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_0_500/epoch40_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_0_500/epoch50_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_0_500/epoch50_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_0_500/epoch50_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_0_500/epoch50_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/best_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/best_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/best_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/best_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "best_checkpoints" 2 | all_model_checkpoint_paths: "best_checkpoints" 3 | -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/epoch30_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/epoch30_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/epoch30_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/epoch30_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/epoch40_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/epoch40_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/epoch40_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/epoch40_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/epoch50_checkpoints.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/epoch50_checkpoints.data-00000-of-00001 -------------------------------------------------------------------------------- /nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/epoch50_checkpoints.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-secure/CoPur/fde4c7a2eb3b21cc7f0eb1082abdd5227b6cd7c9/nus-wide/nus_results/normal_2i_2t_try1/text_500_1000/epoch50_checkpoints.index -------------------------------------------------------------------------------- /nus-wide/nus_server_ae_train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import time 4 | import numpy as np 5 | import copy 6 | from utils.nus_utils import get_local_outputs,load_data 7 | import tensorflow as tf 8 | from models.AE import AE_NUS as AutoEncoder 9 | 10 | 11 | 12 | def parse_path(args): 13 | SavedPaths=[] 14 | 15 | for i in range(args.num_img_clients): 16 | _dir= os.path.join(args.path_predix, 'img_{}_{}'.format(args.img_feature_div[i], args.img_feature_div[i+1])) 17 | SavedPaths.append(_dir) 18 | for i in range(args.num_text_clients): 19 | _dir= os.path.join(args.path_predix, 'text_{}_{}'.format(args.text_feature_div[i], args.text_feature_div[i+1] )) 20 | SavedPaths.append(_dir) 21 | 22 | _dir= os.path.join(args.path_predix,'final_server_AE{}'.format( args.use_ae)) 23 | if not os.path.exists(_dir): 24 | os.makedirs(_dir) 25 | server_savedpath=_dir 26 | print(SavedPaths) 27 | print("server_savedpath", server_savedpath) 28 | return SavedPaths, server_savedpath 29 | 30 | def create_models(args): 31 | from models.LocalModel import VFLPassiveModel 32 | from models.ServerModel import VFLActiveModelWithOneLayer 33 | 34 | LocalModels= [] 35 | for i in range(args.num_clients): 36 | local_model = VFLPassiveModel(emb_dim=args.emb_dim) 37 | local_model.built = True 38 | 39 | local_model.load_weights(os.path.join(args.local_paths[i],'best_checkpoints')) 40 | local_model.trainable = False 41 | LocalModels.append(local_model) 42 | 43 | active_model = VFLActiveModelWithOneLayer(emb_dim=args.emb_dim,class_num= args.num_class) 44 | 45 | return LocalModels, active_model 46 | 47 | def parse_command(): 48 | parser = argparse.ArgumentParser() 49 | 50 | parser.add_argument('--emb_dim', type=int, default=60) 51 | parser.add_argument('--batch_size', type=int, default=256) 52 | parser.add_argument('--epochs', type=int, default=10) 53 | parser.add_argument('--num_class', type=int, default=5) 54 | parser.add_argument('--seed', type=int, default=1) 55 | 56 | parser.add_argument('--num_test_samples', type=int, default=10000) 57 | parser.add_argument('--num_train_samples', type=int, default=60000) 58 | 59 | parser.add_argument('--path_predix', type=str, default='nus_results') 60 | parser.add_argument('--mode', type=str, default='normal', 61 | choices=[ 62 | 'normal', 63 | 64 | ]) 65 | parser.add_argument('--img_feature_div', nargs='+', type=int, default=[0 , 360, 634]) 66 | parser.add_argument('--text_feature_div', nargs='+', type=int, default=[0, 500, 1000]) 67 | 68 | 69 | parser.add_argument('--vis', action='store_true') 70 | parser.add_argument('--use_ae', action='store_true') 71 | 72 | parser.add_argument('--lr', type=float, default=0.001) 73 | 74 | 75 | args = parser.parse_args() 76 | if args.seed is not None: 77 | import random 78 | random.seed(args.seed) 79 | tf.random.set_seed(args.seed) 80 | args.num_img_clients= len(args.img_feature_div)-1 81 | args.num_text_clients= len(args.text_feature_div)-1 82 | args.num_clients = args.num_img_clients+ args.num_text_clients 83 | 84 | args.path_predix_load = args.path_predix 85 | 86 | savefolder= '{}_{}i_{}t_try{}'.format(args.mode, args.num_img_clients, args.num_text_clients, args.seed) 87 | 88 | args.path_predix=os.path.join(args.path_predix, savefolder) 89 | local_paths, sever_path= parse_path(args) 90 | args.local_paths= local_paths 91 | args.sever_path= sever_path 92 | args.top_k= ['buildings', 'grass', 'animal', 'water', 'person'] 93 | print("save to", args.path_predix) 94 | 95 | 96 | return args 97 | 98 | 99 | args = parse_command() 100 | if args.vis: 101 | from tensorboardX import SummaryWriter 102 | import datetime 103 | current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") 104 | log_dir = os.path.join(args.path_predix,current_time) 105 | args.train_writer = tf.summary.create_file_writer(log_dir) 106 | else: 107 | args.train_writer=None 108 | 109 | x_train, (test_images, test_texts), y_train, y_test = load_data(args) 110 | LocalModels, active_model= create_models(args) 111 | ### RAE model 112 | (x_train_images, x_train_texts) = x_train 113 | 114 | if args.use_ae: 115 | print("use AE") 116 | autoencoder_model = AutoEncoder(out_dim= args.num_clients * args.emb_dim) 117 | autoencoder_model.built = True 118 | autoencoder_model.load_weights(os.path.join(args.path_predix,'ae_ckpt.tf')) 119 | 120 | 121 | optimizer_server = tf.keras.optimizers.SGD(learning_rate=args.lr) 122 | 123 | 124 | loss_object = tf.keras.losses.CategoricalCrossentropy() 125 | train_loss = tf.keras.metrics.Mean(name='train_loss') 126 | train_accuracy = tf.keras.metrics.CategoricalAccuracy(name='train_accuracy') 127 | test_loss = tf.keras.metrics.Mean(name='test_loss') 128 | test_accuracy = tf.keras.metrics.CategoricalAccuracy(name='test_accuracy') 129 | acc_train = [] 130 | acc_test = [] 131 | loss_train = [] 132 | loss_test = [] 133 | best_acc=0 134 | 135 | def train(epoch): 136 | 137 | train_ds = tf.data.Dataset.from_tensor_slices( 138 | (x_train, y_train)).shuffle(y_train.shape[0]).batch(args.batch_size) 139 | server_step=0 140 | for (images, texts), labels in train_ds: 141 | server_step+=1 142 | _LocalOutputs= get_local_outputs(args.num_img_clients, images, args.img_feature_div, 143 | args.num_text_clients, texts, args.text_feature_div, LocalModels) 144 | 145 | if args.use_ae: 146 | rae_output,layer_output=autoencoder_model(tf.concat(_LocalOutputs,1)) 147 | LocalOutputs = tf.split(rae_output, args.num_clients, 1 ) 148 | else: 149 | LocalOutputs = _LocalOutputs 150 | with tf.GradientTape() as active_tape: 151 | active_output = active_model(LocalOutputs) 152 | loss = loss_object(labels, active_output) 153 | 154 | [active_model_gradients] = active_tape.gradient(loss, [active_model.trainable_variables]) 155 | optimizer_server.apply_gradients(zip(active_model_gradients, active_model.trainable_variables)) 156 | 157 | train_loss(loss) 158 | train_accuracy(labels, active_output) 159 | if args.vis: 160 | with args.train_writer.as_default(): 161 | tf.summary.scalar('Server_train_loss', loss, step=server_step+epoch*len(train_ds)) 162 | 163 | 164 | def test(epoch): 165 | 166 | _LocalOutputs = get_local_outputs(args.num_img_clients, test_images, args.img_feature_div, 167 | args.num_text_clients, test_texts, args.text_feature_div, LocalModels) 168 | if args.use_ae: 169 | rae_output,layer_output=autoencoder_model(tf.concat(_LocalOutputs,1)) 170 | LocalOutputs = tf.split(rae_output, args.num_clients, 1 ) 171 | else: 172 | LocalOutputs = _LocalOutputs 173 | test_output = active_model(LocalOutputs) 174 | test_loss(loss_object(y_test, test_output)) 175 | test_accuracy(y_test, test_output) 176 | if args.vis: 177 | with args.train_writer.as_default(): 178 | tf.summary.scalar('Server_test_loss', test_loss.result(), step=epoch ) 179 | tf.summary.scalar('Server_test_acc', test_accuracy.result(), step=epoch ) 180 | 181 | 182 | for epoch in range(args.epochs): 183 | start= time.time() 184 | train(epoch) 185 | test(epoch) 186 | 187 | end= time.time() 188 | template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}, Time: {}' 189 | print(template.format(epoch+1, 190 | train_loss.result(), 191 | train_accuracy.result()*100, 192 | test_loss.result(), 193 | test_accuracy.result()*100, 194 | end-start 195 | )) 196 | 197 | acc_train.append(train_accuracy.result()) 198 | acc_test.append(test_accuracy.result()) 199 | loss_train.append(train_loss.result()) 200 | loss_test.append(test_loss.result()) 201 | if best_acc <= test_accuracy.result(): 202 | best_acc= test_accuracy.result() 203 | active_model.save_weights(os.path.join(args.sever_path, 'best_checkpoints')) 204 | train_loss.reset_states() 205 | train_accuracy.reset_states() 206 | test_loss.reset_states() 207 | test_accuracy.reset_states() 208 | active_model.save_weights(os.path.join(args.sever_path, 'checkpoints')) 209 | with open(os.path.join(args.path_predix, 'acc_test.txt'), "w") as outfile: 210 | outfile.write("\n".join("{:.4f}".format(item*100) for item in acc_test)) 211 | 212 | with open(os.path.join(args.path_predix, 'acc_train.txt'), "w") as outfile: 213 | outfile.write("\n".join("{:.4f}".format(item*100) for item in acc_train)) 214 | 215 | -------------------------------------------------------------------------------- /nus-wide/script_nus.sh: -------------------------------------------------------------------------------- 1 | #Please download NUS_WIDE dataset (https://lms.comp.nus.edu.sg/wp-content/uploads/2019/research/nuswide/NUS-WIDE.html) to folder NUS_WIDE 2 | 3 | # The pre-trained Local feature extractors can be found in the folder `nus_results' 4 | # Feature subspace learning 5 | python nus_ae_train.py --ae_epochs 400 --ae_lr 0.001 --img_feature_div 0 360 634 --text_feature_div 0 500 1000 6 | # Fusion center training 7 | python nus_server_ae_train.py --use_ae --lr 0.001 --epochs 200 --img_feature_div 0 360 634 --text_feature_div 0 500 1000 8 | # inference under untargeted attack 9 | python nus_infer_attack.py --use_ae --L_lr 0.001 --img_feature_div 0 360 634 --text_feature_div 0 500 1000 --corruption_amp 10 10 | # inference under targeted attack 11 | python nus_infer_attack.py --use_ae --L_lr 0.001 --img_feature_div 0 360 634 --text_feature_div 0 500 1000 --num_test_samples 1000 --targeted 1 12 | -------------------------------------------------------------------------------- /nus-wide/utils/nus_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from utils.nus_wide_data_util import get_labeled_data 3 | import copy 4 | from models.LocalModel import VFLPassiveModel 5 | from models.ServerModel import VFLActiveModelWithOneLayer,QuanVFLActiveModelWithOneLayer 6 | 7 | import os 8 | 9 | def create_models(args,quan_server=False): 10 | if quan_server== False: 11 | LocalModels= [] 12 | GradientsRes=[] 13 | for i in range(args.num_clients): 14 | local_model = VFLPassiveModel(emb_dim=args.emb_dim) 15 | LocalModels.append(local_model) 16 | GradientsRes.append(None) 17 | active_model = VFLActiveModelWithOneLayer(emb_dim=args.emb_dim,class_num= args.num_class) 18 | return LocalModels, GradientsRes, active_model 19 | else: 20 | local_model = VFLPassiveModel(emb_dim=args.emb_dim) 21 | active_model = QuanVFLActiveModelWithOneLayer(emb_dim=args.emb_dim,class_num= args.num_class) 22 | return local_model, active_model 23 | 24 | def load_models(args,quan_server=False): 25 | if quan_server==False: 26 | LocalModels= [] 27 | for i in range(args.num_clients): 28 | local_model = VFLPassiveModel(emb_dim=args.emb_dim) 29 | local_model.built = True 30 | local_model.load_weights(os.path.join(args.local_paths[i],args.ckpt_name)) 31 | LocalModels.append(local_model) 32 | 33 | active_model = VFLActiveModelWithOneLayer(emb_dim=args.emb_dim,class_num= args.num_class) 34 | active_model.built = True 35 | active_model.load_weights(os.path.join(args.sever_path,args.ckpt_name)) 36 | return LocalModels, active_model 37 | else: 38 | local_model = VFLPassiveModel(emb_dim=args.emb_dim) 39 | local_model.built = True 40 | local_model.load_weights(os.path.join(args.local_paths[0],args.ckpt_name)) 41 | active_model = QuanVFLActiveModelWithOneLayer(emb_dim=args.emb_dim,class_num= args.num_class) 42 | active_model.built = True 43 | active_model.load_weights(os.path.join(args.sever_path,args.ckpt_name)) 44 | return local_model, active_model 45 | 46 | 47 | def create_folder(args): 48 | if not os.path.exists(args.path_predix): 49 | os.makedirs(args.path_predix) 50 | print("save to", args.path_predix) 51 | 52 | SavedPaths=[] 53 | for i in range(args.num_img_clients): 54 | _dir= os.path.join(args.path_predix, 'img_{}_{}'.format(args.img_feature_div[i], args.img_feature_div[i+1])) 55 | if not os.path.exists(_dir): 56 | os.makedirs(_dir) 57 | SavedPaths.append(_dir) 58 | for i in range(args.num_text_clients): 59 | _dir= os.path.join(args.path_predix, 'text_{}_{}'.format(args.text_feature_div[i], args.text_feature_div[i+1])) 60 | if not os.path.exists(_dir): 61 | os.makedirs(_dir) 62 | SavedPaths.append(_dir) 63 | 64 | _dir= os.path.join(args.path_predix,'server') 65 | if not os.path.exists(_dir): 66 | os.makedirs(_dir) 67 | server_savedpath=_dir 68 | 69 | return SavedPaths, server_savedpath 70 | 71 | 72 | def get_local_outputs(num_img_clients, images, img_feature_div, 73 | num_text_clients, texts, text_feature_div, LocalModels): 74 | LocalOutputs =[] 75 | for i in range(num_img_clients): 76 | images_div = images[:,img_feature_div[i]:img_feature_div[i+1]] 77 | LocalOutputs.append(LocalModels[i](images_div)) 78 | for i in range(num_text_clients): 79 | text_div = texts[:,text_feature_div[i]:text_feature_div[i+1]] 80 | LocalOutputs.append(LocalModels[i+num_img_clients](text_div)) 81 | 82 | return LocalOutputs 83 | 84 | 85 | 86 | def load_data(args, test=False): 87 | 88 | if test==True: 89 | test_X_image, test_X_text, test_Y = get_labeled_data('', args.top_k, args.num_test_samples, 'Test') 90 | print("load test data done") 91 | x_test, y_test = (np.array(test_X_image).astype('float32'), np.array(test_X_text).astype('float32')), np.array(test_Y).astype('float32') 92 | 93 | return x_test, y_test 94 | else: 95 | train_X_image, train_X_text, train_Y = get_labeled_data('', args.top_k, args.num_train_samples, 'Train') 96 | print("load train data done") 97 | test_X_image, test_X_text, test_Y = get_labeled_data('', args.top_k, args.num_test_samples, 'Test') 98 | print("load test data done") 99 | x_train, x_test, y_train, y_test = (np.array(train_X_image).astype('float32'), np.array(train_X_text).astype('float32')), \ 100 | (np.array(test_X_image).astype('float32'), np.array(test_X_text).astype('float32')), \ 101 | np.array(train_Y).astype('float32'), np.array(test_Y).astype('float32') 102 | 103 | return x_train, x_test, y_train, y_test 104 | 105 | -------------------------------------------------------------------------------- /nus-wide/utils/nus_wide_data_util.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | from io import BytesIO 4 | 5 | import numpy as np 6 | import pandas as pd 7 | import requests 8 | from PIL import Image 9 | from sklearn.utils import shuffle 10 | 11 | #from data_util.data_loader import TwoPartyDataLoader 12 | 13 | def balance_X_y(XA, XB, y, seed=5): 14 | np.random.seed(seed) 15 | num_pos = np.sum(y == 1) 16 | # num_neg = np.sum(y == -1) 17 | # pos_indexes = [i for (i, _y) in enumerate(y) if _y > 0] 18 | # neg_indexes = [i for (i, _y) in enumerate(y) if _y < 0] 19 | 20 | num_neg = np.sum(y == 0) 21 | pos_indexes = [i for (i, _y) in enumerate(y) if _y > 0.5] 22 | neg_indexes = [i for (i, _y) in enumerate(y) if _y < 0.5] 23 | 24 | # print("len(pos_indexes)", len(pos_indexes)) 25 | # print("len(neg_indexes)", len(neg_indexes)) 26 | # print("num of samples", len(pos_indexes) + len(neg_indexes)) 27 | # print("num_pos:", num_pos) 28 | # print("num_neg:", num_neg) 29 | 30 | if num_pos < num_neg: 31 | np.random.shuffle(neg_indexes) 32 | # randomly pick negative samples of size equal to that of positive samples 33 | rand_indexes = neg_indexes[:num_pos] 34 | indexes = pos_indexes + rand_indexes 35 | np.random.shuffle(indexes) 36 | y = [y[i] for i in indexes] 37 | XA = [XA[i] for i in indexes] 38 | XB = [XB[i] for i in indexes] 39 | 40 | return np.array(XA), np.array(XB), np.array(y) 41 | 42 | 43 | def get_top_k_labels(data_dir, top_k=5): 44 | data_path = "NUS_WIDE/Groundtruth/AllLabels" 45 | label_counts = {} 46 | for filename in os.listdir(os.path.join(data_dir, data_path)): 47 | file = os.path.join(data_dir, data_path, filename) 48 | # print(file) 49 | if os.path.isfile(file): 50 | label = file[:-4].split("_")[-1] 51 | df = pd.read_csv(os.path.join(data_dir, file)) 52 | df.columns = ['label'] 53 | label_counts[label] = (df[df['label'] == 1].shape[0]) 54 | label_counts = sorted(label_counts.items(), key=lambda x: x[1], reverse=True) 55 | selected = [k for (k, v) in label_counts[:top_k]] 56 | return selected 57 | 58 | 59 | def get_labeled_data(data_dir, selected_label, n_samples, dtype="Train"): 60 | # get labels 61 | data_path = "NUS_WIDE/Groundtruth/TrainTestLabels/" 62 | dfs = [] 63 | for label in selected_label: 64 | file = os.path.join(data_dir, data_path, "_".join(["Labels", label, dtype]) + ".txt") 65 | df = pd.read_csv(file, header=None) 66 | # print("df shape", df.shape) 67 | df.columns = [label] 68 | dfs.append(df) 69 | data_labels = pd.concat(dfs, axis=1) 70 | # print(data_labels) 71 | if len(selected_label) > 1: 72 | selected = data_labels[data_labels.sum(axis=1) == 1] 73 | else: 74 | selected = data_labels 75 | # print(selected.shape) 76 | 77 | # get XA, which are image low level features 78 | features_path = "NUS_WIDE/NUS_WID_Low_Level_Features/Low_Level_Features" 79 | # print("data_dir: {0}".format(data_dir)) 80 | # print("features_path: {0}".format(features_path)) 81 | dfs = [] 82 | if dtype=="Train": 83 | filenamelist= ['Train_Normalized_CM55.dat', 84 | 'Train_Normalized_CH.dat', 85 | 'Train_Normalized_EDH.dat', 86 | 'Train_Normalized_CORR.dat', 87 | 'Train_Normalized_WT.dat' ] 88 | else: 89 | filenamelist= ['Test_Normalized_CM55.dat', 90 | 'Test_Normalized_CH.dat', 91 | 'Test_Normalized_EDH.dat', 92 | 'Test_Normalized_CORR.dat', 93 | 'Test_Normalized_WT.dat' ] 94 | # for file in os.listdir(os.path.join(data_dir, features_path)): 95 | # if file.startswith("_".join([dtype, "Normalized"])): 96 | for file in filenamelist: 97 | # print("get_labeled_data load filename:", file) 98 | df = pd.read_csv(os.path.join(data_dir, features_path, file), header=None, sep=" ") 99 | df.dropna(axis=1, inplace=True) 100 | # print("b datasets features", len(df.columns)) 101 | dfs.append(df) 102 | data_XA = pd.concat(dfs, axis=1) 103 | data_X_image_selected = data_XA.loc[selected.index] 104 | # print("X image shape:", data_X_image_selected.shape) # 634 columns 105 | 106 | # get XB, which are tags 107 | tag_path = "NUS_WIDE/NUS_WID_Tags/" 108 | file = "_".join([dtype, "Tags1k"]) + ".dat" 109 | tagsdf = pd.read_csv(os.path.join(data_dir, tag_path, file), header=None, sep="\t") 110 | tagsdf.dropna(axis=1, inplace=True) 111 | data_X_text_selected = tagsdf.loc[selected.index] 112 | # print("X text shape:", data_X_text_selected.shape) 113 | 114 | if n_samples is None: 115 | return data_X_image_selected.values[:], data_X_text_selected.values[:], selected.values[:] 116 | return data_X_image_selected.values[:n_samples], data_X_text_selected.values[:n_samples], selected.values[:n_samples] 117 | 118 | 119 | def image_and_text_data(data_dir, selected, n_samples=2000): 120 | return get_labeled_data(data_dir, selected, n_samples) 121 | 122 | 123 | def get_images(): 124 | image_urls = "D:/Data/NUS_WIDE/NUS_WIDE/NUS-WIDE-urls/NUS-WIDE-urls.txt" 125 | # df = pd.read_csv(image_urls, header=0, sep=" ") 126 | # print(df.head(10)) 127 | # kkk = df.loc[:, "url_Middle"] 128 | # print(kkk.head(10)) 129 | 130 | read_num_urls = 1 131 | with open(image_urls, "r") as fi: 132 | fi.readline() 133 | reader = csv.reader(fi, delimiter=' ', skipinitialspace=True) 134 | for idx, row in enumerate(reader): 135 | if idx >= read_num_urls: 136 | break 137 | print(row[0], row[2], row[3], row[4]) 138 | if row[3] is not None and row[3] != "null": 139 | url = row[4] 140 | print("{0} url: {1}".format(idx, url)) 141 | 142 | str_array = row[0].split("\\") 143 | print(str_array[3], str_array[4]) 144 | 145 | # img = imageio.imread(url) 146 | # print(type(img), img.shape) 147 | 148 | response = requests.get(url) 149 | print(response.status_code) 150 | img = Image.open(BytesIO(response.content)) 151 | arr = np.array(img) 152 | print(type(img), arr.shape) 153 | # imageio.imwrite("", img) 154 | size = 48, 48 155 | img.thumbnail(size) 156 | img.show() 157 | arr = np.array(img) 158 | print("thumbnail", arr.shape) 159 | 160 | 161 | if __name__ == '__main__': 162 | get_images() 163 | -------------------------------------------------------------------------------- /nus-wide/utils/robustae.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | def l21_rownorm(X): 4 | """ 5 | This function calculates the l21 norm of a matrix X, i.e., \sum ||X[i,:]||_2 6 | Input: 7 | ----- 8 | X: {numpy array} 9 | Output: 10 | ------ 11 | l21_norm: {float} 12 | """ 13 | return (np.sqrt(np.multiply(X, X).sum(1))).sum() 14 | 15 | def l21_colnorm(X): 16 | """ 17 | This function calculates the l21 norm of a matrix X, i.e., \sum ||X[:,j]||_2 18 | Input: 19 | ----- 20 | X: {numpy array} 21 | Output: 22 | ------ 23 | l21_norm: {float} 24 | """ 25 | return (np.sqrt(np.multiply(X, X).sum(0))).sum() 26 | 27 | 28 | def inference_purify_miss(args, LocalOutputs,ae_model,L_optimizer,L_optimizer2, rae_loss_object, vis=False): 29 | h=np.concatenate(tuple(LocalOutputs), axis=1) 30 | rae_output,layer_output=ae_model(h) 31 | L=tf.Variable(rae_output,trainable=True) 32 | purify_epochs= args.purify_epochs 33 | for epoch in range(args.initial_epochs): 34 | with tf.GradientTape() as passive_tape: 35 | rae_output,layer_output=ae_model(L) 36 | rae_output_split= tf.split(rae_output, args.num_clients, 1 ) 37 | 38 | loss=0 39 | for i in range(args.num_clients): 40 | 41 | loss += tf.sqrt(rae_loss_object(rae_output_split[i], LocalOutputs[i])) 42 | L_gradients = passive_tape.gradient(loss,[L]) 43 | L_optimizer.apply_gradients(zip(L_gradients, [L])) 44 | L2 = tf.Variable(rae_output, trainable=True) 45 | for epoch in range(purify_epochs): 46 | with tf.GradientTape() as passive_tape: 47 | rae_output,layer_output=ae_model(L2) 48 | rae_output_split= tf.split(rae_output, args.num_clients, 1 ) 49 | L_split2 = tf.split(L2, args.num_clients, 1 ) 50 | loss2=0 51 | for i in range(args.num_clients): 52 | loss2 += args.tau* tf.sqrt(rae_loss_object(rae_output_split[i], L_split2[i])) 53 | 54 | for i in range(len(args.observe_list)): 55 | loss2 += tf.sqrt(rae_loss_object( LocalOutputs[args.observe_list[i]],L_split2[args.observe_list[i]])) 56 | L_gradients = passive_tape.gradient(loss2,[L2]) 57 | L_optimizer2.apply_gradients(zip(L_gradients, [L2])) 58 | if vis: 59 | print('purify epoch {}, purify L2 Loss2: {}'.format(epoch+1,loss2.numpy())) 60 | 61 | return tf.split(rae_output, args.num_clients, 1 ) 62 | -------------------------------------------------------------------------------- /requirement.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | Pillow 3 | sklearn 4 | matplotlib 5 | tensorboardX --------------------------------------------------------------------------------