├── .gitattributes ├── vgg16.py ├── README.md ├── VBPR_update.py └── VBPR.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /vgg16.py: -------------------------------------------------------------------------------- 1 | from keras.applications.imagenet_utils import preprocess_input 2 | from keras.applications.vgg16 import VGG16 3 | from keras.models import Model 4 | from keras.preprocessing import image 5 | import numpy as np 6 | from PIL import ImageFile 7 | ImageFile.LOAD_TRUNCATED_IMAGES = True 8 | 9 | 10 | base_model = VGG16(weights='imagenet',include_top=True) 11 | # model = VGG16(weights='imagenet') 12 | model = Model( 13 | input=base_model.input, output=base_model.get_layer('fc2').output) 14 | 15 | 16 | def extract_feature(img_path): 17 | 18 | img = image.load_img(img_path, target_size=(224, 224)) # 224,224 19 | # except: 20 | # img_path='./images/000000.jpg' 21 | # img = image.load_img(img_path, target_size=(224, 224)) # 224,224 22 | x = image.img_to_array(img) # (3, 224, 224) 23 | x = np.expand_dims(x, axis=0) # (1, 3, 224, 224) 24 | x = preprocess_input(x) 25 | features = model.predict(x) # fc2 26 | # print(features.shape) 27 | # number = np.argmax(features) 28 | # print(words[number]) 29 | return features 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # VBPR 2 | 3 | 4 | A simple version of VBPR. 5 | 6 | Two files are needed to run "VBPR.py" 7 | 8 | 1. "image_feature.csv" The first column of the csv file is item id, while each line is the corresponding feature of the item. 9 | 10 | This file is to build a dictionary of item_id and item_feature : self.imageFeatures. 11 | 12 | 2. "feedback_file.json" The file stores the feedback data of users & items, which has the structure as follows: 13 | 14 | { "user_id1" : { "item_id_1", "item_id_2", ... }, 15 | "user_id2" : { "item_id_i", "item_id_j", ... }, 16 | ... } 17 | This file is to build the user-item relation map: self.R. 18 | 19 | You need to complete the code in line 113-114 with the path of the above files. 20 | 21 | baidu drive link: https://pan.baidu.com/s/1FIsLca0TZW_I4wGmfDwpDw cue：5j3g 22 | 23 | google drive link: https://drive.google.com/open?id=1yOjxhqI19OL2r8Rsf_SU8XTPBGE4F-Rg 24 | 25 | -------------------------------------------------------------------------------- 26 | 27 | # The update edition 28 | 29 | An end to end VBPR, which is no need to load all image features at begining. 30 | 31 | Three files are needed to run "VBPR_update.py" 32 | 33 | 1. "user_idx.json" & "item_idx.json". Structures are as follows, 34 | 35 | { "user_0": 0, "user_1": 1, ..., "user_n": n-1 } 36 | { "item_0": 0, "item_1": 1, ..., "item_m": m-1 } 37 | 38 | 2. Your own rating file. The interaction between users and items are needed. 39 | 40 | 41 | Step to run "VBPR_update.py" 42 | 43 | 1. model = VBPR(K=?, K2=?) 44 | 45 | 2. model.load_training_data() # filled with "user_idx.json" & "item_idx.json" 46 | 47 | 3. build data fed in placeholders, and feed their to the network. 48 | 49 | 50 | -------------------------------------------------------------------------------- /VBPR_update.py: -------------------------------------------------------------------------------- 1 | # An implement of VBPR using tensorflow # 2 | # 2019.3.16 # 3 | import numpy as np 4 | import random 5 | import json 6 | import tensorflow as tf 7 | import vgg16 8 | 9 | 10 | def get_variable(type, shape, mean, stddev, name): 11 | if type == 'W': 12 | var = tf.get_variable(name=name, shape=shape, dtype=tf.float32, 13 | initializer=tf.random_normal_initializer(mean=mean, stddev=stddev)) 14 | tf.add_to_collection('regular_losses', tf.contrib.layers.l2_regularizer(0.005)(var)) 15 | return var 16 | elif type == 'b': 17 | return tf.get_variable(name=name, shape=shape, dtype=tf.float32, 18 | initializer=tf.zeros_initializer()) 19 | 20 | 21 | class TVBPR: 22 | def __init__(self, K, K2): 23 | self.R = [] # Rating matrix 24 | self.nUsers = 0 25 | self.nItems = 0 26 | self.user_dict = {} 27 | self.item_dict = {} 28 | self.imageFeatures = {} 29 | self.imageFeaMatrix = [] 30 | self.itemFeatureDim = 4096 31 | self.k = K # Latent dimension 32 | self.k2 = K2 # Visual dimension 33 | self.MF_loss = 0 34 | 35 | 36 | def load_training_data(self, user_file, item_file): 37 | ## build item attribute dict 38 | 39 | with open(user_file,'r') as f: 40 | self.user_dict = json.load(f) 41 | with open(item_file,'r') as f: 42 | self.item_dict = json.load(f) 43 | 44 | self.nUsers = len(self.user_dict) 45 | self.nItems = len(self.item_dict) 46 | print('User: %d. Item: %d.'%(self.nUsers, self.nItems)) 47 | 48 | def VBPR(self, user_idx, itemFea_pos, itemFea_neg, pos_item_idx, neg_item_idx): 49 | MF_U = get_variable(type='W', shape=[self.nUsers, self.k], mean=0, stddev=0.01, name='MF_U') 50 | MF_I = get_variable(type='W', shape=[self.nItems, self.k], mean=0, stddev=0.01, name='MF_I') 51 | visual_U = get_variable(type='W', shape=[self.nUsers, self.k2], mean=0, stddev=0.01, 52 | name='visual_U') 53 | 54 | # MF_factor 55 | MF_U_factor = tf.gather(MF_U, user_idx) 56 | MF_U_factor = tf.reshape(MF_U_factor, shape=[1, tf.shape(MF_U_factor)[0]]) ##[1, k] 57 | MF_I_factor_pos = tf.gather(MF_I, pos_item_idx) ## [?, nItem] 58 | MF_I_factor_neg = tf.gather(MF_I, neg_item_idx) ## [?, nItem] 59 | 60 | # feature_factor 61 | visual_U_vector = tf.gather(visual_U, user_idx) 62 | visual_U_vector = tf.reshape(visual_U_vector, shape=[1, tf.shape(visual_U_vector)[0]]) ##[1, k2] 63 | 64 | 65 | itemEmb_W = get_variable(type='W', shape=[self.itemFeatureDim, self.k2], 66 | mean=0, stddev=0.01, name='itemEmb_W') 67 | itemEmb_b = get_variable(type='b', shape=[self.k2], mean=0, stddev=0.01, name='itemEmb_b') 68 | 69 | visual_U_factor = visual_U_vector 70 | visual_I_factor_pos = tf.sigmoid(tf.matmul(itemFea_pos, itemEmb_W) + itemEmb_b) 71 | visual_I_factor_neg = tf.sigmoid(tf.matmul(itemFea_neg, itemEmb_W) + itemEmb_b) 72 | 73 | 74 | BPR_user_factor = tf.concat([MF_U_factor, visual_U_factor], axis=1) 75 | BPR_item_factor_pos = tf.concat([MF_I_factor_pos, visual_I_factor_pos], axis=1) 76 | BPR_item_factor_neg = tf.concat([MF_I_factor_neg, visual_I_factor_neg], axis=1) 77 | 78 | uij = tf.multiply(BPR_user_factor, BPR_item_factor_pos) # (?, concat_Dim) 79 | uik = tf.multiply(BPR_user_factor, BPR_item_factor_neg) # (?, concat_Dim) 80 | self.uij = tf.sigmoid(tf.reshape(tf.reduce_sum(uij, axis=1), shape=[tf.shape(uij)[0], 1])) # (?, 1) 81 | self.uik = tf.sigmoid(tf.reshape(tf.reduce_sum(uik, axis=1), shape=[tf.shape(uik)[0], 1])) # (?, 1) 82 | 83 | def train(self): 84 | self.user_idx = tf.placeholder(dtype=tf.int32, shape=[]) 85 | self.itemFea_pos = tf.placeholder(dtype=tf.float32, shape=[None, self.itemFeatureDim]) 86 | self.itemFea_neg = tf.placeholder(dtype=tf.float32, shape=[None, self.itemFeatureDim]) 87 | self.pos_item_idx = tf.placeholder(dtype=tf.int32, shape=[None, ]) 88 | self.neg_item_idx = tf.placeholder(dtype=tf.int32, shape=[None, ]) 89 | 90 | self.VBPR(self.user_idx, self.itemFea_pos, self.itemFea_neg, self.pos_item_idx, self.neg_item_idx) 91 | 92 | # train 93 | uij_shape = tf.shape(self.uij)[0] 94 | uik_shape = tf.shape(self.uik)[0] 95 | uij = tf.tile(self.uij, [uik_shape, 1]) 96 | uik = tf.reshape(tf.tile(self.uik, [1, uij_shape]), shape=[-1, 1]) 97 | 98 | BPR_loss = tf.reduce_mean(-tf.log(tf.sigmoid(uij - uik))) 99 | self.loss = BPR_loss # + regular_loss 100 | self.accuracy = tf.reduce_sum(tf.round(tf.sigmoid(uij - uik))) / tf.reduce_sum(tf.ceil(tf.sigmoid(uij - uik))) 101 | 102 | self.global_step = tf.Variable(0, dtype=tf.int64, name='global_step', trainable=False) 103 | learning_rate = tf.train.exponential_decay(0.01, self.global_step, decay_steps=5000, decay_rate=0.85, 104 | staircase=False) 105 | self.train_step = tf.train.AdamOptimizer(learning_rate).minimize(self.loss) 106 | 107 | 108 | -------------------------------------------------------------------------------- /VBPR.py: -------------------------------------------------------------------------------- 1 | # An implement of VBPR using tensorflow # 2 | # 2018.11.19 # 3 | import csv 4 | import numpy as np 5 | import random 6 | import json 7 | import tensorflow as tf 8 | 9 | class Processing: 10 | def __init__(self, K, K2): 11 | self.R = [] # Rating matrix 12 | self.nUsers = 0 13 | self.nItems = 0 14 | self.user_dict = {} 15 | self.item_dict = {} 16 | self.imageFeatures = {} 17 | self.imageFeaMatrix = [] 18 | self.imageFeatureDim = 4096 19 | self.k = K # Latent dimension 20 | self.k2 = K2 # Visual dimension 21 | self.MF_loss = 0 22 | 23 | # def load_data(self, image_feature_path, rating_file_path): 24 | # # self.load_image_feature(image_feature_path) 25 | # self.load_training_data(rating_file_path) 26 | 27 | def load_image_feature(self, image_feature_path): 28 | csv_reader=csv.reader(open(image_feature_path)) 29 | for item in csv_reader: 30 | item_id=item[0] 31 | item_feature = item[1:] 32 | item_feature = list(map(float, item_feature)) 33 | self.imageFeatures[item_id] = item_feature 34 | self.imageFeaMatrix=[[0.]*self.imageFeatureDim]*self.nItems 35 | for item in self.imageFeatures: 36 | try: 37 | self.imageFeaMatrix[self.item_dict[item]] = self.imageFeatures[item] 38 | except: 39 | pass 40 | 41 | def load_training_data(self, rating_file_path): 42 | with open(rating_file_path,'r') as f: 43 | data = json.load(f) 44 | 45 | # create user/item idx dictionary 46 | for user_id in data: 47 | if user_id not in self.user_dict.keys(): 48 | self.user_dict[user_id] = self.nUsers 49 | self.nUsers += 1 50 | for item_id in data[user_id]: 51 | if item_id not in self.item_dict.keys(): 52 | self.item_dict[item_id] = self.nItems 53 | self.nItems += 1 54 | self.R = np.array([[0.] * self.nItems] * self.nUsers) 55 | for user_id in data: 56 | for item_id in data[user_id]: 57 | self.R[self.user_dict[user_id], self.item_dict[item_id]] = 10 #data[user_id][item_id][1] 58 | 59 | def get_variable(type, shape, mean, stddev, name): 60 | if type == 'W': 61 | var = tf.get_variable(name=name, shape=shape, dtype=tf.float32, 62 | initializer=tf.random_normal_initializer(mean=mean, stddev=stddev)) 63 | tf.add_to_collection('regular_losses', tf.contrib.layers.l2_regularizer(0.005)(var)) 64 | return var 65 | elif type == 'b': 66 | return tf.get_variable(name=name, shape=shape, dtype=tf.float32, 67 | initializer=tf.zeros_initializer()) 68 | 69 | 70 | def VBPR(itemFea_matrix, userFea_matrix, user_idx, pos_item_idx, neg_item_idx): 71 | MF_U = get_variable(type='W', shape=[model.nUsers, model.k], mean=0, stddev=0.01, name='MF_U') 72 | MF_I = get_variable(type='W', shape=[model.nItems, model.k], mean=0, stddev=0.01, name='MF_I') 73 | visual_U = get_variable(type='W', shape=[model.nUsers, model.k2], mean=0, stddev=0.01, name='visual_U') 74 | visual_I = itemFea_matrix 75 | 76 | MF_U_factor = tf.gather(MF_U, user_idx) 77 | MF_U_factor = tf.reshape(MF_U_factor, shape=[1, tf.shape(MF_U_factor)[0]]) ##[1, k] 78 | MF_I_factor_pos = tf.gather(MF_I, pos_item_idx) ## [?, nItem] 79 | MF_I_factor_neg = tf.gather(MF_I, neg_item_idx) ## [?, nItem] 80 | 81 | visual_U_vector = tf.gather(visual_U, user_idx) 82 | visual_U_vector = tf.reshape(visual_U_vector, shape=[1, tf.shape(visual_U_vector)[0]]) ##[1, k2] 83 | visual_I_matrix_pos = tf.gather(visual_I, pos_item_idx) 84 | visual_I_matrix_neg = tf.gather(visual_I, neg_item_idx) 85 | 86 | itemEmb_W = get_variable(type='W', shape=[model.imageFeatureDim, model.k2], mean=0, stddev=0.01, name='itemEmb_W') 87 | itemEmb_b = get_variable(type='b', shape=[model.k2], mean=0, stddev=0.01, name='itemEmb_b') 88 | 89 | visual_U_factor = visual_U_vector 90 | visual_I_factor_pos = tf.sigmoid(tf.matmul(visual_I_matrix_pos, itemEmb_W) + itemEmb_b) 91 | visual_I_factor_neg = tf.sigmoid(tf.matmul(visual_I_matrix_neg, itemEmb_W) + itemEmb_b) 92 | 93 | BPR_user_factor = tf.concat([MF_U_factor, visual_U_factor], axis=1) 94 | BPR_item_factor_pos = tf.concat([MF_I_factor_pos, visual_I_factor_pos], axis=1) 95 | BPR_item_factor_neg = tf.concat([MF_I_factor_neg, visual_I_factor_neg], axis=1) 96 | 97 | uij = tf.multiply(BPR_user_factor, BPR_item_factor_pos) # (?, concat_Dim) 98 | uik = tf.multiply(BPR_user_factor, BPR_item_factor_neg) # (?, concat_Dim) 99 | uij = tf.sigmoid(tf.reshape(tf.reduce_sum(uij, axis=1), shape=[tf.shape(uij)[0], 1])) # (?, 1) 100 | uik = tf.sigmoid(tf.reshape(tf.reduce_sum(uik, axis=1), shape=[tf.shape(uik)[0], 1])) # (?, 1) 101 | 102 | uij_shape = tf.shape(uij)[0] 103 | uik_shape = tf.shape(uik)[0] 104 | uij = tf.tile(uij, [uik_shape, 1]) 105 | uik = tf.reshape(tf.tile(uik, [1, uij_shape]), shape=[-1, 1]) 106 | 107 | BPR_loss = tf.reduce_mean(-tf.log(tf.sigmoid(uij - uik))) 108 | return BPR_loss, uij, uik 109 | 110 | 111 | 112 | model=Processing(K=64, K2=128) 113 | model.load_training_data('') ## feedback_file.json 114 | model.load_image_feature('') ## image_feature.csv 115 | 116 | itemFea_matrix = tf.placeholder(dtype=tf.float32, shape=[model.nItems, model.imageFeatureDim]) 117 | userFea_matrix = tf.placeholder(dtype=tf.float32, shape=[None, model.nItems]) 118 | user_idx = tf.placeholder(dtype=tf.int32, shape=[]) 119 | pos_item_idx = tf.placeholder(dtype=tf.int32, shape=[None,]) 120 | neg_item_idx = tf.placeholder(dtype=tf.int32, shape=[None,]) 121 | 122 | 123 | BPR_loss, uij, uik = VBPR(itemFea_matrix, userFea_matrix, user_idx, pos_item_idx, neg_item_idx) 124 | tf.add_to_collection('BPR_losses', BPR_loss) 125 | regular_loss = tf.add_n(tf.get_collection('regular_losses')) 126 | loss = BPR_loss #+ regular_loss 127 | 128 | accuracy = tf.reduce_sum(tf.round(tf.sigmoid(uij - uik)))/tf.reduce_sum(tf.ceil(tf.sigmoid(uij - uik))) 129 | 130 | global_step = tf.Variable(0, dtype=tf.int64, name='global_step', trainable=False) 131 | learning_rate = tf.train.exponential_decay(0.001, global_step, decay_steps=1000, decay_rate=0.85, staircase=False) 132 | train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss) 133 | 134 | sess=tf.Session() 135 | sess.run(tf.global_variables_initializer()) 136 | saver = tf.train.Saver(max_to_keep=1) 137 | 138 | max_acc = 0 139 | for step in range(20001): 140 | user_i = random.randint(0, model.nUsers-1) 141 | step_now = step 142 | #create positive & negative item 143 | pos_item = [x for x in range(model.nItems) if model.R[user_i][x] != 0] 144 | neg_item = [] 145 | while len(neg_item)< 5*len(pos_item): 146 | a=random.randint(0, model.nItems-1) 147 | if a not in pos_item: 148 | neg_item.append(a) 149 | 150 | _, los, B_loss, l2_loss, acc, uijj= sess.run([train_step, loss, BPR_loss, regular_loss, accuracy, uij], 151 | feed_dict={itemFea_matrix: model.imageFeaMatrix, 152 | userFea_matrix: model.R, 153 | user_idx: user_i, 154 | pos_item_idx: pos_item, 155 | neg_item_idx: neg_item, 156 | global_step: step_now}) 157 | 158 | print('step-%d, loss : %f(%f %f). Accuracy : %f'%(step_now, los, B_loss, l2_loss, acc)) 159 | if step_now % 1000 == 0 and step_now>0: 160 | final_acc = 0 161 | for i in range(model.nUsers): 162 | valid_pos_item = [x for x in range(model.nItems) if model.R[i][x] != 0] 163 | valid_neg_item = [] 164 | while len(valid_neg_item) < 5 * len(valid_pos_item): 165 | a = random.randint(0, model.nItems - 1) 166 | if a not in valid_pos_item: 167 | valid_neg_item.append(a) 168 | 169 | valid_los, valid_acc = sess.run([loss, accuracy], 170 | feed_dict={itemFea_matrix: model.imageFeaMatrix, 171 | userFea_matrix: model.R, 172 | user_idx: i, 173 | pos_item_idx: valid_pos_item, 174 | neg_item_idx: valid_neg_item}) 175 | final_acc += valid_acc 176 | print('Test Accuracy = %f' % (final_acc / model.nUsers)) 177 | if final_acc/model.nUsers > max_acc: 178 | print('saving model. Accuracy = %f'%(final_acc/model.nUsers)) 179 | saver.save(sess, './model/model_%f.ckpt' % (final_acc/model.nUsers), global_step=step) 180 | max_acc = final_acc/model.nUsers 181 | 182 | 183 | 184 | 185 | 186 | --------------------------------------------------------------------------------