├── LICENSE.md ├── README.md └── factorization_machine.py /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Gregory Sanders 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FactorizationMachine 2 | 3 | Quick and dirty implementation of Factorization Machines in Python/Theano. 4 | 5 | http://www.libfm.org/ 6 | -------------------------------------------------------------------------------- /factorization_machine.py: -------------------------------------------------------------------------------- 1 | import theano 2 | from theano import tensor as T 3 | import numpy as np 4 | from sklearn import cross_validation, datasets 5 | from sklearn.preprocessing import normalize 6 | import theano 7 | import sys 8 | import traceback 9 | import time 10 | import sys 11 | 12 | class fact_machine(): 13 | def __init__(self, num_feats, k_size=3, ltype=2, lamb=.1): 14 | #num_feats = samples.shape[1] 15 | self.lamb = lamb 16 | self.ltype = ltype 17 | #self.num_samples=num_samples 18 | self.num_feats=num_feats 19 | self.k_size=k_size 20 | 21 | w0 = np.random.randn() 22 | self.w0 = theano.shared(value=w0, name='w0')#, borrow=True) 23 | 24 | ws = np.random.randn(num_feats) 25 | self.ws = theano.shared(value=ws, name='ws')#, borrow=True) 26 | 27 | vs = np.random.randn(num_feats, k_size) 28 | self.vs = theano.shared(value=vs, name='vs')#, borrow=True) 29 | 30 | self.input_var = T.matrix() 31 | self.target_var = T.vector() 32 | 33 | #Must be numpy array of float32. Keeps weights, changes other things. 34 | def set_data(self, x, y): 35 | self.shared_x = theano.shared(x) 36 | self.shared_y = theano.shared(y) 37 | 38 | self.givens = { 39 | self.input_var : self.shared_x, 40 | self.target_var : self.shared_y 41 | } 42 | self.set_updates() 43 | self.set_train() 44 | self.set_output() 45 | 46 | #Defines the inference-level objective section. 47 | def factorization_objective(self, samples): 48 | yhat = self.w0+T.dot(samples,self.ws) 49 | for i in range(self.num_feats-1): 50 | for j in range(i+1,self.num_feats): 51 | yhat += T.dot(self.vs[i], self.vs[j])*samples[:,i]*samples[:,j] 52 | return yhat 53 | 54 | #The exponential penalty objective outlined in: http://www.csie.ntu.edu.tw/~r01922136/slides/ffm.pdf 55 | def exp_objective(self): 56 | 57 | total_objective = T.log(1+T.exp(-self.target_var*self.factorization_objective(self.input_var))) 58 | if self.ltype == 2: 59 | total_objective += (self.lamb/2)*T.sum(T.sqr(self.ws)) 60 | total_objective += (self.lamb/2)*T.sum(T.sqr(self.vs)) 61 | elif self.ltype == 1: 62 | total_objective += self.lamb*T.sum(T.abs_(self.ws)) 63 | total_objective += self.lamb*T.sum(T.abs_(self.vs)) 64 | else: 65 | raise Exception('Wrong regularization type, must be 1 or 2: ' + str(ltype)) 66 | return T.mean(total_objective) 67 | 68 | #SGD formmulation 69 | def gen_updates_sgd(self, loss, learning_rate=.1): 70 | all_parameters = [self.w0, self.ws, self.vs] 71 | all_grads = [theano.grad(loss, param) for param in all_parameters] 72 | updates = [] 73 | for param_i, grad_i in zip(all_parameters, all_grads): 74 | updates.append((param_i, param_i - learning_rate * grad_i)) 75 | return updates 76 | 77 | def set_updates(self): 78 | updates = self.gen_updates_sgd(self.error()) 79 | self.updates = updates 80 | 81 | def error(self): 82 | return self.exp_objective() 83 | 84 | def predict(self): 85 | output = self.factorization_objective(self.input_var) 86 | return output 87 | 88 | def set_train(self): 89 | self.train = theano.function([], self.error(), givens=self.givens, updates=self.updates) 90 | 91 | def set_output(self): 92 | self.output = theano.function([], self.predict(), givens=self.givens, on_unused_input='ignore') 93 | ''' 94 | iris = datasets.load_iris() #junk dataset for now 95 | samples = iris.data 96 | samples = normalize(samples) 97 | labels = iris.target 98 | labels[labels > 0] = 1 99 | labels[labels == 0] = -1 100 | 101 | train_x, test_x, train_y, test_y = cross_validation.train_test_split(samples, labels, test_size=0.10) 102 | train_x_f=train_x.astype(np.float32) 103 | train_y_f=train_y.astype(np.float32) 104 | test_x_f=test_x.astype(np.float32) 105 | test_y_f=test_y.astype(np.float32) 106 | 107 | #minibatch_size = 100 108 | mb_size = train_x.shape[0] 109 | print mb_size 110 | num_batches = train_x.shape[0]/mb_size 111 | rand_vec = np.arange(mb_size) 112 | #np.random.shuffle(rand_vec) 113 | 114 | f_m = fact_machine(train_x.shape[1], mb_size, k_size=0) 115 | f_m.set_data(train_x_f, train_y_f) 116 | 117 | 118 | converged = False 119 | last_perf = 10000000000 120 | while not converged: 121 | perf_vec = f_m.train() 122 | print perf_vec 123 | if last_perf <= perf_vec + .000001: 124 | converged = True 125 | last_perf = perf_vec 126 | 127 | b = 3 128 | print "Training Accuracy:", np.count_nonzero(np.sign(f_m.output()-f_m.w0.eval()) == train_y), len(train_y) 129 | f_m.set_data(test_x_f, test_y_f) 130 | print "Testing Accuracy:", np.count_nonzero(np.sign(f_m.output()-f_m.w0.eval()) == test_y), len(test_y) 131 | ''' --------------------------------------------------------------------------------