├── pythonConversion.sh ├── __pycache__ ├── common.cpython-35.pyc └── commonCode.cpython-35.pyc ├── README.md ├── fmmutihot.py ├── fm.py ├── deepfm.py ├── xdeepfm.py ├── ffm.py ├── din.py ├── fmmutihot.ipynb ├── fm.ipynb ├── deepfm.ipynb ├── ffm.ipynb ├── xdeepfm.ipynb ├── din.ipynb ├── common.py └── common.ipynb /pythonConversion.sh: -------------------------------------------------------------------------------- 1 | jupyter nbconvert --to script $1 2 | -------------------------------------------------------------------------------- /__pycache__/common.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qian135/ctr_model_zoo/HEAD/__pycache__/common.cpython-35.pyc -------------------------------------------------------------------------------- /__pycache__/commonCode.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qian135/ctr_model_zoo/HEAD/__pycache__/commonCode.cpython-35.pyc -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ctr_model_zoo 2 | some ctr model, implemented by PyTorch, such as Factorization Machines, Field-aware Factorization Machines, DeepFM, xDeepFM, Deep Interest Network 3 | -------------------------------------------------------------------------------- /fmmutihot.py: -------------------------------------------------------------------------------- 1 | 2 | # coding: utf-8 3 | 4 | # In[ ]: 5 | 6 | 7 | ''' 8 | Author: 9 | Shenxin Zhan,zhanshenxin135@163.com 10 | 11 | Reference: 12 | https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf 13 | ''' 14 | 15 | 16 | # In[ ]: 17 | 18 | 19 | import torch 20 | import torch.nn as nn 21 | 22 | from common import FirstOrderMutiHot, SecondOrderMutiHot 23 | 24 | 25 | # In[ ]: 26 | 27 | 28 | class FMMutiHot(nn.Module): 29 | '''support muti-hot feature for Factorization Machine 30 | 31 | ''' 32 | def __init__(self, params): 33 | super(FMMutiHot, self).__init__() 34 | 35 | self.embedding_size = params['embedding_size'] 36 | self.feature_size = params['feature_size'] 37 | self.device = params['device'] 38 | self.fea_name = params['fea_name'] 39 | self.max_len = params['max_len'] 40 | 41 | self.first_order = FirstOrderMutiHot(params) 42 | self.second_order = SecondOrderMutiHot(params) 43 | 44 | 45 | fm_bias = torch.empty(1, dtype=torch.float32, device=self.device, 46 | requires_grad=True) 47 | nn.init.constant_(fm_bias, 0) 48 | self.fm_bias = nn.Parameter(fm_bias) 49 | 50 | def forward(self, features): 51 | feature_idx = features["feature_idx"] 52 | feature_values = features["feature_values"] 53 | 54 | bias = self.fm_bias 55 | 56 | first_order = self.first_order(feature_values, feature_idx) 57 | first_order = torch.sum(first_order, dim=1) 58 | 59 | second_order = self.second_order(feature_values, feature_idx) 60 | second_order = torch.sum(second_order, dim=1) 61 | 62 | logits = second_order + first_order + bias 63 | 64 | return logits 65 | 66 | -------------------------------------------------------------------------------- /fm.py: -------------------------------------------------------------------------------- 1 | 2 | # coding: utf-8 3 | 4 | # In[ ]: 5 | 6 | 7 | ''' 8 | Author: 9 | Shenxin Zhan,zhanshenxin135@163.com 10 | 11 | Reference: 12 | https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf 13 | 14 | Factorization Machines 15 | ''' 16 | 17 | 18 | # In[10]: 19 | 20 | 21 | import torch 22 | import torch.nn as nn 23 | 24 | from common import FirstOrder, SecondOrder 25 | 26 | 27 | # In[11]: 28 | 29 | 30 | class FM(nn.Module): 31 | '''implement Factorization Machine by PyTorch 32 | 33 | ''' 34 | def __init__(self, params): 35 | super(FMModel, self).__init__() 36 | 37 | # parse params 38 | self.embedding_size = params['embedding_size'] 39 | self.feature_size = params['feature_size'] 40 | self.device = params['device'] 41 | 42 | self.first_order = FirstOrder({'device': self.device, 43 | 'feature_size': self.feature_size}) 44 | self.second_order = SecondOrder({'device': self.device, 45 | 'feature_size': self.feature_size, 46 | 'embedding_size': self.embedding_size}) 47 | 48 | fm_bias = torch.empty(1, dtype=torch.float32, device=self.device, 49 | requires_grad=True) 50 | nn.init.constant_(fm_bias, 0) 51 | self.fm_bias = nn.Parameter(fm_bias) 52 | 53 | def forward(self, features): 54 | feature_idx = features["feature_idx"] 55 | feature_values = features["feature_values"] 56 | 57 | bias = self.fm_bias 58 | 59 | first_order = self.first_order(feature_values, feature_idx) 60 | first_order = torch.sum(first_order, dim=1) 61 | 62 | second_order = self.second_order(feature_values, feature_idx) 63 | second_order = torch.sum(second_order, dim=1) 64 | 65 | logits = second_order + first_order + bias 66 | 67 | return logits 68 | 69 | -------------------------------------------------------------------------------- /deepfm.py: -------------------------------------------------------------------------------- 1 | 2 | # coding: utf-8 3 | 4 | # In[ ]: 5 | 6 | 7 | ''' 8 | Author: 9 | Shenxin Zhan,zhanshenxin135@163.com 10 | 11 | Reference: 12 | https://arxiv.org/abs/1703.04247 13 | DeepFM: A Factorization-Machine based Neural Network for CTR Prediction 14 | ''' 15 | 16 | 17 | # In[1]: 18 | 19 | 20 | import torch 21 | import torch.nn as nn 22 | 23 | from common import MLP 24 | 25 | 26 | # In[2]: 27 | 28 | 29 | class DeepFM(nn.Module): 30 | def __init__(self, params, get_embeddings=True, use_batchnorm=True, 31 | use_dropout=True, use_fm=True, use_deep=True): 32 | super(DeepFM, self).__init__() 33 | self.device = params['device'] 34 | self.mlp_input_dim = params['field_size'] * params['embedding_size'] 35 | self.use_fm = use_fm 36 | self.use_deep = use_deep 37 | 38 | self.first_order = FirstOrder(params) 39 | self.second_order = SecondOrder(params, get_embeddings=get_embeddings) 40 | self.mlp = MLP(params, use_batchnorm=use_batchnorm, use_dropout=use_dropout) 41 | 42 | ## final concat layer 43 | if self.use_fm and self.use_deep: 44 | concat_size = params['field_size'] + params['embedding_size'] + params['hidden_dims'][-1] 45 | elif self.use_deep: 46 | concat_size = params['hidden_dims'][-1] 47 | elif self.use_fm: 48 | concat_size = params['field_size'] + params['embedding_size'] 49 | self.concat_layer = nn.Linear(concat_size, 1).to(self.device) 50 | 51 | def forward(self, features): 52 | # parse features 53 | feature_idx = features["feature_idx"] 54 | feature_values = features["feature_values"] 55 | 56 | ## first order 57 | first_order = self.first_order(feature_values, feature_idx) 58 | 59 | ## second order 60 | second_order, embeddings = self.second_order(feature_values, feature_idx) 61 | 62 | # deep 63 | deepInput = embeddings.reshape(embeddings.shape[0], self.mlp_input_dim) 64 | deepOut = self.mlp(deepInput) 65 | 66 | # concat layer 67 | if self.use_deep and self.use_fm: 68 | concat = torch.cat([first_order, second_order, deepOut], dim=1) 69 | elif self.use_deep: 70 | concat = deepOut 71 | elif self.use_fm: 72 | concat = torch.cat([first_order, second_order], dim=1) 73 | 74 | logits = self.concat_layer(concat) 75 | 76 | return logits 77 | 78 | -------------------------------------------------------------------------------- /xdeepfm.py: -------------------------------------------------------------------------------- 1 | 2 | # coding: utf-8 3 | 4 | # In[ ]: 5 | 6 | 7 | ''' 8 | Author: 9 | Shenxin Zhan,zhanshenxin135@163.com 10 | 11 | Reference: 12 | https://arxiv.org/abs/1803.05170 13 | xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems 14 | ''' 15 | 16 | 17 | # In[ ]: 18 | 19 | 20 | import torch 21 | import torch.nn as nn 22 | 23 | from common import MLP, CIN 24 | 25 | 26 | # In[ ]: 27 | 28 | 29 | class xDeepFM(nn.Module): 30 | 31 | def __init__(self, params, get_embeddings=True, use_batchnorm=True, 32 | use_dropout=True, use_fm_second_order=False): 33 | super(xDeepFM, self).__init__() 34 | self.device = params['device'] 35 | self.mlp_input_dim = params['field_size'] * params['embedding_size'] 36 | self.use_fm_second_order = use_fm_second_order 37 | 38 | self.first_order = FirstOrder(params) 39 | self.second_order = SecondOrder(params, get_embeddings=get_embeddings) 40 | self.mlp = MLP(params, use_batchnorm=use_batchnorm, use_dropout=use_dropout) 41 | self.cin = CIN(params) 42 | if params['split_half']: 43 | cinOutputSize = reduce(lambda x, y: x//2 + y//2, params['cin_hidden_dims']) 44 | else: 45 | cinOutputSize = reduce(lambda x, y: x + y, params['cin_hidden_dims']) 46 | if self.use_fm_second_order: 47 | concat_size = params['field_size'] + params['embedding_size'] + params['hidden_dims'][-1] + cinOutputSize 48 | else: 49 | concat_size = params['field_size'] + params['hidden_dims'][-1] + cinOutputSize 50 | 51 | self.concat_layer = nn.Linear(concat_size, 1).to(self.device) 52 | 53 | 54 | def forward(self, features): 55 | # parse features 56 | feature_idx = features["feature_idx"] 57 | feature_values = features["feature_values"] 58 | 59 | ## first order 60 | first_order = self.first_order(feature_values, feature_idx) 61 | 62 | ## second order 63 | second_order, embeddings = self.second_order(feature_values, feature_idx) 64 | 65 | # deep 66 | mlpInput = embeddings.reshape(embeddings.shape[0], self.mlp_input_dim) 67 | mlpOut = self.mlp(mlpInput) 68 | 69 | # cin 70 | cinOut = self.cin(embeddings) 71 | 72 | # concat layer 73 | if self.use_fm_second_order: 74 | concat = torch.cat([first_order, second_order, mlpOut, cinOut], dim=1) 75 | else: 76 | concat = torch.cat([first_order, mlpOut, cinOut], dim=1) 77 | logits = self.concat_layer(concat) 78 | 79 | return logits 80 | 81 | -------------------------------------------------------------------------------- /ffm.py: -------------------------------------------------------------------------------- 1 | 2 | # coding: utf-8 3 | 4 | # In[ ]: 5 | 6 | 7 | ''' 8 | Author: 9 | Shenxin Zhan,zhanshenxin135@163.com 10 | 11 | Reference: 12 | https://www.csie.ntu.edu.tw/~cjlin/papers/ffm.pdf 13 | 14 | Field-aware Factorization Machines for CTR Prediction 15 | ''' 16 | 17 | 18 | # In[ ]: 19 | 20 | 21 | import torch 22 | import torch.nn as nn 23 | 24 | 25 | # In[ ]: 26 | 27 | 28 | class FFM(nn.Module): 29 | def __init__(self, params): 30 | super(FFM, self).__init__() 31 | 32 | # parse params 33 | self.embedding_size = params['embedding_size'] 34 | self.field_size = params['field_size'] 35 | self.feature_size = params['feature_size'] 36 | self.device = params['device'] 37 | 38 | feature_embeddings = torch.empty(self.feature_size, self.field_size, self.embedding_size, 39 | dtype=torch.float32, device=self.device, 40 | requires_grad=True) 41 | nn.init.normal_(feature_embeddings) 42 | self.feature_embeddings = nn.Parameter(feature_embeddings) 43 | 44 | weights_first_order = torch.empty(self.feature_size, 1, 45 | dtype=torch.float32, device=self.device, 46 | requires_grad=True) 47 | nn.init.normal_(weights_first_order) 48 | self.weights_first_order = nn.Parameter(weights_first_order) 49 | 50 | fm_bias = torch.empty(1, dtype=torch.float32, device=self.device, 51 | requires_grad=True) 52 | nn.init.constant_(fm_bias, 0) 53 | self.fm_bias = nn.Parameter(fm_bias) 54 | 55 | def forward(self, features): 56 | # parse features 57 | feature_idx = features["feature_idx"] 58 | feature_values = features["feature_values"] 59 | 60 | bias = self.fm_bias 61 | 62 | weights_first_order = self.weights_first_order[feature_idx].squeeze() 63 | first_order = torch.mul(feature_values, weights_first_order) 64 | first_order = torch.sum(first_order, dim=1, keepdim=True) 65 | 66 | second_order = torch.tensor([[0]]*feature_idx.shape[0], dtype=torch.float32, device=self.device) 67 | for i in range(self.field_size): 68 | for j in range(i+1, self.field_size): 69 | vifj = self.feature_embeddings[feature_idx[:, i], torch.tensor([j], device=self.device), :] 70 | vjfi = self.feature_embeddings[feature_idx[:, j], torch.tensor([i], device=self.device), :] 71 | second_order += torch.sum(torch.mul(vifj, vjfi), dim=1, keepdim=True) * feature_values[:, i][:, np.newaxis] * feature_values[:, j][:, np.newaxis] 72 | 73 | logits = second_order + first_order + bias 74 | 75 | return logits 76 | 77 | -------------------------------------------------------------------------------- /din.py: -------------------------------------------------------------------------------- 1 | 2 | # coding: utf-8 3 | 4 | # In[ ]: 5 | 6 | 7 | ''' 8 | Author: 9 | Shenxin Zhan,zhanshenxin135@163.com 10 | 11 | Reference: 12 | https://arxiv.org/abs/1706.06978 13 | Deep Interest Network for Click-Through Rate Prediction 14 | ''' 15 | 16 | 17 | # In[ ]: 18 | 19 | 20 | import torch 21 | import torch.nn as nn 22 | 23 | from common import MLP 24 | 25 | 26 | # In[ ]: 27 | 28 | 29 | class DIN(nn.Module): 30 | def __init__(self, params, use_batchnorm=True, use_dropout=True): 31 | super(DIN, self).__init__() 32 | 33 | self.device = params['device'] 34 | self.feature_size = params['feature_size'] 35 | self.embedding_size = params['embedding_size'] 36 | self.userItemDict = params['userItemDict'] 37 | self.hidden_dims = params['hidden_dims'] 38 | self.userItemMaxLen = params['userItemMaxLen'] 39 | 40 | feature_embeddings = torch.empty(self.feature_size+1, self.embedding_size, 41 | dtype=torch.float32, device=self.device, 42 | requires_grad=True) 43 | nn.init.normal_(feature_embeddings) 44 | self.feature_embeddings = nn.Parameter(feature_embeddings) 45 | 46 | self.mlp = MLP(params, use_batchnorm=use_batchnorm, use_dropout=use_dropout) 47 | 48 | self.output_layer = nn.Linear(self.hidden_dims[-1], 1).to(self.device) 49 | 50 | 51 | def forward(self, features): 52 | feature_idx = features["feature_idx"] 53 | 54 | uid = feature_idx[:, 0] 55 | feaName_ix = [('item_id', 2), ('author_id', 3), ('music_id', 6)] 56 | feaName_maxlen = [('item_id', 350), ('author_id', 250), ('music_id', 100)] 57 | feaName = ['item_id', 'author_id', 'music_id'] 58 | ad_idx = {} 59 | for t in feaName_ix: 60 | ad_idx[t[0]] = feature_idx[:, t[1]] 61 | 62 | 63 | hist_idx = self.userItemDict.loc[uid.cpu().numpy()][feaName] 64 | 65 | hist_idx_padded = {} 66 | for temp in feaName_maxlen: 67 | hist_idx_padded[temp[0]] = pad_sequence(list(hist_idx[temp[0]]), batch_first=True, 68 | padding_value=self.feature_size)[:, 0:temp[1]].to(self.device) 69 | user_beha_embeddings = [] 70 | for temp in feaName: 71 | hist_embeddings = self.feature_embeddings[hist_idx_padded[temp], :] 72 | ad_embeddings = self.feature_embeddings[ad_idx[temp], :] 73 | 74 | hist_weight = torch.einsum('blk,bk->bl', (hist_embeddings, ad_embeddings)) 75 | mask = hist_idx_padded[temp] != self.feature_size 76 | hist_weight.masked_fill_(mask == 0, -1e9) 77 | hist_weight = torch.softmax(hist_weight, dim=1) 78 | user_beha_embeddings.append(torch.einsum('blk,bl->bk', (hist_embeddings, hist_weight))) 79 | 80 | user_beha_embeddings = torch.cat(user_beha_embeddings, dim=1) 81 | 82 | ad_embeddings = self.feature_embeddings[feature_idx, :].reshape(feature_idx.shape[0], -1) 83 | 84 | embeddings = torch.cat((user_beha_embeddings, ad_embeddings), dim=1) 85 | 86 | # deep 87 | # deepInput = embeddings.reshape(embeddings.shape[0], self.mlp_input_dim) 88 | deepOut = self.mlp(embeddings) 89 | 90 | logits = self.output_layer(deepOut) 91 | 92 | return logits 93 | 94 | -------------------------------------------------------------------------------- /fmmutihot.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "'''\n", 10 | "Author:\n", 11 | " Shenxin Zhan,zhanshenxin135@163.com\n", 12 | " \n", 13 | "Reference:\n", 14 | " https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf\n", 15 | "'''" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 1, 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "import torch\n", 25 | "import torch.nn as nn\n", 26 | "\n", 27 | "from common import FirstOrderMutiHot, SecondOrderMutiHot" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "class FMMutiHot(nn.Module):\n", 37 | " '''support muti-hot feature for Factorization Machine\n", 38 | " \n", 39 | " '''\n", 40 | " def __init__(self, params):\n", 41 | " super(FMMutiHot, self).__init__()\n", 42 | " \n", 43 | " self.embedding_size = params['embedding_size']\n", 44 | " self.feature_size = params['feature_size']\n", 45 | " self.device = params['device']\n", 46 | " self.fea_name = params['fea_name']\n", 47 | " self.max_len = params['max_len'] \n", 48 | " \n", 49 | " self.first_order = FirstOrderMutiHot(params)\n", 50 | " self.second_order = SecondOrderMutiHot(params)\n", 51 | "\n", 52 | " \n", 53 | " fm_bias = torch.empty(1, dtype=torch.float32, device=self.device, \n", 54 | " requires_grad=True)\n", 55 | " nn.init.constant_(fm_bias, 0)\n", 56 | " self.fm_bias = nn.Parameter(fm_bias)\n", 57 | " \n", 58 | " def forward(self, features):\n", 59 | " feature_idx = features[\"feature_idx\"]\n", 60 | " feature_values = features[\"feature_values\"]\n", 61 | " \n", 62 | " bias = self.fm_bias\n", 63 | "\n", 64 | " first_order = self.first_order(feature_values, feature_idx)\n", 65 | " first_order = torch.sum(first_order, dim=1)\n", 66 | "\n", 67 | " second_order = self.second_order(feature_values, feature_idx)\n", 68 | " second_order = torch.sum(second_order, dim=1)\n", 69 | " \n", 70 | " logits = second_order + first_order + bias\n", 71 | "\n", 72 | " return logits" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": null, 78 | "metadata": {}, 79 | "outputs": [], 80 | "source": [] 81 | } 82 | ], 83 | "metadata": { 84 | "kernelspec": { 85 | "display_name": "Python 3", 86 | "language": "python", 87 | "name": "python3" 88 | }, 89 | "language_info": { 90 | "codemirror_mode": { 91 | "name": "ipython", 92 | "version": 3 93 | }, 94 | "file_extension": ".py", 95 | "mimetype": "text/x-python", 96 | "name": "python", 97 | "nbconvert_exporter": "python", 98 | "pygments_lexer": "ipython3", 99 | "version": "3.5.2" 100 | }, 101 | "toc": { 102 | "base_numbering": 1, 103 | "nav_menu": {}, 104 | "number_sections": true, 105 | "sideBar": true, 106 | "skip_h1_title": false, 107 | "title_cell": "Table of Contents", 108 | "title_sidebar": "Contents", 109 | "toc_cell": false, 110 | "toc_position": {}, 111 | "toc_section_display": true, 112 | "toc_window_display": false 113 | } 114 | }, 115 | "nbformat": 4, 116 | "nbformat_minor": 2 117 | } 118 | -------------------------------------------------------------------------------- /fm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "'''\n", 10 | "Author:\n", 11 | " Shenxin Zhan,zhanshenxin135@163.com\n", 12 | " \n", 13 | "Reference:\n", 14 | " https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf\n", 15 | " \n", 16 | " Factorization Machines\n", 17 | "'''" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 10, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "import torch\n", 27 | "import torch.nn as nn\n", 28 | "\n", 29 | "from common import FirstOrder, SecondOrder" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 11, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "class FM(nn.Module):\n", 39 | " '''implement Factorization Machine by PyTorch\n", 40 | " \n", 41 | " '''\n", 42 | " def __init__(self, params):\n", 43 | " super(FMModel, self).__init__()\n", 44 | " \n", 45 | " # parse params\n", 46 | " self.embedding_size = params['embedding_size']\n", 47 | " self.feature_size = params['feature_size']\n", 48 | " self.device = params['device']\n", 49 | "\n", 50 | " self.first_order = FirstOrder({'device': self.device, \n", 51 | " 'feature_size': self.feature_size})\n", 52 | " self.second_order = SecondOrder({'device': self.device, \n", 53 | " 'feature_size': self.feature_size,\n", 54 | " 'embedding_size': self.embedding_size})\n", 55 | " \n", 56 | " fm_bias = torch.empty(1, dtype=torch.float32, device=self.device, \n", 57 | " requires_grad=True)\n", 58 | " nn.init.constant_(fm_bias, 0)\n", 59 | " self.fm_bias = nn.Parameter(fm_bias)\n", 60 | " \n", 61 | " def forward(self, features):\n", 62 | " feature_idx = features[\"feature_idx\"]\n", 63 | " feature_values = features[\"feature_values\"]\n", 64 | " \n", 65 | " bias = self.fm_bias\n", 66 | "\n", 67 | " first_order = self.first_order(feature_values, feature_idx)\n", 68 | " first_order = torch.sum(first_order, dim=1)\n", 69 | "\n", 70 | " second_order = self.second_order(feature_values, feature_idx)\n", 71 | " second_order = torch.sum(second_order, dim=1)\n", 72 | " \n", 73 | " logits = second_order + first_order + bias\n", 74 | "\n", 75 | " return logits" 76 | ] 77 | } 78 | ], 79 | "metadata": { 80 | "kernelspec": { 81 | "display_name": "Python 3", 82 | "language": "python", 83 | "name": "python3" 84 | }, 85 | "language_info": { 86 | "codemirror_mode": { 87 | "name": "ipython", 88 | "version": 3 89 | }, 90 | "file_extension": ".py", 91 | "mimetype": "text/x-python", 92 | "name": "python", 93 | "nbconvert_exporter": "python", 94 | "pygments_lexer": "ipython3", 95 | "version": "3.5.2" 96 | }, 97 | "toc": { 98 | "base_numbering": 1, 99 | "nav_menu": {}, 100 | "number_sections": true, 101 | "sideBar": true, 102 | "skip_h1_title": false, 103 | "title_cell": "Table of Contents", 104 | "title_sidebar": "Contents", 105 | "toc_cell": false, 106 | "toc_position": {}, 107 | "toc_section_display": true, 108 | "toc_window_display": false 109 | } 110 | }, 111 | "nbformat": 4, 112 | "nbformat_minor": 2 113 | } 114 | -------------------------------------------------------------------------------- /deepfm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "'''\n", 10 | "Author:\n", 11 | " Shenxin Zhan,zhanshenxin135@163.com\n", 12 | " \n", 13 | "Reference:\n", 14 | " https://arxiv.org/abs/1703.04247\n", 15 | " DeepFM: A Factorization-Machine based Neural Network for CTR Prediction\n", 16 | "'''" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 1, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "import torch\n", 26 | "import torch.nn as nn\n", 27 | "\n", 28 | "from common import MLP" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 2, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "class DeepFM(nn.Module):\n", 38 | " def __init__(self, params, get_embeddings=True, use_batchnorm=True, \n", 39 | " use_dropout=True, use_fm=True, use_deep=True):\n", 40 | " super(DeepFM, self).__init__()\n", 41 | " self.device = params['device']\n", 42 | " self.mlp_input_dim = params['field_size'] * params['embedding_size']\n", 43 | " self.use_fm = use_fm\n", 44 | " self.use_deep = use_deep\n", 45 | "\n", 46 | " self.first_order = FirstOrder(params)\n", 47 | " self.second_order = SecondOrder(params, get_embeddings=get_embeddings) \n", 48 | " self.mlp = MLP(params, use_batchnorm=use_batchnorm, use_dropout=use_dropout)\n", 49 | " \n", 50 | " ## final concat layer\n", 51 | " if self.use_fm and self.use_deep:\n", 52 | " concat_size = params['field_size'] + params['embedding_size'] + \\\n", 53 | " params['hidden_dims'][-1]\n", 54 | " elif self.use_deep:\n", 55 | " concat_size = params['hidden_dims'][-1]\n", 56 | " elif self.use_fm:\n", 57 | " concat_size = params['field_size'] + params['embedding_size']\n", 58 | " self.concat_layer = nn.Linear(concat_size, 1).to(self.device)\n", 59 | "\n", 60 | " def forward(self, features):\n", 61 | " # parse features\n", 62 | " feature_idx = features[\"feature_idx\"]\n", 63 | " feature_values = features[\"feature_values\"]\n", 64 | " \n", 65 | " ## first order\n", 66 | " first_order = self.first_order(feature_values, feature_idx)\n", 67 | " \n", 68 | " ## second order\n", 69 | " second_order, embeddings = self.second_order(feature_values, feature_idx)\n", 70 | " \n", 71 | " # deep\n", 72 | " deepInput = embeddings.reshape(embeddings.shape[0], self.mlp_input_dim)\n", 73 | " deepOut = self.mlp(deepInput)\n", 74 | " \n", 75 | " # concat layer\n", 76 | " if self.use_deep and self.use_fm:\n", 77 | " concat = torch.cat([first_order, second_order, deepOut], dim=1)\n", 78 | " elif self.use_deep:\n", 79 | " concat = deepOut\n", 80 | " elif self.use_fm:\n", 81 | " concat = torch.cat([first_order, second_order], dim=1)\n", 82 | " \n", 83 | " logits = self.concat_layer(concat)\n", 84 | " \n", 85 | " return logits" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [] 94 | } 95 | ], 96 | "metadata": { 97 | "kernelspec": { 98 | "display_name": "Python 3", 99 | "language": "python", 100 | "name": "python3" 101 | }, 102 | "language_info": { 103 | "codemirror_mode": { 104 | "name": "ipython", 105 | "version": 3 106 | }, 107 | "file_extension": ".py", 108 | "mimetype": "text/x-python", 109 | "name": "python", 110 | "nbconvert_exporter": "python", 111 | "pygments_lexer": "ipython3", 112 | "version": "3.5.2" 113 | }, 114 | "toc": { 115 | "base_numbering": 1, 116 | "nav_menu": {}, 117 | "number_sections": true, 118 | "sideBar": true, 119 | "skip_h1_title": false, 120 | "title_cell": "Table of Contents", 121 | "title_sidebar": "Contents", 122 | "toc_cell": false, 123 | "toc_position": {}, 124 | "toc_section_display": true, 125 | "toc_window_display": false 126 | } 127 | }, 128 | "nbformat": 4, 129 | "nbformat_minor": 2 130 | } 131 | -------------------------------------------------------------------------------- /ffm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "'''\n", 10 | "Author:\n", 11 | " Shenxin Zhan,zhanshenxin135@163.com\n", 12 | " \n", 13 | "Reference:\n", 14 | " https://www.csie.ntu.edu.tw/~cjlin/papers/ffm.pdf\n", 15 | " \n", 16 | " Field-aware Factorization Machines for CTR Prediction\n", 17 | "'''" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": null, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "import torch\n", 27 | "import torch.nn as nn" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "class FFM(nn.Module):\n", 37 | " def __init__(self, params):\n", 38 | " super(FFM, self).__init__()\n", 39 | " \n", 40 | " # parse params\n", 41 | " self.embedding_size = params['embedding_size']\n", 42 | " self.field_size = params['field_size']\n", 43 | " self.feature_size = params['feature_size']\n", 44 | " self.device = params['device']\n", 45 | " \n", 46 | " feature_embeddings = torch.empty(self.feature_size, self.field_size, self.embedding_size, \n", 47 | " dtype=torch.float32, device=self.device, \n", 48 | " requires_grad=True)\n", 49 | " nn.init.normal_(feature_embeddings)\n", 50 | " self.feature_embeddings = nn.Parameter(feature_embeddings)\n", 51 | " \n", 52 | " weights_first_order = torch.empty(self.feature_size, 1, \n", 53 | " dtype=torch.float32, device=self.device,\n", 54 | " requires_grad=True)\n", 55 | " nn.init.normal_(weights_first_order)\n", 56 | " self.weights_first_order = nn.Parameter(weights_first_order)\n", 57 | " \n", 58 | " fm_bias = torch.empty(1, dtype=torch.float32, device=self.device, \n", 59 | " requires_grad=True)\n", 60 | " nn.init.constant_(fm_bias, 0)\n", 61 | " self.fm_bias = nn.Parameter(fm_bias)\n", 62 | " \n", 63 | " def forward(self, features):\n", 64 | " # parse features\n", 65 | " feature_idx = features[\"feature_idx\"]\n", 66 | " feature_values = features[\"feature_values\"]\n", 67 | " \n", 68 | " bias = self.fm_bias\n", 69 | "\n", 70 | " weights_first_order = self.weights_first_order[feature_idx].squeeze()\n", 71 | " first_order = torch.mul(feature_values, weights_first_order)\n", 72 | " first_order = torch.sum(first_order, dim=1, keepdim=True)\n", 73 | " \n", 74 | " second_order = torch.tensor([[0]]*feature_idx.shape[0], dtype=torch.float32, device=self.device)\n", 75 | " for i in range(self.field_size):\n", 76 | " for j in range(i+1, self.field_size):\n", 77 | " vifj = self.feature_embeddings[feature_idx[:, i], torch.tensor([j], device=self.device), :]\n", 78 | " vjfi = self.feature_embeddings[feature_idx[:, j], torch.tensor([i], device=self.device), :]\n", 79 | " second_order += torch.sum(torch.mul(vifj, vjfi), dim=1, keepdim=True) * \\\n", 80 | " feature_values[:, i][:, np.newaxis] * \\\n", 81 | " feature_values[:, j][:, np.newaxis]\n", 82 | " \n", 83 | " logits = second_order + first_order + bias\n", 84 | " \n", 85 | " return logits" 86 | ] 87 | } 88 | ], 89 | "metadata": { 90 | "kernelspec": { 91 | "display_name": "Python 3", 92 | "language": "python", 93 | "name": "python3" 94 | }, 95 | "language_info": { 96 | "codemirror_mode": { 97 | "name": "ipython", 98 | "version": 3 99 | }, 100 | "file_extension": ".py", 101 | "mimetype": "text/x-python", 102 | "name": "python", 103 | "nbconvert_exporter": "python", 104 | "pygments_lexer": "ipython3", 105 | "version": "3.5.2" 106 | }, 107 | "toc": { 108 | "base_numbering": 1, 109 | "nav_menu": {}, 110 | "number_sections": true, 111 | "sideBar": true, 112 | "skip_h1_title": false, 113 | "title_cell": "Table of Contents", 114 | "title_sidebar": "Contents", 115 | "toc_cell": false, 116 | "toc_position": {}, 117 | "toc_section_display": true, 118 | "toc_window_display": false 119 | } 120 | }, 121 | "nbformat": 4, 122 | "nbformat_minor": 2 123 | } 124 | -------------------------------------------------------------------------------- /xdeepfm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "'''\n", 10 | "Author:\n", 11 | " Shenxin Zhan,zhanshenxin135@163.com\n", 12 | " \n", 13 | "Reference:\n", 14 | " https://arxiv.org/abs/1803.05170\n", 15 | " xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems\n", 16 | "'''" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": null, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "import torch\n", 26 | "import torch.nn as nn\n", 27 | "\n", 28 | "from common import MLP, CIN" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "class xDeepFM(nn.Module):\n", 38 | " \n", 39 | " def __init__(self, params, get_embeddings=True, use_batchnorm=True, \n", 40 | " use_dropout=True, use_fm_second_order=False):\n", 41 | " super(xDeepFM, self).__init__()\n", 42 | " self.device = params['device']\n", 43 | " self.mlp_input_dim = params['field_size'] * params['embedding_size']\n", 44 | " self.use_fm_second_order = use_fm_second_order\n", 45 | " \n", 46 | " self.first_order = FirstOrder(params)\n", 47 | " self.second_order = SecondOrder(params, get_embeddings=get_embeddings)\n", 48 | " self.mlp = MLP(params, use_batchnorm=use_batchnorm, use_dropout=use_dropout)\n", 49 | " self.cin = CIN(params)\n", 50 | " if params['split_half']:\n", 51 | " cinOutputSize = reduce(lambda x, y: x//2 + y//2, params['cin_hidden_dims'])\n", 52 | " else:\n", 53 | " cinOutputSize = reduce(lambda x, y: x + y, params['cin_hidden_dims'])\n", 54 | " if self.use_fm_second_order:\n", 55 | " concat_size = params['field_size'] + params['embedding_size'] + \\\n", 56 | " params['hidden_dims'][-1] + cinOutputSize\n", 57 | " else:\n", 58 | " concat_size = params['field_size'] + params['hidden_dims'][-1] + \\\n", 59 | " cinOutputSize\n", 60 | "\n", 61 | " self.concat_layer = nn.Linear(concat_size, 1).to(self.device)\n", 62 | "\n", 63 | "\n", 64 | " def forward(self, features):\n", 65 | " # parse features\n", 66 | " feature_idx = features[\"feature_idx\"]\n", 67 | " feature_values = features[\"feature_values\"]\n", 68 | " \n", 69 | " ## first order\n", 70 | " first_order = self.first_order(feature_values, feature_idx)\n", 71 | " \n", 72 | " ## second order\n", 73 | " second_order, embeddings = self.second_order(feature_values, feature_idx)\n", 74 | " \n", 75 | " # deep\n", 76 | " mlpInput = embeddings.reshape(embeddings.shape[0], self.mlp_input_dim)\n", 77 | " mlpOut = self.mlp(mlpInput)\n", 78 | " \n", 79 | " # cin\n", 80 | " cinOut = self.cin(embeddings)\n", 81 | " \n", 82 | " # concat layer\n", 83 | " if self.use_fm_second_order:\n", 84 | " concat = torch.cat([first_order, second_order, mlpOut, cinOut], dim=1) \n", 85 | " else:\n", 86 | " concat = torch.cat([first_order, mlpOut, cinOut], dim=1) \n", 87 | " logits = self.concat_layer(concat)\n", 88 | " \n", 89 | " return logits" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": null, 95 | "metadata": {}, 96 | "outputs": [], 97 | "source": [] 98 | } 99 | ], 100 | "metadata": { 101 | "kernelspec": { 102 | "display_name": "Python 3", 103 | "language": "python", 104 | "name": "python3" 105 | }, 106 | "language_info": { 107 | "codemirror_mode": { 108 | "name": "ipython", 109 | "version": 3 110 | }, 111 | "file_extension": ".py", 112 | "mimetype": "text/x-python", 113 | "name": "python", 114 | "nbconvert_exporter": "python", 115 | "pygments_lexer": "ipython3", 116 | "version": "3.5.2" 117 | }, 118 | "toc": { 119 | "base_numbering": 1, 120 | "nav_menu": {}, 121 | "number_sections": true, 122 | "sideBar": true, 123 | "skip_h1_title": false, 124 | "title_cell": "Table of Contents", 125 | "title_sidebar": "Contents", 126 | "toc_cell": false, 127 | "toc_position": {}, 128 | "toc_section_display": true, 129 | "toc_window_display": false 130 | } 131 | }, 132 | "nbformat": 4, 133 | "nbformat_minor": 2 134 | } 135 | -------------------------------------------------------------------------------- /din.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "'''\n", 10 | "Author:\n", 11 | " Shenxin Zhan,zhanshenxin135@163.com\n", 12 | " \n", 13 | "Reference:\n", 14 | " https://arxiv.org/abs/1706.06978\n", 15 | " Deep Interest Network for Click-Through Rate Prediction\n", 16 | "'''" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": null, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "import torch\n", 26 | "import torch.nn as nn\n", 27 | "\n", 28 | "from common import MLP" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "class DIN(nn.Module):\n", 38 | " def __init__(self, params, use_batchnorm=True, use_dropout=True):\n", 39 | " super(DIN, self).__init__()\n", 40 | " \n", 41 | " self.device = params['device']\n", 42 | " self.feature_size = params['feature_size']\n", 43 | " self.embedding_size = params['embedding_size']\n", 44 | " self.userItemDict = params['userItemDict']\n", 45 | " self.hidden_dims = params['hidden_dims']\n", 46 | " self.userItemMaxLen = params['userItemMaxLen']\n", 47 | " \n", 48 | " feature_embeddings = torch.empty(self.feature_size+1, self.embedding_size, \n", 49 | " dtype=torch.float32, device=self.device, \n", 50 | " requires_grad=True)\n", 51 | " nn.init.normal_(feature_embeddings)\n", 52 | " self.feature_embeddings = nn.Parameter(feature_embeddings)\n", 53 | " \n", 54 | " self.mlp = MLP(params, use_batchnorm=use_batchnorm, use_dropout=use_dropout)\n", 55 | " \n", 56 | " self.output_layer = nn.Linear(self.hidden_dims[-1], 1).to(self.device)\n", 57 | "\n", 58 | " \n", 59 | " def forward(self, features):\n", 60 | " feature_idx = features[\"feature_idx\"]\n", 61 | " \n", 62 | " uid = feature_idx[:, 0]\n", 63 | " feaName_ix = [('item_id', 2), ('author_id', 3), ('music_id', 6)]\n", 64 | " feaName_maxlen = [('item_id', 350), ('author_id', 250), ('music_id', 100)]\n", 65 | " feaName = ['item_id', 'author_id', 'music_id']\n", 66 | " ad_idx = {}\n", 67 | " for t in feaName_ix:\n", 68 | " ad_idx[t[0]] = feature_idx[:, t[1]]\n", 69 | " \n", 70 | "\n", 71 | " hist_idx = self.userItemDict.loc[uid.cpu().numpy()][feaName]\n", 72 | " \n", 73 | " hist_idx_padded = {}\n", 74 | " for temp in feaName_maxlen:\n", 75 | " hist_idx_padded[temp[0]] = pad_sequence(list(hist_idx[temp[0]]), batch_first=True, \n", 76 | " padding_value=self.feature_size)[:, 0:temp[1]].to(self.device)\n", 77 | " user_beha_embeddings = []\n", 78 | " for temp in feaName:\n", 79 | " hist_embeddings = self.feature_embeddings[hist_idx_padded[temp], :] \n", 80 | " ad_embeddings = self.feature_embeddings[ad_idx[temp], :]\n", 81 | " \n", 82 | " hist_weight = torch.einsum('blk,bk->bl', (hist_embeddings, ad_embeddings))\n", 83 | " mask = hist_idx_padded[temp] != self.feature_size\n", 84 | " hist_weight.masked_fill_(mask == 0, -1e9)\n", 85 | " hist_weight = torch.softmax(hist_weight, dim=1)\n", 86 | " user_beha_embeddings.append(torch.einsum('blk,bl->bk', (hist_embeddings, hist_weight)))\n", 87 | " \n", 88 | " user_beha_embeddings = torch.cat(user_beha_embeddings, dim=1)\n", 89 | " \n", 90 | " ad_embeddings = self.feature_embeddings[feature_idx, :].reshape(feature_idx.shape[0], -1)\n", 91 | " \n", 92 | " embeddings = torch.cat((user_beha_embeddings, ad_embeddings), dim=1)\n", 93 | " \n", 94 | " # deep\n", 95 | "# deepInput = embeddings.reshape(embeddings.shape[0], self.mlp_input_dim)\n", 96 | " deepOut = self.mlp(embeddings)\n", 97 | " \n", 98 | " logits = self.output_layer(deepOut)\n", 99 | " \n", 100 | " return logits" 101 | ] 102 | } 103 | ], 104 | "metadata": { 105 | "kernelspec": { 106 | "display_name": "Python 3", 107 | "language": "python", 108 | "name": "python3" 109 | }, 110 | "language_info": { 111 | "codemirror_mode": { 112 | "name": "ipython", 113 | "version": 3 114 | }, 115 | "file_extension": ".py", 116 | "mimetype": "text/x-python", 117 | "name": "python", 118 | "nbconvert_exporter": "python", 119 | "pygments_lexer": "ipython3", 120 | "version": "3.5.2" 121 | }, 122 | "toc": { 123 | "base_numbering": 1, 124 | "nav_menu": {}, 125 | "number_sections": true, 126 | "sideBar": true, 127 | "skip_h1_title": false, 128 | "title_cell": "Table of Contents", 129 | "title_sidebar": "Contents", 130 | "toc_cell": false, 131 | "toc_position": {}, 132 | "toc_section_display": true, 133 | "toc_window_display": false 134 | } 135 | }, 136 | "nbformat": 4, 137 | "nbformat_minor": 2 138 | } 139 | -------------------------------------------------------------------------------- /common.py: -------------------------------------------------------------------------------- 1 | 2 | # coding: utf-8 3 | 4 | # In[ ]: 5 | 6 | 7 | ''' 8 | some common code for ctr model 9 | ''' 10 | 11 | 12 | # In[ ]: 13 | 14 | 15 | import torch 16 | import torch.nn as nn 17 | from torch.nn.utils.rnn import pad_sequence 18 | 19 | 20 | # In[ ]: 21 | 22 | 23 | class FirstOrder(nn.Module): 24 | def __init__(self, params): 25 | super(FirstOrder, self).__init__() 26 | # parse params 27 | self.device = params['device'] 28 | self.feature_size = params['feature_size'] 29 | 30 | weights_first_order = torch.empty(self.feature_size, 1, 31 | dtype=torch.float32, device=self.device, 32 | requires_grad=True) 33 | nn.init.normal_(weights_first_order) 34 | self.weights_first_order = nn.Parameter(weights_first_order) 35 | 36 | def forward(self, feature_values, feature_idx): 37 | weights_first_order = self.weights_first_order[feature_idx, :] 38 | first_order = torch.mul(feature_values, weights_first_order.squeeze()) 39 | return first_order 40 | 41 | 42 | # In[ ]: 43 | 44 | 45 | class SecondOrder(nn.Module): 46 | def __init__(self, params, get_embeddings=False): 47 | super(SecondOrder, self).__init__() 48 | # parse params 49 | self.device = params['device'] 50 | self.feature_size = params['feature_size'] 51 | self.embedding_size = params['embedding_size'] 52 | self.get_embeddings = get_embeddings 53 | 54 | feature_embeddings = torch.empty(self.feature_size, self.embedding_size, 55 | dtype=torch.float32, device=self.device, 56 | requires_grad=True) 57 | nn.init.normal_(feature_embeddings) 58 | self.feature_embeddings = nn.Parameter(feature_embeddings) 59 | 60 | def forward(self, feature_values, feature_idx): 61 | embeddings = self.feature_embeddings[feature_idx, :] 62 | ## second order 63 | temp1 = torch.pow(torch.einsum('bf,bfk->bk', (feature_values, embeddings)), 2) 64 | temp2 = torch.einsum('bf,bfk->bk', (torch.pow(feature_values, 2), torch.pow(embeddings, 2))) 65 | second_order = temp1-temp2 66 | if self.get_embeddings: 67 | return second_order, embeddings 68 | else: 69 | return second_order 70 | 71 | 72 | # In[ ]: 73 | 74 | 75 | class FirstOrderMutiHot(nn.Module): 76 | '''support muti-hot feature for fm 77 | 78 | ''' 79 | def __init__(self, params): 80 | super(FirstOrderMutiHot, self).__init__() 81 | # parse params 82 | self.device = params['device'] 83 | self.feature_size = params['feature_size'] 84 | self.field_size = params['field_size'] 85 | self.fea_name = params['fea_name'] 86 | self.max_len = params['max_len'] 87 | 88 | weights_first_order = torch.empty(self.feature_size+2, 1, 89 | dtype=torch.float32, device=self.device, 90 | requires_grad=True) 91 | nn.init.normal_(weights_first_order) 92 | self.weights_first_order = nn.Parameter(weights_first_order) 93 | 94 | def forward(self, feature_values, feature_idx): 95 | batch_size = feature_values.shape[0] 96 | 97 | # feature index and value padding 98 | feature_idx_concat, feature_values_concat = [], [] 99 | for t in self.fea_name: 100 | feature_idx_concat = feature_idx_concat + list(feature_idx[t]) 101 | feature_values_concat = feature_values_concat + list(feature_values[t]) 102 | 103 | seqLen = torch.tensor(list(map(len, feature_idx_concat)), dtype=torch.float32, device=self.device) 104 | seqLen = torch.transpose(seqLen.reshape(self.field_size, batch_size), 0, 1) 105 | feature_idx_padded = pad_sequence(feature_idx_concat, batch_first=True, padding_value=self.feature_size)[:, 0:self.max_len].to(self.device) 106 | feature_values_padded = pad_sequence(feature_values_concat, batch_first=True, padding_value=0)[:, 0:self.max_len].to(self.device) 107 | 108 | # first_order 109 | weights_first_order = self.weights_first_order[feature_idx_padded] 110 | first_order = torch.mul(feature_values_padded, weights_first_order.squeeze()) 111 | first_order = first_order.reshape(self.field_size, batch_size, -1) 112 | first_order = torch.transpose(first_order, 0, 1) 113 | first_order = first_order.sum(dim=2) 114 | first_order = first_order / seqLen 115 | 116 | return first_order 117 | 118 | 119 | # In[ ]: 120 | 121 | 122 | class SecondOrderMutiHot(nn.Module): 123 | '''support muti-hot feature for fm 124 | 125 | ''' 126 | def __init__(self, params, get_embeddings=False): 127 | super(SecondOrderMutiHot, self).__init__() 128 | # parse params 129 | self.device = params['device'] 130 | self.feature_size = params['feature_size'] 131 | self.field_size = params['field_size'] 132 | self.embedding_size = params['embedding_size'] 133 | self.get_embeddings = get_embeddings 134 | self.fea_name = params['fea_name'] 135 | self.max_len = params['max_len'] 136 | 137 | feature_embeddings = torch.empty(self.feature_size+2, self.embedding_size, 138 | dtype=torch.float32, device=self.device, 139 | requires_grad=True) 140 | nn.init.normal_(feature_embeddings) 141 | self.feature_embeddings = nn.Parameter(feature_embeddings) 142 | 143 | def forward(self, feature_values, feature_idx): 144 | batch_size = feature_values.shape[0] 145 | 146 | # feature index padding and mask 147 | feature_idx_concat = [] 148 | for t in self.fea_name: 149 | feature_idx_concat = feature_idx_concat + list(feature_idx[t]) 150 | seqLen = torch.tensor(list(map(len, feature_idx_concat)), dtype=torch.float32, device=self.device) 151 | feature_idx_padded = pad_sequence(feature_idx_concat, batch_first=True, padding_value=self.feature_size)[:, 0:self.max_len].to(self.device) 152 | mask = feature_idx_padded != self.feature_size 153 | feature_weight = torch.ones_like(feature_idx_padded, dtype=torch.float32, device=self.device) 154 | feature_weight.masked_fill_(mask == 0, 0) 155 | 156 | # get embeddings and average 157 | embeddings = self.feature_embeddings[feature_idx_padded, :] 158 | embeddings = torch.einsum('ble,bl->ble', embeddings, feature_weight) 159 | embeddings = embeddings.sum(dim=1) 160 | embeddings = embeddings / seqLen.reshape(embeddings.shape[0], 1) 161 | embeddings = embeddings.reshape(self.field_size, batch_size, -1) 162 | embeddings = torch.transpose(embeddings, 0, 1) 163 | 164 | # feature values padding and average 165 | feature_values_concat = [] 166 | for t in self.fea_name: 167 | feature_values_concat = feature_values_concat + list(feature_values[t]) 168 | feature_values_padded = pad_sequence(feature_values_concat, batch_first=True, padding_value=0)[:, 0:self.max_len].to(self.device) 169 | feature_values_padded = feature_values_padded.reshape(self.field_size, batch_size, -1) 170 | feature_values_padded = torch.transpose(feature_values_padded, 0, 1) 171 | feature_values_padded = feature_values_padded.sum(dim=2) 172 | seqLen = torch.transpose(seqLen.reshape(self.field_size, batch_size), 0, 1) 173 | feature_values = feature_values_padded / seqLen 174 | 175 | # second order 176 | temp1 = torch.pow(torch.einsum('bf,bfk->bk', (feature_values, embeddings)), 2) 177 | temp2 = torch.einsum('bf,bfk->bk', (torch.pow(feature_values, 2), torch.pow(embeddings, 2))) 178 | second_order = temp1-temp2 179 | if self.get_embeddings: 180 | return second_order, embeddings 181 | else: 182 | return second_order 183 | 184 | 185 | # In[ ]: 186 | 187 | 188 | class MLP(nn.Module): 189 | def __init__(self, params, use_batchnorm=True, use_dropout=True): 190 | super(MLP, self).__init__() 191 | 192 | self.embedding_size = params['embedding_size'] 193 | self.field_size = params['field_size'] 194 | self.hidden_dims = params['hidden_dims'] 195 | self.device = params['device'] 196 | self.p = params['p'] 197 | self.use_batchnorm = use_batchnorm 198 | self.use_dropout = use_dropout 199 | 200 | 201 | 202 | self.input_dim = self.field_size * self.embedding_size 203 | self.num_layers = len(self.hidden_dims) 204 | 205 | 206 | ## deep weights 207 | self.deep_layers = nn.Sequential() 208 | 209 | net_dims = [self.input_dim]+self.hidden_dims 210 | for i in range(self.num_layers): 211 | self.deep_layers.add_module('fc%d' % (i+1), nn.Linear(net_dims[i], net_dims[i+1]).to(self.device)) 212 | if self.use_batchnorm: 213 | self.deep_layers.add_module('bn%d' % (i+1), nn.BatchNorm1d(net_dims[i+1]).to(self.device)) 214 | self.deep_layers.add_module('relu%d' % (i+1), nn.ReLU().to(self.device)) 215 | if self.use_dropout: 216 | self.deep_layers.add_module('dropout%d' % (i+1), nn.Dropout(self.p).to(self.device)) 217 | 218 | def forward(self, embeddings): 219 | deepInput = embeddings.reshape(embeddings.shape[0], self.input_dim) 220 | deepOut = self.deep_layers(deepInput) 221 | return deepOut 222 | 223 | 224 | # In[ ]: 225 | 226 | 227 | class CIN(nn.Module): 228 | '''xDeepFM CIN Module 229 | ''' 230 | def __init__(self, params): 231 | super(CIN, self).__init__() 232 | # parse params 233 | self.split_half = params['split_half'] 234 | self.field_size = params['field_size'] 235 | self.hidden_dims = params['cin_hidden_dims'] 236 | self.num_layers = len(self.hidden_dims) 237 | 238 | self.net_dims = [self.field_size]+self.hidden_dims 239 | self.hidden_dims_split_half = [self.field_size] 240 | self.conv1ds = nn.ModuleList() 241 | for i in range(self.num_layers): 242 | # h_weights['h_weight%d' % (i+1)] = torch.empty(net_dims[i], self.field_size) 243 | # nn.init.normal_(h_weights['h_weight%d' % (i+1)]) 244 | self.conv1ds.append(nn.Conv1d(self.net_dims[0]*self.hidden_dims_split_half[-1], self.net_dims[i+1], 1)) 245 | if self.split_half: 246 | self.hidden_dims_split_half.append(self.net_dims[i+1] // 2) 247 | else: 248 | self.hidden_dims_split_half.append(self.net_dims[i+1]) 249 | 250 | def forward(self, inputs): 251 | res = [] 252 | h = [inputs] 253 | for i in range(self.num_layers): 254 | temp = torch.einsum('bhd,bmd->bhmd', h[-1], h[0]) 255 | temp = temp.reshape(inputs.shape[0], h[-1].shape[1]*inputs.shape[1], inputs.shape[2]) 256 | # b * hi * d 257 | temp = self.conv1ds[i](temp) 258 | if self.split_half: 259 | next_hidden, hi = torch.split(temp, 2*[temp.shape[1]//2], 1) 260 | else: 261 | next_hidden, hi = temp, temp 262 | h.append(next_hidden) 263 | res.append(hi) 264 | res = torch.cat(res, dim=1) 265 | # b * (h1 + h2 + ... + hn) 266 | res = torch.sum(res, dim=2) 267 | return res 268 | 269 | -------------------------------------------------------------------------------- /common.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "'''\n", 10 | "some common code for ctr model\n", 11 | "'''" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "import torch\n", 21 | "import torch.nn as nn\n", 22 | "from torch.nn.utils.rnn import pad_sequence" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "class FirstOrder(nn.Module):\n", 32 | " def __init__(self, params):\n", 33 | " super(FirstOrder, self).__init__()\n", 34 | " # parse params\n", 35 | " self.device = params['device']\n", 36 | " self.feature_size = params['feature_size']\n", 37 | " \n", 38 | " weights_first_order = torch.empty(self.feature_size, 1, \n", 39 | " dtype=torch.float32, device=self.device,\n", 40 | " requires_grad=True)\n", 41 | " nn.init.normal_(weights_first_order)\n", 42 | " self.weights_first_order = nn.Parameter(weights_first_order)\n", 43 | " \n", 44 | " def forward(self, feature_values, feature_idx): \n", 45 | " weights_first_order = self.weights_first_order[feature_idx, :]\n", 46 | " first_order = torch.mul(feature_values, weights_first_order.squeeze())\n", 47 | " return first_order" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "class SecondOrder(nn.Module):\n", 57 | " def __init__(self, params, get_embeddings=False):\n", 58 | " super(SecondOrder, self).__init__()\n", 59 | " # parse params\n", 60 | " self.device = params['device']\n", 61 | " self.feature_size = params['feature_size']\n", 62 | " self.embedding_size = params['embedding_size']\n", 63 | " self.get_embeddings = get_embeddings\n", 64 | " \n", 65 | " feature_embeddings = torch.empty(self.feature_size, self.embedding_size, \n", 66 | " dtype=torch.float32, device=self.device, \n", 67 | " requires_grad=True)\n", 68 | " nn.init.normal_(feature_embeddings)\n", 69 | " self.feature_embeddings = nn.Parameter(feature_embeddings)\n", 70 | " \n", 71 | " def forward(self, feature_values, feature_idx): \n", 72 | " embeddings = self.feature_embeddings[feature_idx, :]\n", 73 | " ## second order\n", 74 | " temp1 = torch.pow(torch.einsum('bf,bfk->bk', (feature_values, embeddings)), 2)\n", 75 | " temp2 = torch.einsum('bf,bfk->bk', (torch.pow(feature_values, 2), torch.pow(embeddings, 2)))\n", 76 | " second_order = temp1-temp2\n", 77 | " if self.get_embeddings:\n", 78 | " return second_order, embeddings\n", 79 | " else:\n", 80 | " return second_order" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "class FirstOrderMutiHot(nn.Module):\n", 90 | " '''support muti-hot feature for fm\n", 91 | " \n", 92 | " '''\n", 93 | " def __init__(self, params):\n", 94 | " super(FirstOrderMutiHot, self).__init__()\n", 95 | " # parse params\n", 96 | " self.device = params['device']\n", 97 | " self.feature_size = params['feature_size']\n", 98 | " self.field_size = params['field_size']\n", 99 | " self.fea_name = params['fea_name']\n", 100 | " self.max_len = params['max_len'] \n", 101 | " \n", 102 | " weights_first_order = torch.empty(self.feature_size+2, 1, \n", 103 | " dtype=torch.float32, device=self.device,\n", 104 | " requires_grad=True)\n", 105 | " nn.init.normal_(weights_first_order)\n", 106 | " self.weights_first_order = nn.Parameter(weights_first_order)\n", 107 | " \n", 108 | " def forward(self, feature_values, feature_idx):\n", 109 | " batch_size = feature_values.shape[0]\n", 110 | " \n", 111 | " # feature index and value padding\n", 112 | " feature_idx_concat, feature_values_concat = [], []\n", 113 | " for t in self.fea_name:\n", 114 | " feature_idx_concat = feature_idx_concat + list(feature_idx[t])\n", 115 | " feature_values_concat = feature_values_concat + list(feature_values[t])\n", 116 | "\n", 117 | " seqLen = torch.tensor(list(map(len, feature_idx_concat)), dtype=torch.float32, device=self.device)\n", 118 | " seqLen = torch.transpose(seqLen.reshape(self.field_size, batch_size), 0, 1)\n", 119 | " feature_idx_padded = pad_sequence(feature_idx_concat, batch_first=True, padding_value=self.feature_size)[:, 0:self.max_len].to(self.device)\n", 120 | " feature_values_padded = pad_sequence(feature_values_concat, batch_first=True, padding_value=0)[:, 0:self.max_len].to(self.device)\n", 121 | " \n", 122 | " # first_order\n", 123 | " weights_first_order = self.weights_first_order[feature_idx_padded]\n", 124 | " first_order = torch.mul(feature_values_padded, weights_first_order.squeeze())\n", 125 | " first_order = first_order.reshape(self.field_size, batch_size, -1)\n", 126 | " first_order = torch.transpose(first_order, 0, 1)\n", 127 | " first_order = first_order.sum(dim=2)\n", 128 | " first_order = first_order / seqLen\n", 129 | " \n", 130 | " return first_order" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": null, 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [ 139 | "class SecondOrderMutiHot(nn.Module):\n", 140 | " '''support muti-hot feature for fm\n", 141 | " \n", 142 | " '''\n", 143 | " def __init__(self, params, get_embeddings=False):\n", 144 | " super(SecondOrderMutiHot, self).__init__()\n", 145 | " # parse params\n", 146 | " self.device = params['device']\n", 147 | " self.feature_size = params['feature_size']\n", 148 | " self.field_size = params['field_size']\n", 149 | " self.embedding_size = params['embedding_size']\n", 150 | " self.get_embeddings = get_embeddings\n", 151 | " self.fea_name = params['fea_name']\n", 152 | " self.max_len = params['max_len'] \n", 153 | "\n", 154 | " feature_embeddings = torch.empty(self.feature_size+2, self.embedding_size, \n", 155 | " dtype=torch.float32, device=self.device, \n", 156 | " requires_grad=True)\n", 157 | " nn.init.normal_(feature_embeddings)\n", 158 | " self.feature_embeddings = nn.Parameter(feature_embeddings)\n", 159 | " \n", 160 | " def forward(self, feature_values, feature_idx):\n", 161 | " batch_size = feature_values.shape[0]\n", 162 | " \n", 163 | " # feature index padding and mask\n", 164 | " feature_idx_concat = []\n", 165 | " for t in self.fea_name:\n", 166 | " feature_idx_concat = feature_idx_concat + list(feature_idx[t])\n", 167 | " seqLen = torch.tensor(list(map(len, feature_idx_concat)), dtype=torch.float32, device=self.device)\n", 168 | " feature_idx_padded = pad_sequence(feature_idx_concat, batch_first=True, padding_value=self.feature_size)[:, 0:self.max_len].to(self.device)\n", 169 | " mask = feature_idx_padded != self.feature_size\n", 170 | " feature_weight = torch.ones_like(feature_idx_padded, dtype=torch.float32, device=self.device)\n", 171 | " feature_weight.masked_fill_(mask == 0, 0)\n", 172 | " \n", 173 | " # get embeddings and average\n", 174 | " embeddings = self.feature_embeddings[feature_idx_padded, :]\n", 175 | " embeddings = torch.einsum('ble,bl->ble', embeddings, feature_weight)\n", 176 | " embeddings = embeddings.sum(dim=1)\n", 177 | " embeddings = embeddings / seqLen.reshape(embeddings.shape[0], 1)\n", 178 | " embeddings = embeddings.reshape(self.field_size, batch_size, -1)\n", 179 | " embeddings = torch.transpose(embeddings, 0, 1)\n", 180 | "\n", 181 | " # feature values padding and average\n", 182 | " feature_values_concat = []\n", 183 | " for t in self.fea_name:\n", 184 | " feature_values_concat = feature_values_concat + list(feature_values[t])\n", 185 | " feature_values_padded = pad_sequence(feature_values_concat, batch_first=True, padding_value=0)[:, 0:self.max_len].to(self.device)\n", 186 | " feature_values_padded = feature_values_padded.reshape(self.field_size, batch_size, -1)\n", 187 | " feature_values_padded = torch.transpose(feature_values_padded, 0, 1)\n", 188 | " feature_values_padded = feature_values_padded.sum(dim=2)\n", 189 | " seqLen = torch.transpose(seqLen.reshape(self.field_size, batch_size), 0, 1)\n", 190 | " feature_values = feature_values_padded / seqLen\n", 191 | "\n", 192 | " # second order\n", 193 | " temp1 = torch.pow(torch.einsum('bf,bfk->bk', (feature_values, embeddings)), 2)\n", 194 | " temp2 = torch.einsum('bf,bfk->bk', (torch.pow(feature_values, 2), torch.pow(embeddings, 2)))\n", 195 | " second_order = temp1-temp2\n", 196 | " if self.get_embeddings:\n", 197 | " return second_order, embeddings\n", 198 | " else:\n", 199 | " return second_order" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": null, 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [ 208 | "class MLP(nn.Module):\n", 209 | " def __init__(self, params, use_batchnorm=True, use_dropout=True):\n", 210 | " super(MLP, self).__init__()\n", 211 | " \n", 212 | " self.embedding_size = params['embedding_size']\n", 213 | " self.field_size = params['field_size']\n", 214 | " self.hidden_dims = params['hidden_dims']\n", 215 | " self.device = params['device']\n", 216 | " self.p = params['p']\n", 217 | " self.use_batchnorm = use_batchnorm\n", 218 | " self.use_dropout = use_dropout\n", 219 | "\n", 220 | "\n", 221 | " \n", 222 | " self.input_dim = self.field_size * self.embedding_size\n", 223 | " self.num_layers = len(self.hidden_dims)\n", 224 | "\n", 225 | " \n", 226 | " ## deep weights\n", 227 | " self.deep_layers = nn.Sequential()\n", 228 | "\n", 229 | " net_dims = [self.input_dim]+self.hidden_dims\n", 230 | " for i in range(self.num_layers):\n", 231 | " self.deep_layers.add_module('fc%d' % (i+1), nn.Linear(net_dims[i], net_dims[i+1]).to(self.device))\n", 232 | " if self.use_batchnorm:\n", 233 | " self.deep_layers.add_module('bn%d' % (i+1), nn.BatchNorm1d(net_dims[i+1]).to(self.device))\n", 234 | " self.deep_layers.add_module('relu%d' % (i+1), nn.ReLU().to(self.device)) \n", 235 | " if self.use_dropout:\n", 236 | " self.deep_layers.add_module('dropout%d' % (i+1), nn.Dropout(self.p).to(self.device))\n", 237 | " \n", 238 | " def forward(self, embeddings):\n", 239 | " deepInput = embeddings.reshape(embeddings.shape[0], self.input_dim)\n", 240 | " deepOut = self.deep_layers(deepInput)\n", 241 | " return deepOut" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": null, 247 | "metadata": {}, 248 | "outputs": [], 249 | "source": [ 250 | "class CIN(nn.Module):\n", 251 | " '''xDeepFM CIN Module\n", 252 | " '''\n", 253 | " def __init__(self, params):\n", 254 | " super(CIN, self).__init__()\n", 255 | " # parse params\n", 256 | " self.split_half = params['split_half']\n", 257 | " self.field_size = params['field_size']\n", 258 | " self.hidden_dims = params['cin_hidden_dims']\n", 259 | " self.num_layers = len(self.hidden_dims)\n", 260 | " \n", 261 | " self.net_dims = [self.field_size]+self.hidden_dims\n", 262 | " self.hidden_dims_split_half = [self.field_size]\n", 263 | " self.conv1ds = nn.ModuleList()\n", 264 | " for i in range(self.num_layers):\n", 265 | "# h_weights['h_weight%d' % (i+1)] = torch.empty(net_dims[i], self.field_size)\n", 266 | "# nn.init.normal_(h_weights['h_weight%d' % (i+1)])\n", 267 | " self.conv1ds.append(nn.Conv1d(self.net_dims[0]*self.hidden_dims_split_half[-1], self.net_dims[i+1], 1))\n", 268 | " if self.split_half:\n", 269 | " self.hidden_dims_split_half.append(self.net_dims[i+1] // 2)\n", 270 | " else:\n", 271 | " self.hidden_dims_split_half.append(self.net_dims[i+1])\n", 272 | " \n", 273 | " def forward(self, inputs):\n", 274 | " res = []\n", 275 | " h = [inputs]\n", 276 | " for i in range(self.num_layers):\n", 277 | " temp = torch.einsum('bhd,bmd->bhmd', h[-1], h[0])\n", 278 | " temp = temp.reshape(inputs.shape[0], h[-1].shape[1]*inputs.shape[1], inputs.shape[2])\n", 279 | " # b * hi * d\n", 280 | " temp = self.conv1ds[i](temp)\n", 281 | " if self.split_half:\n", 282 | " next_hidden, hi = torch.split(temp, 2*[temp.shape[1]//2], 1)\n", 283 | " else:\n", 284 | " next_hidden, hi = temp, temp\n", 285 | " h.append(next_hidden)\n", 286 | " res.append(hi)\n", 287 | " res = torch.cat(res, dim=1)\n", 288 | " # b * (h1 + h2 + ... + hn)\n", 289 | " res = torch.sum(res, dim=2)\n", 290 | " return res" 291 | ] 292 | }, 293 | { 294 | "cell_type": "code", 295 | "execution_count": null, 296 | "metadata": {}, 297 | "outputs": [], 298 | "source": [] 299 | } 300 | ], 301 | "metadata": { 302 | "kernelspec": { 303 | "display_name": "Python 3", 304 | "language": "python", 305 | "name": "python3" 306 | }, 307 | "language_info": { 308 | "codemirror_mode": { 309 | "name": "ipython", 310 | "version": 3 311 | }, 312 | "file_extension": ".py", 313 | "mimetype": "text/x-python", 314 | "name": "python", 315 | "nbconvert_exporter": "python", 316 | "pygments_lexer": "ipython3", 317 | "version": "3.5.2" 318 | }, 319 | "toc": { 320 | "base_numbering": 1, 321 | "nav_menu": {}, 322 | "number_sections": true, 323 | "sideBar": true, 324 | "skip_h1_title": false, 325 | "title_cell": "Table of Contents", 326 | "title_sidebar": "Contents", 327 | "toc_cell": false, 328 | "toc_position": { 329 | "height": "calc(100% - 180px)", 330 | "left": "10px", 331 | "top": "150px", 332 | "width": "165px" 333 | }, 334 | "toc_section_display": true, 335 | "toc_window_display": true 336 | } 337 | }, 338 | "nbformat": 4, 339 | "nbformat_minor": 2 340 | } 341 | --------------------------------------------------------------------------------