├── LICENSE
├── Makefile
├── README.md
├── bin
    └── .gitignore
├── lambdafm_predict.cpp
├── lambdafm_train.cpp
└── src
    ├── FTRL
        ├── ftrl_model.h
        ├── ftrl_predictor.h
        └── ftrl_trainer.h
    ├── Frame
        ├── Makefile
        ├── pc_frame.cpp
        ├── pc_frame.h
        ├── pc_task.h
        ├── test_main.cpp
        └── test_task.h
    ├── Sample
        └── fm_sample.h
    └── Utils
        ├── utils.cpp
        └── utils.h


/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2021 Zhang Wei
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | all:
2 | 	g++ -O3 lambdafm_train.cpp src/Frame/pc_frame.cpp src/Utils/utils.cpp -I . -std=c++0x -o bin/lambdafm_train -lpthread
3 | 	g++ -O3 lambdafm_predict.cpp src/Frame/pc_frame.cpp src/Utils/utils.cpp -I . -std=c++0x -o bin/lambdafm_predict -lpthread
4 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # lambdaFM
 2 | ## 前言：
 3 | * lambdaFM是lambdaRank和FM(Factorization Machines)的结合，用于解决排序问题。实现了pairwise和lambdaRank两种训练方法，可通过-rank参数选择。
 4 | lambdaFM是在[alphaFM](https://github.com/CastellanZhang/alphaFM)的代码基础上修改而成，同样是单机多线程版本，同样是FTRL优化算法。<br>
 5 | 
 6 | * 算法原理见我的博客文章：http://castellanzhang.github.io/2017/07/16/lambdafm/
 7 | 
 8 | * lambdaFM和alphaFM类似，同样适用于真实业务中大规模数据、高维稀疏特征的训练。由于采用FTRL优化算法，样本只需过一遍，不占用内存。通过管道的方式接受输入。<br>
 9 | 比如训练样本存储在hdfs上，一个典型的使用方法是这样：<br>
10 | 训练：10个线程计算，factorization的维度是8，最后得到模型文件fm_model.txt<br>
11 | `hadoop fs -cat train_data_hdfs_path | ./lambdafm_train -core 10 -dim 1,8 -rank ndcg -m fm_model.txt`<br>
12 | 测试：10个线程计算，factorization的维度是8，加载模型文件fm_model.txt，最后输出预测结果文件fm_pre.txt<br>
13 | `hadoop fs -cat test_data_hdfs_path | ./lambdafm_predict -core 10 -dim 8 -m fm_model.txt -out fm_pre.txt`<br>
14 | 
15 | * lambdaFM同样支持加载上次模型继续训练，以及通过-fvs参数加强v的稀疏性。<br>
16 | 
17 | * 注意：dim参数有些变化，不再需要指定偏置项，所以只有<k1,k2>两项。当将dim参数设置为1,0时，lambdaFM就退化成lambdaLR。<br>
18 | 
19 | ## 安装方法：
20 | 直接在根目录make即可，编译后会在bin目录下生成两个可执行文件。如果编译失败，请升级gcc版本。
21 | ## 输入文件格式：
22 | 类似于RankLib和SVMrank的格式，但更加灵活：特征编号不局限于整数也可以是字符串；特征值可以是整数或浮点数（特征值最好做归一化处理，否则可能会导致结果为nan），
23 | 特征值为0的项可以省略不写；qid可以是数字也可以是字符串；label（即相关性分数）必须是0,1,2,3等非负整数；#后面是注释，注释也可为空。
24 | 相同qid的数据必须相邻在一起，且按照自然展现的顺序排列。举例如下：<br>
25 | `1 qid:1 sex:1 age:0.3 f1:1 f3:0.9 # 1AAAAA`<br>
26 | `0 qid:1 sex:0 age:0.7 f2:0.4 f5:0.8 f8:1 # 1BBBBB`<br>
27 | `3 qid:1 sex:1 age:0.3 f1:1 f3:0.9 # 1CCCCC`<br>
28 | `2 qid:ab sex:0 age:0.2 f2:0.2 f8:1`<br>
29 | `1 qid:ab sex:1 age:0.5 f1:1 f3:0.3`<br>
30 | `4 qid:ab sex:0 age:0.1 f2:0.7 f5:0.2 f8:1`<br>
31 | `...`<br>
32 | ## 模型文件格式：
33 | `feature_name w v1 v2 ... vf w_n w_z v_n1 v_n2 ... v_nf v_z1 v_z2 ... v_zf`
34 | ## 预测结果格式：
35 | `label qid score`<br>
36 | 
37 | ## 参数说明：
38 | ### lambdafm_train的参数：
39 | -m \<model_path\>: 设置模型文件的输出路径。<br>
40 | -dim \<k1,k2\>: k1为1表示使用w参数，为0表示不使用；k2为v的维度，可以是0。	default:1,8<br>
41 | -init_stdev \<stdev\>: v的初始化使用均值为0的高斯分布，stdev为标准差。	default:0.1<br>
42 | -w_alpha \<w_alpha\>: w0和w的FTRL超参数alpha。	default:0.05<br>
43 | -w_beta \<w_beta\>: w0和w的FTRL超参数beta。	default:1.0<br>
44 | -w_l1 \<w_L1_reg\>: w0和w的L1正则。	default:0.1<br>
45 | -w_l2 \<w_L2_reg\>: w0和w的L2正则。	default:5.0<br>
46 | -v_alpha \<v_alpha\>: v的FTRL超参数alpha。	default:0.05<br>
47 | -v_beta \<v_beta\>: v的FTRL超参数beta。	default:1.0<br>
48 | -v_l1 \<v_L1_reg\>: v的L1正则。	default:0.1<br>
49 | -v_l2 \<v_L2_reg\>: v的L2正则。	default:5.0<br>
50 | -core \<threads_num\>: 计算线程数。	default:1<br>
51 | -im \<initial_model_path\>: 上次模型的路径，用于初始化模型参数。如果是第一次训练则不用设置此参数。<br>
52 | -fvs \<force_v_sparse\>: 为了获得更好的稀疏解。当fvs值为1, 则训练中每当wi = 0，即令vi = 0；当fvs为0时关闭此功能。	default:0<br>
53 | -rank \<ranking_loss\>: pairwise或ndcg，支持两种排序算法，ndcg即lambdaRank算法。	default:pairwise<br>
54 | -fast_mode \<fast_mode\>: 提供了一种快速训练模式，在v较大时速度能有一定提升，但效果可能会变差，谨慎使用。1表示开启，0表示关闭。	default:0<br>
55 | ### lambdafm_predict的参数：
56 | -m \<model_path\>: 模型文件路径。<br>
57 | -dim \<factor_num\>: v的维度。	default:8<br>
58 | -core \<threads_num\>: 计算线程数。	default:1<br>
59 | -out \<predict_path\>: 输出文件路径。<br>
60 | 
61 | ## 评测工具：
62 | * 写了一个python版本的NDCG@k计算工具，具体见：https://github.com/CastellanZhang/NDCG<br>
63 | 
64 | 


--------------------------------------------------------------------------------
/bin/.gitignore:
--------------------------------------------------------------------------------
1 | # Ignore everything in this directory
2 | *
3 | # Except this file !.gitignore
4 | !.gitignore
5 | 


--------------------------------------------------------------------------------
/lambdafm_predict.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <map>
  3 | #include <fstream>
  4 | #include "src/Frame/pc_frame.h"
  5 | #include "src/FTRL/ftrl_predictor.h"
  6 | 
  7 | using namespace std;
  8 | 
  9 | struct Option 
 10 | {
 11 |     Option() : dim(8), threads_num(1) {}
 12 |     string model_path, predict_path;
 13 |     int threads_num, dim;
 14 | };
 15 | 
 16 | string predict_help() 
 17 | {
 18 |     return string(
 19 |             "\nusage: cat sample | ./lambdafm_predict [<options>]"
 20 |             "\n"
 21 |             "\n"
 22 |             "options:\n"
 23 |             "-m <model_path>: set the model path\n"
 24 |             "-dim <factor_num>: dim of 2-way interactions\tdefault:8\n"
 25 |             "-core <threads_num>: set the number of threads\tdefault:1\n"
 26 |             "-out <predict_path>: set the predict path\n"
 27 |     );
 28 | }
 29 | 
 30 | vector<string> argv_to_args(int argc, char* argv[])
 31 | {
 32 |     vector<string> args;
 33 |     for(int i = 1; i < argc; ++i)
 34 |     {
 35 |         args.push_back(string(argv[i]));
 36 |     }
 37 |     return args;
 38 | }
 39 | 
 40 | Option parse_option(const vector<string>& args)
 41 | {
 42 |     int argc = args.size();
 43 |     if(0 == argc) 
 44 |         throw invalid_argument("invalid command\n");
 45 |     Option opt;
 46 | 
 47 |     for(int i = 0; i < argc; ++i) 
 48 |     {
 49 |         if(args[i].compare("-m") == 0) 
 50 |         {
 51 |             if(i == argc - 1)
 52 |                 throw invalid_argument("invalid command\n");
 53 |             opt.model_path = args[++i];
 54 |         }
 55 |         else if(args[i].compare("-dim") == 0)
 56 |         {
 57 |             if(i == argc - 1)
 58 |                 throw invalid_argument("invalid command\n");
 59 |             opt.dim = stoi(args[++i]);
 60 |         }
 61 |         else if(args[i].compare("-core") == 0)
 62 |         {
 63 |             if(i == argc - 1)
 64 |                 throw invalid_argument("invalid command\n");
 65 |             opt.threads_num = stoi(args[++i]);
 66 |         }
 67 |         else if(args[i].compare("-out") == 0)
 68 |         {
 69 |             if(i == argc - 1)
 70 |                 throw invalid_argument("invalid command\n");
 71 |             opt.predict_path = args[++i];
 72 |         }
 73 |         else
 74 |         {
 75 |             break;
 76 |         }
 77 |     }
 78 |     return opt;
 79 | }
 80 | 
 81 | 
 82 | int main(int argc, char* argv[])
 83 | {
 84 |     cin.sync_with_stdio(false);
 85 |     cout.sync_with_stdio(false);
 86 |     Option opt;
 87 |     try
 88 |     {
 89 |         opt = parse_option(argv_to_args(argc, argv));
 90 |     }
 91 |     catch(const invalid_argument& e)
 92 |     {
 93 |         cout << e.what() << endl;
 94 |         cout << predict_help() << endl;
 95 |         return EXIT_FAILURE;
 96 |     }
 97 | 
 98 | 
 99 |     ifstream f_model(opt.model_path.c_str());
100 |     ofstream f_predict(opt.predict_path.c_str(), ofstream::out);
101 | 
102 |     ftrl_predictor predictor(opt.dim, f_model, f_predict);
103 | 
104 |     pc_frame frame;
105 |     frame.init(predictor, opt.threads_num);
106 |     frame.run();
107 | 
108 |     f_model.close();
109 |     f_predict.close();
110 |     return 0;
111 | }
112 | 
113 | 


--------------------------------------------------------------------------------
/lambdafm_train.cpp:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <map>
 3 | #include <fstream>
 4 | #include "src/Frame/pc_frame.h"
 5 | #include "src/FTRL/ftrl_trainer.h"
 6 | 
 7 | using namespace std;
 8 | 
 9 | string train_help() 
10 | {
11 |     return string(
12 |             "\nusage: cat sample | ./lambdafm_train [<options>]"
13 |             "\n"
14 |             "\n"
15 |             "options:\n"
16 |             "-m <model_path>: set the output model path\n"
17 |             "-dim <k1,k2>: k1=use 1-way interactions, k2=dim of 2-way interactions\tdefault:1,8\n"
18 |             "-init_stdev <stdev>: stdev for initialization of 2-way factors\tdefault:0.1\n"
19 |             "-w_alpha <w_alpha>: w is updated via FTRL, alpha is one of the learning rate parameters\tdefault:0.05\n"
20 |             "-w_beta <w_beta>: w is updated via FTRL, beta is one of the learning rate parameters\tdefault:1.0\n"
21 |             "-w_l1 <w_L1_reg>: L1 regularization parameter of w\tdefault:0.1\n"
22 |             "-w_l2 <w_L2_reg>: L2 regularization parameter of w\tdefault:5.0\n"
23 |             "-v_alpha <v_alpha>: v is updated via FTRL, alpha is one of the learning rate parameters\tdefault:0.05\n"
24 |             "-v_beta <v_beta>: v is updated via FTRL, beta is one of the learning rate parameters\tdefault:1.0\n"
25 |             "-v_l1 <v_L1_reg>: L1 regularization parameter of v\tdefault:0.1\n"
26 |             "-v_l2 <v_L2_reg>: L2 regularization parameter of v\tdefault:5.0\n"
27 |             "-core <threads_num>: set the number of threads\tdefault:1\n"
28 |             "-im <initial_model_path>: set the initial value of model\n"
29 |             "-fvs <force_v_sparse>: if fvs is 1, set vi = 0 whenever wi = 0\tdefault:0\n"
30 |             "-rank <ranking_loss>: pairwise or ndcg\tdefault:pairwise\n"
31 |             "-fast_mode <fast_mode>: if 1, use fast mode\tdefault:0\n"
32 |     );
33 | }
34 | 
35 | vector<string> argv_to_args(int argc, char* argv[]) 
36 | {
37 |     vector<string> args;
38 |     for(int i = 1; i < argc; ++i)
39 |     {
40 |         args.push_back(string(argv[i]));
41 |     }
42 |     return args;
43 | }
44 | 
45 | 
46 | int main(int argc, char* argv[])
47 | {
48 |     cin.sync_with_stdio(false);
49 |     cout.sync_with_stdio(false);
50 |     srand(time(NULL));
51 |     trainer_option opt;
52 |     try
53 |     {
54 |         opt.parse_option(argv_to_args(argc, argv));
55 |     }
56 |     catch(const invalid_argument& e)
57 |     {
58 |         cout << "invalid_argument:" << e.what() << endl;
59 |         cout << train_help() << endl;
60 |         return EXIT_FAILURE;
61 |     }
62 | 
63 |     ftrl_trainer trainer(opt);
64 | 
65 |     if(opt.b_init) 
66 |     {
67 |         ifstream f_temp(opt.init_m_path.c_str());
68 |         if(!trainer.loadModel(f_temp)) 
69 |         {
70 |             cout << "wrong model" << endl;
71 |             return EXIT_FAILURE;
72 |         }
73 |         f_temp.close();
74 |     }
75 | 
76 |     pc_frame frame;
77 |     frame.init(trainer, opt.threads_num);
78 |     frame.run();
79 | 
80 |     ofstream f_model(opt.model_path.c_str(), ofstream::out);
81 |     trainer.outputModel(f_model);
82 |     f_model.close();
83 | 
84 |     return 0;
85 | }
86 | 
87 | 


--------------------------------------------------------------------------------
/src/FTRL/ftrl_model.h:
--------------------------------------------------------------------------------
  1 | #ifndef FTRL_MODEL_H_
  2 | #define FTRL_MODEL_H_
  3 | 
  4 | #include <unordered_map>
  5 | #include <string>
  6 | #include <vector>
  7 | #include <mutex>
  8 | #include <iostream>
  9 | #include <cmath>
 10 | #include "../Utils/utils.h"
 11 | 
 12 | using namespace std;
 13 | 
 14 | //每一个特征维度的模型单元
 15 | class ftrl_model_unit
 16 | {
 17 | public:
 18 |     double wi;
 19 |     double w_ni;
 20 |     double w_zi;
 21 |     vector<double> vi;
 22 |     vector<double> v_ni;
 23 |     vector<double> v_zi;
 24 |     mutex mtx;
 25 | public:
 26 |     ftrl_model_unit(int factor_num, double v_mean, double v_stdev)
 27 |     {
 28 |         wi = 0.0;
 29 |         w_ni = 0.0;
 30 |         w_zi = 0.0;
 31 |         vi.resize(factor_num);
 32 |         v_ni.resize(factor_num);
 33 |         v_zi.resize(factor_num);
 34 |         for(int f = 0; f < factor_num; ++f)
 35 |         {
 36 |             vi[f] = utils::gaussian(v_mean, v_stdev);
 37 |             v_ni[f] = 0.0;
 38 |             v_zi[f] = 0.0;
 39 |         }
 40 |     }
 41 | 
 42 |     ftrl_model_unit(int factor_num, const vector<string>& modelLineSeg)
 43 |     {
 44 |         vi.resize(factor_num);
 45 |         v_ni.resize(factor_num);
 46 |         v_zi.resize(factor_num);
 47 |         wi = stod(modelLineSeg[1]);
 48 |         w_ni = stod(modelLineSeg[2 + factor_num]);
 49 |         w_zi = stod(modelLineSeg[3 + factor_num]);
 50 |         for(int f = 0; f < factor_num; ++f)
 51 |         {
 52 |             vi[f] = stod(modelLineSeg[2 + f]);
 53 |             v_ni[f] = stod(modelLineSeg[4 + factor_num + f]);
 54 |             v_zi[f] = stod(modelLineSeg[4 + 2 * factor_num + f]);
 55 |         }
 56 |     }
 57 | 
 58 |     void reinit_vi(double v_mean, double v_stdev)
 59 |     {
 60 |         int size = vi.size();
 61 |         for(int f = 0; f < size; ++f)
 62 |         {
 63 |             vi[f] = utils::gaussian(v_mean, v_stdev);
 64 |         }
 65 |     }
 66 | 
 67 |     friend inline ostream& operator <<(ostream& os, const ftrl_model_unit& mu)
 68 |     {
 69 |         os << mu.wi;
 70 |         for(int f = 0; f < mu.vi.size(); ++f)
 71 |         {
 72 |             os << " " << mu.vi[f];
 73 |         }
 74 |         os << " " << mu.w_ni << " " << mu.w_zi;
 75 |         for(int f = 0; f < mu.v_ni.size(); ++f)
 76 |         {
 77 |             os << " " << mu.v_ni[f];
 78 |         }
 79 |         for(int f = 0; f < mu.v_zi.size(); ++f)
 80 |         {
 81 |             os << " " << mu.v_zi[f];
 82 |         }
 83 |         return os;
 84 |     }
 85 | };
 86 | 
 87 | 
 88 | 
 89 | class ftrl_model
 90 | {
 91 | public:
 92 |     unordered_map<string, ftrl_model_unit*> muMap;
 93 | 
 94 |     int factor_num;
 95 |     double init_stdev;
 96 |     double init_mean;
 97 | 
 98 | public:
 99 |     ftrl_model(double _factor_num);
100 |     ftrl_model(double _factor_num, double _mean, double _stdev);
101 |     ftrl_model_unit* getOrInitModelUnit(string index);
102 | 
103 |     double predict(unordered_map<string, double>& x, unordered_map<string, ftrl_model_unit*>& theta, vector<double>& sum);
104 |     double getScore(unordered_map<string, double>& x, unordered_map<string, ftrl_model_unit*>& theta);
105 |     void outputModel(ofstream& out);
106 |     bool loadModel(ifstream& in);
107 |     void debugPrintModel();
108 | 
109 | private:
110 |     double get_wi(unordered_map<string, ftrl_model_unit*>& theta, const string& index);
111 |     double get_vif(unordered_map<string, ftrl_model_unit*>& theta, const string& index, int f);
112 | private:
113 |     mutex mtx;
114 | };
115 | 
116 | 
117 | ftrl_model::ftrl_model(double _factor_num)
118 | {
119 |     factor_num = _factor_num;
120 |     init_mean = 0.0;
121 |     init_stdev = 0.0;
122 | }
123 | 
124 | ftrl_model::ftrl_model(double _factor_num, double _mean, double _stdev)
125 | {
126 |     factor_num = _factor_num;
127 |     init_mean = _mean;
128 |     init_stdev = _stdev;
129 | }
130 | 
131 | 
132 | ftrl_model_unit* ftrl_model::getOrInitModelUnit(string index)
133 | {
134 |     unordered_map<string, ftrl_model_unit*>::iterator iter = muMap.find(index);
135 |     if(iter == muMap.end())
136 |     {
137 |         mtx.lock();
138 |         ftrl_model_unit* pMU = new ftrl_model_unit(factor_num, init_mean, init_stdev);
139 |         muMap.insert(make_pair(index, pMU));
140 |         mtx.unlock();
141 |         return pMU;
142 |     }
143 |     else
144 |     {
145 |         return iter->second;
146 |     }
147 | }
148 | 
149 | 
150 | double ftrl_model::predict(unordered_map<string, double>& x, unordered_map<string, ftrl_model_unit*>& theta, vector<double>& sum)
151 | {
152 |     double result = 0;
153 |     for(unordered_map<string, double>::iterator iter = x.begin(); iter != x.end(); ++iter)
154 |     {
155 |         const string& index = iter->first;
156 |         result += theta[index]->wi * iter->second;
157 |     }
158 |     double sum_sqr, d;
159 |     for(int f = 0; f < factor_num; ++f)
160 |     {
161 |         sum[f] = sum_sqr = 0.0;
162 |         for(unordered_map<string, double>::iterator iter = x.begin(); iter != x.end(); ++iter)
163 |         {
164 |             const string& index = iter->first;
165 |             d = theta[index]->vi[f] * iter->second;
166 |             sum[f] += d;
167 |             sum_sqr += d * d;
168 |         }
169 |         result += 0.5 * (sum[f] * sum[f] - sum_sqr);
170 |     }
171 |     return result;
172 | }
173 | 
174 | 
175 | double ftrl_model::getScore(unordered_map<string, double>& x, unordered_map<string, ftrl_model_unit*>& theta)
176 | {
177 |     double result = 0;
178 |     for(unordered_map<string, double>::iterator iter = x.begin(); iter != x.end(); ++iter)
179 |     {
180 |         const string& index = iter->first;
181 |         result += get_wi(theta, index) * iter->second;
182 |     }
183 |     double sum, sum_sqr, d;
184 |     for(int f = 0; f < factor_num; ++f)
185 |     {
186 |         sum = sum_sqr = 0.0;
187 |         for(unordered_map<string, double>::iterator iter = x.begin(); iter != x.end(); ++iter)
188 |         {
189 |             const string& index = iter->first;
190 |             d = get_vif(theta, index, f) * iter->second;
191 |             sum += d;
192 |             sum_sqr += d * d;
193 |         }
194 |         result += 0.5 * (sum * sum - sum_sqr);
195 |     }
196 |     //return 1.0/(1.0 + exp(-result));
197 |     return result;
198 | }
199 | 
200 | 
201 | double ftrl_model::get_wi(unordered_map<string, ftrl_model_unit*>& theta, const string& index)
202 | {
203 |     unordered_map<string, ftrl_model_unit*>::iterator iter = theta.find(index);
204 |     if(iter == theta.end())
205 |     {
206 |         return 0.0;
207 |     }
208 |     else
209 |     {
210 |         return iter->second->wi;
211 |     }
212 | }
213 | 
214 | 
215 | double ftrl_model::get_vif(unordered_map<string, ftrl_model_unit*>& theta, const string& index, int f)
216 | {
217 |     unordered_map<string, ftrl_model_unit*>::iterator iter = theta.find(index);
218 |     if(iter == theta.end())
219 |     {
220 |         return 0.0;
221 |     }
222 |     else
223 |     {
224 |         return iter->second->vi[f];
225 |     }
226 | }
227 | 
228 | 
229 | void ftrl_model::outputModel(ofstream& out)
230 | {
231 |     for(unordered_map<string, ftrl_model_unit*>::iterator iter = muMap.begin(); iter != muMap.end(); ++iter)
232 |     {
233 |         out << iter->first << " " << *(iter->second) << endl;
234 |     }
235 | }
236 | 
237 | 
238 | void ftrl_model::debugPrintModel()
239 | {
240 |     for(unordered_map<string, ftrl_model_unit*>::iterator iter = muMap.begin(); iter != muMap.end(); ++iter)
241 |     {
242 |         cout << iter->first << " " << *(iter->second) << endl;
243 |     }
244 | }
245 | 
246 | 
247 | bool ftrl_model::loadModel(ifstream& in)
248 | {
249 |     string line;
250 |     vector<string> strVec;
251 |     while(getline(in, line))
252 |     {
253 |         strVec.clear();
254 |         utils::splitString(line, ' ', &strVec);
255 |         if(strVec.size() != 3 * factor_num + 4)
256 |         {
257 |             return false;
258 |         }
259 |         string& index = strVec[0];
260 |         ftrl_model_unit* pMU = new ftrl_model_unit(factor_num, strVec);
261 |         muMap[index] = pMU;
262 |     }
263 |     return true;
264 | }
265 | 
266 | 
267 | 
268 | #endif /*FTRL_MODEL_H_*/
269 | 


--------------------------------------------------------------------------------
/src/FTRL/ftrl_predictor.h:
--------------------------------------------------------------------------------
 1 | #ifndef FTRL_PREDICTOR_H_
 2 | #define FTRL_PREDICTOR_H_
 3 | 
 4 | #include "../Frame/pc_frame.h"
 5 | #include "ftrl_model.h"
 6 | #include "../Sample/fm_sample.h"
 7 | 
 8 | 
 9 | class ftrl_predictor : public pc_task
10 | {
11 | public:
12 |     ftrl_predictor(double _factor_num, ifstream& _fModel, ofstream& _fPredict);
13 |     virtual void run_task(vector<vector<fm_sample> >& dataBuffer);
14 | private:
15 |     ftrl_model* pModel;
16 |     ofstream& fPredict;
17 |     mutex outMtx;
18 | };
19 | 
20 | 
21 | ftrl_predictor::ftrl_predictor(double _factor_num, ifstream& _fModel, ofstream& _fPredict):fPredict(_fPredict)
22 | {
23 |     pModel = new ftrl_model(_factor_num);
24 |     if(!pModel->loadModel(_fModel))
25 |     {
26 |         cout << "load model error!" << endl;
27 |         exit(-1);
28 |     }
29 | }
30 | 
31 | void ftrl_predictor::run_task(vector<vector<fm_sample> >& dataBuffer)
32 | {
33 |     vector<string> outputVec;
34 |     for(int i = 0; i < dataBuffer.size(); ++i)
35 |     {
36 |         for(int j = 0; j < dataBuffer[i].size(); ++j)
37 |         {
38 |             fm_sample& sample = dataBuffer[i][j];
39 |             double score = pModel->getScore(sample.x, pModel->muMap);
40 |             outputVec.push_back(to_string(sample.y) + " " + sample.qid + " " + to_string(score));
41 |         }
42 |     }
43 |     outMtx.lock();
44 |     for(int i = 0; i < outputVec.size(); ++i)
45 |     {
46 |         fPredict << outputVec[i] << endl;
47 |     }
48 |     outMtx.unlock();
49 | }
50 | 
51 | 
52 | #endif /*FTRL_PREDICTOR_H_*/
53 | 


--------------------------------------------------------------------------------
/src/FTRL/ftrl_trainer.h:
--------------------------------------------------------------------------------
  1 | #ifndef FTRL_TRAINER_H_
  2 | #define FTRL_TRAINER_H_
  3 | 
  4 | #include <unordered_set>
  5 | #include <unordered_map>
  6 | #include<algorithm>
  7 | #include "../Frame/pc_frame.h"
  8 | #include "ftrl_model.h"
  9 | #include "../Sample/fm_sample.h"
 10 | #include "../Utils/utils.h"
 11 | 
 12 | using namespace std;
 13 | 
 14 | struct trainer_option
 15 | {
 16 |     trainer_option() : k1(true), factor_num(8), init_mean(0.0), init_stdev(0.1), 
 17 |                w_alpha(0.05), w_beta(1.0), w_l1(0.1), w_l2(5.0), 
 18 |                v_alpha(0.05), v_beta(1.0), v_l1(0.1), v_l2(5.0), 
 19 |                threads_num(1), b_init(false), force_v_sparse(false), 
 20 |                rank("pairwise"), fast_mode(false) {}
 21 |     string model_path, init_m_path;
 22 |     double init_mean, init_stdev;
 23 |     double w_alpha, w_beta, w_l1, w_l2;
 24 |     double v_alpha, v_beta, v_l1, v_l2;
 25 |     int threads_num, factor_num;
 26 |     bool k1, b_init, force_v_sparse;
 27 |     string rank;
 28 |     bool fast_mode;
 29 |     
 30 |     void parse_option(const vector<string>& args) 
 31 |     {
 32 |         int argc = args.size();
 33 |         if(0 == argc) throw invalid_argument("invalid command\n");
 34 |         for(int i = 0; i < argc; ++i)
 35 |         {
 36 |             if(args[i].compare("-m") == 0) 
 37 |             {
 38 |                 if(i == argc - 1)
 39 |                     throw invalid_argument("invalid command\n");
 40 |                 model_path = args[++i];
 41 |             }
 42 |             else if(args[i].compare("-dim") == 0)
 43 |             {
 44 |                 if(i == argc - 1)
 45 |                     throw invalid_argument("invalid command\n");
 46 |                 vector<string> strVec;
 47 |                 string tmpStr = args[++i];
 48 |                 utils::splitString(tmpStr, ',', &strVec);
 49 |                 if(strVec.size() != 2)
 50 |                     throw invalid_argument("invalid command\n");
 51 |                 k1 = 0 == stoi(strVec[0]) ? false : true;
 52 |                 factor_num = stoi(strVec[1]);
 53 |             }
 54 |             else if(args[i].compare("-init_stdev") == 0)
 55 |             {
 56 |                 if(i == argc - 1)
 57 |                     throw invalid_argument("invalid command\n");
 58 |                 init_stdev = stod(args[++i]);
 59 |             }
 60 |             else if(args[i].compare("-w_alpha") == 0)
 61 |             {
 62 |                 if(i == argc - 1)
 63 |                     throw invalid_argument("invalid command\n");
 64 |                 w_alpha = stod(args[++i]);
 65 |             }
 66 |             else if(args[i].compare("-w_beta") == 0)
 67 |             {
 68 |                 if(i == argc - 1)
 69 |                     throw invalid_argument("invalid command\n");
 70 |                 w_beta = stod(args[++i]);
 71 |             }
 72 |             else if(args[i].compare("-w_l1") == 0)
 73 |             {
 74 |                 if(i == argc - 1)
 75 |                     throw invalid_argument("invalid command\n");
 76 |                 w_l1 = stod(args[++i]);
 77 |             }
 78 |             else if(args[i].compare("-w_l2") == 0)
 79 |             {
 80 |                 if(i == argc - 1)
 81 |                     throw invalid_argument("invalid command\n");
 82 |                 w_l2 = stod(args[++i]);
 83 |             }
 84 |             else if(args[i].compare("-v_alpha") == 0)
 85 |             {
 86 |                 if(i == argc - 1)
 87 |                     throw invalid_argument("invalid command\n");
 88 |                 v_alpha = stod(args[++i]);
 89 |             }
 90 |             else if(args[i].compare("-v_beta") == 0)
 91 |             {
 92 |                 if(i == argc - 1)
 93 |                     throw invalid_argument("invalid command\n");
 94 |                 v_beta = stod(args[++i]);
 95 |             }
 96 |             else if(args[i].compare("-v_l1") == 0)
 97 |             {
 98 |                 if(i == argc - 1)
 99 |                     throw invalid_argument("invalid command\n");
100 |                 v_l1 = stod(args[++i]);
101 |             }
102 |             else if(args[i].compare("-v_l2") == 0)
103 |             {
104 |                 if(i == argc - 1)
105 |                     throw invalid_argument("invalid command\n");
106 |                 v_l2 = stod(args[++i]);
107 |             }
108 |             else if(args[i].compare("-core") == 0)
109 |             {
110 |                 if(i == argc - 1)
111 |                     throw invalid_argument("invalid command\n");
112 |                 threads_num = stoi(args[++i]);
113 |             }
114 |             else if(args[i].compare("-im") == 0)
115 |             {
116 |                 if(i == argc - 1)
117 |                     throw invalid_argument("invalid command\n");
118 |                 init_m_path = args[++i];
119 |                 b_init = true; //if im field exits , that means b_init = true !
120 |             }
121 |             else if(args[i].compare("-fvs") == 0)
122 |             {
123 |                 if(i == argc - 1)
124 |                     throw invalid_argument("invalid command\n");
125 |                 int fvs = stoi(args[++i]);
126 |                 force_v_sparse = (1 == fvs) ? true : false;
127 |             }
128 |             else if(args[i].compare("-rank") == 0)
129 |             {
130 |                 if(i == argc - 1)
131 |                     throw invalid_argument("invalid command\n");
132 |                 rank = args[++i];
133 |                 if(rank != "pairwise" && rank != "ndcg")
134 |                     throw invalid_argument("invalid command\n");
135 |             }
136 |             else if(args[i].compare("-fast_mode") == 0)
137 |             {
138 |                 if(i == argc - 1)
139 |                     throw invalid_argument("invalid command\n");
140 |                 int f_m = stoi(args[++i]);
141 |                 fast_mode = (1 == f_m) ? true : false;
142 |             }
143 |             else
144 |             {
145 |                 throw invalid_argument("invalid command\n");
146 |                 break;
147 |             }
148 |         }
149 |     }
150 | 
151 | };
152 | 
153 | 
154 | struct variable_NDCG
155 | {
156 |     vector<double> vecGain;
157 |     vector<double> vecDCG;
158 |     vector<double> vecIdealDCG;
159 |     double DCG;
160 |     double idealDCG;
161 |     void init(int size)
162 |     {
163 |         vecGain.resize(size);
164 |         vecDCG.resize(size);
165 |         vecIdealDCG.resize(size);
166 |     }
167 | };
168 | 
169 | 
170 | class ftrl_trainer : public pc_task
171 | {
172 | public:
173 |     ftrl_trainer(const trainer_option& opt);
174 |     virtual void run_task(vector<vector<fm_sample> >& dataBuffer);
175 |     bool loadModel(ifstream& in);
176 |     void outputModel(ofstream& out);
177 | private:
178 |     void trainGroup(vector<fm_sample>& sampleGrp);
179 |     void trainPair(fm_sample& sample1, fm_sample& sample2, const double deltaNDCG);
180 |     void trainGroupFastMode(vector<fm_sample>& sampleGrp);
181 |     void trainPairFastMode(fm_sample& sample1, fm_sample& sample2, double& pred1, double& pred2,
182 |         map<string, vector<double> >& grad1, map<string, vector<double> >& grad2, const double deltaNDCG);
183 |     //NDCG
184 |     inline double discount(int i);
185 |     inline double gain(int y);
186 |     void calcVariables4NDCG(vector<fm_sample>& sampleGrp, variable_NDCG& v_NDCG);
187 |     inline double calcDeltaNDCG(int i, int j, variable_NDCG& v_NDCG);
188 | private:
189 |     ftrl_model* pModel;
190 |     double w_alpha, w_beta, w_l1, w_l2;
191 |     double v_alpha, v_beta, v_l1, v_l2;
192 |     bool k1;
193 |     bool force_v_sparse;
194 |     string rank;
195 |     bool fast_mode;
196 |     vector<double> vecDiscount;//cache for NDCG
197 |     static const int cacheSize = 5000;//cache size for NDCG
198 | };
199 | 
200 | 
201 | ftrl_trainer::ftrl_trainer(const trainer_option& opt)
202 | {
203 |     w_alpha = opt.w_alpha;
204 |     w_beta = opt.w_beta;
205 |     w_l1 = opt.w_l1;
206 |     w_l2 = opt.w_l2;
207 |     v_alpha = opt.v_alpha;
208 |     v_beta = opt.v_beta;
209 |     v_l1 = opt.v_l1;
210 |     v_l2 = opt.v_l2;
211 |     k1 = opt.k1;
212 |     force_v_sparse = opt.force_v_sparse;
213 |     rank = opt.rank;
214 |     fast_mode = opt.fast_mode;
215 |     pModel = new ftrl_model(opt.factor_num, opt.init_mean, opt.init_stdev);
216 |     vecDiscount.reserve(cacheSize);
217 |     for(int i = 0; i < cacheSize; ++i)
218 |     {
219 |         vecDiscount.push_back(1.0/log2(i + 2.0));// discount(i) = 1/log2(i+1), i from 1 to ... 
220 |     }
221 | }
222 | 
223 | 
224 | void ftrl_trainer::run_task(vector<vector<fm_sample> >& dataBuffer)
225 | {
226 |     for(int i = 0; i < dataBuffer.size(); ++i)
227 |     {
228 |         if(fast_mode)
229 |         {
230 |             trainGroupFastMode(dataBuffer[i]);
231 |         }
232 |         else
233 |         {
234 |             trainGroup(dataBuffer[i]);
235 |         }
236 |     }
237 | }
238 | 
239 | 
240 | void ftrl_trainer::trainGroup(vector<fm_sample>& sampleGrp)
241 | {
242 |     int size = sampleGrp.size();
243 |     variable_NDCG v_NDCG;
244 |     if("ndcg" == rank)
245 |     {
246 |         v_NDCG.init(size);
247 |         calcVariables4NDCG(sampleGrp, v_NDCG);
248 |     }
249 |     for(int i = 0; i < size - 1; ++i)
250 |     {
251 |         for(int j = i + 1; j < size; ++j)
252 |         {
253 |             if(sampleGrp[i].y == sampleGrp[j].y)
254 |             {
255 |                 continue;
256 |             }
257 |             double deltaNDCG = 1.0;
258 |             if("ndcg" == rank)
259 |             {
260 |                 deltaNDCG = calcDeltaNDCG(i, j, v_NDCG);
261 |             }
262 |             if(sampleGrp[i].y > sampleGrp[j].y)
263 |             {
264 |                 trainPair(sampleGrp[i], sampleGrp[j], deltaNDCG);
265 |             }
266 |             else if(sampleGrp[i].y < sampleGrp[j].y)
267 |             {
268 |                 trainPair(sampleGrp[j], sampleGrp[i], deltaNDCG);
269 |             }
270 |         }
271 |     }
272 | }
273 | 
274 | 
275 | void ftrl_trainer::trainPair(fm_sample& sample1, fm_sample& sample2, const double deltaNDCG)
276 | {
277 |     unordered_map<string, ftrl_model_unit*> theta;
278 |     for(unordered_map<string, double>::iterator iter = sample1.x.begin(); iter != sample1.x.end(); ++iter)
279 |     {
280 |         const string& index = iter->first;
281 |         if(theta.find(index) == theta.end())
282 |         {
283 |             theta[index] = pModel->getOrInitModelUnit(index);
284 |         }
285 |     }
286 |     for(unordered_map<string, double>::iterator iter = sample2.x.begin(); iter != sample2.x.end(); ++iter)
287 |     {
288 |         const string& index = iter->first;
289 |         if(theta.find(index) == theta.end())
290 |         {
291 |             theta[index] = pModel->getOrInitModelUnit(index);
292 |         }
293 |     }
294 |     //update w via FTRL
295 |     if(k1)
296 |     {
297 |         for(unordered_map<string, ftrl_model_unit*>::iterator iter = theta.begin(); iter != theta.end(); ++iter)
298 |         {
299 |             ftrl_model_unit& mu = *(iter->second);
300 |             mu.mtx.lock();
301 |             if(fabs(mu.w_zi) <= w_l1)
302 |             {
303 |                 mu.wi = 0.0;
304 |             }
305 |             else
306 |             {
307 |                 if(force_v_sparse && mu.w_ni > 0 && 0.0 == mu.wi)
308 |                 {
309 |                     mu.reinit_vi(pModel->init_mean, pModel->init_stdev);
310 |                 }
311 |                 mu.wi = (-1) *
312 |                     (1 / (w_l2 + (w_beta + sqrt(mu.w_ni)) / w_alpha)) *
313 |                     (mu.w_zi - utils::sgn(mu.w_zi) * w_l1);
314 |             }
315 |             mu.mtx.unlock();
316 |         }
317 |     }
318 |     //update v via FTRL
319 |     for(unordered_map<string, ftrl_model_unit*>::iterator iter = theta.begin(); iter != theta.end(); ++iter)
320 |     {
321 |         ftrl_model_unit& mu = *(iter->second);
322 |         for(int f = 0; f < pModel->factor_num; ++f)
323 |         {
324 |             mu.mtx.lock();
325 |             double& vif = mu.vi[f];
326 |             double& v_nif = mu.v_ni[f];
327 |             double& v_zif = mu.v_zi[f];
328 |             if(v_nif > 0)
329 |             {
330 |                 if(force_v_sparse && 0.0 == mu.wi)
331 |                 {
332 |                     vif = 0.0;
333 |                 }
334 |                 else if(fabs(v_zif) <= v_l1)
335 |                 {
336 |                     vif = 0.0;
337 |                 }
338 |                 else
339 |                 {
340 |                     vif = (-1) *
341 |                         (1 / (v_l2 + (v_beta + sqrt(v_nif)) / v_alpha)) *
342 |                         (v_zif - utils::sgn(v_zif) * v_l1);
343 |                 }
344 |             }
345 |             mu.mtx.unlock();
346 |         }
347 |     }
348 |     vector<double> sum1(pModel->factor_num);
349 |     double p1 = pModel->predict(sample1.x, theta, sum1);
350 |     vector<double> sum2(pModel->factor_num);
351 |     double p2 = pModel->predict(sample2.x, theta, sum2);
352 |     double lambda12 = -1 / (1 + exp(p1 - p2)) * deltaNDCG;
353 |     //update w_n, w_z
354 |     if(k1)
355 |     {
356 |         for(unordered_map<string, ftrl_model_unit*>::iterator iter = theta.begin(); iter != theta.end(); ++iter)
357 |         {
358 |             ftrl_model_unit& mu = *(iter->second);
359 |             const string& index = iter->first;
360 |             unordered_map<string, double>::iterator iter_x1 = sample1.x.find(index);
361 |             double xi1 = (iter_x1 == sample1.x.end()) ? 0 : iter_x1->second;
362 |             unordered_map<string, double>::iterator iter_x2 = sample2.x.find(index);
363 |             double xi2 = (iter_x2 == sample2.x.end()) ? 0 : iter_x2->second;
364 |             double xi = xi1 - xi2;
365 |             if(xi != 0)
366 |             {
367 |                 mu.mtx.lock();
368 |                 double w_gi = lambda12 * xi;
369 |                 double w_si = 1 / w_alpha * (sqrt(mu.w_ni + w_gi * w_gi) - sqrt(mu.w_ni));
370 |                 mu.w_zi += w_gi - w_si * mu.wi;
371 |                 mu.w_ni += w_gi * w_gi;
372 |                 mu.mtx.unlock();
373 |             }
374 |         }
375 |     }
376 |     //update v_n, v_z
377 |     for(unordered_map<string, ftrl_model_unit*>::iterator iter = theta.begin(); iter != theta.end(); ++iter)
378 |     {
379 |         ftrl_model_unit& mu = *(iter->second);
380 |         const string& index = iter->first;
381 |         unordered_map<string, double>::iterator iter_x1 = sample1.x.find(index);
382 |         double xi1 = (iter_x1 == sample1.x.end()) ? 0 : iter_x1->second;
383 |         unordered_map<string, double>::iterator iter_x2 = sample2.x.find(index);
384 |         double xi2 = (iter_x2 == sample2.x.end()) ? 0 : iter_x2->second;
385 |         for(int f = 0; f < pModel->factor_num; ++f)
386 |         {
387 |             mu.mtx.lock();
388 |             double& vif = mu.vi[f];
389 |             double& v_nif = mu.v_ni[f];
390 |             double& v_zif = mu.v_zi[f];
391 |             double v_gif = lambda12 * ((sum1[f] * xi1 - vif * xi1 * xi1) - (sum2[f] * xi2 - vif * xi2 * xi2));
392 |             double v_sif = 1 / v_alpha * (sqrt(v_nif + v_gif * v_gif) - sqrt(v_nif));
393 |             v_zif += v_gif - v_sif * vif;
394 |             v_nif += v_gif * v_gif;
395 |             //有的特征在整个训练集中只出现一次，这里还需要对vif做一次处理
396 |             if(force_v_sparse && v_nif > 0 && 0.0 == mu.wi)
397 |             {
398 |                 vif = 0.0;
399 |             }
400 |             mu.mtx.unlock();
401 |         }
402 |     }
403 | }
404 | 
405 | 
406 | bool ftrl_trainer::loadModel(ifstream& in)
407 | {
408 |     return pModel->loadModel(in);
409 | }
410 | 
411 | 
412 | void ftrl_trainer::outputModel(ofstream& out)
413 | {
414 |     return pModel->outputModel(out);
415 | }
416 | 
417 | 
418 | void ftrl_trainer::trainGroupFastMode(vector<fm_sample>& sampleGrp)
419 | {
420 |     int size = sampleGrp.size();
421 |     variable_NDCG v_NDCG;
422 |     if("ndcg" == rank)
423 |     {
424 |         v_NDCG.init(size);
425 |         calcVariables4NDCG(sampleGrp, v_NDCG);
426 |     }
427 |     int factor_num = pModel->factor_num;
428 |     vector<double> vecPred(size);
429 |     vector<map<string, vector<double> > > vecGradPred2v(size);
430 |     unordered_map<string, ftrl_model_unit*> theta;
431 |     for(int i = 0; i < size; ++i)
432 |     {
433 |         for(unordered_map<string, double>::iterator iter = sampleGrp[i].x.begin(); iter != sampleGrp[i].x.end(); ++iter)
434 |         {
435 |             const string& index = iter->first;
436 |             if(theta.find(index) == theta.end())
437 |             {
438 |                 theta[index] = pModel->getOrInitModelUnit(index);
439 |             }
440 |         }
441 |         vector<double> sum(factor_num);
442 |         vecPred[i] = pModel->predict(sampleGrp[i].x, theta, sum);
443 |         for(unordered_map<string, double>::iterator iter = sampleGrp[i].x.begin(); iter != sampleGrp[i].x.end(); ++iter)
444 |         {
445 |             const string& index = iter->first;
446 |             const double& xi = iter->second;
447 |             vector<double>& vi = theta[index]->vi;
448 |             vecGradPred2v[i][index] = vector<double>(factor_num);
449 |             for(int f = 0; f < factor_num; ++f)
450 |             {
451 |                 vecGradPred2v[i][index][f] = sum[f] * xi - vi[f] * xi * xi;
452 |             }
453 |         }
454 |     }
455 |     for(int i = 0; i < size - 1; ++i)
456 |     {
457 |         for(int j = i + 1; j < size; ++j)
458 |         {
459 |             if(sampleGrp[i].y == sampleGrp[j].y)
460 |             {
461 |                 continue;
462 |             }
463 |             double deltaNDCG = 1.0;
464 |             if("ndcg" == rank)
465 |             {
466 |                 deltaNDCG = calcDeltaNDCG(i, j, v_NDCG);
467 |             }
468 |             if(sampleGrp[i].y > sampleGrp[j].y)
469 |             {
470 |                 trainPairFastMode(sampleGrp[i], sampleGrp[j], vecPred[i], vecPred[j], vecGradPred2v[i], vecGradPred2v[j], deltaNDCG);
471 |             }
472 |             else if(sampleGrp[i].y < sampleGrp[j].y)
473 |             {
474 |                 trainPairFastMode(sampleGrp[j], sampleGrp[i], vecPred[j], vecPred[i], vecGradPred2v[j], vecGradPred2v[i], deltaNDCG);
475 |             }
476 |         }
477 |     }
478 | }
479 | 
480 | 
481 | 
482 | void ftrl_trainer::trainPairFastMode(fm_sample& sample1, fm_sample& sample2, double& pred1, double& pred2,
483 |     map<string, vector<double> >& grad1, map<string, vector<double> >& grad2, const double deltaNDCG)
484 | {
485 |     unordered_map<string, ftrl_model_unit*> theta;
486 |     for(unordered_map<string, double>::iterator iter = sample1.x.begin(); iter != sample1.x.end(); ++iter)
487 |     {
488 |         const string& index = iter->first;
489 |         if(theta.find(index) == theta.end())
490 |         {
491 |             theta[index] = pModel->getOrInitModelUnit(index);
492 |         }
493 |     }
494 |     for(unordered_map<string, double>::iterator iter = sample2.x.begin(); iter != sample2.x.end(); ++iter)
495 |     {
496 |         const string& index = iter->first;
497 |         if(theta.find(index) == theta.end())
498 |         {
499 |             theta[index] = pModel->getOrInitModelUnit(index);
500 |         }
501 |     }
502 |     //update w via FTRL
503 |     if(k1)
504 |     {
505 |         for(unordered_map<string, ftrl_model_unit*>::iterator iter = theta.begin(); iter != theta.end(); ++iter)
506 |         {
507 |             ftrl_model_unit& mu = *(iter->second);
508 |             mu.mtx.lock();
509 |             if(fabs(mu.w_zi) <= w_l1)
510 |             {
511 |                 mu.wi = 0.0;
512 |             }
513 |             else
514 |             {
515 |                 if(force_v_sparse && mu.w_ni > 0 && 0.0 == mu.wi)
516 |                 {
517 |                     mu.reinit_vi(pModel->init_mean, pModel->init_stdev);
518 |                 }
519 |                 mu.wi = (-1) *
520 |                     (1 / (w_l2 + (w_beta + sqrt(mu.w_ni)) / w_alpha)) *
521 |                     (mu.w_zi - utils::sgn(mu.w_zi) * w_l1);
522 |             }
523 |             mu.mtx.unlock();
524 |         }
525 |     }
526 |     //update v via FTRL
527 |     for(unordered_map<string, ftrl_model_unit*>::iterator iter = theta.begin(); iter != theta.end(); ++iter)
528 |     {
529 |         ftrl_model_unit& mu = *(iter->second);
530 |         for(int f = 0; f < pModel->factor_num; ++f)
531 |         {
532 |             mu.mtx.lock();
533 |             double& vif = mu.vi[f];
534 |             double& v_nif = mu.v_ni[f];
535 |             double& v_zif = mu.v_zi[f];
536 |             if(v_nif > 0)
537 |             {
538 |                 if(force_v_sparse && 0.0 == mu.wi)
539 |                 {
540 |                     vif = 0.0;
541 |                 }
542 |                 else if(fabs(v_zif) <= v_l1)
543 |                 {
544 |                     vif = 0.0;
545 |                 }
546 |                 else
547 |                 {
548 |                     vif = (-1) *
549 |                         (1 / (v_l2 + (v_beta + sqrt(v_nif)) / v_alpha)) *
550 |                         (v_zif - utils::sgn(v_zif) * v_l1);
551 |                 }
552 |             }
553 |             mu.mtx.unlock();
554 |         }
555 |     }
556 |     double lambda12 = -1 / (1 + exp(pred1 - pred2)) * deltaNDCG;
557 |     //update w_n, w_z
558 |     if(k1)
559 |     {
560 |         for(unordered_map<string, ftrl_model_unit*>::iterator iter = theta.begin(); iter != theta.end(); ++iter)
561 |         {
562 |             ftrl_model_unit& mu = *(iter->second);
563 |             const string& index = iter->first;
564 |             unordered_map<string, double>::iterator iter_x1 = sample1.x.find(index);
565 |             double xi1 = (iter_x1 == sample1.x.end()) ? 0 : iter_x1->second;
566 |             unordered_map<string, double>::iterator iter_x2 = sample2.x.find(index);
567 |             double xi2 = (iter_x2 == sample2.x.end()) ? 0 : iter_x2->second;
568 |             double xi = xi1 - xi2;
569 |             if(xi != 0)
570 |             {
571 |                 mu.mtx.lock();
572 |                 double w_gi = lambda12 * xi;
573 |                 double w_si = 1 / w_alpha * (sqrt(mu.w_ni + w_gi * w_gi) - sqrt(mu.w_ni));
574 |                 mu.w_zi += w_gi - w_si * mu.wi;
575 |                 mu.w_ni += w_gi * w_gi;
576 |                 mu.mtx.unlock();
577 |             }
578 |         }
579 |     }
580 |     //update v_n, v_z
581 |     vector<double> vecZero(pModel->factor_num, 0);
582 |     for(unordered_map<string, ftrl_model_unit*>::iterator iter = theta.begin(); iter != theta.end(); ++iter)
583 |     {
584 |         ftrl_model_unit& mu = *(iter->second);
585 |         const string& index = iter->first;
586 |         unordered_map<string, double>::iterator iter_x1 = sample1.x.find(index);
587 |         double xi1 = (iter_x1 == sample1.x.end()) ? 0 : iter_x1->second;
588 |         unordered_map<string, double>::iterator iter_x2 = sample2.x.find(index);
589 |         double xi2 = (iter_x2 == sample2.x.end()) ? 0 : iter_x2->second;
590 |         vector<double>* pGrad1Index = &vecZero;
591 |         map<string, vector<double> >::iterator gradIter1 = grad1.find(index);
592 |         if(gradIter1 != grad1.end())
593 |         {
594 |             pGrad1Index = &(gradIter1->second);
595 |         }
596 |         vector<double>* pGrad2Index = &vecZero;
597 |         map<string, vector<double> >::iterator gradIter2 = grad2.find(index);
598 |         if(gradIter2 != grad2.end())
599 |         {
600 |             pGrad2Index = &(gradIter2->second);
601 |         }
602 |         for(int f = 0; f < pModel->factor_num; ++f)
603 |         {
604 |             mu.mtx.lock();
605 |             double& vif = mu.vi[f];
606 |             double& v_nif = mu.v_ni[f];
607 |             double& v_zif = mu.v_zi[f];
608 |             double v_gif = lambda12 * ((*pGrad1Index)[f] - (*pGrad2Index)[f]);
609 |             double v_sif = 1 / v_alpha * (sqrt(v_nif + v_gif * v_gif) - sqrt(v_nif));
610 |             v_zif += v_gif - v_sif * vif;
611 |             v_nif += v_gif * v_gif;
612 |             //有的特征在整个训练集中只出现一次，这里还需要对vif做一次处理
613 |             if(force_v_sparse && v_nif > 0 && 0.0 == mu.wi)
614 |             {
615 |                 vif = 0.0;
616 |             }
617 |             mu.mtx.unlock();
618 |         }
619 |     }
620 | }
621 | 
622 | 
623 | inline double ftrl_trainer::discount(int i)
624 | {
625 |     if(i < cacheSize) return vecDiscount[i];
626 |     return 1.0/log2(i + 2.0);
627 | }
628 | 
629 | 
630 | inline double ftrl_trainer::gain(int y)
631 | {
632 |     return (1<<y) - 1;// gain(y) = 2^y-1
633 | }
634 | 
635 | 
636 | void ftrl_trainer::calcVariables4NDCG(vector<fm_sample>& sampleGrp, variable_NDCG& v_NDCG)
637 | {
638 |     int size = sampleGrp.size();
639 |     v_NDCG.DCG = 0;
640 |     for(int i = 0; i < size; ++i)
641 |     {
642 |         v_NDCG.vecGain[i] = gain(sampleGrp[i].y);
643 |         v_NDCG.vecDCG[i] = v_NDCG.vecGain[i] * discount(i);
644 |         v_NDCG.DCG += v_NDCG.vecDCG[i];
645 |     }
646 |     v_NDCG.vecIdealDCG.assign(v_NDCG.vecGain.begin(), v_NDCG.vecGain.end());
647 |     v_NDCG.idealDCG = 0;
648 |     sort(v_NDCG.vecIdealDCG.begin(), v_NDCG.vecIdealDCG.end(),greater<double>());
649 |     for(int i = 0; i < size; ++i)
650 |     {
651 |         v_NDCG.vecIdealDCG[i] *= discount(i);
652 |         v_NDCG.idealDCG += v_NDCG.vecIdealDCG[i];
653 |     }
654 | }
655 | 
656 | 
657 | inline double ftrl_trainer::calcDeltaNDCG(int i, int j, variable_NDCG& v_NDCG)
658 | {
659 |     double deltaDCG = v_NDCG.vecGain[j] * discount(i) + v_NDCG.vecGain[i] * discount(j) - v_NDCG.vecDCG[i] - v_NDCG.vecDCG[j];
660 |     return fabs(deltaDCG/v_NDCG.idealDCG);
661 | }
662 | 
663 | 
664 | #endif /*FTRL_TRAINER_H_*/
665 | 


--------------------------------------------------------------------------------
/src/Frame/Makefile:
--------------------------------------------------------------------------------
1 | all:
2 | 	g++ test_main.cpp pc_frame.cpp -I . -std=c++0x -o test_main -lpthread
3 | 


--------------------------------------------------------------------------------
/src/Frame/pc_frame.cpp:
--------------------------------------------------------------------------------
  1 | #include "pc_frame.h"
  2 | 
  3 | bool pc_frame::init(pc_task& task, int t_num, int buf_size, int log_num)
  4 | {
  5 |     pTask = &task;
  6 |     threadNum = t_num;
  7 |     bufSize = buf_size;
  8 |     logNum = log_num;
  9 |     sem_init(&semPro, 0, 1);
 10 |     sem_init(&semCon, 0, 0);
 11 |     threadVec.clear();
 12 |     threadVec.push_back(thread(&pc_frame::proThread, this));
 13 |     for(int i = 0; i < threadNum; ++i)
 14 |     {
 15 |         threadVec.push_back(thread(&pc_frame::conThread, this));
 16 |     }
 17 |     return true;
 18 | }
 19 | 
 20 | 
 21 | void pc_frame::run()
 22 | {
 23 |     for(int i = 0; i < threadVec.size(); ++i)
 24 |     {
 25 |         threadVec[i].join();
 26 |     }
 27 | }
 28 | 
 29 | void pc_frame::proThread()
 30 | {
 31 |     string line;
 32 |     int line_num = 0;
 33 |     int i = 0;
 34 |     bool finished_flag = false;
 35 |     bool first_line = true;
 36 |     string last_qid = "";
 37 |     vector<fm_sample> sampleGrp;
 38 |     while(true)
 39 |     {
 40 |         sem_wait(&semPro);
 41 |         bufMtx.lock();
 42 |         for(i = 0; i < bufSize;)
 43 |         {
 44 |             if(!getline(cin, line))
 45 |             {
 46 |                 if(!sampleGrp.empty())
 47 |                 {
 48 |                     buffer.push(sampleGrp);
 49 |                     sampleGrp.clear();
 50 |                 }
 51 |                 finished_flag = true;
 52 |                 break;
 53 |             }
 54 |             line_num++;
 55 |             fm_sample sample(line);
 56 |             if(!first_line && last_qid != sample.qid)
 57 |             {
 58 |                 buffer.push(sampleGrp);
 59 |                 i += sampleGrp.size();
 60 |                 sampleGrp.clear();
 61 |                 last_qid = sample.qid;
 62 |             }
 63 |             sampleGrp.push_back(sample);
 64 |             if(first_line)
 65 |             {
 66 |                 first_line = false;
 67 |                 last_qid = sample.qid;
 68 |             }
 69 |             if(line_num%logNum == 0)
 70 |             {
 71 |                 cout << line_num << " lines have finished" << endl;
 72 |             }
 73 |         }
 74 |         bufMtx.unlock();
 75 |         sem_post(&semCon);
 76 |         if(finished_flag)
 77 |         {
 78 |             break;
 79 |         }
 80 |     }
 81 | }
 82 | 
 83 | 
 84 | void pc_frame::conThread()
 85 | {
 86 |     bool finished_flag = false;
 87 |     vector<vector<fm_sample> > input_vec;
 88 |     while(true)
 89 |     {
 90 |         input_vec.clear();
 91 |         sem_wait(&semCon);
 92 |         bufMtx.lock();
 93 |         for(int i = 0; i < bufSize;)
 94 |         {
 95 |             if(buffer.empty())
 96 |             {
 97 |                 finished_flag = true;
 98 |                 break;
 99 |             }
100 |             input_vec.push_back(buffer.front());
101 |             i += buffer.front().size();
102 |             buffer.pop();
103 |         }
104 |         bufMtx.unlock();
105 |         sem_post(&semPro);
106 |         pTask->run_task(input_vec);
107 |         if(finished_flag)
108 |             break;
109 |     }
110 |     sem_post(&semCon);
111 | }
112 | 
113 | 


--------------------------------------------------------------------------------
/src/Frame/pc_frame.h:
--------------------------------------------------------------------------------
 1 | #ifndef PC_FRAME_H
 2 | #define PC_FRAME_H
 3 | 
 4 | #include <string>
 5 | #include <vector>
 6 | #include <utility>
 7 | #include <unordered_map>
 8 | #include <thread>
 9 | #include <mutex>
10 | #include <queue>
11 | #include <condition_variable>
12 | #include <semaphore.h>
13 | #include <iostream>
14 | #include "pc_task.h"
15 | 
16 | using namespace std;
17 | 
18 | class pc_frame
19 | {
20 | public:
21 |     pc_frame(){}
22 |     bool init(pc_task& task, int t_num, int buf_size = 5000, int log_num = 200000);
23 |     void run();
24 | 
25 | private:
26 |     int threadNum;
27 |     pc_task* pTask;
28 |     mutex bufMtx;
29 |     sem_t semPro, semCon;
30 |     queue<vector<fm_sample> > buffer;
31 |     vector<thread> threadVec;
32 |     int bufSize;
33 |     int logNum;
34 |     void proThread();
35 |     void conThread();
36 | };
37 | 
38 | 
39 | #endif //PC_FRAME_H
40 | 


--------------------------------------------------------------------------------
/src/Frame/pc_task.h:
--------------------------------------------------------------------------------
 1 | #ifndef PC_TASK_H
 2 | #define PC_TASK_H
 3 | 
 4 | #include <vector>
 5 | #include "../Sample/fm_sample.h"
 6 | 
 7 | using std::vector;
 8 | 
 9 | class pc_task
10 | {
11 | public:
12 |     pc_task(){}
13 |     virtual void run_task(vector<vector<fm_sample> >& dataBuffer) = 0;
14 | };
15 | 
16 | 
17 | #endif //PC_TASK_H
18 | 


--------------------------------------------------------------------------------
/src/Frame/test_main.cpp:
--------------------------------------------------------------------------------
 1 | #include "pc_frame.h"
 2 | #include "test_task.h"
 3 | 
 4 | int main()
 5 | {
 6 |     test_task task;
 7 |     pc_frame frame;
 8 |     frame.init(task, 2, 5, 5);
 9 |     frame.run();
10 |     return 0;
11 | }
12 | 
13 | 


--------------------------------------------------------------------------------
/src/Frame/test_task.h:
--------------------------------------------------------------------------------
 1 | #ifndef TEST_TASK_H
 2 | #define TEST_TASK_H
 3 | 
 4 | #include <iostream>
 5 | #include "pc_task.h"
 6 | using namespace std;
 7 | 
 8 | class test_task : public pc_task
 9 | {
10 | public:
11 |     test_task(){}
12 |     virtual void run_task(vector<vector<fm_sample> >& dataBuffer)
13 |     {
14 |         cout << "==========\n";
15 |         for(int i = 0; i < dataBuffer.size(); ++i)
16 |         {
17 |             for(int j = 0; j < dataBuffer[i].size(); ++j)
18 |             {
19 |                 cout << dataBuffer[i][j].y << " " << dataBuffer[i][j].qid << " ";
20 |                 for(int k = 0; k < dataBuffer[i][j].x.size(); ++k)
21 |                 {
22 |                     cout << dataBuffer[i][j].x[k].first << ":" << dataBuffer[i][j].x[k].second << " ";
23 |                 }
24 |                 cout << endl;
25 |             }
26 |         }
27 |         cout << "**********\n";
28 |     }
29 | };
30 | 
31 | 
32 | #endif //TEST_TASK_H
33 | 


--------------------------------------------------------------------------------
/src/Sample/fm_sample.h:
--------------------------------------------------------------------------------
 1 | #ifndef FM_SAMPLE_H_
 2 | #define FM_SAMPLE_H_
 3 | 
 4 | #include <string>
 5 | #include <unordered_map>
 6 | 
 7 | using namespace std;
 8 | 
 9 | const string spliter = " ";
10 | const string innerSpliter = ":";
11 | const char commentTag = '#';
12 | 
13 | class fm_sample
14 | {
15 | public:
16 |     int y;
17 |     string qid;
18 |     unordered_map<string, double> x;
19 | 
20 | public:
21 |     fm_sample(const string& line)
22 |     {
23 |         this->x.clear();
24 |         size_t posb = line.find_first_not_of(spliter, 0);
25 |         size_t pose = line.find_first_of(spliter, posb);
26 |         int label = atoi(line.substr(posb, pose-posb).c_str());
27 |         if(label < 0)
28 |         {
29 |             cout << "wrong line input, label can not be less than 0\n" << line << endl;
30 |             throw "wrong line input";
31 |         }
32 |         this->y = label;
33 |         posb = line.find_first_not_of(spliter, pose);
34 |         pose = line.find_first_of(spliter, posb);
35 |         this->qid = line.substr(posb, pose-posb);
36 |         string key;
37 |         double value;
38 |         while(pose < line.size())
39 |         {
40 |             posb = line.find_first_not_of(spliter, pose);
41 |             if(posb == string::npos)
42 |             {
43 |                 break;
44 |             }
45 |             if(commentTag == line[posb])
46 |             {
47 |                 break;
48 |             }
49 |             pose = line.find_first_of(innerSpliter, posb);
50 |             if(pose == string::npos)
51 |             {
52 |                 cout << "wrong line input\n" << line << endl;
53 |                 throw "wrong line input";
54 |             }
55 |             key = line.substr(posb, pose-posb);
56 |             posb = pose + 1;
57 |             if(posb >= line.size())
58 |             {
59 |                 cout << "wrong line input\n" << line << endl;
60 |                 throw "wrong line input";
61 |             }
62 |             pose = line.find_first_of(spliter, posb);
63 |             value = stod(line.substr(posb, pose-posb));
64 |             if(value != 0)
65 |             {
66 |                 if(this->x.find(key) == this->x.end())
67 |                 {
68 |                     this->x[key] = value;
69 |                 }
70 |                 else
71 |                 {
72 |                     this->x[key] += value;
73 |                 }
74 |             }
75 |         }
76 |     }
77 | };
78 | 
79 | #endif /*FM_SAMPLE_H_*/
80 | 


--------------------------------------------------------------------------------
/src/Utils/utils.cpp:
--------------------------------------------------------------------------------
 1 | #include <cmath>
 2 | #include <stdlib.h>
 3 | #include "utils.h"
 4 | 
 5 | const double kPrecision = 0.0000000001;
 6 | void utils::splitString(string& line, char delimiter, vector<string>* r)
 7 | {
 8 |     int begin = 0;
 9 |     for(int i = 0; i < line.size(); ++i)
10 |     {
11 |         if(line[i] == delimiter)
12 |         {
13 |             (*r).push_back(line.substr(begin, i - begin));
14 |             begin = i + 1;
15 |         }
16 |     }
17 |     if(begin < line.size())
18 |         (*r).push_back(line.substr(begin, line.size() - begin));
19 | }
20 | 
21 | 
22 | int utils::sgn(double x) 
23 | {
24 |     if(x > kPrecision)
25 |         return 1;
26 |     else
27 |         return -1;
28 | }
29 | 
30 | 
31 | double utils::uniform()
32 | {
33 |     return rand()/((double)RAND_MAX + 1.0);
34 | }
35 | 
36 | 
37 | double utils::gaussian() 
38 | {
39 |     double u,v, x, y, Q;
40 |     do
41 |     {
42 |         do 
43 |         {
44 |             u = uniform();
45 |         } while (u == 0.0); 
46 | 
47 |         v = 1.7156 * (uniform() - 0.5);
48 |         x = u - 0.449871;
49 |         y = fabs(v) + 0.386595;
50 |         Q = x * x + y * (0.19600 * y - 0.25472 * x);
51 |     } while (Q >= 0.27597 && (Q > 0.27846 || v * v > -4.0 * u * u * log(u)));
52 |     return v / u;
53 | }
54 | 
55 | 
56 | double utils::gaussian(double mean, double stdev) {
57 |     if(0.0 == stdev)
58 |     {
59 |         return mean;
60 |     }
61 |     else
62 |     {
63 |         return mean + stdev * gaussian();
64 |     }
65 | }
66 | 
67 | 


--------------------------------------------------------------------------------
/src/Utils/utils.h:
--------------------------------------------------------------------------------
 1 | #ifndef FTRL_UTILS_H
 2 | #define FTRL_UTILS_H
 3 | 
 4 | 
 5 | #include <string>
 6 | #include <vector>
 7 | using namespace std;
 8 | 
 9 | class utils 
10 | {
11 | public:
12 |     void static splitString(string& line, char delimiter, vector<string>* r);
13 |     int static sgn(double x);
14 |     double static uniform();
15 |     double static gaussian();
16 |     double static gaussian(double mean, double stdev);
17 | };
18 | 
19 | 
20 | #endif //FTRL_UTILS_H
21 | 


--------------------------------------------------------------------------------