├── FFM
    ├── model
    │   ├── gbdt
    │   │   ├── gbdt
    │   │   ├── src
    │   │   │   ├── timer.h
    │   │   │   ├── timer.cpp
    │   │   │   ├── gbdt.h
    │   │   │   ├── common.h
    │   │   │   ├── train.cpp
    │   │   │   ├── common.cpp
    │   │   │   └── gbdt.cpp
    │   │   ├── Makefile
    │   │   └── README
    │   └── libffm-1.13
    │   │   ├── ffm-train
    │   │   ├── ffm-predict
    │   │   ├── Makefile
    │   │   ├── Makefile.win
    │   │   ├── COPYRIGHT
    │   │   ├── ffm.h
    │   │   ├── ffm-predict.cpp
    │   │   ├── ffm-train.cpp
    │   │   ├── README
    │   │   └── ffm.cpp
    ├── utils
    │   ├── make_submission.py
    │   ├── pre-a.py
    │   ├── pre-b.py
    │   └── data_helpers.py
    ├── run.py
    └── README.md
├── .gitignore
├── CatBoost
    ├── model
    │   └── cb.py
    └── utils
    │   └── data_helpers.py
├── README.md
├── NN
    └── input
    │   ├── validation
    │       └── script
    │       │   └── isrc_process.py
    │   └── training
    │       └── script
    │           ├── isrc_process.py
    │           ├── cnt_log_process.py
    │           └── id_process.py
├── LightGBM
    ├── utils
    │   └── data_helpers.py
    └── model
    │   └── lbgm.py
├── XGBoost
    ├── utils
    │   └── data_helpers.py
    └── model
    │   └── xgb.py
└── LICENSE


/FFM/model/gbdt/gbdt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RandolphVI/Music-Recommendation-System/HEAD/FFM/model/gbdt/gbdt


--------------------------------------------------------------------------------
/FFM/model/libffm-1.13/ffm-train:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RandolphVI/Music-Recommendation-System/HEAD/FFM/model/libffm-1.13/ffm-train


--------------------------------------------------------------------------------
/FFM/model/libffm-1.13/ffm-predict:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RandolphVI/Music-Recommendation-System/HEAD/FFM/model/libffm-1.13/ffm-predict


--------------------------------------------------------------------------------
/FFM/model/gbdt/src/timer.h:
--------------------------------------------------------------------------------
 1 | #include <chrono>
 2 | 
 3 | class Timer
 4 | {
 5 | public:
 6 |     Timer();
 7 |     void reset();
 8 |     void tic();
 9 |     float toc();
10 |     float get();
11 | private:
12 |     std::chrono::high_resolution_clock::time_point begin;
13 |     std::chrono::milliseconds duration;
14 | };
15 | 


--------------------------------------------------------------------------------
/FFM/model/gbdt/Makefile:
--------------------------------------------------------------------------------
 1 | CXX = g++-7
 2 | CXXFLAGS = -Wall -Wconversion -O2 -fPIC -std=c++0x -march=native -fopenmp
 3 | MAIN = gbdt
 4 | FILES = common.cpp timer.cpp gbdt.cpp
 5 | SRCS = $(FILES:%.cpp=src/%.cpp)
 6 | HEADERS = $(FILES:%.cpp=src/%.h)
 7 | 
 8 | all: $(MAIN)
 9 | 
10 | gbdt: src/train.cpp $(SRCS) $(HEADERS)
11 | 	$(CXX) $(CXXFLAGS) -o $@ $< $(SRCS)
12 | 
13 | clean:
14 | 	rm -f $(MAIN)
15 | 


--------------------------------------------------------------------------------
/FFM/model/libffm-1.13/Makefile:
--------------------------------------------------------------------------------
 1 | CXX = g++-7
 2 | CXXFLAGS = -Wall -O3 -std=c++0x -march=native
 3 | 
 4 | # comment the following flags if you do not want to use OpenMP
 5 | DFLAG += -DUSEOMP
 6 | CXXFLAGS += -fopenmp
 7 | 
 8 | all: ffm-train ffm-predict
 9 | 
10 | ffm-train: ffm-train.cpp ffm.o
11 | 	$(CXX) $(CXXFLAGS) -o $@ $^
12 | 
13 | ffm-predict: ffm-predict.cpp ffm.o
14 | 	$(CXX) $(CXXFLAGS) -o $@ $^
15 | 
16 | ffm.o: ffm.cpp ffm.h
17 | 	$(CXX) $(CXXFLAGS) $(DFLAG) -c -o $@ $<
18 | 
19 | clean:
20 | 	rm -f ffm-train ffm-predict ffm.o
21 | 


--------------------------------------------------------------------------------
/FFM/utils/make_submission.py:
--------------------------------------------------------------------------------
 1 | # -*- coding:utf-8 -*-
 2 | __author__ = 'Randolph'
 3 | 
 4 | import argparse
 5 | import sys
 6 | 
 7 | if len(sys.argv) == 1:
 8 |     sys.argv.append('-h')
 9 | 
10 | parser = argparse.ArgumentParser(description='process some integers')
11 | parser.add_argument('out_path', type=str)
12 | parser.add_argument('submission_path', type=str)
13 | args = parser.parse_args()
14 | 
15 | OUT_PATH, SUB_PATH = args.out_path, args.submission_path
16 | 
17 | with open(SUB_PATH, 'w') as f:
18 |     f.write('id,target\n')
19 |     for i, line in enumerate(open(OUT_PATH)):
20 |         f.write('{0},{1}'.format(i, line))
21 | 


--------------------------------------------------------------------------------
/FFM/model/libffm-1.13/Makefile.win:
--------------------------------------------------------------------------------
 1 | CXX = cl.exe
 2 | CFLAGS = /nologo /O2 /EHsc /D "_CRT_SECURE_NO_DEPRECATE" /D "USEOMP" /openmp
 3 | 
 4 | TARGET = windows
 5 | 
 6 | all: $(TARGET) $(TARGET)\ffm-train.exe $(TARGET)\ffm-predict.exe
 7 | 
 8 | $(TARGET)\ffm-predict.exe: ffm.h ffm-predict.cpp ffm.obj
 9 | 	$(CXX) $(CFLAGS) ffm-predict.cpp ffm.obj -Fe$(TARGET)\ffm-predict.exe
10 | 
11 | $(TARGET)\ffm-train.exe: ffm.h ffm-train.cpp ffm.obj
12 | 	$(CXX) $(CFLAGS) ffm-train.cpp ffm.obj -Fe$(TARGET)\ffm-train.exe
13 | 
14 | ffm.obj: ffm.cpp ffm.h
15 | 	$(CXX) $(CFLAGS) -c ffm.cpp
16 | 
17 | .PHONY: $(TARGET)
18 | $(TARGET):
19 | 	-mkdir $(TARGET)
20 | 
21 | clean:
22 | 	-erase /Q *.obj *.exe $(TARGET)\. 
23 | 	-rd $(TARGET)
24 | 


--------------------------------------------------------------------------------
/FFM/model/gbdt/src/timer.cpp:
--------------------------------------------------------------------------------
 1 | #include <string>
 2 | #include "timer.h"
 3 | 
 4 | Timer::Timer()
 5 | {
 6 |     reset();
 7 | }
 8 | 
 9 | void Timer::reset()
10 | {
11 |     begin = std::chrono::high_resolution_clock::now();
12 |     duration = 
13 |         std::chrono::duration_cast<std::chrono::milliseconds>(begin-begin);
14 | }
15 | 
16 | void Timer::tic()
17 | {
18 |     begin = std::chrono::high_resolution_clock::now();
19 | }
20 | 
21 | float Timer::toc()
22 | {
23 |     duration += std::chrono::duration_cast<std::chrono::milliseconds>
24 |                     (std::chrono::high_resolution_clock::now()-begin);
25 |     return (float)duration.count()/1000;
26 | }
27 | 
28 | float Timer::get()
29 | {
30 |     float time = toc();
31 |     tic();
32 |     return time;
33 | }
34 | 


--------------------------------------------------------------------------------
/FFM/model/gbdt/src/gbdt.h:
--------------------------------------------------------------------------------
 1 | #include <vector>
 2 | #include <memory>
 3 | #include <mutex>
 4 | 
 5 | #include "common.h"
 6 | 
 7 | struct TreeNode
 8 | {
 9 |     TreeNode() : idx(0), feature(-1), threshold(0), gamma(0) {} 
10 |     uint32_t idx;
11 |     int32_t feature;
12 |     float threshold, gamma;
13 | };
14 | 
15 | class CART 
16 | {
17 | public:
18 |     CART() : tnodes(max_tnodes)
19 |     {
20 |         for(uint32_t i = 1; i <= max_tnodes; ++i)
21 |             tnodes[i].idx = i;
22 |     }
23 |     void fit(Problem const &prob, std::vector<float> const &R, 
24 |         std::vector<float> &F1);
25 |     std::pair<uint32_t, float> predict(float const * const x) const;
26 | 
27 |     static uint32_t max_depth, max_tnodes;
28 | 
29 | private:
30 |     static std::mutex mtx;
31 |     static bool verbose;
32 |     std::vector<TreeNode> tnodes;
33 | };
34 | 
35 | class GBDT
36 | {
37 | public:
38 |     GBDT(uint32_t const nr_tree) : trees(nr_tree), bias(0) {}
39 |     void fit(Problem const &Tr, Problem const &Va);
40 |     float predict(float const * const x) const;
41 |     std::vector<uint32_t> get_indices(float const * const x) const;
42 | 
43 | private:
44 |     std::vector<CART> trees;
45 |     float bias;
46 | };
47 | 


--------------------------------------------------------------------------------
/FFM/model/gbdt/README:
--------------------------------------------------------------------------------
 1 | Data Format
 2 | ===========
 3 | The input of this GBDT solver consists of a label vector (y), a dense matrix
 4 | (XD), and a binary sparse matrix (XS). The input format of these two matrices
 5 | are introduced in the following two sections.
 6 | 
 7 | Dense Matrix
 8 | ------------
 9 | The input format is:
10 | 
11 | <label> <value_1> <value_2> ... 
12 | .
13 | .
14 | .
15 | 
16 | Note that to represent a dense matrix, we do not have to give indices. For
17 | example,
18 | 
19 | 1 32 91 27 44
20 | 0 13 25 55 83
21 | 0 32 11 78 99
22 | 
23 | represents:
24 | 
25 | y        XD
26 | 1   32 91 27 44
27 | 0   13 25 55 83
28 | 0   32 11 78 99
29 | 
30 | Binary Sparse Matrix
31 | --------------------
32 | The input format is:
33 | 
34 | <label> <index_1> <index_2> ... 
35 | .
36 | .
37 | .
38 | 
39 | To represent a binary sparse matrix, we only need to know where non-zero
40 | elements are, so values are not specified.
41 | 
42 | For example, 
43 | 
44 | 1 2 9 5
45 | 0 1 3 7
46 | 0 4 8 2
47 | 
48 | represents:
49 | 
50 | y          XS
51 | 1   0 1 0 0 1 0 0 0 1
52 | 0   1 0 1 0 0 0 1 0 0
53 | 0   0 1 0 1 0 0 0 1 0
54 | 
55 | Note that the labels in binary sparse matrix are just dummies. They do not have
56 | pratical use; please specify correct labels in dense matrix.
57 | 


--------------------------------------------------------------------------------
/FFM/model/libffm-1.13/COPYRIGHT:
--------------------------------------------------------------------------------
 1 | 
 2 | Copyright (c) 2015 The LIBFFM Project.
 3 | All rights reserved.
 4 | 
 5 | Redistribution and use in source and binary forms, with or without
 6 | modification, are permitted provided that the following conditions
 7 | are met:
 8 | 
 9 | 1. Redistributions of source code must retain the above copyright
10 | notice, this list of conditions and the following disclaimer.
11 | 
12 | 2. Redistributions in binary form must reproduce the above copyright
13 | notice, this list of conditions and the following disclaimer in the
14 | documentation and/or other materials provided with the distribution.
15 | 
16 | 3. Neither name of copyright holders nor the names of its contributors
17 | may be used to endorse or promote products derived from this software
18 | without specific prior written permission.
19 | 
20 | 
21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 | ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 | A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR
25 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
26 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | ### Compiled source ###
  2 | *.com
  3 | *.class
  4 | *.dll
  5 | *.exe
  6 | *.o
  7 | *.so
  8 | 
  9 | ### Packages ###
 10 | # it's better to unpack these files and commit the raw source
 11 | # git has its own built in compression methods
 12 | *.7z
 13 | *.dmg
 14 | *.gz
 15 | *.iso
 16 | *.jar
 17 | *.rar
 18 | *.tar
 19 | *.zip
 20 | 
 21 | ### Logs and databases ###
 22 | *.log
 23 | *.sql
 24 | *.sqlite
 25 | 
 26 | ### Mac OS generated files ###
 27 | .DS_Store
 28 | .DS_Store?
 29 | ._*
 30 | .Spotlight-V100
 31 | .Trashes
 32 | ehthumbs.db
 33 | Thumbs.db
 34 | 
 35 | ### JetBrain config files ###
 36 | .idea
 37 | 
 38 | ### Python ###
 39 | # Byte-compiled / optimized / DLL files
 40 | *.npy
 41 | runs/
 42 | __pycache__/
 43 | *.py[cod]
 44 | *$py.class
 45 | 
 46 | # Distribution / packaging
 47 | .Python
 48 | env/
 49 | build/
 50 | develop-eggs/
 51 | dist/
 52 | downloads/
 53 | eggs/
 54 | .eggs/
 55 | lib/
 56 | lib64/
 57 | parts/
 58 | sdist/
 59 | var/
 60 | *.egg-info/
 61 | .installed.cfg
 62 | *.egg
 63 | 
 64 | # PyInstaller
 65 | #  Usually these files are written by a python script from a template
 66 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 67 | *.manifest
 68 | *.spec
 69 | 
 70 | # Installer logs
 71 | pip-log.txt
 72 | pip-delete-this-directory.txt
 73 | 
 74 | # Unit test / coverage reports
 75 | htmlcov/
 76 | .tox/
 77 | .coverage
 78 | .coverage.*
 79 | .cache
 80 | nosetests.xml
 81 | coverage.xml
 82 | *,cover
 83 | 
 84 | # Translations
 85 | *.mo
 86 | *.pot
 87 | 
 88 | # Django stuff:
 89 | *.log
 90 | 
 91 | # Sphinx documentation
 92 | docs/_build/
 93 | 
 94 | # PyBuilder
 95 | target/
 96 | 
 97 | 
 98 | ### IPythonNotebook ###
 99 | # Temporary data
100 | .ipynb_checkpoints/
101 | 
102 | ###Private Data info###
103 | *.txt
104 | *.rtf
105 | *.png
106 | *.graffle
107 | *.json
108 | *.csv
109 | *.numbers
110 | *.model
111 | 
112 | *.sparse
113 | *.dense
114 | *.out
115 | *.ffm
116 | *.hdf
117 | 
118 | randolph
119 | temp.py
120 | Icon?


--------------------------------------------------------------------------------
/FFM/model/gbdt/src/common.h:
--------------------------------------------------------------------------------
 1 | #pragma GCC diagnostic ignored "-Wunused-result"
 2 | 
 3 | #ifndef _COMMON_H_
 4 | #define _COMMON_H_
 5 | 
 6 | #define flag { printf("\nLINE: %d\n", __LINE__); fflush(stdout); }
 7 | 
 8 | #include <cstdio>
 9 | #include <string>
10 | #include <vector>
11 | #include <cmath>
12 | 
13 | #include <pmmintrin.h>
14 | 
15 | struct Node
16 | {
17 |     Node() : i(0), v(0) {}
18 |     Node(uint32_t const i, float const v) : i(i), v(v) {}
19 |     uint32_t i;
20 |     float v;
21 | };
22 | 
23 | struct Problem
24 | {
25 |     Problem() : nr_instance(0), nr_field(0), nr_sparse_field(0) {}
26 |     Problem(uint32_t const nr_instance, uint32_t const nr_field) 
27 |         : nr_instance(nr_instance), nr_field(nr_field), nr_sparse_field(0),
28 |           X(nr_field, std::vector<Node>(nr_instance)), 
29 |           Z(nr_field, std::vector<Node>(nr_instance)), 
30 |           Y(nr_instance) {}
31 |     uint32_t const nr_instance, nr_field;
32 |     uint32_t nr_sparse_field;
33 |     std::vector<std::vector<Node>> X, Z;
34 |     std::vector<uint32_t> SI, SJ;
35 |     std::vector<uint64_t> SIP, SJP;
36 |     std::vector<float> Y;
37 | };
38 | 
39 | inline std::vector<float> 
40 | construct_instance(Problem const &prob, uint32_t const i)
41 | {
42 |     uint32_t const nr_field = prob.nr_field; 
43 |     uint32_t const nr_sparse_field = prob.nr_sparse_field;
44 |     std::vector<uint32_t> const &SJ = prob.SJ;
45 |     std::vector<uint64_t> const &SJP = prob.SJP;
46 | 
47 |     std::vector<float> x(nr_field+nr_sparse_field, 0);
48 |     for(uint32_t j = 0; j < prob.nr_field; ++j)
49 |         x[j] = prob.Z[j][i].v;
50 |     for(uint64_t p = SJP[i]; p < SJP[i+1]; ++p)
51 |         x[SJ[p]+nr_field] = 1;
52 | 
53 |     return x;
54 | }
55 | 
56 | Problem read_data(std::string const &dense_path, 
57 |     std::string const &sparse_path);
58 | 
59 | FILE *open_c_file(std::string const &path, std::string const &mode);
60 | 
61 | std::vector<std::string> 
62 | argv_to_args(int const argc, char const * const * const argv);
63 | 
64 | #endif // _COMMON_H_
65 | 


--------------------------------------------------------------------------------
/FFM/model/libffm-1.13/ffm.h:
--------------------------------------------------------------------------------
 1 | #ifndef _LIBFFM_H
 2 | #define _LIBFFM_H
 3 | 
 4 | #ifdef __cplusplus
 5 | extern "C" 
 6 | {
 7 | 
 8 | namespace ffm
 9 | {
10 | #endif
11 | 
12 | typedef float ffm_float;
13 | typedef double ffm_double;
14 | typedef int ffm_int;
15 | typedef long long ffm_long;
16 | 
17 | struct ffm_node
18 | {
19 |     ffm_int f;
20 |     ffm_int j;
21 |     ffm_float v;
22 | };
23 | 
24 | struct ffm_problem
25 | {
26 |     ffm_int n;
27 |     ffm_int l;
28 |     ffm_int m;
29 |     ffm_node *X;
30 |     ffm_long *P;
31 |     ffm_float *Y;
32 | };
33 | 
34 | struct ffm_model
35 | {
36 |     ffm_int n;
37 |     ffm_int m;
38 |     ffm_int k;
39 |     ffm_float *W;
40 |     bool normalization;
41 | };
42 | 
43 | struct ffm_parameter
44 | {
45 |     ffm_float eta;
46 |     ffm_float lambda;
47 |     ffm_int nr_iters;
48 |     ffm_int k;
49 |     ffm_int nr_threads;
50 |     bool quiet;
51 |     bool normalization;
52 |     bool random;
53 |     bool auto_stop;
54 | };
55 | 
56 | ffm_problem* ffm_read_problem(char const *path);
57 | 
58 | int ffm_read_problem_to_disk(char const *txt_path, char const *bin_path);
59 | 
60 | void ffm_destroy_problem(struct ffm_problem **prob);
61 | 
62 | ffm_int ffm_save_model(ffm_model *model, char const *path);
63 | 
64 | ffm_model* ffm_load_model(char const *path);
65 | 
66 | void ffm_destroy_model(struct ffm_model **model);
67 | 
68 | ffm_parameter ffm_get_default_param();
69 | 
70 | ffm_model* ffm_train_with_validation(struct ffm_problem *Tr, struct ffm_problem *Va, struct ffm_parameter param);
71 | 
72 | ffm_model* ffm_train(struct ffm_problem *prob, struct ffm_parameter param);
73 | 
74 | ffm_model* ffm_train_with_validation_on_disk(char const *Tr_path, char const *Va_path, struct ffm_parameter param);
75 | 
76 | ffm_model* ffm_train_on_disk(char const *path, struct ffm_parameter param);
77 | 
78 | ffm_float ffm_predict(ffm_node *begin, ffm_node *end, ffm_model *model);
79 | 
80 | ffm_float ffm_cross_validation(struct ffm_problem *prob, ffm_int nr_folds, struct ffm_parameter param);
81 | 
82 | #ifdef __cplusplus
83 | } // namespace ffm
84 | 
85 | } // extern "C"
86 | #endif
87 | 
88 | #endif // _LIBFFM_H
89 | 


--------------------------------------------------------------------------------
/FFM/run.py:
--------------------------------------------------------------------------------
 1 | # -*- coding:utf-8 -*-
 2 | 
 3 | import subprocess
 4 | import time
 5 | 
 6 | start = time.time()
 7 | 
 8 | # cmd = 'python3 utils/count.py data/train.csv > data/fc.trva.top.txt'
 9 | # subprocess.call(cmd, shell=True)
10 | 
11 | # cmd = 'python3 utils/data_helpers.py data/fc.trva.top.txt'
12 | # subprocess.call(cmd, shell=True)
13 | 
14 | # cmd = 'python3 utils/pre-a.py data/train.csv data/tr.gbdt.dense data/tr.gbdt.sparse'
15 | # subprocess.call(cmd, shell=True)
16 | 
17 | # cmd = 'python3 utils/pre-a.py data/test.csv data/te.gbdt.dense data/te.gbdt.sparse'
18 | # subprocess.call(cmd, shell=True)
19 | 
20 | # gbdt make
21 | # cmd = 'make -C model/gbdt'
22 | # subprocess.call(cmd, shell=True)
23 | 
24 | # cmd = 'model/gbdt/gbdt -t 35 -s 1 data/te.gbdt.dense data/te.gbdt.sparse data/tr.gbdt.dense data/tr.gbdt.sparse data/te.gbdt.out data/tr.gbdt.out'
25 | # subprocess.call(cmd, shell=True)
26 | 
27 | # cmd = 'python3 utils/pre-b.py data/train.csv data/tr.gbdt.out data/tr.ffm'
28 | # subprocess.call(cmd, shell=True)
29 | 
30 | # cmd = 'python3 utils/pre-b.py data/test.csv data/te.gbdt.out data/te.ffm'
31 | # subprocess.call(cmd, shell=True)
32 | 
33 | # libffm
34 | # `ffm-train'
35 | # usage: ffm-train [options] training_set_file [model_file]
36 | # options:
37 | # -l <lambda>: set regularization parameter (default 0.00002)
38 | # -k <factor>: set number of latent factors (default 4)
39 | # -t <iteration>: set number of iterations (default 15)
40 | # -r <eta>: set learning rate (default 0.2)
41 | # -s <nr_threads>: set number of threads (default 1)
42 | # -p <path>: set path to the validation set
43 | # --quiet: quiet model (no output)
44 | # --no-norm: disable instance-wise normalization
45 | # --auto-stop: stop at the iteration that achieves the best validation loss (must be used with -p)
46 | 
47 | cmd = 'model/libffm-1.13/ffm-train -k 8 -t 30 -s 2 --auto-stop -p data/te.ffm data/tr.ffm data/model'
48 | subprocess.call(cmd, shell=True)
49 | 
50 | # cmd = 'model/libffm-1.13/ffm-predict data/te.ffm data/model data/te.out'
51 | # subprocess.call(cmd, shell=True)
52 | 
53 | # cmd = 'python3 utils/make_submission.py data/te.out data/submission.csv'
54 | # subprocess.call(cmd, shell=True)
55 | 
56 | print('time used = {0:.0f} sec'.format(time.time()-start))
57 | 


--------------------------------------------------------------------------------
/CatBoost/model/cb.py:
--------------------------------------------------------------------------------
 1 | # -*- coding:utf-8 -*-
 2 | 
 3 | import os
 4 | import time
 5 | import xgboost
 6 | import catboost
 7 | 
 8 | # from catboost import CatBoostClassifier
 9 | from utils import data_helpers as dh
10 | 
11 | data_path = '../data/'
12 | result_path = '../result/'
13 | logs_path = '../logs/'
14 | 
15 | logger = dh.logger_fn('catlog', (logs_path + 'cat-{}.log').format(time.asctime()))
16 | 
17 | 
18 | def main():
19 |     # Preprocessing Data
20 |     # ==================================================
21 |     logger.info('Preprocessing data...')
22 | 
23 |     if os.path.exists(data_path + 'train.hdf') and os.path.exists(data_path + 'test.hdf'):
24 |         pass
25 |     else:
26 |         dh.preprocess()
27 | 
28 |     logger.info('Done preprocessing...')
29 | 
30 |     # Creating Features
31 |     # ==================================================
32 |     logger.info('Creating Features...')
33 | 
34 |     file = ['Xtrain0.hdf', 'Xtrain1.hdf', 'ytrain0.hdf', 'ytrain1.hdf', 'Xtest.hdf']
35 |     for item in file:
36 |         if os.path.exists(data_path + item):
37 |             pass
38 |         else:
39 |             dh.create_features()
40 |             break
41 | 
42 |     logger.info('Done creating...')
43 | 
44 |     # Fitting
45 |     # ==================================================
46 |     logger.info('Training XGBoost & CatBoost model...')
47 | 
48 |     model_xgb = xgboost.XGBClassifier(
49 |         learning_rate=0.03,
50 |         max_depth=7,
51 |         nthread=50,
52 |         seed=1,
53 |         n_estimators=750
54 |     )
55 | 
56 |     model_cb = catboost.CatBoostClassifier(
57 |         iterations=2000,
58 |         learning_rate=0.03,
59 |         depth=7,
60 |         loss_function='Logloss',
61 |         thread_count=50,
62 |         random_seed=1
63 |     )
64 | 
65 |     dh.train_model(model_xgb, model_cb)
66 |     logger.info('Done training...')
67 | 
68 |     # Model Blending
69 |     # ==================================================
70 |     logger.info('Start models blending...')
71 | 
72 |     p = dh.blend(pct1=0.6, pct2=0.4)
73 | 
74 |     dh.make_submission(data=p)
75 | 
76 |     logger.info('Done saving model predictions...')
77 |     logger.info('All Finished!')
78 | 
79 | 
80 | if __name__ == '__main__':
81 |     main()
82 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # ALL WE NEED IS Tree 🌲
 2 | 
 3 | 该项目代码为本人首次参加 Kaggle 比赛的模型代码。参加的 Kaggle 比赛内容为[音乐推荐系统](https://www.kaggle.com/c/kkbox-music-recommendation-challenge)。比赛的任务要求如下：
 4 | 
 5 | >  The main objective of the project is to predict the chances of a user listening to a song repetitively after the first observable listening event within a time window was triggered. If there are recurring listening event(s) triggered within a month after the user’s very first observable listening event, its target is marked 1, and 0 otherwise in the training set. The same rule applies to the testing set.
 6 | 
 7 | 在比赛结束时，我所在的队伍在 Public Leaderboard 排名为 62 名，在 Private Leaderboard 排名为 59 名（参赛队伍为 1172 支）。
 8 | 
 9 | ![](https://farm5.staticflickr.com/4727/25262275988_b4b3986aef_o.png)
10 | 
11 | ## Requirements
12 | 
13 | - Python 3.x
14 | - Numpy
15 | - **CatBoost**
16 | - **XGBoost**
17 | - **LightBGM**
18 | - **GBDT**
19 | - **Libffm**
20 | - **g++ (with C++11 and OpenMP support)**
21 | 
22 | ## Dataset
23 | 
24 | [Get the research data](https://www.kaggle.com/c/kkbox-music-recommendation-challenge)
25 | 
26 | 数据来源为 Kaggle 上的比赛数据集，内容为音乐推荐。
27 | 
28 | ## How to Kick the Ass 👾
29 | 
30 | 
31 | ### Step 1 & 2: EDA & FE
32 | Exploratory Data Analysis 以及 Feature Engineering 的工作部分在此不进行展开，如果对如何系统合理地处理数据感兴趣，可以参考我个人博客中的该篇文章[「Music Recommendation Challenge」](http://randolph.pro/2017/12/17/%E3%80%8CKaggle%E3%80%8DMusic%20Recommendation%20Challenge/)。
33 | 
34 | ### Step 3: Choose the Model
35 | 
36 | #### LIBFFM + GBDT
37 | 
38 | This model is called **Field-aware Factorization Machines**. If you want to use this model, please download [LIBFFM](http://www.csie.ntu.edu.tw/~r01922136/libffm) first.
39 | 
40 | #### CatBoost
41 | 
42 | 
43 | #### XGBoost
44 | 
45 | #### LightBGM
46 | 
47 | 
48 | 
49 | 
50 | ## About Me
51 | 
52 | 黄威，Randolph
53 | 
54 | SCU SE Bachelor; USTC CS Ph.D.
55 | 
56 | Email: chinawolfman@hotmail.com
57 | 
58 | My Blog: [randolph.pro](http://randolph.pro)
59 | 
60 | LinkedIn: [randolph's linkedin](https://www.linkedin.com/in/randolph-%E9%BB%84%E5%A8%81/)
61 | 
62 | ## Reference
63 | 
64 | - [Field-aware Factorization Machines for CTR Prediction](http://ntucsu.csie.ntu.edu.tw/~cjlin/papers/ffm.pdf)
65 | - [Field-aware Factorization Machines in a Real-world Online Advertising System](https://arxiv.org/pdf/1701.04099.pdf)
66 | 


--------------------------------------------------------------------------------
/FFM/utils/pre-a.py:
--------------------------------------------------------------------------------
 1 | # -*- coding:utf-8 -*-
 2 | __author__ = 'Randolph'
 3 | 
 4 | import csv
 5 | import sys
 6 | import argparse
 7 | from utils import data_helpers as dh
 8 | 
 9 | 
10 | if len(sys.argv) == 1:
11 |     sys.argv.append('-h')
12 | 
13 | parser = argparse.ArgumentParser()
14 | parser.add_argument('csv_path', type=str)
15 | parser.add_argument('dense_path', type=str)
16 | parser.add_argument('sparse_path', type=str)
17 | args = vars(parser.parse_args())
18 | 
19 | num_count = 0
20 | cat_count = 0
21 | 
22 | for i, row in enumerate(csv.DictReader(open(args['csv_path'])), start=1):
23 |     if i == 1:
24 |         for item in row:
25 |             if 'I' in item:
26 |                 num_count += 1
27 |             if 'C' in item:
28 |                 cat_count += 1
29 |     break
30 | 
31 | print("Numerical feature: {0}; Categorical feature: {1}".format(num_count, cat_count))
32 | 
33 | # These features are dense enough (they appear in the dataset more than threshold=k times), so we include them in GBDT
34 | target_cat_feats = dh.extract_topKfeature('./data/fc.txt', topK=50)
35 | 
36 | with open(args['dense_path'], 'w') as f_d, open(args['sparse_path'], 'w') as f_s:
37 |     for row in csv.DictReader(open(args['csv_path'])):
38 |         feats = []
39 |         # 对数值类型特征进行 dense 数据格式处理
40 |         for j in range(1, num_count+1):
41 |             val = row['I{0}'.format(j)]
42 |             # 对数值型中的缺省值标注为 -10
43 |             if val == '':
44 |                 val = -10
45 |             feats.append('{0}'.format(val))
46 |         if 'tr' in args['dense_path']:
47 |             f_d.write(row['target'] + ' ' + ' '.join(feats) + '\n')
48 |         if 'te' in args['dense_path']:
49 |             f_d.write('0' + ' ' + ' '.join(feats) + '\n')
50 | 
51 |         cat_feats = set()
52 |         # 对类别型特征，如果其出现在经过 GBDT 筛选后得到的显著特征当中，那么就进行 sparse 数据格式处理
53 |         for j in range(1, cat_count+1):
54 |             field = 'C{0}'.format(j)
55 |             key = field + '-' + row[field]
56 |             cat_feats.add(key)
57 | 
58 |         feats = []
59 |         for j, feat in enumerate(target_cat_feats, start=1):
60 |             if feat in cat_feats:
61 |                 feats.append(str(j))
62 |         if 'tr' in args['sparse_path']:
63 |             f_s.write(row['target'] + ' ' + ' '.join(feats) + '\n')
64 |         if 'te' in args['sparse_path']:
65 |             f_s.write('0' + ' ' + ' '.join(feats) + '\n')
66 | 


--------------------------------------------------------------------------------
/NN/input/validation/script/isrc_process.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | import pandas as pd
 3 | 
 4 | from sklearn.preprocessing import LabelEncoder
 5 | 
 6 | ## load the data
 7 | train = pd.read_csv('../temporal_data/train_id.csv')
 8 | test = pd.read_csv('../temporal_data/test_id.csv')
 9 | song = pd.read_csv('../temporal_data/songs_id_cnt.csv')
10 | 
11 | data = train[['msno', 'song_id']].append(test[['msno', 'song_id']])
12 | 
13 | print('Data loaded.')
14 | 
15 | ## isrc process
16 | isrc = song['isrc']
17 | song['cc'] = isrc.str.slice(0, 2)
18 | song['xxx'] = isrc.str.slice(2, 5)
19 | song['yy'] = isrc.str.slice(5, 7).astype(float)
20 | song['yy'] = song['yy'].apply(lambda x: 2000+x if x < 18 else 1900+x)
21 | 
22 | song['cc'] = LabelEncoder().fit_transform(song['cc'])
23 | song['xxx'] = LabelEncoder().fit_transform(song['xxx'])
24 | song['isrc_missing'] = (song['cc'] == 0) * 1.0
25 | 
26 | ## song_cnt
27 | song_cc_cnt = song.groupby(by='cc').count()['song_id'].to_dict()
28 | song_cc_cnt[0] = None
29 | song['cc_song_cnt'] = song['cc'].apply(lambda x: song_cc_cnt[x] if not np.isnan(x) else None)
30 | 
31 | song_xxx_cnt = song.groupby(by='xxx').count()['song_id'].to_dict()
32 | song_xxx_cnt[0] = None
33 | song['xxx_song_cnt'] = song['xxx'].apply(lambda x: song_xxx_cnt[x] if not np.isnan(x) else None)
34 | 
35 | song_yy_cnt = song.groupby(by='yy').count()['song_id'].to_dict()
36 | song_yy_cnt[0] = None
37 | song['yy_song_cnt'] = song['yy'].apply(lambda x: song_yy_cnt[x] if not np.isnan(x) else None)
38 | 
39 | data = data.merge(song, on='song_id', how='left')
40 | 
41 | song_cc_cnt = data.groupby(by='cc').count()['msno'].to_dict()
42 | song_cc_cnt[0] = None
43 | song['cc_rec_cnt'] = song['cc'].apply(lambda x: song_cc_cnt[x] if not np.isnan(x) else None)
44 | 
45 | song_xxx_cnt = data.groupby(by='xxx').count()['msno'].to_dict()
46 | song_xxx_cnt[0] = None
47 | song['xxx_rec_cnt'] = song['xxx'].apply(lambda x: song_xxx_cnt[x] if not np.isnan(x) else None)
48 | 
49 | song_yy_cnt = data.groupby(by='yy').count()['msno'].to_dict()
50 | song_yy_cnt[0] = None
51 | song['yy_rec_cnt'] = song['yy'].apply(lambda x: song_yy_cnt[x] if not np.isnan(x) else None)
52 | 
53 | ## to_csv
54 | features = ['cc_song_cnt', 'xxx_song_cnt', 'yy_song_cnt', 'cc_rec_cnt', \
55 |         'xxx_rec_cnt', 'yy_rec_cnt']
56 | for feat in features:
57 |     song[feat] = np.log1p(song[feat])
58 | 
59 | song.drop(['name', 'isrc'], axis=1, inplace=True)
60 | song.to_csv('../temporal_data/songs_id_cnt_isrc.csv', index=False)
61 | 


--------------------------------------------------------------------------------
/NN/input/training/script/isrc_process.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | import pandas as pd
 3 | 
 4 | from sklearn.preprocessing import LabelEncoder
 5 | 
 6 | ## load the data
 7 | train = pd.read_csv('../temporal_data/train_id.csv')
 8 | test = pd.read_csv('../temporal_data/test_id.csv')
 9 | song = pd.read_csv('../temporal_data/songs_id_cnt.csv')
10 | 
11 | data = train[['msno', 'song_id']].append(test[['msno', 'song_id']])
12 | 
13 | print('Data loaded.')
14 | 
15 | ## isrc process
16 | isrc = song['isrc']
17 | song['cc'] = isrc.str.slice(0, 2)
18 | song['xxx'] = isrc.str.slice(2, 5)
19 | song['yy'] = isrc.str.slice(5, 7).astype(float)
20 | song['yy'] = song['yy'].apply(lambda x: 2000+x if x < 18 else 1900+x)
21 | 
22 | song['cc'] = LabelEncoder().fit_transform(song['cc'].astype(str))
23 | song['xxx'] = LabelEncoder().fit_transform(song['xxx'].astype(str))
24 | song['isrc_missing'] = (song['cc'] == 0) * 1.0
25 | 
26 | ## song_cnt
27 | # 根据歌曲国家划分歌曲
28 | song_cc_cnt = song.groupby(by='cc').count()['song_id'].to_dict()
29 | song_cc_cnt[0] = None
30 | song['cc_song_cnt'] = song['cc'].apply(lambda x: song_cc_cnt[x] if not np.isnan(x) else None)
31 | 
32 | # 根据歌曲出版商划分歌曲
33 | song_xxx_cnt = song.groupby(by='xxx').count()['song_id'].to_dict()
34 | song_xxx_cnt[0] = None
35 | song['xxx_song_cnt'] = song['xxx'].apply(lambda x: song_xxx_cnt[x] if not np.isnan(x) else None)
36 | 
37 | # 根据歌曲年份划分歌曲
38 | song_yy_cnt = song.groupby(by='yy').count()['song_id'].to_dict()
39 | song_yy_cnt[0] = None
40 | song['yy_song_cnt'] = song['yy'].apply(lambda x: song_yy_cnt[x] if not np.isnan(x) else None)
41 | 
42 | data = data.merge(song, on='song_id', how='left')
43 | 
44 | # 每个国家被多少用户听过
45 | song_cc_cnt = data.groupby(by='cc').count()['msno'].to_dict()
46 | song_cc_cnt[0] = None
47 | song['cc_rec_cnt'] = song['cc'].apply(lambda x: song_cc_cnt[x] if not np.isnan(x) else None)
48 | 
49 | # 每个出版商被多少用户听过
50 | song_xxx_cnt = data.groupby(by='xxx').count()['msno'].to_dict()
51 | song_xxx_cnt[0] = None
52 | song['xxx_rec_cnt'] = song['xxx'].apply(lambda x: song_xxx_cnt[x] if not np.isnan(x) else None)
53 | 
54 | # 每个年份被多少用户听过
55 | song_yy_cnt = data.groupby(by='yy').count()['msno'].to_dict()
56 | song_yy_cnt[0] = None
57 | song['yy_rec_cnt'] = song['yy'].apply(lambda x: song_yy_cnt[x] if not np.isnan(x) else None)
58 | 
59 | ## to_csv
60 | features = ['cc_song_cnt', 'xxx_song_cnt', 'yy_song_cnt', 'cc_rec_cnt', 'xxx_rec_cnt', 'yy_rec_cnt']
61 | for feat in features:
62 |     song[feat] = np.log1p(song[feat])
63 | 
64 | # 删去歌曲 'name' 与 'isrc' 特征
65 | song.drop(['name', 'isrc'], axis=1, inplace=True)
66 | song.to_csv('../temporal_data/songs_id_cnt_isrc.csv', index=False)
67 | 


--------------------------------------------------------------------------------
/FFM/model/libffm-1.13/ffm-predict.cpp:
--------------------------------------------------------------------------------
  1 | #include <cstring>
  2 | #include <fstream>
  3 | #include <iostream>
  4 | #include <string>
  5 | #include <iomanip>
  6 | #include <memory>
  7 | #include <cmath>
  8 | #include <stdexcept>
  9 | #include <vector>
 10 | #include <cstdlib>
 11 | 
 12 | #include "ffm.h"
 13 | 
 14 | using namespace std;
 15 | using namespace ffm;
 16 | 
 17 | struct Option
 18 | {
 19 |     string test_path, model_path, output_path;
 20 | };
 21 | 
 22 | string predict_help()
 23 | {
 24 |     return string(
 25 | "usage: ffm-predict test_file model_file output_file\n");
 26 | }
 27 | 
 28 | Option parse_option(int argc, char **argv)
 29 | {
 30 |     vector<string> args;
 31 |     for(int i = 0; i < argc; i++)
 32 |         args.push_back(string(argv[i]));
 33 | 
 34 |     if(argc == 1)
 35 |         throw invalid_argument(predict_help());
 36 | 
 37 |     Option option;
 38 | 
 39 |     if(argc != 4)
 40 |         throw invalid_argument("cannot parse argument");
 41 | 
 42 |     option.test_path = string(args[1]);
 43 |     option.model_path = string(args[2]);
 44 |     option.output_path = string(args[3]);
 45 | 
 46 |     return option;
 47 | }
 48 | 
 49 | void predict(string test_path, string model_path, string output_path)
 50 | {
 51 |     int const kMaxLineSize = 1000000;
 52 | 
 53 |     FILE *f_in = fopen(test_path.c_str(), "r");
 54 |     ofstream f_out(output_path);
 55 |     char line[kMaxLineSize];
 56 | 
 57 |     ffm_model *model = ffm_load_model(model_path.c_str());
 58 | 
 59 |     ffm_double loss = 0;
 60 |     vector<ffm_node> x;
 61 |     ffm_int i = 0;
 62 | 
 63 |     for(; fgets(line, kMaxLineSize, f_in) != nullptr; i++)
 64 |     {
 65 |         x.clear();
 66 |         char *y_char = strtok(line, " \t");
 67 |         ffm_float y = (atoi(y_char)>0)? 1.0f : -1.0f;
 68 | 
 69 |         while(true)
 70 |         {
 71 |             char *field_char = strtok(nullptr,":");
 72 |             char *idx_char = strtok(nullptr,":");
 73 |             char *value_char = strtok(nullptr," \t");
 74 |             if(field_char == nullptr || *field_char == '\n')
 75 |                 break;
 76 | 
 77 |             ffm_node N;
 78 |             N.f = atoi(field_char);
 79 |             N.j = atoi(idx_char);
 80 |             N.v = atof(value_char);
 81 | 
 82 |             x.push_back(N);
 83 |         }
 84 | 
 85 |         ffm_float y_bar = ffm_predict(x.data(), x.data()+x.size(), model);
 86 | 
 87 |         loss -= y==1? log(y_bar) : log(1-y_bar);
 88 | 
 89 |         f_out << y_bar << "\n";
 90 |     }
 91 | 
 92 |     loss /= i;
 93 | 
 94 |     cout << "logloss = " << fixed << setprecision(5) << loss << endl;
 95 | 
 96 |     ffm_destroy_model(&model);
 97 | 
 98 | }
 99 | 
100 | int main(int argc, char **argv)
101 | {
102 |     Option option;
103 |     try
104 |     {
105 |         option = parse_option(argc, argv);
106 |     }
107 |     catch(invalid_argument const &e)
108 |     {
109 |         cout << e.what() << endl;
110 |         return 1;
111 |     }
112 | 
113 |     predict(option.test_path, option.model_path, option.output_path);
114 | }
115 | 


--------------------------------------------------------------------------------
/LightGBM/utils/data_helpers.py:
--------------------------------------------------------------------------------
 1 | # -*- coding:utf-8 -*-
 2 | __author__ = 'Randolph'
 3 | 
 4 | import logging
 5 | 
 6 | 
 7 | def logger_fn(name, file, level=logging.INFO):
 8 |     logger = logging.getLogger(name)
 9 |     logger.setLevel(level)
10 |     fh = logging.FileHandler(file, mode='w')
11 |     ch = logging.StreamHandler()
12 |     ch.setLevel(level)
13 |     formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
14 |     fh.setFormatter(formatter)
15 |     ch.setFormatter(formatter)
16 |     logger.addHandler(fh)
17 |     logger.addHandler(ch)
18 |     return logger
19 | 
20 | 
21 | def song_lang_boolean(x):
22 |     # '3', '10', '24', '59' -> Chinese
23 |     if '17.0' in str(x) or '45.0' in str(x):
24 |         return 1
25 |     return 0
26 | 
27 | 
28 | def deal_bd_info(x):
29 |     if int(x) >= 80 or int(x) <= 5:
30 |         return 28
31 |     else:
32 |         return int(x)
33 | 
34 | 
35 | def deal_source_system_tab_info(x):
36 |     if str(x) in ['my library', 'search']:
37 |         return 0
38 |     elif str(x) in ['discover', 'explore', 'radio']:
39 |         return 1
40 |     elif str(x) in ['null', '', 'notification', 'settings']:
41 |         return 2
42 |     else:
43 |         return 3
44 | 
45 | 
46 | def deal_source_screen_name_info(x):
47 |     if str(x) in ['Payment', 'My library', 'My library_Search', 'Local playlist more', 'Search']:
48 |         return 0
49 |     elif str(x) in ['Album more', 'Artist more', 'Concert', 'Discover Chart', 'Discover Feature', 'Discover Genre',
50 |                     'Discover New', 'Explore', 'Radio']:
51 |         return 1
52 |     elif str(x) in ['People global', 'People local', 'Search Home', 'Search Trends', ' Self Profile more']:
53 |         return 2
54 |     else:
55 |         return 3
56 | 
57 | 
58 | def deal_source_type_info(x):
59 |     if str(x) in ['local-library', 'local-playlist']:
60 |         return 0
61 |     elif str(x) in ['artist', 'album', 'my-daily-playlist', 'online-playlist', 'radio', 'song-based-playlist',
62 |                     'top-hits-for-artist', 'topic-article-playlist', 'song']:
63 |         return 1
64 |     else:
65 |         return 2
66 | 
67 | 
68 | def check_missing_values(df):
69 |     print(df.isnull().values.any())
70 |     columns_with_nan = []
71 |     if df.isnull().values.any():
72 |         columns_with_nan = df.columns[df.isnull().any()].tolist()
73 |     print(columns_with_nan)
74 |     for col in columns_with_nan:
75 |         print("{0} : {1}".format(col, df[col].isnull().sum()))
76 | 
77 | 
78 | def fillin_nan_value(train_merged_df=None, test_merged_df=None):
79 |     if train_merged_df is not None:
80 |         train_merged_df.fillna({'genre_ids': 'no_genre_id',
81 |                                 'artist': 'no_artist',
82 |                                 'composer': 'no_composer',
83 |                                 'lyricist': 'no_lyricist',
84 |                                 'gender': 'unknown'}, inplace=True)
85 |     if test_merged_df is not None:
86 |         test_merged_df.fillna({'genre_ids': 'no_genre_id',
87 |                                'artist': 'no_artist',
88 |                                'composer': 'no_composer',
89 |                                'lyricist': 'no_lyricist',
90 |                                'gender': 'unknown'}, inplace=True)
91 | 


--------------------------------------------------------------------------------
/XGBoost/utils/data_helpers.py:
--------------------------------------------------------------------------------
 1 | # -*- coding:utf-8 -*-
 2 | __author__ = 'Randolph'
 3 | 
 4 | import logging
 5 | 
 6 | 
 7 | def logger_fn(name, file, level=logging.INFO):
 8 |     logger = logging.getLogger(name)
 9 |     logger.setLevel(level)
10 |     fh = logging.FileHandler(file, mode='w')
11 |     ch = logging.StreamHandler()
12 |     ch.setLevel(level)
13 |     formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
14 |     fh.setFormatter(formatter)
15 |     ch.setFormatter(formatter)
16 |     logger.addHandler(fh)
17 |     logger.addHandler(ch)
18 |     return logger
19 | 
20 | 
21 | def song_lang_boolean(x):
22 |     # '3', '10', '24', '59' -> Chinese
23 |     if '17.0' in str(x) or '45.0' in str(x):
24 |         return 1
25 |     return 0
26 | 
27 | 
28 | def deal_bd_info(x):
29 |     if int(x) >= 80 or int(x) <= 5:
30 |         return 28
31 |     else:
32 |         return int(x)
33 | 
34 | 
35 | def deal_source_system_tab_info(x):
36 |     if str(x) in ['my library', 'search']:
37 |         return 0
38 |     elif str(x) in ['discover', 'explore', 'radio']:
39 |         return 1
40 |     elif str(x) in ['null', '', 'notification', 'settings']:
41 |         return 2
42 |     else:
43 |         return 3
44 | 
45 | 
46 | def deal_source_screen_name_info(x):
47 |     if str(x) in ['Payment', 'My library', 'My library_Search', 'Local playlist more', 'Search']:
48 |         return 0
49 |     elif str(x) in ['Album more', 'Artist more', 'Concert', 'Discover Chart', 'Discover Feature', 'Discover Genre',
50 |                     'Discover New', 'Explore', 'Radio']:
51 |         return 1
52 |     elif str(x) in ['People global', 'People local', 'Search Home', 'Search Trends', ' Self Profile more']:
53 |         return 2
54 |     else:
55 |         return 3
56 | 
57 | 
58 | def deal_source_type_info(x):
59 |     if str(x) in ['local-library', 'local-playlist']:
60 |         return 0
61 |     elif str(x) in ['artist', 'album', 'my-daily-playlist', 'online-playlist', 'radio', 'song-based-playlist',
62 |                     'top-hits-for-artist', 'topic-article-playlist', 'song']:
63 |         return 1
64 |     else:
65 |         return 2
66 | 
67 | 
68 | def check_missing_values(df):
69 |     print(df.isnull().values.any())
70 |     columns_with_nan = []
71 |     if df.isnull().values.any():
72 |         columns_with_nan = df.columns[df.isnull().any()].tolist()
73 |     print(columns_with_nan)
74 |     for col in columns_with_nan:
75 |         print("{0} : {1}".format(col, df[col].isnull().sum()))
76 | 
77 | 
78 | def fillin_nan_value(train_merged_df=None, test_merged_df=None):
79 |     if train_merged_df is not None:
80 |         train_merged_df.fillna({'genre_ids': 'no_genre_id',
81 |                                 'artist': 'no_artist',
82 |                                 'composer': 'no_composer',
83 |                                 'lyricist': 'no_lyricist',
84 |                                 'gender': 'unknown'}, inplace=True)
85 |     if test_merged_df is not None:
86 |         test_merged_df.fillna({'genre_ids': 'no_genre_id',
87 |                                'artist': 'no_artist',
88 |                                'composer': 'no_composer',
89 |                                'lyricist': 'no_lyricist',
90 |                                'gender': 'unknown'}, inplace=True)
91 | 


--------------------------------------------------------------------------------
/FFM/model/gbdt/src/train.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <omp.h>
  3 | 
  4 | #include "common.h"
  5 | #include "timer.h"
  6 | #include "gbdt.h"
  7 | 
  8 | namespace {
  9 | 
 10 | struct Option
 11 | {
 12 |     Option() : nr_tree(30), nr_thread(1) {}
 13 |     std::string Tr_path, TrS_path, Va_path, VaS_path, Va_out_path, Tr_out_path;
 14 |     uint32_t nr_tree, nr_thread;
 15 | };
 16 | 
 17 | std::string train_help()
 18 | {
 19 |     return std::string(
 20 | "usage: gbdt [<options>] <dense_validation_path> <sparse_validation_path> <dense_train_path> <sparse_train_path> <validation_output_path> <train_output_path>\n"
 21 | "\n"
 22 | "options:\n"
 23 | "-d <depth>: set the maximum depth of a tree\n"
 24 | "-s <nr_thread>: set the maximum number of threads\n"
 25 | "-t <nr_tree>: set the number of trees\n");
 26 | }
 27 | 
 28 | Option parse_option(std::vector<std::string> const &args)
 29 | {
 30 |     uint32_t const argc = static_cast<uint32_t>(args.size());
 31 | 
 32 |     if(argc == 0)
 33 |         throw std::invalid_argument(train_help());
 34 | 
 35 |     Option opt; 
 36 | 
 37 |     uint32_t i = 0;
 38 |     for(; i < argc; ++i)
 39 |     {
 40 |         if(args[i].compare("-d") == 0)
 41 |         {
 42 |             if(i == argc-1)
 43 |                 throw std::invalid_argument("invalid command");
 44 |             CART::max_depth = std::stoi(args[++i]);
 45 |         }
 46 |         else if(args[i].compare("-t") == 0)
 47 |         {
 48 |             if(i == argc-1)
 49 |                 throw std::invalid_argument("invalid command");
 50 |             opt.nr_tree = std::stoi(args[++i]);
 51 |         }
 52 |         else if(args[i].compare("-s") == 0)
 53 |         {
 54 |             if(i == argc-1)
 55 |                 throw std::invalid_argument("invalid command");
 56 |             opt.nr_thread = std::stoi(args[++i]);
 57 |         }
 58 |         else
 59 |         {
 60 |             break;
 61 |         }
 62 |     }
 63 | 
 64 |     if(i != argc-6)
 65 |         throw std::invalid_argument("invalid command");
 66 | 
 67 |     opt.Va_path = args[i++];
 68 |     opt.VaS_path = args[i++];
 69 |     opt.Tr_path = args[i++];
 70 |     opt.TrS_path = args[i++];
 71 |     opt.Va_out_path = args[i++];
 72 |     opt.Tr_out_path = args[i++];
 73 | 
 74 |     return opt;
 75 | }
 76 | 
 77 | void write(Problem const &prob, GBDT const &gbdt, std::string const &path)
 78 | {
 79 |     FILE *f = open_c_file(path, "w");
 80 | 
 81 |     for(uint32_t i = 0; i < prob.nr_instance; ++i)
 82 |     {
 83 |         std::vector<float> x = construct_instance(prob, i);
 84 |         std::vector<uint32_t> indices = gbdt.get_indices(x.data());
 85 | 
 86 |         fprintf(f, "%d", static_cast<int>(prob.Y[i]));
 87 |         for(uint32_t t = 0; t < indices.size(); ++t)
 88 |             fprintf(f, " %d", indices[t]);
 89 |         fprintf(f, "\n");
 90 |     }
 91 | 
 92 |     fclose(f);
 93 | }
 94 | 
 95 | } //unnamed namespace
 96 | 
 97 | int main(int const argc, char const * const * const argv)
 98 | {
 99 |     Option opt;
100 |     try
101 |     {
102 |         opt = parse_option(argv_to_args(argc, argv));
103 |     }
104 |     catch(std::invalid_argument const &e)
105 |     {
106 |         std::cout << e.what();
107 |         return EXIT_FAILURE;
108 |     }
109 | 
110 |     std::cout << "reading data..." << std::flush;
111 |     Problem const Tr = read_data(opt.Tr_path, opt.TrS_path);
112 |     Problem const Va = read_data(opt.Va_path, opt.VaS_path);
113 |     std::cout << "done\n" << std::flush;
114 | 
115 | 	omp_set_num_threads(static_cast<int>(opt.nr_thread));
116 | 
117 |     GBDT gbdt(opt.nr_tree);
118 |     gbdt.fit(Tr, Va);
119 | 
120 |     write(Tr, gbdt, opt.Tr_out_path);
121 |     write(Va, gbdt, opt.Va_out_path);
122 | 
123 |     return EXIT_SUCCESS;
124 | }
125 | 


--------------------------------------------------------------------------------
/FFM/utils/pre-b.py:
--------------------------------------------------------------------------------
  1 | # -*- coding:utf-8 -*-
  2 | __author__ = 'Randolph'
  3 | 
  4 | import sys
  5 | import csv
  6 | import math
  7 | import argparse
  8 | from utils import data_helpers as dh
  9 | 
 10 | if len(sys.argv) == 1:
 11 |     sys.argv.append('-h')
 12 | 
 13 | parser = argparse.ArgumentParser()
 14 | # nr_bins 为指数级的数，如果将所有的类型特征进行 one-hot 编码之后会得到近百万的维度，对其进行 hash trick
 15 | # 具体做法是将三种特征（类别型特征（包括频繁项与非频繁项），数值型特征，GBDT 筛选得到的显著特征）与对应的特征值结合后的特征名称进行 hash
 16 | # 然后与 nr_bins = 10^6 取余，目的是进行降维
 17 | parser.add_argument('-n', '--nr_bins', type=int, default=int(1e+6))
 18 | # threshold 为频繁特征的阈值，默认值为 100
 19 | parser.add_argument('-t', '--threshold', type=int, default=int(100))
 20 | parser.add_argument('csv_path', type=str)
 21 | parser.add_argument('gbdt_path', type=str)
 22 | parser.add_argument('out_path', type=str)
 23 | args = vars(parser.parse_args())
 24 | 
 25 | 
 26 | num_count = 0
 27 | cat_count = 0
 28 | 
 29 | for i, row in enumerate(csv.DictReader(open(args['csv_path'])), start=1):
 30 |     if i == 1:
 31 |         for item in row:
 32 |             if 'I' in item:
 33 |                 num_count += 1
 34 |             if 'C' in item:
 35 |                 cat_count += 1
 36 |     break
 37 | 
 38 | print("Numerical feature: {0}; Categorical feature: {1}".format(num_count, cat_count))
 39 | 
 40 | 
 41 | def gen_hashed_fm_feats(feats, nr_bins):
 42 |     feats = ['{0}:{1}:1'.format(field-1, dh.hashstr(feat, nr_bins)) for (field, feat) in feats]
 43 |     return feats
 44 | 
 45 | 
 46 | def gen_feats(row):
 47 |     feats = []
 48 |     # 对数值类型特征进行处理
 49 |     for j in range(1, num_count+1):
 50 |         field = 'I{0}'.format(j)
 51 |         value = row[field]
 52 |         # 对数值型中的缺省值标注为 -10
 53 |         if value != '':
 54 |             value = float(value)
 55 |             # 如果特征值超过 2，那么对其进行求自然对数，然后平方，最后向上取整，与该特征所在列进行组合，生成特征名称
 56 |             if value > 2:
 57 |                 value = int(math.log(float(value))**2)
 58 |             else:
 59 |                 value = 'SP' + str(value)
 60 |         key = field + '-' + str(value)
 61 |         feats.append(key)
 62 |     # 对类别型特征进行处理
 63 |     for j in range(1, cat_count+1):
 64 |         field = 'C{0}'.format(j)
 65 |         value = row[field]
 66 |         key = field + '-' + value
 67 |         feats.append(key)
 68 |     return feats
 69 | 
 70 | 
 71 | # 对于那些出现次数超过 threshold（默认为 100） 的类别性特征标记为频繁特征
 72 | frequent_feats = dh.read_freqent_feats(args['threshold'])
 73 | 
 74 | 
 75 | with open(args['out_path'], 'w') as f:
 76 |     for row, line_gbdt in zip(csv.DictReader(open(args['csv_path'])), open(args['gbdt_path'])):
 77 |         feats = []
 78 |         # 对原始数据的数值型特征与类别型特征分别进行处理
 79 |         for feat in gen_feats(row):
 80 |             field = feat.split('-')[0]
 81 |             type, field = field[0], int(field[1:])
 82 |             # 如果特征是类别型特征，并不属于频繁特征，即出现次数少于 threshold 的类别型特征，则统一都标记成 'less'
 83 |             if type == 'C' and feat not in frequent_feats:
 84 |                 feat = feat.split('-')[0] + 'less'
 85 |             # 如果特征是类别型特征，并属于频繁特征，那么其中 field 需要加上数值型特征的种类个数，作为偏移，这里为 num_count
 86 |             if type == 'C':
 87 |                 field += num_count
 88 |             # 如果特征是数值型特征
 89 |             feats.append((field, feat))
 90 | 
 91 |         # 对于 GBDT 所构造的显著特征进行处理
 92 |         for i, feat in enumerate(line_gbdt.strip().split()[1:], start=1):
 93 |             # 其中 field 需要加上数值型特征与类别型特征的种类个数和，作为偏移，这里为 num_count + cat_count
 94 |             field = i + num_count + cat_count
 95 |             feats.append((field, str(i) + ":" + feat))
 96 | 
 97 |         feats = gen_hashed_fm_feats(feats, args['nr_bins'])
 98 |         if 'tr' in args['out_path']:
 99 |             f.write(row['target'] + ' ' + ' '.join(feats) + '\n')
100 |         if 'te' in args['out_path']:
101 |             f.write('0' + ' ' + ' '.join(feats) + '\n')
102 | 


--------------------------------------------------------------------------------
/XGBoost/model/xgb.py:
--------------------------------------------------------------------------------
  1 | # -*- coding:utf-8 -*-
  2 | __author__ = 'Randolph'
  3 | 
  4 | import gc
  5 | import time
  6 | import numpy as np
  7 | import pandas as pd
  8 | import xgboost as xgb
  9 | 
 10 | from tqdm import *
 11 | from sklearn.model_selection import train_test_split
 12 | from sklearn.preprocessing import LabelEncoder
 13 | from utils import data_helpers as dh
 14 | 
 15 | data_path = '../data/'
 16 | result_path = '../result/'
 17 | logs_path = '../logs/'
 18 | 
 19 | logger = dh.logger_fn('xgblog', (logs_path + 'xgb-{0}.log').format(time.asctime()))
 20 | 
 21 | # Loading Data
 22 | # ==================================================
 23 | 
 24 | logger.info('Loading data...')
 25 | 
 26 | train = pd.read_csv(data_path + 'train.csv')
 27 | test = pd.read_csv(data_path + 'test.csv')
 28 | 
 29 | logger.info('Done loading...')
 30 | 
 31 | # Checking nan value
 32 | # ==================================================
 33 | 
 34 | logger.info('Checking nan value...')
 35 | dh.fillin_nan_value(train_merged_df=train, test_merged_df=test)
 36 | logger.info('Done checking...')
 37 | 
 38 | # Drop feature
 39 | # ==================================================
 40 | 
 41 | logger.info('Dropping data...')
 42 | drop_features = []
 43 | 
 44 | train = train.drop(drop_features, axis=1)
 45 | test = test.drop(drop_features, axis=1)
 46 | 
 47 | logger.info('Done dropping...')
 48 | 
 49 | # Add Features
 50 | # ==================================================
 51 | 
 52 | logger.info('Adding new features...')
 53 | 
 54 | logger.info('Done adding features...')
 55 | 
 56 | # Split Dataset
 57 | # ==================================================
 58 | 
 59 | cols = list(train.columns)
 60 | cols.remove('target')
 61 | 
 62 | for col in tqdm(cols):
 63 |     if train[col].dtype == 'object':
 64 |         train[col] = train[col].apply(str)
 65 |         test[col] = test[col].apply(str)
 66 | 
 67 |         le = LabelEncoder()
 68 |         train_vals = list(train[col].unique())
 69 |         test_vals = list(test[col].unique())
 70 |         le.fit(train_vals + test_vals)
 71 |         train[col] = le.transform(train[col])
 72 |         test[col] = le.transform(test[col])
 73 | 
 74 |         print(col + ': ' + str(len(train_vals)) + ', ' + str(len(test_vals)))
 75 | 
 76 | X = np.array(train.drop(['target'], axis=1))
 77 | y = train['target'].values
 78 | 
 79 | X_test = np.array(test.drop(['id'], axis=1))
 80 | ids = test['id'].values
 81 | 
 82 | X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1, random_state=0)
 83 | 
 84 | d_train = xgb.DMatrix(X_train, label=y_train)
 85 | d_valid = xgb.DMatrix(X_valid, label=y_valid)
 86 | d_test = xgb.DMatrix(X_test)
 87 | 
 88 | eval_set = [(X_train, y_train), (X_valid, y_valid)]
 89 | watchlist = [(d_train, 'train'), (d_valid, 'valid')]
 90 | 
 91 | del train, test
 92 | gc.collect()
 93 | 
 94 | # Training Data
 95 | # ==================================================
 96 | 
 97 | logger.info('Training XGBoost model...')
 98 | 
 99 | params_xgboost = {
100 |     'objective': 'binary:logistic',
101 |     'eta': 0.1,
102 |     'max_depth': 10,
103 |     'silent': 1,
104 |     'eval_metric': 'auc'
105 | }
106 | 
107 | logger.info(params_xgboost)
108 | 
109 | model = xgb.train(params_xgboost, d_train, 105, watchlist, early_stopping_rounds=20, maximize=True, verbose_eval=10)
110 | 
111 | logger.info('Done training XGBoost model and make predictions next...')
112 | 
113 | p_test = model.predict(d_test)
114 | 
115 | # Preparing submission
116 | # ==================================================
117 | 
118 | subm = pd.DataFrame()
119 | subm['id'] = ids
120 | subm['target'] = p_test
121 | print(len(ids), len(p_test))
122 | subm.to_csv(result_path + 'submission_xgboost.csv', index=False)
123 | 
124 | logger.info('Done saving XGBoost model predictions...')
125 | logger.info('All Finished!')
126 | 


--------------------------------------------------------------------------------
/FFM/utils/data_helpers.py:
--------------------------------------------------------------------------------
  1 | # -*- coding:utf-8 -*-
  2 | __author__ = 'Randolph'
  3 | 
  4 | import csv
  5 | import collections
  6 | import hashlib
  7 | 
  8 | 
  9 | data_path = './data/'
 10 | 
 11 | train_file = data_path + 'train.csv'
 12 | fc_trva_file = data_path + 'fc.trva.txt'
 13 | fc_file = data_path + 'fc.txt'
 14 | 
 15 | 
 16 | def count(input_file, output_file):
 17 |     num_count = 0
 18 |     cat_count = 0
 19 | 
 20 |     for i, row in enumerate(csv.DictReader(open(input_file)), start=1):
 21 |         if i == 1:
 22 |             for item in row:
 23 |                 if 'I' in item:
 24 |                     num_count += 1
 25 |                 if 'C' in item:
 26 |                     cat_count += 1
 27 |         break
 28 | 
 29 |     print("Numerical feature: {0}; Categorical feature: {1}\n".format(num_count, cat_count))
 30 | 
 31 |     counts = collections.defaultdict(lambda: [0, 0, 0])
 32 | 
 33 |     for i, row in enumerate(csv.DictReader(open(input_file)), start=1):
 34 |         label = row['target']
 35 |         for j in range(1, cat_count + 1):
 36 |             field = 'C{0}'.format(j)
 37 |             value = row[field]
 38 |             if label == '0':
 39 |                 counts[field + ',' + value][0] += 1
 40 |             else:
 41 |                 counts[field + ',' + value][1] += 1
 42 |             counts[field + ',' + value][2] += 1
 43 |         if i % 1000000 == 0:
 44 |             print('{0}m'.format(int(i / 1000000)))
 45 | 
 46 |     with open(output_file, 'w') as fout:
 47 |         fout.write('Field,Value,Neg,Pos,Total,Deviation\n')
 48 |         content = []
 49 |         for key, (neg, pos, total) in sorted(counts.items(), key=lambda x: x[1][2]):
 50 |             if total < 10:
 51 |                 continue
 52 |             deviation = round((float(pos / total) - 0.5) * 100, 2)
 53 |             content.append(key + ',' + str(neg) + ',' + str(pos) + ',' + str(total) + ',' + str(deviation) + '\n')
 54 |         content.reverse()
 55 |         for eachline in content:
 56 |             fout.write(eachline)
 57 | 
 58 | 
 59 | def extra_feature_pair(input_file, output_file, threshold1=200000, threshold2=1):
 60 |     with open(input_file, 'r') as fin, open(output_file, 'w') as fout:
 61 |         for index, eachline in enumerate(fin, start=1):
 62 |             if index > 2:
 63 |                 line = eachline.strip().split(',')
 64 |                 total = line[4]
 65 |                 deviation = line[5]
 66 |                 if int(total) >= threshold1:
 67 |                     fout.write(eachline)
 68 | 
 69 | 
 70 | def extract_topKfeature(input_file, topK, threshold=40):
 71 |     topK_feature = []
 72 |     with open(input_file, 'r') as fin:
 73 |         for index, eachline in enumerate(fin, start=1):
 74 |             line = eachline.strip().split(',')
 75 |             field = line[0]
 76 |             attribute = line[1]
 77 |             total = line[4]
 78 |             if total.isdigit():
 79 |                 total = int(int(line[4])/10000)
 80 |                 if topK:
 81 |                     if index > topK:
 82 |                         break
 83 |                     out_str = field + '-' + attribute
 84 |                     topK_feature.append(out_str)
 85 |                 else:
 86 |                     if int(total) > threshold:
 87 |                         out_str = field + '-' + attribute
 88 |                         topK_feature.append(out_str)
 89 |     print(topK_feature)
 90 |     print('The topK features number is: {0}'.format(len(topK_feature)))
 91 |     return topK_feature
 92 | 
 93 | 
 94 | def read_freqent_feats(threshold=100):
 95 |     frequent_feats = set()
 96 |     for row in csv.DictReader(open(fc_trva_file)):
 97 |         if int(row['Total']) < threshold:
 98 |             continue
 99 |         frequent_feats.add(row['Field'] + '-' + row['Value'])
100 |     return frequent_feats
101 | 
102 | 
103 | def hashstr(str, nr_bins):
104 |     return int(hashlib.md5(str.encode('utf8')).hexdigest(), 16) % (nr_bins - 1) + 1
105 | 
106 | 
107 | if __name__ == '__main__':
108 |     count(train_file, fc_trva_file)
109 |     extra_feature_pair(fc_trva_file, fc_file, threshold1=1000000)
110 | 


--------------------------------------------------------------------------------
/FFM/README.md:
--------------------------------------------------------------------------------
  1 | # Field-aware Factorization Machine for Music Recommendation Challenge
  2 | 
  3 | **FFM**（Field-aware Factorization Machine）最初的概念来自Yu-Chin Juan（阮毓钦，毕业于中国台湾大学）与其比赛队员，是他们借鉴了来自 Michael Jahrer 的论文中的 **field** 概念提出了 FM 的升级版模型。通过引入 **field** 的概念，FFM 把相同性质的特征归于同一个 **field**。 FFM 起初就是用于 CTR（广告点击率预测）任务的，通过分析该比赛数据特征，该比赛任务的数据特征同 CTR 任务的特点接近，因此可以套用 FFM 模型。
  4 | 
  5 | Ps：FFM 的作者，也就是 Yu-Chin Juan 也参加了这个比赛（成绩比我们低 🙃）。
  6 | 
  7 | 具体的数据处理流程以及模型的训练预测均参考 [Yu-Chin Juan 之前在 Kaggle 比赛上 CTR 任务上的方法](https://github.com/guestwalk/kaggle-2014-criteo)。
  8 | 
  9 | 
 10 | ## Requirement
 11 | 
 12 | - 64-bit Unix-like operating system (My code based on macOS High Sierra 10.13)
 13 | 
 14 | - Python3
 15 | 
 16 | - **g++ (with C++11 and OpenMP support)**
 17 | 
 18 | 
 19 | ## Dataset
 20 | 
 21 | [Get the research data](https://www.kaggle.com/c/kkbox-music-recommendation-challenge)
 22 | 
 23 | 原始数据来源为 Kaggle 上的比赛数据集，但还需要将原始数据 merge 成最终的 `train.csv` 与 `test.csv` 作为下一步的输入（该模型不涉及原始数据合并代码）。
 24 | 
 25 | 其中对于合并好的 `train.csv` 以及 `test.csv`  还要进行额外的列名 rename 处理，需要处理成：
 26 | 
 27 | ```
 28 | <train.csv>
 29 | target,I1,I2,...,I7,C1,C2,...,C20
 30 | 1,28,2013,...,206471,male,方大同,...,HK
 31 | 0,27,2017,...,284584,female,周杰伦,...,TW
 32 | ```
 33 | 
 34 | ```
 35 | <test.csv>
 36 | id,I1,I2,...,I7,C1,C2,...,C20
 37 | 0,30,2014,...,224130,male,孙燕姿,...,TW
 38 | 1,17,2015,...,320470,unkown,张悬,...,TW
 39 | ```
 40 | 
 41 | 其中 `train.csv` 有 `target` (label)这一特征属性，而 `test.csv` 则不存在 `target` 特征属性。
 42 | 
 43 | `I1、I2、I3` 这种以 **I(Integer)** 开头的表示该特征属性数值型特征，而 `C1、C2、C3` 这种以 **C(Category)** 开头的表示该特征属性类别型特征。之所以将两者区分开，是因为后续步骤会对两者进行不同的处理方式。
 44 | 
 45 | 
 46 | ## Step-by-step
 47 | 
 48 | ### Step 1 Libffm + GBDT
 49 | 
 50 | 首先要先成功编译和安装  [libffm](https://github.com/guestwalk/libffm) 与 GBDT，安装的前提是需要机器上有支持 **OpenMP** 的编译器。如果你使用 OS X 的系统，需要将代码中 `model`  文件夹下 libffm 与 GBDT 模型里面的 `Makefile` 中的第一行代码都改成：
 51 | 
 52 | ```C
 53 | CXX = g++-x
 54 | ```
 55 | 
 56 | 其中 x 取决于你新安装的 g++ 版本（我安装的是 g++ 7），因为 OS X 自带的编译器是不支持 **OpenMP**。
 57 | 
 58 | 之后，在代码主目录下输入：
 59 | 
 60 | ```shell
 61 | make -C model/gbdt
 62 | make -C model/libffm-1.13
 63 | ```
 64 | 
 65 | 如果没有出现 Error 信息，则说明已经成功编译和安装 libffm 与 GBDT。
 66 | 
 67 | ### Step 2  
 68 | 
 69 | 可以直接运行 `run.py` 代码，或者分别运行各流程的代码。
 70 | 
 71 | #### 2.1 Make the `fc.trva.top.txt` file 
 72 | 
 73 | ```shell
 74 | python3 utils/count.py data/train.csv > data/fc.trva.top.txt
 75 | python3 utils/data_helpers.py
 76 | ```
 77 | 
 78 | 目的是生成 `fc.trva.top.txt` 文件，该文件包含了关于 `train.csv` 所有类别型特征与该特征下的各个 value 的组合的计数情况，以便后续步骤找出我们需要的 TopK 高频组合。文件大致如下所示：
 79 | 
 80 | ```
 81 | <fc.trva.top.txt>
 82 | Field,Value,Neg,Pos,Total,Ratio
 83 | 
 84 | ```
 85 | 
 86 | 
 87 | 
 88 | 值得一提的是，在 Yu-Chin Juan 的解决方案中，该文件的 `Ratio` 是按照组合 target =1
 89 | 
 90 | #### 2.2 
 91 | 
 92 | ```
 93 | python3 utils/data_helpers.py data/fc.trva.top.txt
 94 | ```
 95 | 
 96 | 
 97 | 
 98 | 
 99 | 
100 | 
101 | 
102 | Miscellaneous
103 | =============
104 | 
105 | 1. By default we use only one thread, so it may take a long time to train the
106 |    model. If you have multi-core CPUs, you may want to set NR_THREAD in run.py
107 |    to use more cores. 
108 | 
109 | 2. Our algorithms is non-deterministic when multiple threads are used. That
110 |    is, the results can be slightly different when you run the script two or
111 |    more times. In our experience, the variances of logloss generally do not 
112 |    exceed 0.0001.
113 | 
114 | 3. This script generates a prediction with around 0.44510 / 0.44500 on 
115 |    public / private leaderboards, which are slightly worse than what we had 
116 |    during the competition. The difference is due to some minor changes.
117 | 
118 |    If you want to reproduce our best results, please do:
119 | 
120 |      $ git checkout v1.0
121 | 
122 |    For detailed instructions, please follow README in that version.
123 | 
124 | 
125 | ## About Me
126 | 
127 | 黄威，Randolph
128 | 
129 | SCU SE Bachelor; USTC CS Master
130 | 
131 | Email: chinawolfman@hotmail.com
132 | 
133 | My Blog: [randolph.pro](http://randolph.pro)
134 | 
135 | LinkedIn: [randolph's linkedin](https://www.linkedin.com/in/randolph-%E9%BB%84%E5%A8%81/)
136 | 
137 | ## Reference
138 | 
139 | - [Field-aware Factorization Machines for CTR Prediction](http://ntucsu.csie.ntu.edu.tw/~cjlin/papers/ffm.pdf)
140 | - [Field-aware Factorization Machines in a Real-world Online Advertising System](https://arxiv.org/pdf/1701.04099.pdf)
141 | 


--------------------------------------------------------------------------------
/FFM/model/gbdt/src/common.cpp:
--------------------------------------------------------------------------------
  1 | #include <stdexcept>
  2 | #include <cstring>
  3 | #include <cassert>
  4 | #include <algorithm>
  5 | #include <omp.h>
  6 | 
  7 | #include "common.h"
  8 | 
  9 | namespace {
 10 | 
 11 | uint32_t const kMaxLineSize = 1000000;
 12 | 
 13 | uint32_t get_nr_line(std::string const &path)
 14 | {
 15 |     FILE *f = open_c_file(path.c_str(), "r");
 16 |     char line[kMaxLineSize];
 17 | 
 18 |     uint32_t nr_line = 0;
 19 |     while(fgets(line, kMaxLineSize, f) != nullptr)
 20 |         ++nr_line;
 21 | 
 22 |     fclose(f);
 23 | 
 24 |     return nr_line;
 25 | }
 26 | 
 27 | uint32_t get_nr_field(std::string const &path)
 28 | {
 29 |     FILE *f = open_c_file(path.c_str(), "r");
 30 |     char line[kMaxLineSize];
 31 | 
 32 |     fgets(line, kMaxLineSize, f);
 33 |     strtok(line, " \t");
 34 | 
 35 |     uint32_t nr_field = 0;
 36 |     while(1)
 37 |     {
 38 |         char *val_char = strtok(nullptr," \t");
 39 |         if(val_char == nullptr || *val_char == '\n')
 40 |             break;
 41 |         ++nr_field;
 42 |     }
 43 | 
 44 |     fclose(f);
 45 | 
 46 |     return nr_field;
 47 | }
 48 | 
 49 | void read_dense(Problem &prob, std::string const &path)
 50 | {
 51 |     char line[kMaxLineSize];
 52 | 
 53 |     FILE *f = open_c_file(path.c_str(), "r");
 54 |     for(uint32_t i = 0; fgets(line, kMaxLineSize, f) != nullptr; ++i)
 55 |     {
 56 |         char *p = strtok(line, " \t");
 57 |         prob.Y[i] = (atoi(p)>0)? 1.0f : -1.0f;
 58 |         for(uint32_t j = 0; j < prob.nr_field; ++j)
 59 |         {
 60 |             char *val_char = strtok(nullptr," \t");
 61 | 
 62 |             float const val = static_cast<float>(atof(val_char));
 63 | 
 64 |             prob.X[j][i] = Node(i, val);
 65 |         }
 66 |     }
 67 | 
 68 |     fclose(f);
 69 | }
 70 | 
 71 | void sort_problem(Problem &prob)
 72 | {
 73 |     struct sort_by_v
 74 |     {
 75 |         bool operator() (Node const lhs, Node const rhs)
 76 |         {
 77 |             return lhs.v < rhs.v;
 78 |         }
 79 |     };
 80 | 
 81 |     #pragma omp parallel for schedule(static)
 82 |     for(uint32_t j = 0; j < prob.nr_field; ++j)
 83 |     {
 84 |         std::vector<Node> &X1 = prob.X[j];
 85 |         std::vector<Node> &Z1 = prob.Z[j];
 86 |         std::sort(X1.begin(), X1.end(), sort_by_v());
 87 |         for(uint32_t i = 0; i < prob.nr_instance; ++i)
 88 |             Z1[X1[i].i] = Node(i, X1[i].v);
 89 |     }
 90 | }
 91 | 
 92 | void read_sparse(Problem &prob, std::string const &path)
 93 | {
 94 |     char line[kMaxLineSize];
 95 | 
 96 |     FILE *f = open_c_file(path.c_str(), "r");
 97 | 
 98 |     std::vector<std::vector<uint32_t>> buffer;
 99 | 
100 |     uint64_t nnz = 0; 
101 |     uint32_t nr_instance = 0;
102 |     prob.SJP.push_back(0);
103 |     for(; fgets(line, kMaxLineSize, f) != nullptr; ++nr_instance)
104 |     {
105 |         strtok(line, " \t");
106 |         for( ; ; ++nnz)
107 |         {
108 |             char *idx_char = strtok(nullptr," \t");
109 |             if(idx_char == nullptr || *idx_char == '\n')
110 |                 break;
111 | 
112 |             uint32_t const idx = atoi(idx_char);
113 |             if(idx > buffer.size())
114 |                 buffer.resize(idx);
115 |             buffer[idx-1].push_back(nr_instance);
116 |             prob.SJ.push_back(idx-1);
117 |         }
118 |         prob.SJP.push_back(prob.SJ.size());
119 |     }
120 |     prob.SJ.shrink_to_fit();
121 |     prob.SJP.shrink_to_fit();
122 | 
123 |     prob.nr_sparse_field = static_cast<uint32_t>(buffer.size());
124 |     prob.SI.resize(nnz);
125 |     prob.SIP.resize(prob.nr_sparse_field+1);
126 |     prob.SIP[0] = 0;
127 | 
128 |     uint64_t p = 0;
129 |     for(uint32_t j = 0; j < prob.nr_sparse_field; ++j)
130 |     {
131 |         for(auto i : buffer[j]) 
132 |             prob.SI[p++] = i;
133 |         prob.SIP[j+1] = p;
134 |     }
135 | 
136 |     fclose(f);
137 | 
138 |     sort_problem(prob);
139 | }
140 | 
141 | } //unamed namespace
142 | 
143 | Problem read_data(std::string const &dense_path, std::string const &sparse_path)
144 | {
145 |     Problem prob(get_nr_line(dense_path), get_nr_field(dense_path));
146 | 
147 |     read_dense(prob, dense_path);
148 | 
149 |     read_sparse(prob, sparse_path);
150 | 
151 |     return prob;
152 | }
153 | 
154 | FILE *open_c_file(std::string const &path, std::string const &mode)
155 | {
156 |     FILE *f = fopen(path.c_str(), mode.c_str());
157 |     if(!f)
158 |         throw std::runtime_error(std::string("cannot open ")+path);
159 |     return f;
160 | }
161 | 
162 | std::vector<std::string> 
163 | argv_to_args(int const argc, char const * const * const argv)
164 | {
165 |     std::vector<std::string> args;
166 |     for(int i = 1; i < argc; ++i)
167 |         args.emplace_back(argv[i]);
168 |     return args;
169 | }
170 | 


--------------------------------------------------------------------------------
/NN/input/training/script/cnt_log_process.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import pandas as pd
  3 | 
  4 | train = pd.read_csv('../temporal_data/train_id.csv')
  5 | test = pd.read_csv('../temporal_data/test_id.csv')
  6 | member = pd.read_csv('../temporal_data/members_id.csv')
  7 | song_origin = pd.read_csv('../temporal_data/songs_id.csv')
  8 | song_extra = pd.read_csv('../temporal_data/songs_extra_id.csv')
  9 | 
 10 | song = pd.DataFrame({'song_id': range(max(train.song_id.max(), test.song_id.max())+1)})
 11 | song = song.merge(song_origin, on='song_id', how='left')
 12 | song = song.merge(song_extra, on='song_id', how='left')
 13 | 
 14 | data = train[['msno', 'song_id']].append(test[['msno', 'song_id']])
 15 | 
 16 | ## member_cnt
 17 | # 每个用户听过的歌曲数
 18 | mem_rec_cnt = data.groupby(by='msno').count()['song_id'].to_dict()
 19 | member['msno_rec_cnt'] = member['msno'].apply(lambda x: mem_rec_cnt[x])
 20 | 
 21 | member['bd'] = member['bd'].apply(lambda x: np.nan if x <= 0 or x >= 75 else x)
 22 | 
 23 | ## song_cnt
 24 | # 每个歌手的歌曲数
 25 | artist_song_cnt = song.groupby(by='artist_name').count()['song_id'].to_dict()
 26 | song['artist_song_cnt'] = song['artist_name'].apply(lambda x: artist_song_cnt[x] if not np.isnan(x) else np.nan)
 27 | 
 28 | # 每个作曲家的歌曲数
 29 | composer_song_cnt = song.groupby(by='composer').count()['song_id'].to_dict()
 30 | composer_song_cnt[0] = np.nan
 31 | song['composer_song_cnt'] = song['composer'].apply(lambda x: composer_song_cnt[x] if not np.isnan(x) else np.nan)
 32 | 
 33 | # 每个作词家的歌曲数
 34 | lyricist_song_cnt = song.groupby(by='lyricist').count()['song_id'].to_dict()
 35 | lyricist_song_cnt[0] = np.nan
 36 | song['lyricist_song_cnt'] = song['lyricist'].apply(lambda x: lyricist_song_cnt[x] if not np.isnan(x) else np.nan)
 37 | 
 38 | # 每个「第一」流派的歌曲数
 39 | genre_song_cnt = song.groupby(by='first_genre_id').count()['song_id'].to_dict()
 40 | genre_song_cnt[0] = np.nan
 41 | song['genre_song_cnt'] = song['first_genre_id'].apply(lambda x: genre_song_cnt[x] if not np.isnan(x) else np.nan)
 42 | 
 43 | # =============================================================================
 44 | 
 45 | data = data.merge(song, on='song_id', how='left')
 46 | 
 47 | # 每首歌被多少用户听过
 48 | song_rec_cnt = data.groupby(by='song_id').count()['msno'].to_dict()
 49 | song['song_rec_cnt'] = song['song_id'].apply(lambda x: song_rec_cnt[x] if not np.isnan(x) else np.nan)
 50 | 
 51 | # 每个歌手被多少用户听过
 52 | artist_rec_cnt = data.groupby(by='artist_name').count()['msno'].to_dict()
 53 | song['artist_rec_cnt'] = song['artist_name'].apply(lambda x: artist_rec_cnt[x] if not np.isnan(x) else np.nan)
 54 | 
 55 | # 每个作曲家被多少用户听过
 56 | composer_rec_cnt = data.groupby(by='composer').count()['msno'].to_dict()
 57 | composer_rec_cnt[0] = np.nan
 58 | song['composer_rec_cnt'] = song['composer'].apply(lambda x: composer_rec_cnt[x] if not np.isnan(x) else np.nan)
 59 | 
 60 | # 每个作词家被多少用户听过
 61 | lyricist_rec_cnt = data.groupby(by='lyricist').count()['msno'].to_dict()
 62 | lyricist_rec_cnt[0] = np.nan
 63 | song['lyricist_rec_cnt'] = song['lyricist'].apply(lambda x: lyricist_rec_cnt[x] if not np.isnan(x) else np.nan)
 64 | 
 65 | # 每个「第一」流派被多少用户听过
 66 | genre_rec_cnt = data.groupby(by='first_genre_id').count()['msno'].to_dict()
 67 | genre_rec_cnt[0] = np.nan
 68 | song['genre_rec_cnt'] = song['first_genre_id'].apply(lambda x: genre_rec_cnt[x] if not np.isnan(x) else np.nan)
 69 | 
 70 | ## msno context features
 71 | # 三个 dummy 特征
 72 | dummy_feat = ['source_system_tab', 'source_screen_name', 'source_type']
 73 | concat = train.drop('target', axis=1).append(test.drop('id', axis=1))
 74 | 
 75 | # 计算三个 dummy 特征的 mean 
 76 | for feat in dummy_feat:
 77 |     feat_dummies = pd.get_dummies(concat[feat])
 78 |     feat_dummies.columns = ['msno_{0}_{1}'.format(feat, col) for col in feat_dummies.columns]
 79 |     feat_dummies['msno'] = concat['msno'].values
 80 |     feat_dummies = feat_dummies.groupby('msno').mean()
 81 |     feat_dummies['msno'] = feat_dummies.index
 82 |     member = member.merge(feat_dummies, on='msno', how='left')
 83 | 
 84 | train_temp = train.merge(member, on='msno', how='left')
 85 | test_temp = test.merge(member, on='msno', how='left')
 86 | 
 87 | train['msno_source_system_tab_prob'] = train_temp[[col for col in train_temp.columns if 'source_system_tab' in col]].apply(lambda x: \
 88 |     x['msno_source_system_tab_{0}'.format(int(x['source_system_tab']))], axis=1)
 89 |     
 90 | test['msno_source_system_tab_prob'] = test_temp[[col for col in test_temp.columns if 'source_system_tab' in col]].apply(lambda x: \
 91 |         x['msno_source_system_tab_{0}'.format(int(x['source_system_tab']))], axis=1)
 92 | 
 93 | train['msno_source_screen_name_prob'] = train_temp[[col for col in train_temp.columns if 'source_screen_name' in col]].apply(lambda x: \
 94 |         x['msno_source_screen_name_{0}'.format(int(x['source_screen_name']))], axis=1)
 95 |         
 96 | test['msno_source_screen_name_prob'] = test_temp[[col for col in test_temp.columns if 'source_screen_name' in col]].apply(lambda x: \
 97 |         x['msno_source_screen_name_{0}'.format(int(x['source_screen_name']))], axis=1)
 98 | 
 99 | train['msno_source_type_prob'] = train_temp[[col for col in train_temp.columns if 'source_type' in col]].apply(lambda x: \
100 |         x['msno_source_type_{0}'.format(int(x['source_type']))], axis=1)
101 |         
102 | test['msno_source_type_prob'] = test_temp[[col for col in test_temp.columns if 'source_type' in col]].apply(lambda x: \
103 |         x['msno_source_type_{0}'.format(int(x['source_type']))], axis=1)
104 | 
105 | ## to_csv
106 | features = ['msno_rec_cnt']
107 | for feat in features:
108 |     member[feat] = np.log1p(member[feat])
109 | member.to_csv('../temporal_data/members_id_cnt.csv', index=False)
110 | 
111 | features = ['song_length', 'song_rec_cnt', 'artist_song_cnt', 'composer_song_cnt', \
112 |         'lyricist_song_cnt', 'genre_song_cnt', 'artist_rec_cnt', \
113 |         'composer_rec_cnt', 'lyricist_rec_cnt', 'genre_rec_cnt']
114 | for feat in features:
115 |     song[feat] = np.log1p(song[feat])
116 | #song['song_length'] = np.log1p(song['song_length'])
117 | song.to_csv('../temporal_data/songs_id_cnt.csv', index=False)
118 | train.to_csv('../temporal_data/train_id_cnt.csv', index=False)
119 | test.to_csv('../temporal_data/test_id_cnt.csv', index=False)
120 | 


--------------------------------------------------------------------------------
/NN/input/training/script/id_process.py:
--------------------------------------------------------------------------------
  1 | import time
  2 | import numpy as np
  3 | import pandas as pd
  4 | 
  5 | from sklearn.preprocessing import LabelEncoder
  6 | 
  7 | ## load the data
  8 | members = pd.read_csv('../source_data/members.csv')
  9 | songs = pd.read_csv('../source_data/songs.csv')
 10 | songs_extra = pd.read_csv('../source_data/song_extra_info.csv')
 11 | train = pd.read_csv('../source_data/train.csv')
 12 | test = pd.read_csv('../source_data/test.csv')
 13 | 
 14 | song_id_set = set(train['song_id'].append(test['song_id']))
 15 | 
 16 | songs['appeared'] = songs['song_id'].apply(lambda x: True if x in song_id_set else False)
 17 | songs = songs[songs.appeared]
 18 | songs.drop('appeared', axis=1, inplace=True)
 19 | 
 20 | songs_extra['appeared'] = songs_extra['song_id'].apply(lambda x: True if x in song_id_set else False)
 21 | songs_extra = songs_extra[songs_extra.appeared]
 22 | songs_extra.drop('appeared', axis=1, inplace=True)
 23 | 
 24 | msno_set = set(train['msno'].append(test['msno']))
 25 | 
 26 | members['appeared'] = members['msno'].apply(lambda x: True if x in msno_set else False)
 27 | members = members[members.appeared]
 28 | members.drop('appeared', axis=1, inplace=True)
 29 | 
 30 | print('Data loaded.')
 31 | 
 32 | ## preprocess msno and song_id
 33 | msno_encoder = LabelEncoder()
 34 | msno_encoder.fit(members['msno'].values)
 35 | members['msno'] = msno_encoder.transform(members['msno'])
 36 | train['msno'] = msno_encoder.transform(train['msno'])
 37 | test['msno'] = msno_encoder.transform(test['msno'])
 38 | 
 39 | print('MSNO done.')
 40 | 
 41 | song_id_encoder = LabelEncoder()
 42 | song_id_encoder.fit(train['song_id'].append(test['song_id']))
 43 | songs['song_id'] = song_id_encoder.transform(songs['song_id'])
 44 | songs_extra['song_id'] = song_id_encoder.transform(songs_extra['song_id'])
 45 | train['song_id'] = song_id_encoder.transform(train['song_id'])
 46 | test['song_id'] = song_id_encoder.transform(test['song_id'])
 47 | 
 48 | print('Song_id done.')
 49 | 
 50 | ## preprocess the features in train.csv & test.csv
 51 | columns = ['source_system_tab', 'source_screen_name', 'source_type']
 52 | for column in columns:
 53 |     column_encoder = LabelEncoder()
 54 |     column_encoder.fit((train[column].append(test[column])).astype(str))
 55 |     train[column] = column_encoder.transform(train[column].astype(str))
 56 |     test[column] = column_encoder.transform(test[column].astype(str))
 57 | 
 58 | print('Source information done.')
 59 | 
 60 | ## preprocess the features in members.csv
 61 | columns = ['city', 'gender', 'registered_via']
 62 | for column in columns:
 63 |     column_encoder = LabelEncoder()
 64 |     column_encoder.fit(members[column].astype(str))
 65 |     members[column] = column_encoder.transform(members[column].astype(str))
 66 | 
 67 | members['registration_init_time'] = members['registration_init_time'].apply(lambda x: \
 68 |         time.mktime(time.strptime(str(x),'%Y%m%d')))
 69 | members['expiration_date'] = members['expiration_date'].apply(lambda x: \
 70 |         time.mktime(time.strptime(str(x),'%Y%m%d')))
 71 | 
 72 | print('Members information done.')
 73 | 
 74 | ## preprocess the features in songs.csv
 75 | genre_id = np.zeros((len(songs), 4))
 76 | for i in range(len(songs)):
 77 |     if not isinstance(songs['genre_ids'].values[i], str):
 78 |         continue
 79 |     ids = str(songs['genre_ids'].values[i]).split('|')
 80 |     if len(ids) > 2:
 81 |         genre_id[i, 0] = int(ids[0])
 82 |         genre_id[i, 1] = int(ids[1])
 83 |         genre_id[i, 2] = int(ids[2])
 84 |     elif len(ids) > 1:
 85 |         genre_id[i, 0] = int(ids[0])
 86 |         genre_id[i, 1] = int(ids[1])
 87 |     elif len(ids) == 1:
 88 |         genre_id[i, 0] = int(ids[0])
 89 |     genre_id[i, 3] = len(ids)
 90 | songs['first_genre_id'] = genre_id[:, 0]
 91 | songs['second_genre_id'] = genre_id[:, 1]
 92 | songs['third_genre_id'] = genre_id[:, 2]
 93 | songs['genre_id_cnt'] = genre_id[:, 3].astype(np.int8)
 94 | 
 95 | genre_encoder = LabelEncoder()
 96 | genre_encoder.fit((songs.first_genre_id.append(songs.second_genre_id)).append(songs.third_genre_id))
 97 | songs['first_genre_id'] = genre_encoder.transform(songs['first_genre_id'])
 98 | songs['second_genre_id'] = genre_encoder.transform(songs['second_genre_id'])
 99 | songs['third_genre_id'] = genre_encoder.transform(songs['third_genre_id'])
100 | songs.drop('genre_ids', axis=1, inplace=True)
101 | 
102 | def artist_count(x):
103 |     return x.count('and') + x.count(',') + x.count(' feat') + x.count('&') + 1
104 | 
105 | songs['artist_cnt'] = songs['artist_name'].apply(artist_count).astype(np.int8)
106 | 
107 | def get_count(x):
108 |     try:
109 |         return sum(map(x.count, ['|', '/', '\\', ';'])) + 1
110 |     except:
111 |         return 0
112 | 
113 | songs['lyricist_cnt'] = songs['lyricist'].apply(get_count).astype(np.int8)
114 | songs['composer_cnt'] = songs['composer'].apply(get_count).astype(np.int8)
115 | 
116 | songs['is_featured'] = songs['artist_name'].apply(lambda x: 1 if ' feat' \
117 |         in str(x) else 0).astype(np.int8)
118 | 
119 | def get_first_artist(x):
120 |     if x.count('and') > 0:
121 |         x = x.split('and')[0]
122 |     if x.count(',') > 0:
123 |         x = x.split(',')[0]
124 |     if x.count(' feat') > 0:
125 |         x = x.split(' feat')[0]
126 |     if x.count('&') > 0:
127 |         x = x.split('&')[0]
128 |     return x.strip()
129 | 
130 | songs['artist_name'] = songs['artist_name'].apply(get_first_artist)
131 |     
132 | def get_first_term(x):
133 |     try:
134 |         if x.count('|') > 0:
135 |             x = x.split('|')[0]
136 |         if x.count('/') > 0:
137 |             x = x.split('/')[0]
138 |         if x.count('\\') > 0:
139 |             x = x.split('\\')[0]
140 |         if x.count(';') > 0:
141 |             x = x.split(';')[0]
142 |         return x.strip()
143 |     except:
144 |         return x
145 | 
146 | songs['lyricist'] = songs['lyricist'].apply(get_first_term)
147 | songs['composer'] = songs['composer'].apply(get_first_term)        
148 | 
149 | songs['language'] = songs['language'].fillna(-1)
150 | columns = ['artist_name', 'lyricist', 'composer', 'language']
151 | for column in columns:
152 |     column_encoder = LabelEncoder()
153 |     column_encoder.fit(songs[column].astype(str))
154 |     songs[column] = column_encoder.transform(songs[column].astype(str))
155 | 
156 | ## save files
157 | members.to_csv('../temporal_data/members_id.csv', index=False)
158 | songs.to_csv('../temporal_data/songs_id.csv', index=False)
159 | songs_extra.to_csv('../temporal_data/songs_extra_id.csv', index=False)
160 | train.to_csv('../temporal_data/train_id.csv', index=False)
161 | test.to_csv('../temporal_data/test_id.csv', index=False)
162 | 
163 | 


--------------------------------------------------------------------------------
/FFM/model/libffm-1.13/ffm-train.cpp:
--------------------------------------------------------------------------------
  1 | #pragma GCC diagnostic ignored "-Wunused-result" 
  2 | #include <algorithm>
  3 | #include <cstring>
  4 | #include <iostream>
  5 | #include <stdexcept>
  6 | #include <string>
  7 | #include <vector>
  8 | #include <cstdlib>
  9 | 
 10 | #include "ffm.h"
 11 | 
 12 | using namespace std;
 13 | using namespace ffm;
 14 | 
 15 | string train_help()
 16 | {
 17 |     return string(
 18 | "usage: ffm-train [options] training_set_file [model_file]\n"
 19 | "\n"
 20 | "options:\n"
 21 | "-l <lambda>: set regularization parameter (default 0.00002)\n"
 22 | "-k <factor>: set number of latent factors (default 4)\n"
 23 | "-t <iteration>: set number of iterations (default 15)\n"
 24 | "-r <eta>: set learning rate (default 0.2)\n"
 25 | "-s <nr_threads>: set number of threads (default 1)\n"
 26 | "-p <path>: set path to the validation set\n"
 27 | "-v <fold>: set the number of folds for cross-validation\n"
 28 | "--quiet: quiet model (no output)\n"
 29 | "--no-norm: disable instance-wise normalization\n"
 30 | "--no-rand: disable random update\n"
 31 | "--on-disk: perform on-disk training (a temporary file <training_set_file>.bin will be generated)\n"
 32 | "--auto-stop: stop at the iteration that achieves the best validation loss (must be used with -p)\n");
 33 | }
 34 | 
 35 | struct Option
 36 | {
 37 |     Option() : param(ffm_get_default_param()), nr_folds(1), do_cv(false), on_disk(false) {}
 38 |     string tr_path, va_path, model_path;
 39 |     ffm_parameter param;
 40 |     ffm_int nr_folds;
 41 |     bool do_cv, on_disk;
 42 | };
 43 | 
 44 | string basename(string path)
 45 | {
 46 |     const char *ptr = strrchr(&*path.begin(), '/');
 47 |     if(!ptr)
 48 |         ptr = path.c_str();
 49 |     else
 50 |         ptr++;
 51 |     return string(ptr);
 52 | }
 53 | 
 54 | Option parse_option(int argc, char **argv)
 55 | {
 56 |     vector<string> args;
 57 |     for(int i = 0; i < argc; i++)
 58 |         args.push_back(string(argv[i]));
 59 | 
 60 |     if(argc == 1)
 61 |         throw invalid_argument(train_help());
 62 | 
 63 |     Option opt;
 64 | 
 65 |     ffm_int i = 1;
 66 |     for(; i < argc; i++)
 67 |     {
 68 |         if(args[i].compare("-t") == 0)
 69 |         {
 70 |             if(i == argc-1)
 71 |                 throw invalid_argument("need to specify number of iterations after -t");
 72 |             i++;
 73 |             opt.param.nr_iters = atoi(args[i].c_str());
 74 |             if(opt.param.nr_iters <= 0)
 75 |                 throw invalid_argument("number of iterations should be greater than zero");
 76 |         }
 77 |         else if(args[i].compare("-k") == 0)
 78 |         {
 79 |             if(i == argc-1)
 80 |                 throw invalid_argument("need to specify number of factors after -k");
 81 |             i++;
 82 |             opt.param.k = atoi(args[i].c_str());
 83 |             if(opt.param.k <= 0)
 84 |                 throw invalid_argument("number of factors should be greater than zero");
 85 |         }
 86 |         else if(args[i].compare("-r") == 0)
 87 |         {
 88 |             if(i == argc-1)
 89 |                 throw invalid_argument("need to specify eta after -r");
 90 |             i++;
 91 |             opt.param.eta = atof(args[i].c_str());
 92 |             if(opt.param.eta <= 0)
 93 |                 throw invalid_argument("learning rate should be greater than zero");
 94 |         }
 95 |         else if(args[i].compare("-l") == 0)
 96 |         {
 97 |             if(i == argc-1)
 98 |                 throw invalid_argument("need to specify lambda after -l");
 99 |             i++;
100 |             opt.param.lambda = atof(args[i].c_str());
101 |             if(opt.param.lambda < 0)
102 |                 throw invalid_argument("regularization cost should not be smaller than zero");
103 |         }
104 |         else if(args[i].compare("-s") == 0)
105 |         {
106 |             if(i == argc-1)
107 |                 throw invalid_argument("need to specify number of threads after -s");
108 |             i++;
109 |             opt.param.nr_threads = atoi(args[i].c_str());
110 |             if(opt.param.nr_threads <= 0)
111 |                 throw invalid_argument("number of threads should be greater than zero");
112 |         }
113 |         else if(args[i].compare("-v") == 0)
114 |         {
115 |             if(i == argc-1)
116 |                 throw invalid_argument("need to specify number of folds after -v");
117 |             i++;
118 |             opt.nr_folds = atoi(args[i].c_str());
119 |             if(opt.nr_folds <= 1)
120 |                 throw invalid_argument("number of folds should be greater than one");
121 |             opt.do_cv = true;
122 |         }
123 |         else if(args[i].compare("-p") == 0)
124 |         {
125 |             if(i == argc-1)
126 |                 throw invalid_argument("need to specify path after -p");
127 |             i++;
128 |             opt.va_path = args[i];
129 |         }
130 |         else if(args[i].compare("--no-norm") == 0)
131 |         {
132 |             opt.param.normalization = false;
133 |         }
134 |         else if(args[i].compare("--quiet") == 0)
135 |         {
136 |             opt.param.quiet = true;
137 |         }
138 |         else if(args[i].compare("--no-rand") == 0)
139 |         {
140 |             opt.param.random = false;
141 |         }
142 |         else if(args[i].compare("--on-disk") == 0)
143 |         {
144 |             opt.on_disk = true;
145 |         }
146 |         else if(args[i].compare("--auto-stop") == 0)
147 |         {
148 |             opt.param.auto_stop = true;
149 |         }
150 |         else
151 |         {
152 |             break;
153 |         }
154 |     }
155 | 
156 |     if(i != argc-2 && i != argc-1)
157 |         throw invalid_argument("cannot parse command\n");
158 | 
159 |     opt.tr_path = args[i];
160 |     i++;
161 | 
162 |     if(i < argc)
163 |     {
164 |         opt.model_path = string(args[i]);
165 |     }
166 |     else if(i == argc)
167 |     {
168 |         opt.model_path = basename(opt.tr_path) + ".model";
169 |     }
170 |     else
171 |     {
172 |         throw invalid_argument("cannot parse argument");
173 |     }
174 | 
175 |     return opt;
176 | }
177 | 
178 | int train(Option opt)
179 | {
180 |     ffm_problem *tr = ffm_read_problem(opt.tr_path.c_str());
181 |     if(tr == nullptr)
182 |     {
183 |         cerr << "cannot load " << opt.tr_path << endl << flush;
184 |         return 1;
185 |     }
186 | 
187 |     ffm_problem *va = nullptr;
188 |     if(!opt.va_path.empty())
189 |     {
190 |         va = ffm_read_problem(opt.va_path.c_str());
191 |         if(va == nullptr)
192 |         {
193 |             ffm_destroy_problem(&tr);
194 |             cerr << "cannot load " << opt.va_path << endl << flush;
195 |             return 1;
196 |         }
197 |     }
198 | 
199 |     int status = 0;
200 |     if(opt.do_cv)
201 |     {
202 |         ffm_cross_validation(tr, opt.nr_folds, opt.param);
203 |     }
204 |     else
205 |     {
206 |         ffm_model *model = ffm_train_with_validation(tr, va, opt.param);
207 | 
208 |         status = ffm_save_model(model, opt.model_path.c_str());
209 | 
210 |         ffm_destroy_model(&model);
211 |     }
212 | 
213 |     ffm_destroy_problem(&tr);
214 |     ffm_destroy_problem(&va);
215 | 
216 |     return status;
217 | }
218 | 
219 | int train_on_disk(Option opt)
220 | {
221 |     if(opt.param.random)
222 |     {
223 |         cout << "Random update is not allowed in disk-level training. Please use `--no-rand' to disable." << endl;
224 |         return 1;
225 |     }
226 | 
227 |     if(opt.do_cv)
228 |     {
229 |         cout << "Cross-validation is not yet implemented in disk-level training." << endl;
230 |         return 1;
231 |     }
232 | 
233 |     string tr_bin_path = basename(opt.tr_path) + ".bin";
234 |     string va_bin_path = opt.va_path.empty()? "" : basename(opt.va_path) + ".bin";
235 | 
236 |     ffm_read_problem_to_disk(opt.tr_path.c_str(), tr_bin_path.c_str());
237 |     if(!opt.va_path.empty())
238 |         ffm_read_problem_to_disk(opt.va_path.c_str(), va_bin_path.c_str());
239 | 
240 |     ffm_model *model = ffm_train_with_validation_on_disk(tr_bin_path.c_str(), va_bin_path.c_str(), opt.param);
241 | 
242 |     ffm_int status = ffm_save_model(model, opt.model_path.c_str());
243 |     if(status != 0)
244 |     {
245 |         ffm_destroy_model(&model);
246 | 
247 |         return 1;
248 |     }
249 | 
250 |     ffm_destroy_model(&model);
251 | 
252 |     return 0;
253 | }
254 | 
255 | int main(int argc, char **argv)
256 | {
257 |     Option opt;
258 |     try
259 |     {
260 |         opt = parse_option(argc, argv);
261 |     }
262 |     catch(invalid_argument &e)
263 |     {
264 |         cout << e.what() << endl;
265 |         return 1;
266 |     }
267 | 
268 |     if(opt.on_disk)
269 |     {
270 |         return train_on_disk(opt);
271 |     }
272 |     else
273 |     {
274 |         return train(opt);
275 |     }
276 | }
277 | 


--------------------------------------------------------------------------------
/LightGBM/model/lbgm.py:
--------------------------------------------------------------------------------
  1 | # -*- coding:utf-8 -*-
  2 | __author__ = 'Randolph'
  3 | 
  4 | import gc
  5 | import time
  6 | import numpy as np
  7 | import pandas as pd
  8 | import lightgbm as lgb
  9 | from utils import data_helpers as dh
 10 | 
 11 | data_path = '../data/'
 12 | result_path = '../result/'
 13 | logs_path = '../logs/'
 14 | 
 15 | logger = dh.logger_fn('lgbmlog', (logs_path + 'lgbm-{0}.log').format(time.asctime()))
 16 | 
 17 | # Loading Data
 18 | # ==================================================
 19 | 
 20 | logger.info('Loading data...')
 21 | 
 22 | train = pd.read_csv(data_path + 'train_lgbm.csv', dtype={'target': np.uint8,
 23 |                                                          'bd': np.uint8,
 24 |                                                          'membership_days': np.uint16,
 25 |                                                          'song_length': np.uint16,
 26 |                                                          'genre_count': np.uint8,
 27 |                                                          'artist_count': np.uint8,
 28 |                                                          'composer_count': np.uint8,
 29 |                                                          'lyricist_count': np.uint8,
 30 |                                                          'count_song_played': np.uint32,
 31 |                                                          'count_artist_played': np.uint32,
 32 |                                                          'count_genre_played': np.uint32,
 33 |                                                          'listen_count': np.uint32,
 34 |                                                          'user_id': 'category',
 35 |                                                          'song_id': 'category',
 36 |                                                          'source_system_tab': 'category',
 37 |                                                          'source_screen_name': 'category',
 38 |                                                          'source_type': 'category',
 39 |                                                          'city': 'category',
 40 |                                                          'gender': 'category',
 41 |                                                          'registered_via': 'category',
 42 |                                                          'registration_year': 'category',
 43 |                                                          'registration_month': 'category',
 44 |                                                          'registration_date': 'category',
 45 |                                                          'expiration_year': 'category',
 46 |                                                          'expiration_month': 'category',
 47 |                                                          'expiration_date': 'category',
 48 |                                                          'genre_ids': 'category',
 49 |                                                          'artist': 'category',
 50 |                                                          'composer': 'category',
 51 |                                                          'lyricist': 'category',
 52 |                                                          'language': 'category',
 53 |                                                          'smaller_song': 'category',
 54 |                                                          'artist_composer': 'category',
 55 |                                                          'artist_composer_lyricist': 'category',
 56 |                                                          'song_country': 'category',
 57 |                                                          'song_publisher': 'category',
 58 |                                                          'song_year': 'category'})
 59 | 
 60 | test = pd.read_csv(data_path + 'test_lgbm.csv', dtype={'bd': np.uint8,
 61 |                                                        'membership_days': np.uint16,
 62 |                                                        'song_length': np.uint16,
 63 |                                                        'genre_count': np.uint8,
 64 |                                                        'artist_count': np.uint8,
 65 |                                                        'composer_count': np.uint8,
 66 |                                                        'lyricist_count': np.uint8,
 67 |                                                        'count_song_played': np.uint32,
 68 |                                                        'count_artist_played': np.uint32,
 69 |                                                        'count_genre_played': np.uint32,
 70 |                                                        'listen_count': np.uint32,
 71 |                                                        'user_id': 'category',
 72 |                                                        'song_id': 'category',
 73 |                                                        'source_system_tab': 'category',
 74 |                                                        'source_screen_name': 'category',
 75 |                                                        'source_type': 'category',
 76 |                                                        'city': 'category',
 77 |                                                        'gender': 'category',
 78 |                                                        'registered_via': 'category',
 79 |                                                        'registration_year': 'category',
 80 |                                                        'registration_month': 'category',
 81 |                                                        'registration_date': 'category',
 82 |                                                        'expiration_year': 'category',
 83 |                                                        'expiration_month': 'category',
 84 |                                                        'expiration_date': 'category',
 85 |                                                        'genre_ids': 'category',
 86 |                                                        'artist': 'category',
 87 |                                                        'composer': 'category',
 88 |                                                        'lyricist': 'category',
 89 |                                                        'language': 'category',
 90 |                                                        'smaller_song': 'category',
 91 |                                                        'artist_composer': 'category',
 92 |                                                        'artist_composer_lyricist': 'category',
 93 |                                                        'song_country': 'category',
 94 |                                                        'song_publisher': 'category',
 95 |                                                        'song_year': 'category'})
 96 | 
 97 | logger.info('Done loading...')
 98 | 
 99 | # Checking nan value
100 | # ==================================================
101 | 
102 | logger.info('Checking nan value...')
103 | dh.fillin_nan_value(train_merged_df=train, test_merged_df=test)
104 | logger.info('Done checking...')
105 | 
106 | # Drop feature
107 | # ==================================================
108 | 
109 | logger.info('Dropping data...')
110 | drop_features = []
111 | 
112 | train = train.drop(drop_features, axis=1)
113 | test = test.drop(drop_features, axis=1)
114 | 
115 | logger.info('Done dropping...')
116 | 
117 | # Add Features
118 | # ==================================================
119 | 
120 | logger.info('Adding new features...')
121 | 
122 | logger.info('Done adding features...')
123 | 
124 | # Split Dataset
125 | # ==================================================
126 | 
127 | logger.info("Train test and validation sets...")
128 | for col in train.columns:
129 |     if train[col].dtype == object:
130 |         train[col] = train[col].astype('category')
131 |         test[col] = test[col].astype('category')
132 | 
133 | X_train = train.drop(['target'], axis=1)
134 | y_train = train['target'].values
135 | 
136 | X_test = test.drop(['id'], axis=1)
137 | ids = test['id'].values
138 | 
139 | logger.info('Done splitting...')
140 | 
141 | del train, test
142 | gc.collect()
143 | 
144 | # Training Data
145 | # ==================================================
146 | 
147 | logger.info('Training data...')
148 | 
149 | d_train_final = lgb.Dataset(data=X_train, label=y_train, max_bin=256)
150 | watchlist_final = lgb.Dataset(data=X_train, label=y_train, max_bin=256)
151 | 
152 | NUM_ROUNDS = 3000
153 | 
154 | # Training GBDT model
155 | logger.info('Training GBDT model...')
156 | 
157 | params_gbdt = {
158 |     'objective': 'binary',
159 |     'metric': {'binary_logloss', 'auc'},
160 |     'boosting': 'gbdt',
161 |     'learning_rate': 0.1,
162 |     'verbose': 0,
163 |     'num_leaves': 512,
164 |     'bagging_fraction': 0.95,
165 |     'bagging_freq': 1,
166 |     'bagging_seed': 1,
167 |     'feature_fraction': 0.9,
168 |     'feature_fraction_seed': 1,
169 |     'max_bin': 256,
170 |     'max_depth': 10,
171 |     'num_rounds': NUM_ROUNDS
172 | }
173 | 
174 | logger.info(params_gbdt)
175 | 
176 | model_f1 = lgb.train(params_gbdt, train_set=d_train_final, valid_sets=watchlist_final, verbose_eval=5)
177 | 
178 | logger.info('Done training GBDT model and make predictions next...')
179 | 
180 | p_test_1 = model_f1.predict(X_test)
181 | 
182 | subm_gbdt = pd.DataFrame()
183 | subm_gbdt['id'] = ids
184 | subm_gbdt['target'] = p_test_1
185 | subm_gbdt.to_csv(result_path + 'submission_lgbm_gbdt_' + str(params_gbdt['num_leaves']) + '_'
186 |                  + str(NUM_ROUNDS) + '.csv', index=False, float_format='%.5f')
187 | 
188 | logger.info('Done saving GBDT model predictions...')
189 | 
190 | # Training DART model
191 | logger.info('Training DART model...')
192 | 
193 | params_dart = {
194 |     'objective': 'binary',
195 |     'metric': {'binary_logloss', 'auc'},
196 |     'boosting': 'dart',
197 |     'learning_rate': 0.1,
198 |     'verbose': 0,
199 |     'num_leaves': 512,
200 |     'bagging_fraction': 0.95,
201 |     'bagging_freq': 1,
202 |     'bagging_seed': 1,
203 |     'feature_fraction': 0.9,
204 |     'feature_fraction_seed': 1,
205 |     'max_bin': 256,
206 |     'max_depth': 10,
207 |     'num_rounds': NUM_ROUNDS
208 | }
209 | 
210 | logger.info(params_dart)
211 | 
212 | model_f2 = lgb.train(params_dart, train_set=d_train_final, valid_sets=watchlist_final, verbose_eval=5)
213 | 
214 | logger.info('Done training DART model and make predictions next...')
215 | 
216 | p_test_2 = model_f2.predict(X_test)
217 | 
218 | subm_dart = pd.DataFrame()
219 | subm_dart['id'] = ids
220 | subm_dart['target'] = p_test_2
221 | subm_dart.to_csv(result_path + 'submission_lgbm_dart_' + str(params_dart['num_leaves']) + '_'
222 |                  + str(NUM_ROUNDS) + '.csv', index=False, float_format='%.5f')
223 | 
224 | logger.info('Done saving DART model predictions...')
225 | 
226 | # Averaging two model predictions
227 | # ==================================================
228 | 
229 | p_test_avg = np.mean([p_test_1, p_test_2], axis=0)
230 | 
231 | subm_avg = pd.DataFrame()
232 | subm_avg['id'] = ids
233 | subm_avg['target'] = p_test_avg
234 | subm_avg.to_csv(result_path + 'submission_lgbm_avg_' + str(params_dart['num_leaves']) + '_'
235 |                 + str(NUM_ROUNDS) + '.csv', index=False, float_format='%.5f')
236 | 
237 | logger.info('Done saving avg predictions...')
238 | logger.info('All Finished!')
239 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "{}"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright {yyyy} {name of copyright owner}
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------
/FFM/model/libffm-1.13/README:
--------------------------------------------------------------------------------
  1 | LIBFFM is a library for field-aware factorization machine. For the formulation it solves, please check:
  2 | 
  3 |     http://www.csie.ntu.edu.tw/~r01922136/slides/ffm.pdf
  4 | 
  5 | 
  6 | 
  7 | Table of Contents
  8 | =================
  9 | 
 10 | - Overfitting and Early Stopping
 11 | - Installation
 12 | - Data Format
 13 | - Command Line Usage
 14 | - Examples
 15 | - Library Usage
 16 | - OpenMP
 17 | - Building Windows Binaries
 18 | 
 19 | 
 20 | 
 21 | Overfitting and Early Stopping
 22 | ==============================
 23 | 
 24 | FFM is prone to overfitting, and the solution we have so far is early stopping. See how FFM behaves on a certain data
 25 | set:
 26 | 
 27 |     > ffm-train -p va.ffm -l 0.00002 tr.ffm
 28 |     iter   tr_logloss   va_logloss
 29 |        1      0.49738      0.48776
 30 |        2      0.47383      0.47995
 31 |        3      0.46366      0.47480
 32 |        4      0.45561      0.47231
 33 |        5      0.44810      0.47034
 34 |        6      0.44037      0.47003
 35 |        7      0.43239      0.46952
 36 |        8      0.42362      0.46999
 37 |        9      0.41394      0.47088
 38 |       10      0.40326      0.47228
 39 |       11      0.39156      0.47435
 40 |       12      0.37886      0.47683
 41 |       13      0.36522      0.47975
 42 |       14      0.35079      0.48321
 43 |       15      0.33578      0.48703
 44 | 
 45 | 
 46 | We see the best validation loss is achieved at 7th iteration. If we keep training, then overfitting begins. It is worth
 47 | noting that increasing regularization parameter do not help:
 48 | 
 49 |     > ffm-train -p va.ffm -l 0.0002 -t 50 -s 12 tr.ffm
 50 |     iter   tr_logloss   va_logloss
 51 |        1      0.50532      0.49905
 52 |        2      0.48782      0.49242
 53 |        3      0.48136      0.48748
 54 |                  ...
 55 |       29      0.42183      0.47014
 56 |                  ...
 57 |       48      0.37071      0.47333
 58 |       49      0.36767      0.47374
 59 |       50      0.36472      0.47404
 60 | 
 61 | 
 62 | To avoid overfitting, we recommend always provide a validation set with option `-p.' You can use option `--auto-stop' to
 63 | stop at the iteration that reaches the best validation loss:
 64 | 
 65 |     > ffm-train -p va.ffm -l 0.00002 --auto-stop tr.ffm
 66 |     iter   tr_logloss   va_logloss
 67 |        1      0.49738      0.48776
 68 |        2      0.47383      0.47995
 69 |        3      0.46366      0.47480
 70 |        4      0.45561      0.47231
 71 |        5      0.44810      0.47034
 72 |        6      0.44037      0.47003
 73 |        7      0.43239      0.46952
 74 |        8      0.42362      0.46999
 75 |     Auto-stop. Use model at 7th iteration.
 76 | 
 77 | 
 78 | Installation
 79 | ============
 80 | 
 81 | Requirement: LIBFFM is written in C++. It requires C++11 and OpenMP supports. If OpenMP is not available on your
 82 | platform, please refer to section `OpenMP.' 
 83 | 
 84 | - Unix-like systems:
 85 |   To compile on Unix-like systems, type `make' in the command line.
 86 | 
 87 | - OS X:
 88 |   The built-in compiler should be able to compile LIBFFM. However, OpenMP may
 89 |   not be supported. In this case you have to compile without OpenMP. See
 90 |   section `OpenMP' for detail.
 91 | 
 92 | - Windows:
 93 |   See `Building Windows Binaries' to compile.
 94 | 
 95 | 
 96 | 
 97 | Data Format
 98 | ===========
 99 | 
100 | The data format of LIBFFM is:
101 | 
102 | <label> <field1>:<index1>:<value1> <field2>:<index2>:<value2> ...
103 | .
104 | .
105 | .
106 | 
107 | `field' and `index' should be non-negative integers. See an example
108 | `bigdata.tr.txt.'
109 | 
110 | 
111 | 
112 | Command Line Usage
113 | ==================
114 | 
115 | -   `ffm-train'
116 | 
117 |     usage: ffm-train [options] training_set_file [model_file]
118 | 
119 |     options:
120 |     -l <lambda>: set regularization parameter (default 0.00002)
121 |     -k <factor>: set number of latent factors (default 4)
122 |     -t <iteration>: set number of iterations (default 15)
123 |     -r <eta>: set learning rate (default 0.2)
124 |     -s <nr_threads>: set number of threads (default 1)
125 |     -p <path>: set path to the validation set
126 |     -v <fold>: set the number of folds for cross-validation
127 |     --quiet: quiet model (no output)
128 |     --no-norm: disable instance-wise normalization
129 |     --no-rand: disable random update
130 |     --on-disk: perform on-disk training (a temporary file <training_set_file>.bin will be generated)
131 |     --auto-stop: stop at the iteration that achieves the best validation loss (must be used with -p)
132 | 
133 |     By default we do instance-wise normalization. That is, we normalize the 2-norm of each instance to 1. You can use
134 |     `--no-norm' to disable this function.
135 |     
136 |     By default, our algorithm randomly select an instance for update in each inner iteration. On some datasets you may
137 |     want to do update in the original order. You can do it by using `--no-rand' together with `-s 1.'
138 | 
139 |     If you do not have enough memory, then you can use `--on-disk' to do disk-level training. Two restrictions when you
140 |     use this mode:
141 |         
142 |         1. So far we do not allow random update in the mode, so please use
143 |            `--no-rand' if you want to do on-disk training. 
144 |            
145 |         2. Cross-validation in this mode is not yet supported.
146 | 
147 |     A binary file `training_set_file.bin' will be generated to store the data in binary format.
148 | 
149 |     Because FFM usually need early stopping for better test performance, we provide an option `--auto-stop' to stop at
150 |     the iteration that achieves the best validation loss. Note that you need to provide a validation set with `-p' when
151 |     you use this option.
152 | 
153 | 
154 | -   `ffm-predict'
155 | 
156 |     usage: ffm-predict test_file model_file output_file
157 | 
158 | 
159 | 
160 | Examples
161 | ========
162 | 
163 | > ffm-train bigdata.tr.txt model
164 | 
165 | train a model using the default parameters
166 | 
167 | > ffm-train -l 0.001 -k 16 -t 30 -r 0.05 -s 4 bigdata.tr.txt model
168 | 
169 | train a model using the following parameters:
170 | 
171 |     regularization cost = 0.001
172 |     latent factors = 16
173 |     iterations = 30
174 |     learning rate = 0.05
175 |     threads = 4
176 | 
177 | > ffm-train -p bigdata.te.txt bigdata.tr.txt model
178 | 
179 | use bigdata.te.txt as validation set
180 | 
181 | > ffm-train -v 5 bigdata.tr.txt
182 | 
183 | do five fold cross validation
184 | 
185 | > ffm-train --quiet bigdata.tr.txt
186 | 
187 | do not print message to screen
188 | 
189 | > ffm-predict bigdata.te.txt model output
190 | 
191 | do prediction
192 | 
193 | > ffm-train --no-rand --on-disk bigdata.tr.txt
194 | 
195 | perform on-disk training
196 | 
197 | > ffm-train -p bigdata.te.txt -t 100 --auto-stop bigdata.tr.txt
198 | 
199 | use auto-stop to stop at the best iteration according to validation loss
200 | 
201 | Library Usage
202 | =============
203 | 
204 | These structures and functions are declared in the header file `ffm.h.' You need to #include `ffm.h' in your C/C++
205 | source files and link your program with `ffm.cpp.' You can see `ffm-train.cpp' and `ffm-predict.cpp' for examples
206 | showing how to use them.
207 | 
208 | There are four public data structures in LIBFFM.
209 | 
210 | 
211 | -   struct ffm_node
212 |     {
213 |         ffm_int f;    // field index
214 |         ffm_int j;    // column index
215 |         ffm_float v;  // value
216 |     };
217 | 
218 |     Each `ffm_node' represents a non-zero element in a sparse matrix.
219 | 
220 | -   struct ffm_problem
221 |     {
222 |         ffm_int n;      // number of features
223 |         ffm_int l;      // number of instances
224 |         ffm_int m;      // number of fields
225 |         ffm_node *X;    // non-zero elements
226 |         ffm_long *P;    // row pointers
227 |         ffm_float *Y;   // labels
228 |     };
229 | 
230 | -   struct ffm_parameter
231 |     {
232 |         ffm_float eta;
233 |         ffm_float lambda;
234 |         ffm_int nr_iters;
235 |         ffm_int k;
236 |         ffm_int nr_threads;
237 |         bool quiet;
238 |         bool normalization;
239 |         bool random;
240 |         bool auto_stop;
241 |     };
242 | 
243 |     `ffm_parameter' represents the parameters used for training. The meaning of
244 |     each variable is:
245 | 
246 |     variable         meaning                             default
247 |     ============================================================
248 |     eta              learning rate                           0.1
249 |     lambda           regularization cost                       0
250 |     nr_iters         number of iterations                     15
251 |     k                number of latent factors                  4
252 |     nr_threads       number of threads used                    1
253 |     quiet            no outputs to stdout                  false
254 |     normalization    instance-wise normalization           false
255 |     random           randomly select instance in SG         true
256 |     auto_stop        auto stop at the best iteration       false
257 | 
258 |     To obtain a parameter object with default values, use the function
259 |     `ffm_get_default_param.'
260 | 
261 | 
262 | -   struct ffm_model
263 |     {
264 |         ffm_int n;              // number of features
265 |         ffm_int m;              // number of fields
266 |         ffm_int k;              // number of latent factors
267 |         ffm_float *W;           // store model values
268 |         bool normalization;     // do instance-wise normalization
269 |     };
270 | 
271 | 
272 | 
273 | Functions available in LIBFFM include:
274 | 
275 | 
276 | -   ffm_parameter ffm_get_default_param();
277 | 
278 |     Get default parameters.
279 | 
280 | -   ffm_int ffm_save_model(struct ffm_model const *model, char const *path);
281 |     
282 |     Save a model. It returns 0 on sucess and 1 on failure.
283 | 
284 | -   struct ffm_model* ffm_load_model(char const *path);
285 | 
286 |     Load a model. If the model could not be loaded, a nullptr is returned.
287 | 
288 | -   void ffm_destroy_model(struct ffm_model **model);
289 |     
290 |     Destroy a model.
291 | 
292 | -   struct ffm_model* ffm_train(struct ffm_problem const *prob, ffm_parameter param);
293 | 
294 |     Train a model.
295 | 
296 | -   struct ffm_model* ffm_train_with_validation(struct ffm_problem const *Tr, struct ffm_problem const *Va, ffm_parameter param);
297 | 
298 |     Train a model with training set `Tr' and validation set `Va.' The logloss of the validation set is printed at each
299 |     iteration.
300 |     
301 | -   ffm_float ffm_cross_validation(struct ffm_problem const *prob, ffm_int nr_folds, ffm_parameter param);
302 | 
303 |     Do cross validation with `nr_folds' folds.
304 | 
305 | -   ffm_float ffm_predict(ffm_node *begin, ffm_node *end, ffm_model *model);
306 | 
307 |     Do prediction. `begin' and `end' are pointers to specify the beginning and ending position of the instance to be
308 |     predicted.
309 | 
310 | 
311 | 
312 | OpenMP
313 | ======
314 | 
315 | We use OpenMP to do parallelization. If OpenMP is not available on your
316 | platform, then please comment out the following lines in Makefile.
317 | 
318 |     DFLAG += -DUSEOMP
319 |     CXXFLAGS += -fopenmp
320 | 
321 | Note: Please always run `make clean all' if these flags are changed.
322 | 
323 | 
324 | 
325 | Building Windows Binaries
326 | =========================
327 | 
328 | To build them via command-line tools of Visual C++, use the following steps:
329 | 
330 | 1. Open a DOS command box (or Developer Command Prompt for Visual Studio) and
331 | go to LIBFFM directory. If environment variables of VC++ have not been set,
332 | type
333 | 
334 | "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\bin\amd64\vcvars64.bat"
335 | 
336 | You may have to modify the above command according which version of VC++ or
337 | where it is installed.
338 | 
339 | 2. Type
340 | 
341 | nmake -f Makefile.win clean all
342 | 
343 | 
344 | 
345 | Contributors
346 | ============
347 | 
348 | Yu-Chin Juan, Wei-Sheng Chin, and Yong Zhuang
349 | 
350 | For questions, comments, feature requests, or bug report, please send your email to 
351 | 
352 |     Yu-Chin (guestwalk@gmail.com)
353 | 


--------------------------------------------------------------------------------
/FFM/model/gbdt/src/gbdt.cpp:
--------------------------------------------------------------------------------
  1 | #include <limits>
  2 | #include <numeric>
  3 | #include <algorithm>
  4 | #include <thread>
  5 | #include <omp.h>
  6 | 
  7 | #include "gbdt.h"
  8 | #include "timer.h"
  9 | 
 10 | /*
 11 | This implementation is based on Algo 5 in the following paper:
 12 | 
 13 |    http://statweb.stanford.edu/~jhf/ftp/trebst.pdf
 14 | 
 15 | We do not have a plan to maintain this part of code, so there is no documents.
 16 | 
 17 | We recommend you either check the above document, or use some exisiting packages such as xgboost.
 18 | */
 19 | 
 20 | namespace {
 21 | 
 22 | float calc_bias(std::vector<float> const &Y)
 23 | {
 24 |     double y_bar = std::accumulate(Y.begin(), Y.end(), 0.0);
 25 |     y_bar /= static_cast<double>(Y.size());
 26 |     return static_cast<float>(log((1.0+y_bar)/(1.0-y_bar)));
 27 | }
 28 | 
 29 | struct Location
 30 | {
 31 |     Location() : tnode_idx(1), r(0), shrinked(false) {}
 32 |     uint32_t tnode_idx;
 33 |     float r;
 34 |     bool shrinked;
 35 | };
 36 | 
 37 | struct Meta
 38 | {
 39 |     Meta() : sl(0), s(0), nl(0), n(0), v(0.0f/0.0f) {}
 40 |     double sl, s;
 41 |     uint32_t nl, n;
 42 |     float v;
 43 | };
 44 | 
 45 | struct Defender
 46 | {
 47 |     Defender() : ese(0), threshold(0) {}
 48 |     double ese;
 49 |     float threshold;
 50 | };
 51 | 
 52 | void scan(
 53 |     Problem const &prob,
 54 |     std::vector<Location> const &locations,
 55 |     std::vector<Meta> const &metas0,
 56 |     std::vector<Defender> &defenders,
 57 |     uint32_t const offset,
 58 |     bool const forward)
 59 | {
 60 |     uint32_t const nr_field = prob.nr_field;
 61 |     uint32_t const nr_instance = prob.nr_instance;
 62 | 
 63 |     #pragma omp parallel for schedule(dynamic)
 64 |     for(uint32_t j = 0; j < nr_field; ++j)
 65 |     {
 66 |         std::vector<Meta> metas = metas0;
 67 | 
 68 |         for(uint32_t i_bar = 0; i_bar < nr_instance; ++i_bar)
 69 |         {
 70 |             uint32_t const i = forward? i_bar : nr_instance-i_bar-1;
 71 | 
 72 |             Node const &dnode = prob.X[j][i];
 73 |             Location const &location = locations[dnode.i];
 74 |             if(location.shrinked)
 75 |                 continue;
 76 | 
 77 |             uint32_t const f = location.tnode_idx-offset;
 78 |             Meta &meta = metas[f];
 79 | 
 80 |             if(dnode.v != meta.v)
 81 |             {
 82 |                 double const sr = meta.s - meta.sl;
 83 |                 uint32_t const nr = meta.n - meta.nl;
 84 |                 double const current_ese = 
 85 |                     (meta.sl*meta.sl)/static_cast<double>(meta.nl) + 
 86 |                     (sr*sr)/static_cast<double>(nr);
 87 | 
 88 |                 Defender &defender = defenders[f*nr_field+j];
 89 |                 double &best_ese = defender.ese;
 90 |                 if(current_ese > best_ese)
 91 |                 {
 92 |                     best_ese = current_ese;
 93 |                     defender.threshold = forward? dnode.v : meta.v;
 94 |                 }
 95 |                 if(i_bar > nr_instance/2)
 96 |                     break;
 97 |             }
 98 | 
 99 |             meta.sl += location.r;
100 |             ++meta.nl;
101 |             meta.v = dnode.v;
102 |         }
103 |     }
104 | }
105 | 
106 | void scan_sparse(
107 |     Problem const &prob,
108 |     std::vector<Location> const &locations,
109 |     std::vector<Meta> const &metas0,
110 |     std::vector<Defender> &defenders,
111 |     uint32_t const offset,
112 |     bool const forward)
113 | {
114 |     uint32_t const nr_sparse_field = prob.nr_sparse_field;
115 |     uint32_t const nr_leaf = offset;
116 | 
117 |     #pragma omp parallel for schedule(dynamic)
118 |     for(uint32_t j = 0; j < nr_sparse_field; ++j)
119 |     {
120 |         std::vector<Meta> metas = metas0;
121 |         for(uint64_t p = prob.SIP[j]; p < prob.SIP[j+1]; ++p)
122 |         {
123 |             Location const &location = locations[prob.SI[p]];
124 |             if(location.shrinked)
125 |                 continue;
126 |             Meta &meta = metas[location.tnode_idx-offset];
127 |             meta.sl += location.r;
128 |             ++meta.nl;
129 |         }
130 | 
131 |         for(uint32_t f = 0; f < nr_leaf; ++f)
132 |         {
133 |             Meta const &meta = metas[f];
134 |             if(meta.nl == 0)
135 |                 continue;
136 |             
137 |             double const sr = meta.s - meta.sl;
138 |             uint32_t const nr = meta.n - meta.nl;
139 |             double const current_ese = 
140 |                 (meta.sl*meta.sl)/static_cast<double>(meta.nl) + 
141 |                 (sr*sr)/static_cast<double>(nr);
142 | 
143 |             Defender &defender = defenders[f*nr_sparse_field+j];
144 |             double &best_ese = defender.ese;
145 |             if(current_ese > best_ese)
146 |             {
147 |                 best_ese = current_ese;
148 |                 defender.threshold = 1;
149 |             }
150 |         }
151 |     }
152 | }
153 | 
154 | } //unnamed namespace
155 | 
156 | uint32_t CART::max_depth = 7;
157 | uint32_t CART::max_tnodes = static_cast<uint32_t>(pow(2, CART::max_depth+1));
158 | std::mutex CART::mtx;
159 | bool CART::verbose = false;
160 | 
161 | void CART::fit(Problem const &prob, std::vector<float> const &R, 
162 |     std::vector<float> &F1)
163 | {
164 |     uint32_t const nr_field = prob.nr_field;
165 |     uint32_t const nr_sparse_field = prob.nr_sparse_field;
166 |     uint32_t const nr_instance = prob.nr_instance;
167 | 
168 |     std::vector<Location> locations(nr_instance);
169 |     #pragma omp parallel for schedule(static)
170 |     for(uint32_t i = 0; i < nr_instance; ++i)
171 |         locations[i].r = R[i];
172 |     for(uint32_t d = 0, offset = 1; d < max_depth; ++d, offset *= 2)
173 |     {
174 |         uint32_t const nr_leaf = static_cast<uint32_t>(pow(2, d));
175 |         std::vector<Meta> metas0(nr_leaf);
176 | 
177 |         for(uint32_t i = 0; i < nr_instance; ++i)
178 |         {
179 |             Location &location = locations[i];
180 |             if(location.shrinked)
181 |                 continue;
182 | 
183 |             Meta &meta = metas0[location.tnode_idx-offset];
184 |             meta.s += location.r;
185 |             ++meta.n;
186 |         }
187 | 
188 |         std::vector<Defender> defenders(nr_leaf*nr_field);
189 |         std::vector<Defender> defenders_sparse(nr_leaf*nr_sparse_field);
190 |         for(uint32_t f = 0; f < nr_leaf; ++f)
191 |         {
192 |             Meta const &meta = metas0[f];
193 |             double const ese = meta.s*meta.s/static_cast<double>(meta.n);
194 |             for(uint32_t j = 0; j < nr_field; ++j)
195 |                 defenders[f*nr_field+j].ese = ese;
196 |             for(uint32_t j = 0; j < nr_sparse_field; ++j)
197 |                 defenders_sparse[f*nr_sparse_field+j].ese = ese;
198 |         }
199 |         std::vector<Defender> defenders_inv = defenders;
200 | 
201 |         std::thread thread_f(scan, std::ref(prob), std::ref(locations),
202 |             std::ref(metas0), std::ref(defenders), offset, true);
203 |         std::thread thread_b(scan, std::ref(prob), std::ref(locations),
204 |             std::ref(metas0), std::ref(defenders_inv), offset, false);
205 |         scan_sparse(prob, locations, metas0, defenders_sparse, offset, true);
206 |         thread_f.join();
207 |         thread_b.join();
208 | 
209 |         for(uint32_t f = 0; f < nr_leaf; ++f)
210 |         {
211 |             Meta const &meta = metas0[f];
212 |             double best_ese = meta.s*meta.s/static_cast<double>(meta.n);
213 |             TreeNode &tnode = tnodes[f+offset];
214 |             for(uint32_t j = 0; j < nr_field; ++j)
215 |             {
216 |                 Defender defender = defenders[f*nr_field+j];
217 |                 if(defender.ese > best_ese)
218 |                 {
219 |                     best_ese = defender.ese;
220 |                     tnode.feature = j;
221 |                     tnode.threshold = defender.threshold;
222 |                 }
223 | 
224 |                 defender = defenders_inv[f*nr_field+j];
225 |                 if(defender.ese > best_ese)
226 |                 {
227 |                     best_ese = defender.ese;
228 |                     tnode.feature = j;
229 |                     tnode.threshold = defender.threshold;
230 |                 }
231 |             }
232 |             for(uint32_t j = 0; j < nr_sparse_field; ++j)
233 |             {
234 |                 Defender defender = defenders_sparse[f*nr_sparse_field+j];
235 |                 if(defender.ese > best_ese)
236 |                 {
237 |                     best_ese = defender.ese;
238 |                     tnode.feature = nr_field + j;
239 |                     tnode.threshold = defender.threshold;
240 |                 }
241 |             }
242 |         }
243 | 
244 |         #pragma omp parallel for schedule(static)
245 |         for(uint32_t i = 0; i < nr_instance; ++i)
246 |         {
247 |             Location &location = locations[i];
248 |             if(location.shrinked)
249 |                 continue;
250 | 
251 |             uint32_t &tnode_idx = location.tnode_idx;
252 |             TreeNode &tnode = tnodes[tnode_idx];
253 |             if(tnode.feature == -1)
254 |             {
255 |                 location.shrinked = true;
256 |             }
257 |             else if(static_cast<uint32_t>(tnode.feature) < nr_field)
258 |             {
259 |                 if(prob.Z[tnode.feature][i].v < tnode.threshold)
260 |                     tnode_idx = 2*tnode_idx; 
261 |                 else
262 |                     tnode_idx = 2*tnode_idx+1; 
263 |             }
264 |             else
265 |             {
266 |                 uint32_t const target_feature 
267 |                     = static_cast<uint32_t>(tnode.feature-nr_field);
268 |                 bool is_one = false;
269 |                 for(uint64_t p = prob.SJP[i]; p < prob.SJP[i+1]; ++p) 
270 |                 {
271 |                     if(prob.SJ[p] == target_feature)
272 |                     {
273 |                         is_one = true;
274 |                         break;
275 |                     }
276 |                 }
277 |                 if(!is_one)
278 |                     tnode_idx = 2*tnode_idx; 
279 |                 else
280 |                     tnode_idx = 2*tnode_idx+1; 
281 |             }
282 |         }
283 |     }
284 | 
285 |     std::vector<std::pair<double, double>> 
286 |         tmp(max_tnodes, std::make_pair(0, 0));
287 |     for(uint32_t i = 0; i < nr_instance; ++i)
288 |     {
289 |         float const r = locations[i].r;
290 |         uint32_t const tnode_idx = locations[i].tnode_idx;
291 |         tmp[tnode_idx].first += r;
292 |         tmp[tnode_idx].second += fabs(r)*(1-fabs(r));
293 |     }
294 | 
295 |     for(uint32_t tnode_idx = 1; tnode_idx <= max_tnodes; ++tnode_idx)
296 |     {
297 |         double a, b;
298 |         std::tie(a, b) = tmp[tnode_idx];
299 |         tnodes[tnode_idx].gamma = (b <= 1e-12)? 0 : static_cast<float>(a/b);
300 |     }
301 | 
302 |     #pragma omp parallel for schedule(static)
303 |     for(uint32_t i = 0; i < nr_instance; ++i)
304 |         F1[i] = tnodes[locations[i].tnode_idx].gamma;
305 | }
306 | 
307 | std::pair<uint32_t, float> CART::predict(float const * const x) const
308 | {
309 |     uint32_t tnode_idx = 1;
310 |     for(uint32_t d = 0; d <= max_depth; ++d)
311 |     {
312 |         TreeNode const &tnode = tnodes[tnode_idx];
313 |         if(tnode.feature == -1)
314 |             return std::make_pair(tnode.idx, tnode.gamma);
315 | 
316 |         if(x[tnode.feature] < tnode.threshold)
317 |             tnode_idx = tnode_idx*2;
318 |         else
319 |             tnode_idx = tnode_idx*2+1;
320 |     }
321 | 
322 |     return std::make_pair(-1, -1);
323 | }
324 | 
325 | void GBDT::fit(Problem const &Tr, Problem const &Va)
326 | {
327 |     bias = calc_bias(Tr.Y);
328 | 
329 |     std::vector<float> F_Tr(Tr.nr_instance, bias), F_Va(Va.nr_instance, bias);
330 | 
331 |     Timer timer;
332 |     printf("iter     time    tr_loss    va_loss\n");
333 |     for(uint32_t t = 0; t < trees.size(); ++t)
334 |     {
335 |         timer.tic();
336 | 
337 |         std::vector<float> const &Y = Tr.Y;
338 |         std::vector<float> R(Tr.nr_instance), F1(Tr.nr_instance);
339 | 
340 |         #pragma omp parallel for schedule(static)
341 |         for(uint32_t i = 0; i < Tr.nr_instance; ++i) 
342 |             R[i] = static_cast<float>(Y[i]/(1+exp(Y[i]*F_Tr[i])));
343 | 
344 |         trees[t].fit(Tr, R, F1);
345 | 
346 |         double Tr_loss = 0;
347 |         #pragma omp parallel for schedule(static) reduction(+: Tr_loss)
348 |         for(uint32_t i = 0; i < Tr.nr_instance; ++i) 
349 |         {
350 |             F_Tr[i] += F1[i];
351 |             Tr_loss += log(1+exp(-Y[i]*F_Tr[i]));
352 |         }
353 |         Tr_loss /= static_cast<double>(Tr.nr_instance);
354 | 
355 |         #pragma omp parallel for schedule(static)
356 |         for(uint32_t i = 0; i < Va.nr_instance; ++i)
357 |         {
358 |             std::vector<float> x = construct_instance(Va, i);
359 |             F_Va[i] += trees[t].predict(x.data()).second;
360 |         }
361 | 
362 |         double Va_loss = 0;
363 |         #pragma omp parallel for schedule(static) reduction(+: Va_loss)
364 |         for(uint32_t i = 0; i < Va.nr_instance; ++i) 
365 |             Va_loss += log(1+exp(-Va.Y[i]*F_Va[i]));
366 |         Va_loss /= static_cast<double>(Va.nr_instance);
367 | 
368 |         printf("%4d %8.1f %10.5f %10.5f\n", t, timer.toc(), Tr_loss, Va_loss);
369 |         fflush(stdout);
370 |     }
371 | }
372 | 
373 | float GBDT::predict(float const * const x) const
374 | {
375 |     float s = bias;
376 |     for(auto &tree : trees)
377 |         s += tree.predict(x).second;
378 |     return s;
379 | }
380 | 
381 | std::vector<uint32_t> GBDT::get_indices(float const * const x) const
382 | {
383 |     uint32_t const nr_tree = static_cast<uint32_t>(trees.size());
384 | 
385 |     std::vector<uint32_t> indices(nr_tree);
386 |     for(uint32_t t = 0; t < nr_tree; ++t)
387 |         indices[t] = trees[t].predict(x).first;
388 |     return indices;
389 | }
390 | 


--------------------------------------------------------------------------------
/CatBoost/utils/data_helpers.py:
--------------------------------------------------------------------------------
  1 | # -*- coding:utf-8 -*-
  2 | 
  3 | import joblib
  4 | import logging
  5 | import numpy as np
  6 | import pandas as pd
  7 | import scipy.sparse as sp
  8 | 
  9 | 
 10 | from itertools import combinations
 11 | from sklearn.preprocessing import LabelEncoder
 12 | from scipy.sparse import coo_matrix
 13 | from lightfm import LightFM
 14 | 
 15 | data_path = '../data/'
 16 | result_path = '../result/'
 17 | logs_path = '../logs/'
 18 | 
 19 | date_columns = ['expiration_date', 'registration_init_time']
 20 | not_categorical_columns = ['target', 'song_length', 'registration_init_time', 'expiration_date', 'time', 'bd']
 21 | 
 22 | 
 23 | def logger_fn(name, file, level=logging.INFO):
 24 |     logger = logging.getLogger(name)
 25 |     logger.setLevel(level)
 26 |     fh = logging.FileHandler(file, mode='w')
 27 |     ch = logging.StreamHandler()
 28 |     ch.setLevel(level)
 29 |     formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
 30 |     fh.setFormatter(formatter)
 31 |     ch.setFormatter(formatter)
 32 |     logger.addHandler(fh)
 33 |     logger.addHandler(ch)
 34 |     return logger
 35 | 
 36 | 
 37 | def preprocess():
 38 |     train = pd.read_csv(data_path + "train.csv")
 39 |     test = pd.read_csv(data_path + "test.csv", index_col=0)
 40 |     song_data = pd.read_csv(data_path + "songs.csv")
 41 |     user_data = pd.read_csv(data_path + "members.csv", parse_dates=date_columns)
 42 | 
 43 |     all_data = pd.concat([train, test])
 44 |     all_data = all_data.merge(song_data, on='song_id', how='left')
 45 |     all_data = all_data.merge(user_data, on='msno', how='left')
 46 | 
 47 |     enc = LabelEncoder()
 48 | 
 49 |     for col in [
 50 |         'msno', 'song_id', 'source_screen_name', 'source_system_tab', 'source_type',
 51 |         'genre_ids', 'artist_name', 'composer', 'lyricist', 'gender'
 52 |     ]:
 53 |         all_data[col] = enc.fit_transform(all_data[col].fillna('nan'))
 54 | 
 55 |     for col in ['language', 'city', 'registered_via']:
 56 |         all_data[col] = enc.fit_transform(all_data[col].fillna(-2))
 57 | 
 58 |     all_data['time'] = all_data.index / len(all_data)
 59 | 
 60 |     n = len(train)
 61 |     train_data = all_data[:n]
 62 |     test_data = all_data[n:]
 63 | 
 64 |     train_data.to_hdf(data_path + 'train.hdf', key='wsdm')
 65 |     test_data.to_hdf(data_path + 'test.hdf', key='wsdm')
 66 | 
 67 | 
 68 | def create_features():
 69 |     train_data = pd.read_hdf(data_path + 'train.hdf', parse_dates=date_columns)
 70 |     test_data = pd.read_hdf(data_path + 'test.hdf', parse_dates=date_columns)
 71 | 
 72 |     all_data = pd.concat([train_data, test_data])
 73 |     df_test = test_data
 74 |     df_history_test = train_data
 75 | 
 76 |     df_trains = []
 77 |     df_history_trains = []
 78 | 
 79 |     n = len(test_data)
 80 |     shift = int(0.05 * len(train_data))
 81 | 
 82 |     for i in range(2):
 83 |         m = - (i * shift)
 84 |         if m == 0:
 85 |             m = None
 86 |         df_trains.append(train_data[-(n + i * shift):m])
 87 |         df_history_trains.append(train_data[:-(n + i * shift)])
 88 | 
 89 |     categorical_columns = all_data.columns.difference(not_categorical_columns)
 90 | 
 91 |     orders = {}
 92 |     for col in categorical_columns:
 93 |         orders[col] = 10 ** (int(np.log(all_data[col].max() + 1) / np.log(10)) + 1)
 94 | 
 95 |     print('Orders computing finished...')
 96 | 
 97 |     # =================================================
 98 |     def get_group(df, cols):
 99 |         group = df[cols[0]].copy()
100 |         for col in cols[1:]:
101 |             group = group * orders[col] + df[col]
102 |         return group
103 | 
104 |     def mean(df, df_history, cols):
105 |         group = get_group(df, cols)
106 |         group_history = get_group(df_history, cols)
107 | 
108 |         mean_map = df_history.groupby(group_history)['target'].mean()
109 | 
110 |         return group.map(mean_map).fillna(-1)
111 | 
112 |     def count(all_data, df, cols):
113 |         group = get_group(df, cols)
114 |         group_all = get_group(all_data, cols)
115 | 
116 |         count_map = group_all.value_counts()
117 |         return group.map(count_map).fillna(0)
118 | 
119 |     def regression(df, df_history, cols):
120 |         group = get_group(df, cols)
121 |         group_history = get_group(df_history, cols)
122 | 
123 |         targets = {}
124 |         times = {}
125 |         for (y, t), u in zip(df_history[['target', 'time']].values, group_history):
126 |             if u not in targets:
127 |                 targets[u] = [y]
128 |                 times[u] = [t]
129 |             else:
130 |                 targets[u].append(y)
131 |                 times[u].append(t)
132 | 
133 |         linal_user = {}
134 |         for u in times:
135 |             if len(times[u]) > 1:
136 |                 A = np.vstack([times[u], np.ones(len(times[u]))]).T
137 |                 linal_user[u] = np.linalg.inv(A.T.dot(A)).dot(A.T).dot(targets[u])
138 | 
139 |         result = []
140 | 
141 |         for t, u in zip(df['time'], group):
142 |             if u not in times:
143 |                 result.append(0.5)
144 |             else:
145 |                 if len(times[u]) < 2:
146 |                     result.append(0.5)
147 |                 else:
148 |                     result.append(linal_user[u].dot([t, 1]))
149 | 
150 |         return result
151 | 
152 |     def time_from_prev_heard(df, df_history, cols):
153 |         group = get_group(df, cols)
154 |         group_history = get_group(df_history, cols)
155 | 
156 |         last_heard = df_history.groupby(group_history)['time'].last().to_dict()
157 | 
158 |         result = []
159 | 
160 |         for t, g in zip(df.time, group):
161 |             if g in last_heard:
162 |                 result.append(t - last_heard[g])
163 |             else:
164 |                 result.append(-1)
165 |             last_heard[g] = t
166 |         return result
167 | 
168 |     def time_to_next_heard(df, df_history, cols):
169 |         result = []
170 |         df_reverse = df.sort_index(ascending=False)
171 |         group = get_group(df_reverse, cols)
172 | 
173 |         next_heard = {}
174 | 
175 |         for g, t in zip(group, df_reverse['time']):
176 |             if g in next_heard:
177 |                 result.append(t - next_heard[g])
178 |             else:
179 |                 result.append(-1)
180 |             next_heard[g] = t
181 | 
182 |         result.reverse()
183 |         return result
184 | 
185 |     def count_from_future(df, df_history, cols):
186 |         result = []
187 |         df_reverse = df.sort_index(ascending=False)
188 |         group = get_group(df_reverse, cols)
189 | 
190 |         count = {}
191 |         for g in group.values:
192 |             if g in count:
193 |                 result.append(count[g])
194 |                 count[g] += 1
195 |             else:
196 |                 result.append(0)
197 |                 count[g] = 1
198 |         result.reverse()
199 |         return result
200 | 
201 |     def last_time_diff(df, df_history, cols):
202 |         group = get_group(df, cols)
203 |         last_time = df.groupby(group)['time'].last()
204 |         return group.map(last_time) - df.time
205 | 
206 |     def count_from_past(df, df_history, cols):
207 |         group = get_group(df, cols)
208 |         count = {}
209 |         result = []
210 |         for g in group.values:
211 |             if g not in count:
212 |                 count[g] = 0
213 |             else:
214 |                 count[g] += 1
215 |             result.append(count[g])
216 |         return result
217 | 
218 |     def part_of_unique_song(df):
219 |         group = get_group(all_data, ['msno', 'artist_name'])
220 |         group_df = get_group(df, ['msno', 'artist_name'])
221 | 
222 |         num_song_by_artist = all_data.groupby('artist_name')['song_id'].nunique()
223 |         num_song_by_user_artist = all_data.groupby(group)['song_id'].nunique()
224 | 
225 |         s1 = df['artist_name'].map(num_song_by_artist)
226 |         s2 = group_df.map(num_song_by_user_artist)
227 | 
228 |         return s2 / s1
229 | 
230 |     def matrix_factorization(df, df_history):
231 |         cols = ['msno', 'source_type']
232 |         group = get_group(df, cols)
233 |         group_history = get_group(df_history, cols)
234 | 
235 |         encoder = LabelEncoder()
236 |         encoder.fit(pd.concat([group, group_history]))
237 | 
238 |         df['user_id'] = encoder.transform(group)
239 |         df_history['uesr_id'] = encoder.transform(group_history)
240 | 
241 |         num_users = max(df['user_id'].max(), df_history['user_id'].max()) + 1
242 |         num_items = max(df['song_id'].max(), df_history['song_id'].max()) + 1
243 |         num_msno = max(df['msno'].max(), df_history['msno'].max()) + 1
244 | 
245 |         M = coo_matrix(
246 |             (df_history['target'], (df_history['user_id'], df_history['song_id'])),
247 |             shape=(num_users, num_items)
248 |         )
249 | 
250 |         user_features = pd.concat([df, df_history])[['msno', 'user_id']].drop_duplicates()
251 | 
252 |         user_features = coo_matrix(
253 |             (np.ones(len(user_features)), (user_features['user_id'], user_features['msno'])),
254 |             shape=[num_users, num_msno]
255 |         )
256 | 
257 |         user_features = sp.hstack([sp.eye(num_users), user_features])
258 | 
259 |         model = LightFM(no_components=50, learning_rate=0.1)
260 | 
261 |         model.fit(
262 |             M,
263 |             epochs=2,
264 |             num_threads=50,
265 |             user_features=user_features
266 |         )
267 | 
268 |         result = model.predict(
269 |             df['user_id'].values,
270 |             df['song_id'].values,
271 |             user_features=user_features
272 |         )
273 | 
274 |         return result
275 | 
276 |     def col_name(cols, func):
277 |         return '_'.join(cols) + '_' + func.__name__
278 | 
279 |     def feature_engineer(df, df_history):
280 |         X = pd.DataFrame()
281 | 
282 |         print('Feature_Engineer Begins...')
283 |         print('Totally 6 steps.')
284 | 
285 |         for num_col in [1, 2]:
286 |             for cols in combinations(categorical_columns, num_col):
287 |                 for func in [mean, count, time_to_next_heard, count_from_future,
288 |                              last_time_diff, count_from_past]:
289 |                     print(col_name(cols, func) + ' Begins!')
290 |                     X[col_name(cols, func)] = func(df, df_history, list(cols))
291 |                     print(col_name(cols, func) + ' Finished!')
292 | 
293 |         print('Step 1 Finished! [1/6]')
294 | 
295 |         for cols in combinations(categorical_columns, 3):
296 |             for func in [mean, count]:
297 |                 X[col_name(cols, func)] = func(df, df_history, list(cols))
298 |             if 'msno' in cols:
299 |                 for func in [time_to_next_heard, last_time_diff, count_from_past]:
300 |                     X[col_name(cols, func)] = func(df, df_history, list(cols))
301 | 
302 |         print('Step 2 Finished! [2/6]')
303 | 
304 |         for cols in [
305 |             ['msno'],
306 |             ['msno', 'source_type'],
307 |             ['msno', 'genre_ids'],
308 |             ['msno', 'artist_name'],
309 |             ['msno', 'composer'],
310 |             ['msno', 'language'],
311 |             ['song_id']
312 |         ]:
313 |             X[col_name(cols, regression)] = regression(df, df_history, cols)
314 | 
315 |         print('Step 3 Finished! [3/6]')
316 | 
317 |         for cols in [
318 |             ['msno'],
319 |             ['msno', 'genre_ids'],
320 |             ['msno', 'composer'],
321 |             ['msno', 'language'],
322 |             ['msno', 'artist_name']
323 |         ]:
324 |             X[col_name(cols, time_from_prev_heard)] = time_from_prev_heard(df, df_history, cols)
325 | 
326 |         print('Step 4 Finished! [4/6]')
327 | 
328 |         for col in ['song_length', 'bd']:
329 |             X[col] = df[col]
330 | 
331 |         for col in ['expiration_date', 'registration_init_time']:
332 |             X[col] = df[col].apply(lambda x: x.toordinal())
333 | 
334 |         X['part_song_listened'] = df['song_length'] / X['msno_time_to_next_heard']
335 |         X['time_from_test_period'] = np.arange(len(df))
336 |         X['part_of_unique_song'] = part_of_unique_song(df)
337 | 
338 |         X['matrix_factorization'] = matrix_factorization(df, df_history)
339 | 
340 |         print('Step 5 Finished! [5/6]')
341 | 
342 |         for i in [500000, 2000000]:
343 |             for cols in [
344 |                 ['msno'],
345 |                 ['msno', 'source_type'],
346 |                 ['msno', 'genre_ids'],
347 |                 ['msno', 'artist_name'],
348 |                 ['msno', 'composer'],
349 |                 ['msno', 'language'],
350 |                 ['song_id']
351 |             ]:
352 |                 X[col_name(cols, mean) + str(i)] = mean(df_history[-i:], df, cols)
353 | 
354 |         print('Step 6 Finished! [6/6]')
355 | 
356 |         return X
357 | 
358 |     Xtest = feature_engineer(df_test, df_history_test)
359 |     Xtrain0 = feature_engineer(df_trains[0], df_history_trains[0])
360 |     Xtrain1 = feature_engineer(df_trains[1], df_history_trains[1])
361 | 
362 |     print('Feature Engineer Finished!')
363 | 
364 |     Xtest.to_hdf(data_path + 'Xtest.hdf', key='abc')
365 |     Xtrain0.to_hdf(data_path + 'Xtrain0.hdf', key='abc')
366 |     Xtrain1.to_hdf(data_path + 'Xtrain1.hdf', key='abc')
367 | 
368 |     df_trains[0]['target'].to_hdf(data_path + 'ytrain0.hdf', key='abc')
369 |     df_trains[1]['target'].to_hdf(data_path + 'ytrain1.hdf', key='abc')
370 | 
371 | 
372 | def train_model(model_xgb, model_cb):
373 |     Xtrain0 = pd.read_hdf(data_path + 'Xtrain0.hdf')
374 |     ytrain0 = pd.read_hdf(data_path + 'ytrain0.hdf')
375 |     Xtrain1 = pd.read_hdf(data_path + 'Xtrain1.hdf')
376 |     ytrain1 = pd.read_hdf(data_path + 'ytrain1.hdf')
377 |     Xtest = pd.read_hdf(data_path + 'Xtest.hdf')
378 | 
379 |     # Running XGBoost Model
380 |     model_xgb.fit(Xtrain0, ytrain0)
381 |     p = model_xgb.predict_proba(Xtest)[:, 1]
382 |     joblib.dump(p, data_path + 'p0_xgb_mf')
383 | 
384 |     model_xgb.fit(Xtrain0.drop('matrix_factorization', axis=1), ytrain0)
385 |     p = model_xgb.predict_proba(Xtest.drop('matrix_factorization', axis=1))[:, 1]
386 |     joblib.dump(p, data_path + 'p0_xgb')
387 | 
388 |     model_xgb.fit(Xtrain1, ytrain1)
389 |     p = model_xgb.predict_proba(Xtest)[:, 1]
390 |     joblib.dump(p, data_path + 'p1_xgb_mf')
391 | 
392 |     model_xgb.fit(Xtrain1.drop('matrix_factorization', axis=1), ytrain1)
393 |     p = model_xgb.predict_proba(Xtest.drop('matrix_factorization', axis=1))[:, 1]
394 |     joblib.dump(p, data_path + 'p1_xgb')
395 | 
396 |     # Running CatBoost Model
397 |     model_cb.fit(Xtrain0, ytrain0)
398 |     p = model_cb.predict_proba(Xtest)[:, 1]
399 |     joblib.dump(p, data_path + 'p0_cb_mf')
400 | 
401 |     model_cb.fit(Xtrain0.drop('matrix_factorization', axis=1), ytrain0)
402 |     p = model_cb.predict_proba(Xtest.drop('matrix_factorization', axis=1))[:, 1]
403 |     joblib.dump(p, data_path + 'p0_cb')
404 | 
405 |     model_cb.fit(Xtrain1, ytrain1)
406 |     p = model_cb.predict_proba(Xtest)[:, 1]
407 |     joblib.dump(p, data_path + 'p1_cb_mf')
408 | 
409 |     model_cb.fit(Xtrain1.drop('matrix_factorization', axis=1), ytrain1)
410 |     p = model_cb.predict_proba(Xtest.drop('matrix_factorization', axis=1))[:, 1]
411 |     joblib.dump(p, data_path + 'p1_cb')
412 | 
413 | 
414 | def blend(pct1, pct2):
415 |     p0_xgb_mf = joblib.load(data_path + 'p0_xgb_mf')
416 |     p0_xgb = joblib.load(data_path + 'p0_xgb')
417 |     p1_xgb_mf = joblib.load(data_path + 'p1_xgb_mf')
418 |     p1_xgb = joblib.load(data_path + 'p1_xgb')
419 | 
420 |     p0_cb_mf = joblib.load(data_path + 'p0_cb_mf')
421 |     p0_cb = joblib.load(data_path + 'p0_cb')
422 |     p1_cb_mf = joblib.load(data_path + 'p1_cb_mf')
423 |     p1_cb = joblib.load(data_path + 'p1_cb')
424 | 
425 |     p_xgb = pct1 * p0_xgb + pct2 * p1_xgb
426 |     p_xgb_mf = pct1 * p0_xgb_mf + pct2 * p1_xgb_mf
427 |     p_cb = pct1 * p0_cb + pct2 * p1_cb
428 |     p_cb_mf = pct1 * p0_cb_mf + pct2 * p1_cb_mf
429 | 
430 |     p_x = pct1 * p_xgb_mf + pct2 * p_xgb
431 |     p_c = pct1 * p_cb_mf + pct2 * p_cb
432 | 
433 |     p = pct1 * p_c + pct2 * p_x
434 | 
435 |     return p
436 | 
437 | 
438 | def make_submission(data):
439 |     sub = pd.DataFrame(data)
440 |     sub = sub.reset_index()
441 |     sub.columns = ['id', 'target']
442 |     sub.to_csv(result_path + 'submission.csv', index=False)
443 | 
444 | 
445 | if __name__ == '__main__':
446 |     create_features()
447 | 


--------------------------------------------------------------------------------
/FFM/model/libffm-1.13/ffm.cpp:
--------------------------------------------------------------------------------
   1 | #pragma GCC diagnostic ignored "-Wunused-result" 
   2 | #include <algorithm>
   3 | #include <cmath>
   4 | #include <iostream>
   5 | #include <iomanip>
   6 | #include <fstream>
   7 | #include <new>
   8 | #include <memory>
   9 | #include <random>
  10 | #include <stdexcept>
  11 | #include <string>
  12 | #include <cstring>
  13 | #include <vector>
  14 | #include <pmmintrin.h>
  15 | 
  16 | #if defined USEOMP
  17 | #include <omp.h>
  18 | #endif
  19 | 
  20 | #include "ffm.h"
  21 | 
  22 | namespace ffm {
  23 | 
  24 | namespace {
  25 | 
  26 | using namespace std;
  27 | 
  28 | ffm_int const kALIGNByte = 16;
  29 | ffm_int const kALIGN = kALIGNByte/sizeof(ffm_float);
  30 | ffm_int const kCHUNK_SIZE = 10000000;
  31 | ffm_int const kMaxLineSize = 100000;
  32 | 
  33 | inline ffm_float wTx(
  34 |     ffm_node *begin,
  35 |     ffm_node *end,
  36 |     ffm_float r,
  37 |     ffm_model &model, 
  38 |     ffm_float kappa=0, 
  39 |     ffm_float eta=0, 
  40 |     ffm_float lambda=0, 
  41 |     bool do_update=false)
  42 | {
  43 |     ffm_long align0 = (ffm_long)model.k*2;
  44 |     ffm_long align1 = (ffm_long)model.m*align0;
  45 | 
  46 |     __m128 XMMkappa = _mm_set1_ps(kappa);
  47 |     __m128 XMMeta = _mm_set1_ps(eta);
  48 |     __m128 XMMlambda = _mm_set1_ps(lambda);
  49 | 
  50 |     __m128 XMMt = _mm_setzero_ps();
  51 | 
  52 |     for(ffm_node *N1 = begin; N1 != end; N1++)
  53 |     {
  54 |         ffm_int j1 = N1->j;
  55 |         ffm_int f1 = N1->f;
  56 |         ffm_float v1 = N1->v;
  57 |         if(j1 >= model.n || f1 >= model.m)
  58 |             continue;
  59 | 
  60 |         for(ffm_node *N2 = N1+1; N2 != end; N2++)
  61 |         {
  62 |             ffm_int j2 = N2->j;
  63 |             ffm_int f2 = N2->f;
  64 |             ffm_float v2 = N2->v;
  65 |             if(j2 >= model.n || f2 >= model.m)
  66 |                 continue;
  67 | 
  68 |             ffm_float *w1 = model.W + j1*align1 + f2*align0;
  69 |             ffm_float *w2 = model.W + j2*align1 + f1*align0;
  70 | 
  71 |             __m128 XMMv = _mm_set1_ps(v1*v2*r);
  72 | 
  73 |             if(do_update)
  74 |             {
  75 |                 __m128 XMMkappav = _mm_mul_ps(XMMkappa, XMMv);
  76 | 
  77 |                 ffm_float *wg1 = w1 + model.k;
  78 |                 ffm_float *wg2 = w2 + model.k;
  79 |                 for(ffm_int d = 0; d < model.k; d += 4)
  80 |                 {
  81 |                     __m128 XMMw1 = _mm_load_ps(w1+d);
  82 |                     __m128 XMMw2 = _mm_load_ps(w2+d);
  83 | 
  84 |                     __m128 XMMwg1 = _mm_load_ps(wg1+d);
  85 |                     __m128 XMMwg2 = _mm_load_ps(wg2+d);
  86 | 
  87 |                     __m128 XMMg1 = _mm_add_ps(
  88 |                                    _mm_mul_ps(XMMlambda, XMMw1),
  89 |                                    _mm_mul_ps(XMMkappav, XMMw2));
  90 |                     __m128 XMMg2 = _mm_add_ps(
  91 |                                    _mm_mul_ps(XMMlambda, XMMw2),
  92 |                                    _mm_mul_ps(XMMkappav, XMMw1));
  93 | 
  94 |                     XMMwg1 = _mm_add_ps(XMMwg1, _mm_mul_ps(XMMg1, XMMg1));
  95 |                     XMMwg2 = _mm_add_ps(XMMwg2, _mm_mul_ps(XMMg2, XMMg2));
  96 | 
  97 |                     XMMw1 = _mm_sub_ps(XMMw1, _mm_mul_ps(XMMeta, 
  98 |                             _mm_mul_ps(_mm_rsqrt_ps(XMMwg1), XMMg1)));
  99 |                     XMMw2 = _mm_sub_ps(XMMw2, _mm_mul_ps(XMMeta, 
 100 |                             _mm_mul_ps(_mm_rsqrt_ps(XMMwg2), XMMg2)));
 101 | 
 102 |                     _mm_store_ps(w1+d, XMMw1);
 103 |                     _mm_store_ps(w2+d, XMMw2);
 104 | 
 105 |                     _mm_store_ps(wg1+d, XMMwg1);
 106 |                     _mm_store_ps(wg2+d, XMMwg2);
 107 |                 }
 108 |             }
 109 |             else
 110 |             {
 111 |                 for(ffm_int d = 0; d < model.k; d += 4)
 112 |                 {
 113 |                     __m128  XMMw1 = _mm_load_ps(w1+d);
 114 |                     __m128  XMMw2 = _mm_load_ps(w2+d);
 115 | 
 116 |                     XMMt = _mm_add_ps(XMMt, 
 117 |                            _mm_mul_ps(_mm_mul_ps(XMMw1, XMMw2), XMMv));
 118 |                 }
 119 |             }
 120 |         }
 121 |     }
 122 | 
 123 |     if(do_update)
 124 |         return 0;
 125 | 
 126 |     XMMt = _mm_hadd_ps(XMMt, XMMt);
 127 |     XMMt = _mm_hadd_ps(XMMt, XMMt);
 128 |     ffm_float t;
 129 |     _mm_store_ss(&t, XMMt);
 130 | 
 131 |     return t;
 132 | }
 133 | 
 134 | ffm_float* malloc_aligned_float(ffm_long size)
 135 | {
 136 |     void *ptr;
 137 | 
 138 | #ifdef _WIN32
 139 |     ptr = _aligned_malloc(size*sizeof(ffm_float), kALIGNByte);
 140 |     if(ptr == nullptr)
 141 |         throw bad_alloc();
 142 | #else
 143 |     int status = posix_memalign(&ptr, kALIGNByte, size*sizeof(ffm_float));
 144 |     if(status != 0)
 145 |         throw bad_alloc();
 146 | #endif
 147 |     
 148 |     return (ffm_float*)ptr;
 149 | }
 150 | 
 151 | ffm_model* init_model(ffm_int n, ffm_int m, ffm_parameter param)
 152 | {
 153 |     ffm_int k_aligned = (ffm_int)ceil((ffm_double)param.k/kALIGN)*kALIGN;
 154 | 
 155 |     ffm_model *model = new ffm_model;
 156 |     model->n = n;
 157 |     model->k = k_aligned;
 158 |     model->m = m;
 159 |     model->W = nullptr;
 160 |     model->normalization = param.normalization;
 161 |     
 162 |     try
 163 |     {
 164 |         model->W = malloc_aligned_float((ffm_long)n*m*k_aligned*2);
 165 |     }
 166 |     catch(bad_alloc const &e)
 167 |     {
 168 |         ffm_destroy_model(&model);
 169 |         throw;
 170 |     }
 171 | 
 172 |     ffm_float coef = 1.0f/sqrt(param.k);
 173 |     ffm_float *w = model->W;
 174 | 
 175 |     default_random_engine generator;
 176 |     uniform_real_distribution<ffm_float> distribution(0.0, 1.0);
 177 | 
 178 |     for(ffm_int j = 0; j < model->n; j++)
 179 |     {
 180 |         for(ffm_int f = 0; f < model->m; f++)
 181 |         {
 182 |             for(ffm_int d = 0; d < param.k; d++, w++)
 183 |                 *w = coef*distribution(generator);
 184 |             for(ffm_int d = param.k; d < k_aligned; d++, w++)
 185 |                 *w = 0;
 186 |             for(ffm_int d = k_aligned; d < 2*k_aligned; d++, w++)
 187 |                 *w = 1;
 188 |         }
 189 |     }
 190 | 
 191 |     return model;
 192 | }
 193 | 
 194 | void shrink_model(ffm_model &model, ffm_int k_new)
 195 | {
 196 |     for(ffm_int j = 0; j < model.n; j++)
 197 |     {
 198 |         for(ffm_int f = 0; f < model.m; f++)
 199 |         {
 200 |             ffm_float *src = model.W + ((ffm_long)j*model.m+f)*model.k*2;
 201 |             ffm_float *dst = model.W + ((ffm_long)j*model.m+f)*k_new;
 202 |             copy(src, src+k_new, dst);
 203 |         }
 204 |     }
 205 | 
 206 |     model.k = k_new;
 207 | }
 208 | 
 209 | vector<ffm_float> normalize(ffm_problem &prob)
 210 | {
 211 |     vector<ffm_float> R(prob.l);
 212 | #if defined USEOMP
 213 | #pragma omp parallel for schedule(static)
 214 | #endif
 215 |     for(ffm_int i = 0; i < prob.l; i++)
 216 |     {
 217 |         ffm_float norm = 0;
 218 |         for(ffm_long p = prob.P[i]; p < prob.P[i+1]; p++)
 219 |             norm += prob.X[p].v*prob.X[p].v;
 220 |         R[i] = 1/norm;
 221 |     }
 222 | 
 223 |     return R;
 224 | }
 225 | 
 226 | shared_ptr<ffm_model> train(
 227 |     ffm_problem *tr, 
 228 |     vector<ffm_int> &order, 
 229 |     ffm_parameter param, 
 230 |     ffm_problem *va=nullptr)
 231 | {
 232 | #if defined USEOMP
 233 |     ffm_int old_nr_threads = omp_get_num_threads();
 234 |     omp_set_num_threads(param.nr_threads);
 235 | #endif
 236 | 
 237 |     shared_ptr<ffm_model> model = 
 238 |         shared_ptr<ffm_model>(init_model(tr->n, tr->m, param),
 239 |             [] (ffm_model *ptr) { ffm_destroy_model(&ptr); });
 240 | 
 241 |     vector<ffm_float> R_tr, R_va;
 242 |     if(param.normalization)
 243 |     {
 244 |         R_tr = normalize(*tr);
 245 |         if(va != nullptr)
 246 |             R_va = normalize(*va);
 247 |     }
 248 |     else
 249 |     {
 250 |         R_tr = vector<ffm_float>(tr->l, 1);
 251 |         if(va != nullptr)
 252 |             R_va = vector<ffm_float>(va->l, 1);
 253 |     }
 254 | 
 255 |     bool auto_stop = param.auto_stop && va != nullptr && va->l != 0;
 256 | 
 257 |     ffm_int k_aligned = (ffm_int)ceil((ffm_double)param.k/kALIGN)*kALIGN;
 258 |     ffm_long w_size = (ffm_long)model->n * model->m * k_aligned * 2;
 259 |     vector<ffm_float> prev_W;
 260 |     if(auto_stop)
 261 |         prev_W.assign(w_size, 0);
 262 |     ffm_double best_va_loss = numeric_limits<ffm_double>::max();
 263 | 
 264 |     if(!param.quiet)
 265 |     {
 266 |         if(param.auto_stop && (va == nullptr || va->l == 0))
 267 |             cerr << "warning: ignoring auto-stop because there is no validation set" << endl;
 268 | 
 269 |         cout.width(4);
 270 |         cout << "iter";
 271 |         cout.width(13);
 272 |         cout << "tr_logloss";
 273 |         if(va != nullptr && va->l != 0)
 274 |         {
 275 |             cout.width(13);
 276 |             cout << "va_logloss";
 277 |         }
 278 |         cout << endl;
 279 |     }
 280 | 
 281 |     for(ffm_int iter = 1; iter <= param.nr_iters; iter++)
 282 |     {
 283 |         ffm_double tr_loss = 0;
 284 |         if(param.random)
 285 |             random_shuffle(order.begin(), order.end());
 286 | #if defined USEOMP
 287 | #pragma omp parallel for schedule(static) reduction(+: tr_loss)
 288 | #endif
 289 |         for(ffm_int ii = 0; ii < tr->l; ii++)
 290 |         {
 291 |             ffm_int i = order[ii];
 292 | 
 293 |             ffm_float y = tr->Y[i];
 294 |             
 295 |             ffm_node *begin = &tr->X[tr->P[i]];
 296 | 
 297 |             ffm_node *end = &tr->X[tr->P[i+1]];
 298 | 
 299 |             ffm_float r = R_tr[i];
 300 | 
 301 |             ffm_float t = wTx(begin, end, r, *model);
 302 | 
 303 |             ffm_float expnyt = exp(-y*t);
 304 | 
 305 |             tr_loss += log(1+expnyt);
 306 |                
 307 |             ffm_float kappa = -y*expnyt/(1+expnyt);
 308 | 
 309 |             wTx(begin, end, r, *model, kappa, param.eta, param.lambda, true);
 310 |         }
 311 | 
 312 |         if(!param.quiet)
 313 |         {
 314 |             tr_loss /= tr->l;
 315 | 
 316 |             cout.width(4);
 317 |             cout << iter;
 318 |             cout.width(13);
 319 |             cout << fixed << setprecision(5) << tr_loss;
 320 |             if(va != nullptr && va->l != 0)
 321 |             {
 322 |                 ffm_double va_loss = 0;
 323 | #if defined USEOMP
 324 | #pragma omp parallel for schedule(static) reduction(+:va_loss)
 325 | #endif
 326 |                 for(ffm_int i = 0; i < va->l; i++)
 327 |                 {
 328 |                     ffm_float y = va->Y[i];
 329 | 
 330 |                     ffm_node *begin = &va->X[va->P[i]];
 331 | 
 332 |                     ffm_node *end = &va->X[va->P[i+1]];
 333 | 
 334 |                     ffm_float r = R_va[i];
 335 | 
 336 |                     ffm_float t = wTx(begin, end, r, *model);
 337 |                     
 338 |                     ffm_float expnyt = exp(-y*t);
 339 | 
 340 |                     va_loss += log(1+expnyt);
 341 |                 }
 342 |                 va_loss /= va->l;
 343 | 
 344 |                 cout.width(13);
 345 |                 cout << fixed << setprecision(5) << va_loss;
 346 | 
 347 |                 if(auto_stop)
 348 |                 {
 349 |                     if(va_loss > best_va_loss)
 350 |                     {
 351 |                         memcpy(model->W, prev_W.data(), w_size*sizeof(ffm_float));
 352 |                         cout << endl << "Auto-stop. Use model at " << iter-1 << "th iteration." << endl;
 353 |                         break;
 354 |                     }
 355 |                     else
 356 |                     {
 357 |                         memcpy(prev_W.data(), model->W, w_size*sizeof(ffm_float));
 358 |                         best_va_loss = va_loss; 
 359 |                     }
 360 |                 }
 361 |             }
 362 |             cout << endl;
 363 |         }
 364 |     }
 365 | 
 366 |     shrink_model(*model, param.k);
 367 | 
 368 | #if defined USEOMP
 369 |     omp_set_num_threads(old_nr_threads);
 370 | #endif
 371 | 
 372 |     return model;
 373 | }
 374 | 
 375 | // TODO: This function will be merged with train().
 376 | shared_ptr<ffm_model> train_on_disk(
 377 |     string tr_path,
 378 |     string va_path,
 379 |     ffm_parameter param)
 380 | {
 381 | #if defined USEOMP
 382 |     ffm_int old_nr_threads = omp_get_num_threads();
 383 |     omp_set_num_threads(param.nr_threads);
 384 | #endif
 385 | 
 386 |     FILE *f_tr = fopen(tr_path.c_str(), "rb");
 387 |     FILE *f_va = nullptr;
 388 |     if(!va_path.empty())
 389 |         f_va = fopen(va_path.c_str(), "rb");
 390 | 
 391 |     ffm_int m, n, max_l;
 392 |     ffm_long max_nnz;
 393 |     fread(&m, sizeof(ffm_int), 1, f_tr);
 394 |     fread(&n, sizeof(ffm_int), 1, f_tr);
 395 |     fread(&max_l, sizeof(ffm_int), 1, f_tr);
 396 |     fread(&max_nnz, sizeof(ffm_long), 1, f_tr);
 397 | 
 398 |     shared_ptr<ffm_model> model = 
 399 |         shared_ptr<ffm_model>(init_model(n, m, param),
 400 |             [] (ffm_model *ptr) { ffm_destroy_model(&ptr); });
 401 | 
 402 |     vector<ffm_float> Y;
 403 |     Y.reserve(max_l);
 404 |     vector<ffm_float> R;
 405 |     R.reserve(max_l);
 406 |     vector<ffm_long> P;
 407 |     P.reserve(max_l+1);
 408 |     vector<ffm_node> X;
 409 |     X.reserve(max_nnz);
 410 | 
 411 |     bool auto_stop = param.auto_stop && !va_path.empty();
 412 | 
 413 |     ffm_int k_aligned = (ffm_int)ceil((ffm_double)param.k/kALIGN)*kALIGN;
 414 |     ffm_long w_size = (ffm_long)model->n * model->m * k_aligned * 2;
 415 |     vector<ffm_float> prev_W;
 416 |     if(auto_stop)
 417 |         prev_W.assign(w_size, 0);
 418 |     ffm_double best_va_loss = numeric_limits<ffm_double>::max();
 419 | 
 420 |     if(!param.quiet)
 421 |     {
 422 |         if(param.auto_stop && va_path.empty())
 423 |             cerr << "warning: ignoring auto-stop because there is no validation set" << endl;
 424 |         cout.width(4);
 425 |         cout << "iter";
 426 |         cout.width(13);
 427 |         cout << "tr_logloss";
 428 |         if(!va_path.empty())
 429 |         {
 430 |             cout.width(13);
 431 |             cout << "va_logloss";
 432 |         }
 433 |         cout << endl;
 434 |     }
 435 | 
 436 |     for(ffm_int iter = 1; iter <= param.nr_iters; iter++)
 437 |     {
 438 |         ffm_double tr_loss = 0;
 439 | 
 440 |         fseek(f_tr, 3*sizeof(ffm_int)+sizeof(ffm_long), SEEK_SET);
 441 | 
 442 |         ffm_int tr_l = 0;
 443 |         while(true)
 444 |         {
 445 |             ffm_int l;
 446 |             fread(&l, sizeof(ffm_int), 1, f_tr);
 447 |             tr_l += l;
 448 |             if(l == 0)
 449 |                 break;
 450 | 
 451 |             Y.resize(l);
 452 |             fread(Y.data(), sizeof(ffm_float), l, f_tr);
 453 | 
 454 |             R.resize(l);
 455 |             fread(R.data(), sizeof(ffm_float), l, f_tr);
 456 | 
 457 |             P.resize(l+1);
 458 |             fread(P.data(), sizeof(ffm_long), l+1, f_tr);
 459 | 
 460 |             X.resize(P[l]);
 461 |             fread(X.data(), sizeof(ffm_node), P[l], f_tr);
 462 | 
 463 | #if defined USEOMP
 464 | #pragma omp parallel for schedule(static) reduction(+: tr_loss)
 465 | #endif
 466 |             for(ffm_int i = 0; i < l; i++)
 467 |             {
 468 |                 ffm_float y = Y[i];
 469 |                 
 470 |                 ffm_node *begin = &X[P[i]];
 471 | 
 472 |                 ffm_node *end = &X[P[i+1]];
 473 | 
 474 |                 ffm_float r = param.normalization? R[i] : 1;
 475 | 
 476 |                 ffm_float t = wTx(begin, end, r, *model);
 477 | 
 478 |                 ffm_float expnyt = exp(-y*t);
 479 | 
 480 |                 tr_loss += log(1+expnyt);
 481 |                    
 482 |                 ffm_float kappa = -y*expnyt/(1+expnyt);
 483 | 
 484 |                 wTx(begin, end, r, *model, kappa, param.eta, param.lambda, true);
 485 |             }
 486 |         }
 487 | 
 488 |         if(!param.quiet)
 489 |         {
 490 |             tr_loss /= tr_l;
 491 | 
 492 |             cout.width(4);
 493 |             cout << iter;
 494 |             cout.width(13);
 495 |             cout << fixed << setprecision(5) << tr_loss;
 496 | 
 497 |             if(f_va != nullptr)
 498 |             {
 499 |                 fseek(f_va, 3*sizeof(ffm_int)+sizeof(ffm_long), SEEK_SET);
 500 | 
 501 |                 ffm_int va_l = 0;
 502 |                 ffm_double va_loss = 0;
 503 |                 while(true)
 504 |                 {
 505 |                     ffm_int l;
 506 |                     fread(&l, sizeof(ffm_int), 1, f_va);
 507 |                     va_l += l;
 508 |                     if(l == 0)
 509 |                         break;
 510 | 
 511 |                     vector<ffm_float> Y(l);
 512 |                     fread(Y.data(), sizeof(ffm_float), l, f_va);
 513 | 
 514 |                     vector<ffm_float> R(l);
 515 |                     fread(R.data(), sizeof(ffm_float), l, f_va);
 516 | 
 517 |                     vector<ffm_long> P(l+1);
 518 |                     fread(P.data(), sizeof(ffm_long), l+1, f_va);
 519 | 
 520 |                     vector<ffm_node> X(P[l]);
 521 |                     fread(X.data(), sizeof(ffm_node), P[l], f_va);
 522 | 
 523 | #if defined USEOMP
 524 | #pragma omp parallel for schedule(static) reduction(+: va_loss)
 525 | #endif
 526 |                     for(ffm_int i = 0; i < l; i++)
 527 |                     {
 528 |                         ffm_float y = Y[i];
 529 |                         
 530 |                         ffm_node *begin = &X[P[i]];
 531 | 
 532 |                         ffm_node *end = &X[P[i+1]];
 533 | 
 534 |                         ffm_float r = param.normalization? R[i] : 1;
 535 | 
 536 |                         ffm_float t = wTx(begin, end, r, *model);
 537 | 
 538 |                         ffm_float expnyt = exp(-y*t);
 539 | 
 540 |                         va_loss += log(1+expnyt);
 541 |                     }
 542 |                 }
 543 |                 va_loss /= va_l;
 544 | 
 545 |                 cout.width(13);
 546 |                 cout << fixed << setprecision(5) << va_loss;
 547 | 
 548 |                 if(auto_stop)
 549 |                 {
 550 |                     if(va_loss > best_va_loss)
 551 |                     {
 552 |                         memcpy(model->W, prev_W.data(), w_size*sizeof(ffm_float));
 553 |                         cout << endl << "Auto-stop. Use model at " << iter-1 << "th iteration." << endl;
 554 |                         break;
 555 |                     }
 556 |                     else
 557 |                     {
 558 |                         memcpy(prev_W.data(), model->W, w_size*sizeof(ffm_float));
 559 |                         best_va_loss = va_loss; 
 560 |                     }
 561 |                 }
 562 |             }
 563 |             cout << endl;
 564 |         }
 565 |     }
 566 | 
 567 |     shrink_model(*model, param.k);
 568 | 
 569 |     fclose(f_tr);
 570 |     if(!va_path.empty())
 571 |         fclose(f_va);
 572 | 
 573 | #if defined USEOMP
 574 |     omp_set_num_threads(old_nr_threads);
 575 | #endif
 576 | 
 577 |     return model;
 578 | }
 579 | 
 580 | } // unnamed namespace
 581 | 
 582 | ffm_problem* ffm_read_problem(char const *path)
 583 | {
 584 |     if(strlen(path) == 0)
 585 |         return nullptr;
 586 | 
 587 |     FILE *f = fopen(path, "r");
 588 |     if(f == nullptr)
 589 |         return nullptr;
 590 | 
 591 |     ffm_problem *prob = new ffm_problem;
 592 |     prob->l = 0;
 593 |     prob->n = 0;
 594 |     prob->m = 0;
 595 |     prob->X = nullptr;
 596 |     prob->P = nullptr;
 597 |     prob->Y = nullptr;
 598 | 
 599 |     char line[kMaxLineSize];
 600 | 
 601 |     ffm_long nnz = 0;
 602 |     for(; fgets(line, kMaxLineSize, f) != nullptr; prob->l++)
 603 |     {
 604 |         strtok(line, " \t");
 605 |         for(; ; nnz++)
 606 |         {
 607 |             char *ptr = strtok(nullptr," \t");
 608 |             if(ptr == nullptr || *ptr == '\n')
 609 |                 break;
 610 |         }
 611 |     }
 612 |     rewind(f);
 613 | 
 614 |     prob->X = new ffm_node[nnz];
 615 |     prob->P = new ffm_long[prob->l+1];
 616 |     prob->Y = new ffm_float[prob->l];
 617 | 
 618 |     ffm_long p = 0;
 619 |     prob->P[0] = 0;
 620 |     for(ffm_int i = 0; fgets(line, kMaxLineSize, f) != nullptr; i++)
 621 |     {
 622 |         char *y_char = strtok(line, " \t");
 623 |         ffm_float y = (atoi(y_char)>0)? 1.0f : -1.0f;
 624 |         prob->Y[i] = y;
 625 | 
 626 |         for(; ; p++)
 627 |         {
 628 |             char *field_char = strtok(nullptr,":");
 629 |             char *idx_char = strtok(nullptr,":");
 630 |             char *value_char = strtok(nullptr," \t");
 631 |             if(field_char == nullptr || *field_char == '\n')
 632 |                 break;
 633 | 
 634 |             ffm_int field = atoi(field_char);
 635 |             ffm_int idx = atoi(idx_char);
 636 |             ffm_float value = atof(value_char);
 637 | 
 638 |             prob->m = max(prob->m, field+1);
 639 |             prob->n = max(prob->n, idx+1);
 640 |             
 641 |             prob->X[p].f = field;
 642 |             prob->X[p].j = idx;
 643 |             prob->X[p].v = value;
 644 |         }
 645 |         prob->P[i+1] = p;
 646 |     }
 647 | 
 648 |     fclose(f);
 649 | 
 650 |     return prob;
 651 | }
 652 | 
 653 | int ffm_read_problem_to_disk(char const *txt_path, char const *bin_path)
 654 | {
 655 |     FILE *f_txt = fopen(txt_path, "r");
 656 |     if(f_txt == nullptr)
 657 |         return 1;
 658 | 
 659 |     FILE *f_bin = fopen(bin_path, "wb");
 660 |     if(f_bin == nullptr)
 661 |         return 1;
 662 | 
 663 |     vector<char> line(kMaxLineSize);
 664 | 
 665 |     ffm_int m = 0;
 666 |     ffm_int n = 0;
 667 |     ffm_int max_l = 0;
 668 |     ffm_long max_nnz = 0;
 669 |     ffm_long p = 0;
 670 | 
 671 |     vector<ffm_float> Y;
 672 |     vector<ffm_float> R;
 673 |     vector<ffm_long> P(1, 0);
 674 |     vector<ffm_node> X;
 675 | 
 676 |     auto write_chunk = [&] ()
 677 |     {
 678 |         ffm_int l = Y.size();
 679 |         ffm_long nnz = P[l];
 680 | 
 681 |         max_l = max(max_l, l);
 682 |         max_nnz = max(max_nnz, nnz);
 683 | 
 684 |         fwrite(&l, sizeof(ffm_int), 1, f_bin);
 685 |         fwrite(Y.data(), sizeof(ffm_float), l, f_bin);
 686 |         fwrite(R.data(), sizeof(ffm_float), l, f_bin);
 687 |         fwrite(P.data(), sizeof(ffm_long), l+1, f_bin);
 688 |         fwrite(X.data(), sizeof(ffm_node), nnz, f_bin);
 689 | 
 690 |         Y.clear();
 691 |         R.clear();
 692 |         P.assign(1, 0);
 693 |         X.clear();
 694 |         p = 0;
 695 |     };
 696 | 
 697 |     fwrite(&m, sizeof(ffm_int), 1, f_bin);
 698 |     fwrite(&n, sizeof(ffm_int), 1, f_bin);
 699 |     fwrite(&max_l, sizeof(ffm_int), 1, f_bin);
 700 |     fwrite(&max_nnz, sizeof(ffm_long), 1, f_bin);
 701 | 
 702 |     while(fgets(line.data(), kMaxLineSize, f_txt))
 703 |     {
 704 |         char *y_char = strtok(line.data(), " \t");
 705 | 
 706 |         ffm_float y = (atoi(y_char)>0)? 1.0f : -1.0f;
 707 | 
 708 |         ffm_float scale = 0;
 709 |         for(; ; p++)
 710 |         {
 711 |             char *field_char = strtok(nullptr,":");
 712 |             char *idx_char = strtok(nullptr,":");
 713 |             char *value_char = strtok(nullptr," \t");
 714 |             if(field_char == nullptr || *field_char == '\n')
 715 |                 break;
 716 | 
 717 |             ffm_node N;
 718 |             N.f = atoi(field_char);
 719 |             N.j = atoi(idx_char);
 720 |             N.v = atof(value_char);
 721 | 
 722 |             X.push_back(N);
 723 | 
 724 |             m = max(m, N.f+1);
 725 |             n = max(n, N.j+1);
 726 | 
 727 |             scale += N.v*N.v;
 728 |         }
 729 |         scale = 1/scale;
 730 | 
 731 |         Y.push_back(y);
 732 |         R.push_back(scale);
 733 |         P.push_back(p);
 734 | 
 735 |         if(X.size() > (size_t)kCHUNK_SIZE)
 736 |             write_chunk(); 
 737 |     }
 738 |     write_chunk(); 
 739 |     write_chunk(); 
 740 | 
 741 |     rewind(f_bin);
 742 |     fwrite(&m, sizeof(ffm_int), 1, f_bin);
 743 |     fwrite(&n, sizeof(ffm_int), 1, f_bin);
 744 |     fwrite(&max_l, sizeof(ffm_int), 1, f_bin);
 745 |     fwrite(&max_nnz, sizeof(ffm_long), 1, f_bin);
 746 | 
 747 |     fclose(f_bin);
 748 |     fclose(f_txt);
 749 | 
 750 |     return 0;
 751 | }
 752 | 
 753 | void ffm_destroy_problem(ffm_problem **prob)
 754 | {
 755 |     if(prob == nullptr || *prob == nullptr)
 756 |         return;
 757 |     delete[] (*prob)->X;
 758 |     delete[] (*prob)->P;
 759 |     delete[] (*prob)->Y;
 760 |     delete *prob;
 761 |     *prob = nullptr;
 762 | }
 763 | 
 764 | ffm_int ffm_save_model(ffm_model *model, char const *path)
 765 | {
 766 |     ofstream f_out(path);
 767 |     if(!f_out.is_open())
 768 |         return 1;
 769 | 
 770 |     f_out << "n " << model->n << "\n";
 771 |     f_out << "m " << model->m << "\n";
 772 |     f_out << "k " << model->k << "\n";
 773 |     f_out << "normalization " << model->normalization << "\n";
 774 | 
 775 |     ffm_float *ptr = model->W;
 776 |     for(ffm_int j = 0; j < model->n; j++)
 777 |     {
 778 |         for(ffm_int f = 0; f < model->m; f++)
 779 |         {
 780 |             f_out << "w" << j << "," << f << " ";
 781 |             for(ffm_int d = 0; d < model->k; d++, ptr++)
 782 |                 f_out << *ptr << " ";
 783 |             f_out << "\n";
 784 |         }
 785 |     }
 786 | 
 787 |     return 0;
 788 | }
 789 | 
 790 | ffm_model* ffm_load_model(char const *path)
 791 | {
 792 |     ifstream f_in(path);
 793 |     if(!f_in.is_open())
 794 |         return nullptr;
 795 | 
 796 |     string dummy;
 797 | 
 798 |     ffm_model *model = new ffm_model;
 799 |     model->W = nullptr;
 800 | 
 801 |     f_in >> dummy >> model->n >> dummy >> model->m >> dummy >> model->k 
 802 |          >> dummy >> model->normalization;
 803 | 
 804 |     try
 805 |     {
 806 |         model->W = malloc_aligned_float((ffm_long)model->m*model->n*model->k);
 807 |     }
 808 |     catch(bad_alloc const &e)
 809 |     {
 810 |         ffm_destroy_model(&model);
 811 |         return nullptr;
 812 |     }
 813 | 
 814 |     ffm_float *ptr = model->W;
 815 |     for(ffm_int j = 0; j < model->n; j++)
 816 |     {
 817 |         for(ffm_int f = 0; f < model->m; f++)
 818 |         {
 819 |             f_in >> dummy;
 820 |             for(ffm_int d = 0; d < model->k; d++, ptr++)
 821 |                 f_in >> *ptr;
 822 |         }
 823 |     }
 824 | 
 825 |     return model;
 826 | }
 827 | 
 828 | void ffm_destroy_model(ffm_model **model)
 829 | {
 830 |     if(model == nullptr || *model == nullptr)
 831 |         return;
 832 | #ifdef _WIN32
 833 |     _aligned_free((*model)->W);
 834 | #else
 835 |     free((*model)->W);
 836 | #endif
 837 |     delete *model;
 838 |     *model = nullptr;
 839 | }
 840 | 
 841 | ffm_parameter ffm_get_default_param()
 842 | {
 843 |     ffm_parameter param;
 844 | 
 845 |     param.eta = 0.2;
 846 |     param.lambda = 0.00002;
 847 |     param.nr_iters = 15;
 848 |     param.k = 4;
 849 |     param.nr_threads = 1;
 850 |     param.quiet = false;
 851 |     param.normalization = true;
 852 |     param.random = true;
 853 |     param.auto_stop = false;
 854 | 
 855 |     return param;
 856 | }
 857 | 
 858 | ffm_model* ffm_train_with_validation(ffm_problem *tr, ffm_problem *va, ffm_parameter param)
 859 | {
 860 |     vector<ffm_int> order(tr->l);
 861 |     for(ffm_int i = 0; i < tr->l; i++)
 862 |         order[i] = i;
 863 | 
 864 |     shared_ptr<ffm_model> model = train(tr, order, param, va);
 865 | 
 866 |     ffm_model *model_ret = new ffm_model;
 867 | 
 868 |     model_ret->n = model->n;
 869 |     model_ret->m = model->m;
 870 |     model_ret->k = model->k;
 871 |     model_ret->normalization = model->normalization;
 872 | 
 873 |     model_ret->W = model->W;
 874 |     model->W = nullptr;
 875 | 
 876 |     return model_ret;
 877 | }
 878 | 
 879 | ffm_model* ffm_train(ffm_problem *prob, ffm_parameter param)
 880 | {
 881 |     return ffm_train_with_validation(prob, nullptr, param);
 882 | }
 883 | 
 884 | ffm_model* ffm_train_with_validation_on_disk(
 885 |     char const *tr_path,
 886 |     char const *va_path,
 887 |     ffm_parameter param)
 888 | {
 889 |     shared_ptr<ffm_model> model = train_on_disk(tr_path, va_path, param);
 890 | 
 891 |     ffm_model *model_ret = new ffm_model;
 892 | 
 893 |     model_ret->n = model->n;
 894 |     model_ret->m = model->m;
 895 |     model_ret->k = model->k;
 896 |     model_ret->normalization = model->normalization;
 897 | 
 898 |     model_ret->W = model->W;
 899 |     model->W = nullptr;
 900 | 
 901 |     return model_ret;
 902 | }
 903 | 
 904 | ffm_model* ffm_train_on_disk(char const *prob_path, ffm_parameter param)
 905 | {
 906 |     return ffm_train_with_validation_on_disk(prob_path, "", param);
 907 | }
 908 | 
 909 | ffm_float ffm_predict(ffm_node *begin, ffm_node *end, ffm_model *model)
 910 | {
 911 |     ffm_float r = 1;
 912 |     if(model->normalization)
 913 |     {
 914 |         r = 0;
 915 |         for(ffm_node *N = begin; N != end; N++)
 916 |             r += N->v*N->v; 
 917 |         r = 1/r;
 918 |     }
 919 | 
 920 |     ffm_long align0 = (ffm_long)model->k;
 921 |     ffm_long align1 = (ffm_long)model->m*align0;
 922 | 
 923 |     ffm_float t = 0;
 924 |     for(ffm_node *N1 = begin; N1 != end; N1++)
 925 |     {
 926 |         ffm_int j1 = N1->j;
 927 |         ffm_int f1 = N1->f;
 928 |         ffm_float v1 = N1->v;
 929 |         if(j1 >= model->n || f1 >= model->m)
 930 |             continue;
 931 | 
 932 |         for(ffm_node *N2 = N1+1; N2 != end; N2++)
 933 |         {
 934 |             ffm_int j2 = N2->j;
 935 |             ffm_int f2 = N2->f;
 936 |             ffm_float v2 = N2->v;
 937 |             if(j2 >= model->n || f2 >= model->m)
 938 |                 continue;
 939 | 
 940 |             ffm_float *w1 = model->W + j1*align1 + f2*align0;
 941 |             ffm_float *w2 = model->W + j2*align1 + f1*align0;
 942 | 
 943 |             ffm_float v = v1*v2*r;
 944 | 
 945 |             for(ffm_int d = 0; d < model->k; d++)
 946 |                 t += w1[d]*w2[d]*v;
 947 |         }
 948 |     }
 949 | 
 950 |     return 1/(1+exp(-t));
 951 | }
 952 | 
 953 | ffm_float ffm_cross_validation(
 954 |     ffm_problem *prob, 
 955 |     ffm_int nr_folds,
 956 |     ffm_parameter param)
 957 | {
 958 | #if defined USEOMP
 959 |     ffm_int old_nr_threads = omp_get_num_threads();
 960 |     omp_set_num_threads(param.nr_threads);
 961 | #endif
 962 | 
 963 |     bool quiet = param.quiet;
 964 |     param.quiet = true;
 965 | 
 966 |     vector<ffm_int> order(prob->l);
 967 |     for(ffm_int i = 0; i < prob->l; i++)
 968 |         order[i] = i;
 969 |     random_shuffle(order.begin(), order.end());
 970 | 
 971 |     if(!quiet)
 972 |     {
 973 |         cout.width(4);
 974 |         cout << "fold";
 975 |         cout.width(13);
 976 |         cout << "logloss";
 977 |         cout << endl;
 978 |     }
 979 | 
 980 |     ffm_double loss = 0;
 981 |     ffm_int nr_instance_per_fold = prob->l/nr_folds;
 982 |     for(ffm_int fold = 0; fold < nr_folds; fold++)
 983 |     {
 984 |         ffm_int begin = fold*nr_instance_per_fold;
 985 |         ffm_int end = min(begin + nr_instance_per_fold, prob->l);
 986 | 
 987 |         vector<ffm_int> order1;
 988 |         for(ffm_int i = 0; i < begin; i++)
 989 |             order1.push_back(order[i]);
 990 |         for(ffm_int i = end; i < prob->l; i++)
 991 |             order1.push_back(order[i]);
 992 | 
 993 |         shared_ptr<ffm_model> model = train(prob, order1, param);
 994 | 
 995 |         ffm_double loss1 = 0;
 996 | #if defined USEOMP
 997 | #pragma omp parallel for schedule(static) reduction(+: loss1)
 998 | #endif
 999 |         for(ffm_int ii = begin; ii < end; ii++)
1000 |         {
1001 |             ffm_int i = order[ii];
1002 | 
1003 |             ffm_float y = prob->Y[i];
1004 |             
1005 |             ffm_node *begin = &prob->X[prob->P[i]];
1006 | 
1007 |             ffm_node *end = &prob->X[prob->P[i+1]];
1008 | 
1009 |             ffm_float y_bar = ffm_predict(begin, end, model.get());
1010 | 
1011 |             loss1 -= y==1? log(y_bar) : log(1-y_bar);
1012 |         }
1013 |         loss += loss1;
1014 | 
1015 |         if(!quiet)
1016 |         {
1017 |             cout.width(4);
1018 |             cout << fold;
1019 |             cout.width(13);
1020 |             cout << fixed << setprecision(4) << loss1 / (end-begin);
1021 |             cout << endl;
1022 |         }
1023 |     }
1024 | 
1025 |     if(!quiet)
1026 |     {
1027 |         cout.width(17);
1028 |         cout.fill('=');
1029 |         cout << "" << endl;
1030 |         cout.fill(' ');
1031 |         cout.width(4);
1032 |         cout << "avg";
1033 |         cout.width(13);
1034 |         cout << fixed << setprecision(4) << loss/prob->l;
1035 |         cout << endl;
1036 |     }
1037 | 
1038 | #if defined USEOMP
1039 |     omp_set_num_threads(old_nr_threads);
1040 | #endif
1041 | 
1042 |     return loss/prob->l;
1043 | }
1044 | 
1045 | } // namespace ffm
1046 | 


--------------------------------------------------------------------------------