├── CMakeLists.txt ├── conf ├── config.json ├── dist_config.json ├── dist_nn_config.json ├── nn_config.json └── test.json ├── data └── uci_data │ ├── bank │ ├── test │ └── train │ └── dist_bank │ ├── train_0 │ └── train_1 ├── include ├── all.h ├── dataload.h ├── dense_layer.h ├── dist_optimizer.h ├── eigen_func.h ├── generator.h ├── hash_method.h ├── init.h ├── instance.h ├── ioc.h ├── loss_func.h ├── lr.h ├── matrix_val.h ├── network.h ├── optimizer.h └── tools.h ├── readme.txt ├── run.sh ├── run_build.sh ├── script ├── kill.sh ├── local.sh └── ps_local.sh ├── src ├── lr.cpp ├── lr_uci.cpp ├── lr_uci_dist.cpp ├── network.cpp └── network_dist.cpp └── test ├── dataload.cpp ├── eigen.cpp ├── fibonacci.cpp ├── lr_test.cpp ├── range.cpp └── test_json.cpp /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 2.6) 2 | project (YCDL) 3 | 4 | # config gloal output 5 | SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin/output) 6 | 7 | # c++ 11 8 | set(CMAKE_CXX_STANDARD 11) 9 | 10 | set(CMAKE_CXX_FLAGS "-pthread -std=c++11 -g") 11 | include_directories ( 12 | "${PROJECT_SOURCE_DIR}/include" 13 | "${PROJECT_SOURCE_DIR}/thirdparty/ps-lite/include" 14 | "${PROJECT_SOURCE_DIR}/thirdparty/eigen-3.3.7" 15 | "${PROJECT_SOURCE_DIR}/thirdparty/json/include" 16 | ) 17 | link_directories( 18 | "${PROJECT_BINARY_DIR}" 19 | "${PROJECT_BINARY_DIR}/thirdparty/ps-lite" 20 | ) 21 | 22 | add_subdirectory(thirdparty/ps-lite) 23 | 24 | set(STA_DEPS pslite protobuf zmq) 25 | 26 | # add src executable 27 | file(GLOB example_sources "src/*.cpp") 28 | foreach( example_source_file ${example_sources} ) 29 | get_filename_component(filename ${example_source_file} NAME) 30 | string(REPLACE ".cpp" "" example_name ${filename} ) 31 | add_executable(${example_name} ${example_source_file}) 32 | target_link_libraries(${example_name} ${STA_DEPS}) 33 | set_target_properties(${example_name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY 34 | ${PROJECT_SOURCE_DIR}/bin/output ) 35 | endforeach() 36 | 37 | # add test executable 38 | file(GLOB example_sources "test/*.cpp") 39 | foreach( example_source_file ${example_sources} ) 40 | get_filename_component(filename ${example_source_file} NAME) 41 | string(REPLACE ".cpp" "" example_name ${filename} ) 42 | add_executable(${example_name} ${example_source_file}) 43 | target_link_libraries(${example_name} ${STA_DEPS}) 44 | set_target_properties(${example_name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY 45 | ${PROJECT_SOURCE_DIR}/bin/test) 46 | endforeach() 47 | -------------------------------------------------------------------------------- /conf/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "epoch": 1000, 3 | "batch_size": 50, 4 | "shuffle_num": 10, 5 | "test_file": "./data/train0", 6 | "train_file": "./data/test0", 7 | "optimizer": { 8 | "name": "Adagrad", 9 | "learning_rate": 0.5, 10 | "dim": 3, 11 | "eps": 1e-7 12 | }, 13 | "loss_func_name": "sigmoid_cross_entroy_with_logits" 14 | } 15 | -------------------------------------------------------------------------------- /conf/dist_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "epoch": 1000, 3 | "batch_size": 50, 4 | "shuffle_num": 10, 5 | "test_file": "./data/train0", 6 | "train_file": "./data/test0", 7 | "optimizer": { 8 | "name": "DistSGD", 9 | "learning_rate": 0.5, 10 | "dim": 3, 11 | "eps": 1e-7, 12 | "sync_mode": 1 13 | }, 14 | "loss_func_name": "sigmoid_cross_entroy_with_logits" 15 | } 16 | -------------------------------------------------------------------------------- /conf/dist_nn_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "optimizer": { 3 | "name": "DistAdagrad", 4 | "learning_rate": 0.1, 5 | "eps": 1e-7, 6 | "sync_mode": 1 7 | }, 8 | "epoch": 100, 9 | "batch_size": 50, 10 | "shuffle_num": 10, 11 | "test_file": "./data/uci_data/bank/test", 12 | "train_file": "./data/uci_data/bank/train", 13 | "input_params": { 14 | "slots": "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15", 15 | "dim": 2, 16 | "input_name": "input", 17 | "label_name": "label" 18 | }, 19 | "params": [ 20 | { 21 | "name": "weight1", 22 | "row": 32, 23 | "col": 50, 24 | "need_gradient": true 25 | }, 26 | { 27 | "name": "weight2", 28 | "row": 50, 29 | "col": 1, 30 | "need_gradient": true 31 | } 32 | ], 33 | "layers": [ 34 | { 35 | "name": "Dense", 36 | "input1": "input", 37 | "input2": "weight1", 38 | "output": { 39 | "name": "out1" 40 | } 41 | }, 42 | { 43 | "name": "Activation", 44 | "input": "out1", 45 | "output": { 46 | "name": "out2" 47 | }, 48 | "act": "sigmoid" 49 | }, 50 | { 51 | "name": "Dense", 52 | "input1": "out2", 53 | "input2": "weight2", 54 | "output": { 55 | "name": "out22" 56 | } 57 | }, 58 | { 59 | "name": "Activation", 60 | "input": "out22", 61 | "output": { 62 | "name": "out11" 63 | }, 64 | "act": "sigmoid" 65 | }, 66 | { 67 | "name": "Loss", 68 | "input": "out11", 69 | "label": "label", 70 | "loss": "mse" 71 | } 72 | ] 73 | } 74 | -------------------------------------------------------------------------------- /conf/nn_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "optimizer": { 3 | "name": "Adagrad", 4 | "learning_rate": 0.1, 5 | "eps": 1e-7 6 | }, 7 | "epoch": 100, 8 | "batch_size": 50, 9 | "shuffle_num": 10, 10 | "test_file": "./data/uci_data/bank/test", 11 | "train_file": "./data/uci_data/bank/train", 12 | "input_params": { 13 | "slots": "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15", 14 | "dim": 2, 15 | "input_name": "input", 16 | "label_name": "label" 17 | }, 18 | "params": [ 19 | { 20 | "name": "weight1", 21 | "row": 32, 22 | "col": 50, 23 | "need_gradient": true 24 | }, 25 | { 26 | "name": "weight2", 27 | "row": 50, 28 | "col": 1, 29 | "need_gradient": true 30 | } 31 | ], 32 | "layers": [ 33 | { 34 | "name": "Dense", 35 | "input1": "input", 36 | "input2": "weight1", 37 | "output": { 38 | "name": "out1" 39 | } 40 | }, 41 | { 42 | "name": "Activation", 43 | "input": "out1", 44 | "output": { 45 | "name": "out2" 46 | }, 47 | "act": "sigmoid" 48 | }, 49 | { 50 | "name": "Dense", 51 | "input1": "out2", 52 | "input2": "weight2", 53 | "output": { 54 | "name": "out22" 55 | } 56 | }, 57 | { 58 | "name": "Activation", 59 | "input": "out22", 60 | "output": { 61 | "name": "out11" 62 | }, 63 | "act": "sigmoid" 64 | }, 65 | { 66 | "name": "Loss", 67 | "input": "out11", 68 | "label": "label", 69 | "loss": "mse" 70 | } 71 | ] 72 | } 73 | -------------------------------------------------------------------------------- /conf/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "optimizer": { 3 | "name": "Adagrad", 4 | "learning_rate": 0.1, 5 | "eps": 1e-7 6 | }, 7 | "global": { 8 | "epoch": 100, 9 | "batch_size": 50, 10 | "shuffle_num": 10, 11 | "test_file": "./data/uci_data/bank/test", 12 | "train_file": "./data/uci_data/bank/train" 13 | }, 14 | "layers": [ 15 | { 16 | "name": "Dense", 17 | "input1": "input", 18 | "input2": "weight1", 19 | "output": { 20 | "name": "out1" 21 | } 22 | }, 23 | { 24 | "name": "Loss", 25 | "input": "out11", 26 | "label": "label", 27 | "loss": "mse" 28 | } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /include/all.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "eigen_func.h" 24 | #include "nlohmann/json.hpp" 25 | #include 26 | #include "hash_method.h" 27 | #include "ps/ps.h" 28 | 29 | namespace YCDL { 30 | typedef uint64_t ull; 31 | using SLOT_ID_FEAS = std::pair>; 32 | 33 | inline nlohmann::json &global_conf() { 34 | static nlohmann::json conf; 35 | return conf; 36 | } 37 | 38 | class Layer {}; 39 | } 40 | -------------------------------------------------------------------------------- /include/dataload.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "all.h" 3 | #include "tools.h" 4 | #include "instance.h" 5 | #include 6 | 7 | namespace YCDL { 8 | Generator > 9 | dataload(std::string path, int epoch, int batch_size, bool is_train, int shuffle_num = 1) { 10 | return Generator < std::vector < Instance > > ([=](Yield > &yield) { 11 | std::string s; 12 | int cnt = 0, pool_sz = shuffle_num * batch_size; 13 | std::vector instances, ans; 14 | int nums = epoch; 15 | while (nums) { 16 | std::ifstream file; 17 | file.open(path); 18 | nums--; 19 | while (getline(file, s)) { 20 | cnt += 1; 21 | std::vector segs; 22 | line_split(s, "\t", segs); 23 | Instance ins; 24 | ins.label = boost::lexical_cast(segs[0]); 25 | for (int i = 1; i < segs.size(); i++) { 26 | ins.feas.push_back(boost::lexical_cast(segs[i])); 27 | ins.vals.push_back(1.0); 28 | } 29 | ins.feas.push_back(0); 30 | ins.vals.push_back(1.0); 31 | instances.emplace_back(std::move(ins)); 32 | if (cnt == pool_sz) { 33 | cnt = 0; 34 | if (is_train) { 35 | std::random_shuffle(instances.begin(), instances.end()); 36 | } 37 | for (int i = 0; i < pool_sz; i += batch_size) { 38 | ans.assign(instances.begin() + i, instances.begin() + i + batch_size); 39 | yield(ans); 40 | } 41 | instances.clear(); 42 | } 43 | } 44 | file.close(); 45 | } 46 | if (cnt != pool_sz && instances.size() != 0) { 47 | if (is_train) { 48 | std::random_shuffle(instances.begin(), instances.end()); 49 | } 50 | for (int i = 0; i < cnt; i += batch_size) { 51 | if (i + batch_size < cnt) { 52 | ans.assign(instances.begin() + i, instances.begin() + i + batch_size); 53 | } else { 54 | ans.clear(); 55 | ans.assign(instances.begin() + i, instances.end()); 56 | } 57 | yield(ans); 58 | } 59 | instances.clear(); 60 | } 61 | }); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /include/dense_layer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "all.h" 3 | #include "matrix_val.h" 4 | #include "eigen_func.h" 5 | 6 | namespace YCDL { 7 | class NNLayer : public Layer { 8 | public: 9 | virtual void load_conf(const nlohmann::json &conf) = 0; 10 | 11 | virtual void forward() = 0; 12 | 13 | virtual void backward() = 0; 14 | }; 15 | 16 | class Concat: public NNLayer { 17 | public: 18 | void load_conf(const nlohmann::json &conf) override { 19 | auto inputs = conf["input"]; 20 | for (int i = 0; i < inputs.size(); i++) { 21 | auto name = inputs[i]; 22 | _input_vec.push_back(global_matrix_value().get_matrix_value(name)); 23 | } 24 | 25 | _output = std::make_shared(); 26 | std::string out_name = conf["output"]["name"]; 27 | global_matrix_value().add_matrix_value(out_name, _output); 28 | } 29 | 30 | void forward() override { 31 | int input_size = _input_vec.size(); 32 | _output->_has_grad = false; 33 | _output->_trainable = false; 34 | int row = 0, col = 0; 35 | for (int i = 0; i < input_size; i++) { 36 | row = _input_vec[i]->_val.rows(); 37 | col += _input_vec[i]->_val.cols(); 38 | if (_input_vec[i]->_trainable) { 39 | _output->_trainable = true; 40 | } 41 | } 42 | 43 | Eigen::MatrixXf &out_val = _output->_val; 44 | out_val.setZero(row, col); 45 | 46 | int idx = 0; 47 | for (int i = 0; i < input_size; i++) { 48 | auto &input_val = _input_vec[i]->_val; 49 | int input_col = input_val.cols(); 50 | for (int j = 0; j < row; j++) { 51 | for (int k = 0; k < input_col; k++) { 52 | out_val(j, idx + k) = input_val(j, k); 53 | } 54 | } 55 | idx += input_col; 56 | } 57 | } 58 | 59 | void backward() override { 60 | if (!_output->_has_grad) { 61 | return; 62 | } 63 | int input_size = _input_vec.size(); 64 | Eigen::MatrixXf &out_grad = _output->_grad; 65 | 66 | int row = out_grad.rows(); 67 | int col = out_grad.cols(); 68 | 69 | int idx = 0; 70 | for (int i = 0; i < input_size; i++) { 71 | if (!_input_vec[i]->_trainable) { 72 | continue; 73 | } 74 | int input_col = _input_vec[i]->_val.cols(); 75 | auto &input_grad = _input_vec[i]->_grad; 76 | if (!_input_vec[i]->_has_grad) { 77 | _input_vec[i]->_grad.resize(row, input_col); 78 | } 79 | for (int j = 0; j < row; j++) { 80 | for (int k = 0; k < input_col; k++) { 81 | if (_input_vec[i]->_has_grad) { 82 | input_grad(j, k) += out_grad(j, idx + k); 83 | } else { 84 | input_grad(j, k) = out_grad(j, idx + k); 85 | } 86 | } 87 | } 88 | idx += input_col; 89 | } 90 | } 91 | 92 | std::vector > _input_vec; 93 | std::shared_ptr _output; 94 | }; 95 | 96 | class Activation : public NNLayer { 97 | public: 98 | void load_conf(const nlohmann::json &conf) override { 99 | _input = global_matrix_value().get_matrix_value(conf["input"]); 100 | 101 | _output = std::make_shared(); 102 | std::string out_name = conf["output"]["name"]; 103 | global_matrix_value().add_matrix_value(out_name, _output); 104 | 105 | _act = conf["act"]; 106 | 107 | } 108 | 109 | void forward() override { 110 | Eigen::MatrixXf val = _input->_val; 111 | Eigen::MatrixXf &out_val = _output->_val; 112 | if (_act == "sigmoid") { 113 | // cout << "sigmoid\n"; 114 | out_val = 1.0 / (1.0 + (-val.array()).exp()); 115 | //out_val = val; 116 | } 117 | _output->_has_grad = false; 118 | _output->_trainable = _input->_trainable; 119 | } 120 | 121 | void backward() override { 122 | if (!_output->_has_grad) { 123 | return; 124 | } 125 | Eigen::MatrixXf &val = _input->_val; 126 | Eigen::MatrixXf out_val = _output->_val; 127 | Eigen::MatrixXf &out_grad = _output->_grad; 128 | 129 | if (_input->_trainable) { 130 | Eigen::MatrixXf &grad = _input->_grad; 131 | Eigen::MatrixXf val2; 132 | if (_act == "sigmoid") { 133 | //val2 = out_grad; 134 | 135 | Eigen::MatrixXf ones; 136 | ones.setOnes(out_val.rows(), out_val.cols()); 137 | val2 = out_val.cwiseProduct(ones - out_val); 138 | val2 = val2.cwiseProduct(out_grad); 139 | } 140 | if (_input->_has_grad) { 141 | grad += val2; 142 | } else { 143 | grad = val2; 144 | } 145 | _input->_has_grad = true; 146 | } 147 | } 148 | 149 | std::shared_ptr _input; 150 | std::shared_ptr _output = std::make_shared(); 151 | std::string _act; 152 | }; 153 | 154 | 155 | class Dense : public NNLayer { 156 | public: 157 | 158 | void load_conf(const nlohmann::json &conf) override { 159 | _input1 = global_matrix_value().get_matrix_value(conf["input1"]); 160 | _input2 = global_matrix_value().get_matrix_value(conf["input2"]); 161 | 162 | _output = std::make_shared(); 163 | std::string out_name = conf["output"]["name"]; 164 | global_matrix_value().add_matrix_value(out_name, _output); 165 | 166 | if (conf.find("trans1") != conf.end()) { 167 | _trans1 = conf["trans1"]; 168 | } else { 169 | _trans1 = false; 170 | } 171 | if (conf.find("trans2") != conf.end()) { 172 | _trans2 = conf["trans2"]; 173 | } else { 174 | _trans2 = false; 175 | } 176 | if (conf.find("has_bias") != conf.end()) { 177 | _has_bias = conf["has_bias"]; 178 | } else { 179 | _has_bias = false; 180 | } 181 | 182 | if (_has_bias) { 183 | _concat_layer = std::make_shared(); 184 | _input_bias = std::make_shared(); 185 | _concat_layer->_output = std::make_shared(); 186 | std::string out_name_bias = out_name + "_bias"; 187 | global_matrix_value().add_matrix_value(out_name_bias, _concat_layer->_output); 188 | } 189 | } 190 | 191 | void forward() override { 192 | Eigen::MatrixXf &val1 = _input1->_val; 193 | 194 | if (_has_bias) { 195 | auto &bias_val = _input_bias->_val; 196 | int row = _input1->_val.rows(); 197 | bias_val.setOnes(row, 1); 198 | _concat_layer->_input_vec.clear(); 199 | _concat_layer->_input_vec.push_back(_input1); 200 | _concat_layer->_input_vec.push_back(_input_bias); 201 | 202 | _concat_layer->forward(); 203 | val1 = _concat_layer->_output->_val; 204 | } 205 | 206 | Eigen::MatrixXf &val2 = _input2->_val; 207 | Eigen::MatrixXf &out_val = _output->_val; 208 | 209 | matrix_multi(_trans1, _trans2, false, val1, val2, out_val); 210 | _output->_has_grad = false; 211 | if (_input1->_trainable || _input2->_trainable) { 212 | _output->_trainable = true; 213 | } else { 214 | _output->_trainable = false; 215 | } 216 | } 217 | 218 | void backward() override { 219 | if (!_output->_has_grad) { 220 | return; 221 | } 222 | 223 | auto input1_tmp = _input1; 224 | 225 | if (_has_bias) { 226 | input1_tmp = _concat_layer->_output; 227 | } 228 | 229 | Eigen::MatrixXf &val1 = input1_tmp->_val; 230 | Eigen::MatrixXf &val2 = _input2->_val; 231 | Eigen::MatrixXf &out_grad = _output->_grad; 232 | 233 | if (input1_tmp->_trainable) { 234 | Eigen::MatrixXf &grad1 = input1_tmp->_grad; 235 | if (input1_tmp->_has_grad) { 236 | matrix_multi_addition(false, !_trans2, _trans1, out_grad, val2, grad1); 237 | } else { 238 | matrix_multi(false, !_trans2, _trans1, out_grad, val2, grad1); 239 | } 240 | input1_tmp->_has_grad = true; 241 | } 242 | 243 | if (_input2->_trainable) { 244 | Eigen::MatrixXf &grad2 = _input2->_grad; 245 | if (_input2->_has_grad) { 246 | matrix_multi_addition(!_trans1, false, _trans2, val1, out_grad, grad2); 247 | } else { 248 | matrix_multi(!_trans1, false, _trans2, val1, out_grad, grad2); 249 | } 250 | _input2->_has_grad = true; 251 | } 252 | 253 | if (_has_bias) { 254 | _concat_layer->backward(); 255 | } 256 | } 257 | 258 | std::shared_ptr _input1; 259 | std::shared_ptr _input2; 260 | std::shared_ptr _output; 261 | bool _trans1 = false; 262 | bool _trans2 = false; 263 | bool _has_bias = false; 264 | std::shared_ptr _concat_layer; 265 | std::shared_ptr _input_bias; 266 | }; 267 | 268 | class Loss : public NNLayer { 269 | public: 270 | void load_conf(const nlohmann::json &conf) override { 271 | _label = global_matrix_value().get_matrix_value(conf["label"]); 272 | _input = global_matrix_value().get_matrix_value(conf["input"]); 273 | _loss_func = conf["loss"]; 274 | 275 | // for auc stat 276 | global_matrix_value().add_matrix_value("predict", _input); 277 | } 278 | 279 | void forward() override { 280 | } 281 | 282 | void backward() override { 283 | if (!_input->_trainable) { 284 | return; 285 | } 286 | if (!_input->_has_grad) { 287 | _input->_grad.setZero(_input->_val.rows(), _input->_val.cols()); 288 | _input->_has_grad = true; 289 | } 290 | if (_loss_func == "logloss") { 291 | auto tmp = _input->_val; 292 | tmp = 1.0 / (1.0 + (-_input->_val.array()).exp()); 293 | _input->_grad += tmp - _label->_val; 294 | } else if (_loss_func == "mse") { 295 | _input->_grad += _input->_val - _label->_val; 296 | } 297 | } 298 | 299 | std::shared_ptr _label; 300 | std::shared_ptr _input; 301 | std::string _loss_func; 302 | }; 303 | } 304 | -------------------------------------------------------------------------------- /include/dist_optimizer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "ps/ps.h" 4 | #include "all.h" 5 | #include "optimizer.h" 6 | #include 7 | 8 | namespace YCDL { 9 | 10 | // dist worker 11 | class DistWorker : public Optimizer { 12 | public: 13 | DistWorker() { 14 | _kv = new ps::KVWorker(0, 0); 15 | } 16 | 17 | virtual void push(std::vector keys, std::vector> &vals) override { 18 | int n = keys.size(); 19 | 20 | // std::cout << "Push请求 " << ps::MyRank() << std::endl; 21 | // sorted keys 22 | std::vector sorted_keys = keys; 23 | std::sort(sorted_keys.begin(), sorted_keys.end()); 24 | sorted_keys.erase(std::unique(sorted_keys.begin(), sorted_keys.end()), sorted_keys.end()); 25 | int sorted_keys_len = sorted_keys.size(); 26 | std::unordered_map sorted_keys_mp; 27 | for (int i = 0; i < sorted_keys_len; i++) { 28 | sorted_keys_mp[sorted_keys[i]] = i; 29 | } 30 | 31 | // aggregate sorted vals 32 | std::vector> sorted_vals(sorted_keys_len, std::vector()); 33 | std::vector dup_cnt(sorted_keys_len, 0); 34 | for (int i = 0; i < n; i++) { 35 | int k = sorted_keys_mp[keys[i]]; 36 | dup_cnt[k]++; 37 | if (sorted_vals[k].empty()) { 38 | sorted_vals[k] = vals[i]; 39 | } else { 40 | for (int j = 0; j < vals[i].size(); j++) { 41 | sorted_vals[k][j] += vals[i][j]; 42 | } 43 | } 44 | } 45 | for (int i = 0; i < sorted_keys_len; i++) { 46 | for (int j = 0; j < sorted_vals[i].size(); j++) { 47 | sorted_vals[i][j] /= dup_cnt[i]; 48 | } 49 | } 50 | 51 | // prepare push values 52 | std::vector values; 53 | std::vector lens; 54 | for (auto &val : sorted_vals) { 55 | lens.push_back(val.size()); 56 | for (auto x : val) { 57 | values.push_back(x); 58 | } 59 | } 60 | _kv->Wait(_kv->Push(sorted_keys, values, lens)); 61 | } 62 | 63 | virtual int pull(std::vector keys, std::vector> &vals, bool is_train = true, int dim = 1) override { 64 | 65 | int n = keys.size(); 66 | 67 | // sorted keys 68 | std::vector sorted_keys = keys; 69 | std::sort(sorted_keys.begin(), sorted_keys.end()); 70 | sorted_keys.erase(std::unique(sorted_keys.begin(), sorted_keys.end()), sorted_keys.end()); 71 | int sorted_keys_len = sorted_keys.size(); 72 | std::unordered_map sorted_keys_mp; 73 | for (int i = 0; i < sorted_keys_len; i++) { 74 | sorted_keys_mp[sorted_keys[i]] = i; 75 | } 76 | 77 | std::vector values, values_init; 78 | std::vector lens; 79 | for (auto &key : sorted_keys) { 80 | lens.push_back(dim); 81 | for (int i = 0; i < dim; i++) { 82 | values_init.push_back(0.0); 83 | } 84 | } 85 | 86 | if (is_train) { 87 | // init value 88 | //std::cout << "Push请求 init " << ps::MyRank() << std::endl; 89 | _kv->Wait(_kv->Push(sorted_keys, values_init, lens)); 90 | } 91 | 92 | //std::cout << "Pull请求 " << ps::MyRank() << std::endl; 93 | _kv->Wait(_kv->Pull(sorted_keys, &(values), &(lens))); 94 | //std::cout << "Pull请求 返回" << std::endl; 95 | 96 | for (int i = 0; i < n; i++) { 97 | int k = sorted_keys_mp[keys[i]]; 98 | for (int j = 0; j < dim; j++) { 99 | vals[i][j] = values[dim * k + j]; 100 | } 101 | } 102 | return 0; 103 | } 104 | 105 | ps::KVWorker *_kv; 106 | }; 107 | 108 | 109 | // dist server 110 | class PushReq { 111 | public: 112 | PushReq(const ps::KVMeta &req_meta_tmp, const ps::KVPairs &req_data_tmp) { 113 | req_meta = req_meta_tmp; 114 | req_data = req_data_tmp; 115 | } 116 | 117 | ps::KVMeta req_meta; 118 | ps::KVPairs req_data; 119 | }; 120 | 121 | class SGDServer : public Layer { 122 | public: 123 | SGDServer(int sync_mode = 1, float learning_rate = 0.01) { 124 | _sync_mode = sync_mode; 125 | _learning_rate = learning_rate; 126 | _ps_server = new ps::KVServer(0); 127 | _ps_server->set_request_handle( 128 | std::bind(&SGDServer::DataHandle, this, std::placeholders::_1, std::placeholders::_2, 129 | std::placeholders::_3)); 130 | } 131 | 132 | ~SGDServer() {} 133 | 134 | void DataHandle(const ps::KVMeta &req_meta, 135 | const ps::KVPairs &req_data, 136 | ps::KVServer *server) { 137 | 138 | size_t n = req_data.keys.size(); 139 | 140 | if (req_meta.push) { 141 | //std::cout << ps::MyRank() << " server get push request " << n << " keys\n"; 142 | 143 | for (int i = 0; i < n; i++) { 144 | auto &key = req_data.keys[i]; 145 | int length = req_data.lens[i]; 146 | 147 | //init value 148 | if (_weights.count(key) == 0) { 149 | // use lens[n] represent is_train 150 | std::vector w; 151 | for (int j = 0; j < length; j++) { 152 | w.push_back(get_random_double()); 153 | } 154 | _weights[key] = std::move(w); 155 | } 156 | } 157 | 158 | // deal func 159 | auto deal_push_req = [&](const ps::KVMeta &req_meta_tmp, const ps::KVPairs &req_data_tmp) { 160 | int idx = 0; 161 | size_t n = req_data_tmp.keys.size(); 162 | for (size_t i = 0; i < n; ++i) { 163 | auto &key = req_data_tmp.keys[i]; 164 | auto &weights = _weights[key]; 165 | int length = req_data_tmp.lens[i]; 166 | for (int j = 0; j < length; j++) { 167 | weights[j] -= _learning_rate * req_data_tmp.vals[idx + j]; 168 | } 169 | idx += length; 170 | } 171 | }; 172 | if (!_sync_mode) { 173 | // async push 174 | deal_push_req(req_meta, req_data); 175 | server->Response(req_meta); 176 | } else { 177 | 178 | //std::cout << "server:" << ps::MyRank() << " " << req_meta.sender << std::endl; 179 | _mutex.lock(); 180 | if (_push_req.size() == ps::NumWorkers() - 1) { 181 | deal_push_req(req_meta, req_data); 182 | for (auto &x : _push_req) { 183 | deal_push_req(x.req_meta, x.req_data); 184 | } 185 | server->Response(req_meta); 186 | for (auto &x : _push_req) { 187 | server->Response(x.req_meta); 188 | } 189 | _push_req.clear(); 190 | } else { 191 | _push_req.push_back({req_meta, req_data}); 192 | } 193 | _mutex.unlock(); 194 | } 195 | } else { // pull 196 | //std::cout << "server get pull request " << n << " keys\n"; 197 | ps::KVPairs response; 198 | response.keys = req_data.keys; 199 | int dim = 0; 200 | for (size_t i = 0; i < n; ++i) { 201 | auto &key = req_data.keys[i]; 202 | std::vector weights(dim, 0.0); 203 | if (_weights.count(key)) { 204 | weights = _weights[key]; 205 | } 206 | dim = weights.size(); 207 | response.lens.push_back(weights.size()); 208 | for (auto x: weights) { 209 | response.vals.push_back(x); 210 | } 211 | } 212 | server->Response(req_meta, response); 213 | //std::cout << "server send pull response\n"; 214 | } 215 | //std::cout << "server out :" << ps::MyRank() << " " << _push_req.size() << std::endl; 216 | } 217 | 218 | int _sync_mode; 219 | float _learning_rate; 220 | 221 | std::vector _push_req; 222 | std::mutex _mutex; 223 | 224 | std::unordered_map> _weights; 225 | ps::KVServer *_ps_server; 226 | 227 | }; 228 | 229 | 230 | class AdagradServer : public Layer { 231 | public: 232 | explicit AdagradServer(int sync_mode = 1, float learning_rate = 0.1) { 233 | _sync_mode = sync_mode; 234 | _learning_rate = learning_rate; 235 | _ps_server = new ps::KVServer(0); 236 | _eps = 1e-7; 237 | _ps_server->set_request_handle( 238 | std::bind(&AdagradServer::DataHandle, this, std::placeholders::_1, std::placeholders::_2, 239 | std::placeholders::_3)); 240 | } 241 | 242 | void DataHandle(const ps::KVMeta &req_meta, 243 | const ps::KVPairs &req_data, 244 | ps::KVServer *server) { 245 | 246 | size_t n = req_data.keys.size(); 247 | 248 | if (req_meta.push) { 249 | //std::cout << ps::MyRank() << " server get push request " << n << " keys\n"; 250 | 251 | for (int i = 0; i < n; i++) { 252 | auto &key = req_data.keys[i]; 253 | int length = req_data.lens[i]; 254 | 255 | //init value 256 | if (_weights.count(key) == 0) { 257 | // use lens[n] represent is_train 258 | std::vector w, w2; 259 | for (int j = 0; j < length; j++) { 260 | w.push_back(get_random_double()); 261 | w2.push_back(0.0); 262 | } 263 | _weights[key] = std::move(w); 264 | _weights2[key] = std::move(w2); 265 | } 266 | } 267 | 268 | // deal func 269 | auto deal_push_req = [&](const ps::KVMeta &req_meta_tmp, const ps::KVPairs &req_data_tmp) { 270 | int idx = 0; 271 | size_t n = req_data_tmp.keys.size(); 272 | for (size_t i = 0; i < n; ++i) { 273 | auto &key = req_data_tmp.keys[i]; 274 | auto &weights = _weights[key]; 275 | auto &weights2 = _weights2[key]; 276 | int length = req_data_tmp.lens[i]; 277 | for (int j = 0; j < length; j++) { 278 | weights2[j] += req_data_tmp.vals[idx + j] * req_data_tmp.vals[idx + j]; 279 | weights[j] -= _learning_rate * req_data_tmp.vals[idx + j] / (_eps + std::sqrt(weights2[j])); 280 | } 281 | idx += length; 282 | } 283 | }; 284 | if (!_sync_mode) { 285 | // async push 286 | deal_push_req(req_meta, req_data); 287 | server->Response(req_meta); 288 | } else { 289 | 290 | //std::cout << "server:" << ps::MyRank() << " " << req_meta.sender << std::endl; 291 | _mutex.lock(); 292 | if (_push_req.size() == ps::NumWorkers() - 1) { 293 | deal_push_req(req_meta, req_data); 294 | for (auto &x : _push_req) { 295 | deal_push_req(x.req_meta, x.req_data); 296 | } 297 | server->Response(req_meta); 298 | for (auto &x : _push_req) { 299 | server->Response(x.req_meta); 300 | } 301 | _push_req.clear(); 302 | } else { 303 | _push_req.push_back({req_meta, req_data}); 304 | } 305 | _mutex.unlock(); 306 | } 307 | } else { // pull 308 | //std::cout << "server get pull request " << n << " keys\n"; 309 | ps::KVPairs response; 310 | response.keys = req_data.keys; 311 | int dim = 0; 312 | for (size_t i = 0; i < n; ++i) { 313 | auto &key = req_data.keys[i]; 314 | std::vector weights(dim, 0.0); 315 | if (_weights.count(key)) { 316 | weights = _weights[key]; 317 | } 318 | dim = weights.size(); 319 | response.lens.push_back(weights.size()); 320 | for (auto x: weights) { 321 | response.vals.push_back(x); 322 | } 323 | } 324 | server->Response(req_meta, response); 325 | //std::cout << "server send pull response\n"; 326 | } 327 | //std::cout << "server out :" << ps::MyRank() << " " << _push_req.size() << std::endl; 328 | } 329 | 330 | double _eps; 331 | std::unordered_map> _weights2; 332 | 333 | int _sync_mode; 334 | float _learning_rate; 335 | 336 | std::vector _push_req; 337 | std::mutex _mutex; 338 | 339 | std::unordered_map> _weights; 340 | ps::KVServer *_ps_server; 341 | }; 342 | } 343 | -------------------------------------------------------------------------------- /include/eigen_func.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | namespace YCDL { 5 | template 6 | void matrix_multi(bool transa, bool transb, bool transc, const A &a, const B &b, C &c) { 7 | if (transc) { 8 | return matrix_multi(!transb, !transa, false, b, a, c); 9 | } 10 | if (!transa) { 11 | if (!transb) { 12 | assert(a.cols() == b.rows()); 13 | c.noalias() = a * b; 14 | } else { 15 | assert(a.cols() == b.cols()); 16 | c.noalias() = a * b.transpose(); 17 | } 18 | } else { 19 | if (!transb) { 20 | assert(a.rows() == b.rows()); 21 | c.noalias() = a.transpose() * b; 22 | } else { 23 | assert(a.rows() == b.cols()); 24 | c.noalias() = a.transpose() * b.transpose(); 25 | } 26 | } 27 | } 28 | 29 | template 30 | void matrix_multi_addition(bool transa, bool transb, bool transc, const A &a, const B &b, C &c) { 31 | if (transc) { 32 | return matrix_multi_addition(!transb, !transa, false, b, a, c); 33 | } 34 | if (!transa) { 35 | if (!transb) { 36 | assert(a.cols() == b.rows()); 37 | c.noalias() += a * b; 38 | } else { 39 | assert(a.cols() == b.cols()); 40 | c.noalias() += a * b.transpose(); 41 | } 42 | } else { 43 | if (!transb) { 44 | assert(a.rows() == b.rows()); 45 | c.noalias() += a.transpose() * b; 46 | } else { 47 | assert(a.rows() == b.cols()); 48 | c.noalias() += a.transpose() * b.transpose(); 49 | } 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /include/generator.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | namespace YCDL { 9 | 10 | template 11 | class Generator; 12 | 13 | template 14 | class Yield { 15 | 16 | enum State { 17 | WAITING, READY, TERMINATED, FINISHED 18 | } state = WAITING; 19 | 20 | friend Generator; 21 | std::promise promise; 22 | std::mutex mutex; 23 | std::condition_variable ready_wait; 24 | 25 | struct FinishedException : public std::exception { 26 | const char *what() const throw() override { return "FinishedException"; } 27 | }; 28 | 29 | struct TerminatedException : public std::exception { 30 | const char *what() const throw() override { return "TerminatedException"; } 31 | }; 32 | 33 | void terminate() { 34 | std::lock_guard guard(mutex); 35 | if (state != FINISHED) { 36 | state = TERMINATED; 37 | ready_wait.notify_one(); 38 | } 39 | } 40 | 41 | void finish() { 42 | std::unique_lock lock(mutex); 43 | while (state == WAITING) ready_wait.wait(lock); 44 | if (state == TERMINATED) return; 45 | state = FINISHED; 46 | throw FinishedException(); 47 | } 48 | 49 | template 50 | void set_exception(E e) { 51 | std::unique_lock lock(mutex); 52 | while (state == WAITING) ready_wait.wait(lock); 53 | if (state == TERMINATED) return; 54 | promise.set_exception(e); 55 | } 56 | 57 | std::future get_future() { 58 | std::lock_guard guard(mutex); 59 | promise = std::promise(); 60 | ready_wait.notify_one(); 61 | state = READY; 62 | return promise.get_future(); 63 | } 64 | 65 | Yield() {} 66 | 67 | public: 68 | 69 | void operator()(const T &value) { 70 | std::unique_lock lock(mutex); 71 | while (state == WAITING) ready_wait.wait(lock); 72 | if (state == TERMINATED) throw TerminatedException(); 73 | promise.set_value(value); 74 | state = WAITING; 75 | } 76 | 77 | }; 78 | 79 | template 80 | class Generator { 81 | std::function &)> generator_function; 82 | 83 | public: 84 | Generator(const std::function &)> &gf) : generator_function(gf) {} 85 | 86 | class const_iterator : public std::iterator { 87 | friend Generator; 88 | 89 | struct Data { 90 | Yield yield; 91 | T current_value; 92 | std::thread thread; 93 | 94 | ~Data() { 95 | yield.terminate(); 96 | thread.join(); 97 | } 98 | }; 99 | 100 | std::unique_ptr data; 101 | 102 | public: 103 | 104 | T &operator*() const { 105 | return data->current_value; 106 | } 107 | 108 | T *operator->() const { 109 | return &data->current_value; 110 | } 111 | 112 | const_iterator &operator++() { 113 | try { 114 | data->current_value = data->yield.get_future().get(); 115 | } 116 | catch (typename Yield::FinishedException) { 117 | data.reset(); 118 | } 119 | return *this; 120 | } 121 | 122 | bool operator!=(const const_iterator &other) const { return data != other.data; } 123 | }; 124 | 125 | const_iterator begin() const { 126 | const_iterator it; 127 | it.data.reset(new typename const_iterator::Data); 128 | auto data = it.data.get(); 129 | 130 | data->thread = std::thread([this, data]() { 131 | try { 132 | generator_function(data->yield); 133 | data->yield.finish(); 134 | } 135 | catch (typename Yield::TerminatedException) { 136 | 137 | } 138 | catch (...) { 139 | data->yield.set_exception(std::current_exception()); 140 | } 141 | }); 142 | 143 | ++it; 144 | 145 | return it; 146 | } 147 | 148 | const_iterator end() const { 149 | return const_iterator(); 150 | } 151 | }; 152 | } 153 | 154 | -------------------------------------------------------------------------------- /include/hash_method.h: -------------------------------------------------------------------------------- 1 | #pragma 2 | #include 3 | 4 | namespace YCDL { 5 | uint64_t BKDRHash(std::string s) { 6 | uint64_t seed = 131; // 31 131 1313 13131 131313 etc.. 7 | uint64_t hash = 0; 8 | 9 | for (auto str : s) { 10 | hash = hash * seed + str; 11 | } 12 | return (hash & ULLONG_MAX); 13 | } 14 | } -------------------------------------------------------------------------------- /include/init.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "all.h" 3 | #include "ioc.h" 4 | #include "loss_func.h" 5 | #include "optimizer.h" 6 | #include "dist_optimizer.h" 7 | #include "dense_layer.h" 8 | 9 | namespace YCDL { 10 | void initialize() { 11 | RegisterLayer("sigmoid_cross_entroy_with_logits"); 12 | RegisterLayer("mse"); 13 | RegisterLayer("sigmoid_mse"); 14 | RegisterLayer("SGD"); 15 | RegisterLayer("Adagrad"); 16 | RegisterLayer("DistWorker"); 17 | RegisterLayer("Dense"); 18 | RegisterLayer("Loss"); 19 | RegisterLayer("Activation"); 20 | RegisterLayer2("DistAdagrad"); 21 | RegisterLayer2("DistSGD"); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /include/instance.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "all.h" 3 | 4 | namespace YCDL { 5 | struct Instance { 6 | int label; 7 | std::vector feas; 8 | std::vector vals; 9 | std::vector slot_feas; 10 | float pre; 11 | }; 12 | } -------------------------------------------------------------------------------- /include/ioc.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "all.h" 4 | 5 | namespace YCDL { 6 | template 7 | class IocContainer { 8 | public: 9 | using FuncType = std::function()>; 10 | using FuncType2 = std::function; 11 | 12 | 13 | //注册一个key对应的类型 14 | template 15 | void registerType(std::string key) { 16 | FuncType func = [] { return std::make_shared(); }; 17 | registerType(key, func); 18 | } 19 | 20 | std::shared_ptr resolveShared(std::string key) { 21 | if (m_map.find(key) == m_map.end()) { 22 | return nullptr; 23 | } 24 | auto func = m_map[key]; 25 | return std::shared_ptr(func()); 26 | } 27 | 28 | //注册一个key对应的类型 29 | template 30 | void registerType2(std::string key) { 31 | FuncType2 func = [] { return new ClassType; }; 32 | registerType2(key, func); 33 | } 34 | 35 | T* resolveShared2(std::string key) { 36 | if (m_map2.find(key) == m_map2.end()) { 37 | return nullptr; 38 | } 39 | auto func = m_map2[key]; 40 | return func(); 41 | } 42 | 43 | private: 44 | void registerType(std::string key, FuncType type) { 45 | if (m_map.find(key) != m_map.end()) { 46 | throw std::invalid_argument("this key has exist"); 47 | } 48 | m_map.emplace(key, type); 49 | } 50 | void registerType2(std::string key, FuncType2 type) { 51 | if (m_map2.find(key) != m_map2.end()) { 52 | throw std::invalid_argument("this key has exist"); 53 | } 54 | m_map2.emplace(key, type); 55 | } 56 | 57 | private: 58 | std::map m_map; 59 | std::map m_map2; 60 | 61 | }; 62 | 63 | template 64 | inline IocContainer &global_layer_factory() { 65 | static IocContainer f; 66 | return f; 67 | } 68 | 69 | template 70 | inline std::shared_ptr MakeLayer(std::string name) { 71 | return global_layer_factory().resolveShared(name); 72 | } 73 | template 74 | inline T* MakeLayer2(std::string name) { 75 | return global_layer_factory().resolveShared2(name); 76 | } 77 | 78 | template 79 | inline void RegisterLayer(std::string name) { 80 | global_layer_factory().template registerType(name); 81 | } 82 | template 83 | inline void RegisterLayer2(std::string name) { 84 | global_layer_factory().template registerType2(name); 85 | } 86 | 87 | struct ICar { 88 | virtual ~ICar() {} 89 | 90 | virtual void test() const = 0; 91 | }; 92 | 93 | struct Car : ICar { 94 | void test() const { 95 | std::cout << "Car test" << std::endl; 96 | } 97 | }; 98 | 99 | struct Bus : ICar { 100 | void test() const { 101 | std::cout << "Bus test" << std::endl; 102 | } 103 | }; 104 | 105 | void test() { 106 | IocContainer ioc; 107 | ioc.registerType("bus"); 108 | ioc.registerType("car"); 109 | std::shared_ptr bus = ioc.resolveShared("bus"); 110 | bus->test(); 111 | return; 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /include/loss_func.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "all.h" 4 | 5 | namespace YCDL { 6 | class loss_func_layer : public Layer { 7 | public: 8 | virtual double forward(double logit, int label) = 0; 9 | 10 | virtual double backward(double logit, int label) = 0; 11 | }; 12 | 13 | class sigmoid_cross_entroy_with_logits : public loss_func_layer { 14 | public: 15 | double forward(double logit, int label) { 16 | double pre = sigmoid(logit); 17 | double loss = -(label * log(pre) + (1 - label) * log(1 - pre)); 18 | return loss; 19 | } 20 | 21 | double backward(double logit, int label) { 22 | double pre = sigmoid(logit); 23 | double grad = pre - label; 24 | return grad; 25 | } 26 | }; 27 | 28 | class mse : public loss_func_layer { 29 | public: 30 | double forward(double logit, int label) { 31 | double loss = (label - logit) * (label - logit) / 2; 32 | return loss; 33 | } 34 | 35 | double backward(double logit, int label) { 36 | double grad = logit - label; 37 | return grad; 38 | } 39 | }; 40 | 41 | class sigmoid_mse : public loss_func_layer { 42 | public: 43 | double forward(double logit, int label) { 44 | double pre = sigmoid(logit); 45 | double loss = (label - pre) * (label - pre) / 2; 46 | return loss; 47 | } 48 | 49 | double backward(double logit, int label) { 50 | double pre = sigmoid(logit); 51 | double grad = (pre - label) * pre * (1 - pre); 52 | return grad; 53 | } 54 | }; 55 | } -------------------------------------------------------------------------------- /include/lr.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "all.h" 3 | #include "instance.h" 4 | #include "optimizer.h" 5 | #include "loss_func.h" 6 | 7 | namespace YCDL { 8 | class LRmodel { 9 | public: 10 | std::shared_ptr _sparse_weight_sgd; 11 | std::shared_ptr _loss_layer; 12 | int _dim; 13 | 14 | LRmodel(std::shared_ptr &sparse_weight_sgd, std::shared_ptr &loss_layer, 15 | int len = 1) { 16 | _dim = len; 17 | _sparse_weight_sgd = sparse_weight_sgd; 18 | _loss_layer = loss_layer; 19 | } 20 | 21 | void forward(std::vector &instances, bool is_train = true) { 22 | for (int j = 0; j < instances.size(); j++) { 23 | double logit = 0.0; 24 | Instance &ins = instances[j]; 25 | std::vector> weight(ins.feas.size(), std::vector(_dim, 0.0)); 26 | _sparse_weight_sgd->pull(ins.feas, weight, is_train, _dim); 27 | for (int k = 0; k < ins.feas.size(); k++) { 28 | for (int l = 0; l < _dim; l++) { 29 | logit += weight[k][l] * ins.vals[k]; 30 | } 31 | } 32 | ins.pre = logit; 33 | } 34 | } 35 | 36 | void backward(std::vector &instances) { 37 | for (int j = 0; j < instances.size(); j++) { 38 | Instance &ins = instances[j]; 39 | double grad_label = _loss_layer->backward(ins.pre, ins.label); 40 | std::vector> grads(ins.feas.size(), std::vector(_dim, 0.0)); 41 | for (int k = 0; k < ins.feas.size(); k++) { 42 | for (int l = 0; l < _dim; l++) { 43 | grads[k][l] = grad_label * ins.vals[k]; 44 | } 45 | } 46 | _sparse_weight_sgd->push(ins.feas, grads); 47 | } 48 | } 49 | 50 | void stat(std::vector &instances) { 51 | // stat 52 | double loss = 0.0; 53 | double pre_avg = 0.0; 54 | double label_avg = 0.0; 55 | std::vector > label_pre; 56 | forward(instances, false); 57 | for (int j = 0; j < instances.size(); j++) { 58 | Instance &ins = instances[j]; 59 | double logit = ins.pre; 60 | double predict = sigmoid(logit); 61 | label_pre.push_back(std::make_pair(ins.label, predict)); 62 | loss += _loss_layer->forward(logit, ins.label); 63 | pre_avg += predict; 64 | label_avg += ins.label; 65 | } 66 | pre_avg /= instances.size(); 67 | label_avg /= instances.size(); 68 | loss /= instances.size(); 69 | std::vector out = calc(label_pre, pre_avg); 70 | std::cout << "auc: " << calc_auc(label_pre) << ", loss: " << loss << ", acc: " << out[0] 71 | << ", pre: " << out[1] << ", recall: " << out[2] << ", pre_avg: " << pre_avg << ", label_avg: " 72 | << label_avg << std::endl; 73 | } 74 | }; 75 | } 76 | -------------------------------------------------------------------------------- /include/matrix_val.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "all.h" 4 | #include "optimizer.h" 5 | 6 | namespace YCDL { 7 | class MatrixValue { 8 | public: 9 | virtual void pull(bool is_train = true) {} 10 | 11 | virtual void pull(std::vector> &lines, bool is_train = true) {} 12 | 13 | virtual void push() {} 14 | 15 | virtual void push(std::vector> &lines) {} 16 | 17 | virtual void push_over(std::vector val) {} 18 | 19 | virtual void 20 | init(std::string name, int row, int col, std::shared_ptr opt, bool need_gradient) {} 21 | 22 | virtual void 23 | init(std::vector slot_vec, int dim, std::shared_ptr opt, bool need_gradient) {} 24 | 25 | bool update_value(std::vector &vec, int n, int m) { 26 | if (vec.size() != n * m) { 27 | return false; 28 | } 29 | _val.setZero(n, m); 30 | for (int i = 0; i < n * m; i++) { 31 | _val(i / m, i % m) = vec[i]; 32 | } 33 | return true; 34 | } 35 | 36 | bool _trainable = false; 37 | bool _has_grad = false; 38 | Eigen::MatrixXf _val, _grad; 39 | }; 40 | 41 | class DenseMatrixValue : public MatrixValue { 42 | public: 43 | void init(std::string name, int row, int col, std::shared_ptr opt, 44 | bool trainable) override { 45 | _name = name; 46 | _key = BKDRHash(name); 47 | _row = row; 48 | _col = col; 49 | _optimizer = opt; 50 | _trainable = trainable; 51 | } 52 | 53 | void pull(bool is_train = true) override { 54 | std::vector feas{_key}; 55 | std::vector> vals(1, std::vector(_row * _col, 0.0)); 56 | _optimizer->pull(feas, vals, is_train, _row * _col); 57 | _val.setZero(_row, _col); 58 | _has_grad = false; 59 | for (int i = 0; i < _row * _col; i++) { 60 | _val(i / _col, i % _col) = vals[0][i]; 61 | } 62 | } 63 | 64 | void push() override { 65 | if (!_trainable || !_has_grad) { 66 | return; 67 | } 68 | std::vector val; 69 | for (int i = 0; i < _row; i++) { 70 | for (int j = 0; j < _col; j++) { 71 | val.push_back(_grad(i, j)); 72 | } 73 | } 74 | std::vector feas{_key}; 75 | std::vector> vals(1, val); 76 | _optimizer->push(feas, vals); 77 | } 78 | 79 | void push_over(std::vector val) override { 80 | std::vector feas{_key}; 81 | std::vector> vals(1, val); 82 | _optimizer->push_over(feas, vals); 83 | } 84 | 85 | std::shared_ptr _optimizer; 86 | int _row; 87 | int _col; 88 | std::string _name; 89 | ull _key; 90 | }; 91 | 92 | class SparseMatrixValue : public MatrixValue { 93 | public: 94 | void 95 | init(std::vector slot_vec, int dim, std::shared_ptr opt, bool trainable) override { 96 | _slot_vec = slot_vec; 97 | _dim = dim; 98 | _optimizer = opt; 99 | _trainable = trainable; 100 | for (int i = 0; i < slot_vec.size(); i++) { 101 | _slot_id_to_idx_mp[slot_vec[i]] = i; 102 | } 103 | } 104 | 105 | void pull(std::vector> &lines, bool is_train = true) override { 106 | 107 | int batch_size = lines.size(); 108 | // prepare feas 109 | std::vector feas; 110 | for (auto &line : lines) { 111 | for (auto &slot_pair : line) { 112 | int slot_id = slot_pair.first; 113 | std::vector &feas_str = slot_pair.second; 114 | for (auto fea_str : feas_str) { 115 | std::string slot_fea = boost::lexical_cast(slot_id) + ":" + fea_str; 116 | ull tmp_key = BKDRHash(slot_fea); 117 | feas.push_back(tmp_key); 118 | } 119 | } 120 | } 121 | 122 | // pull feas 123 | std::vector> vals(feas.size(), std::vector(_dim, 0.0)); 124 | _optimizer->pull(feas, vals, is_train, _dim); 125 | 126 | // matrix value init 127 | int col = _dim * _slot_vec.size(); 128 | _val.setZero(batch_size, col); 129 | _has_grad = false; 130 | 131 | // transform feas to matrix value 132 | int line_id = 0; 133 | int feas_idx = 0; 134 | for (auto &line : lines) { 135 | for (auto &slot_pair : line) { 136 | int slot_id = slot_pair.first; 137 | std::vector &feas_str = slot_pair.second; 138 | 139 | if (!_slot_id_to_idx_mp.count(slot_id)) { 140 | std::cerr << "ERROR slot_id: " << slot_id << std::endl; 141 | return; 142 | } 143 | int slot_idx = _slot_id_to_idx_mp[slot_id]; 144 | 145 | for (auto fea_str : feas_str) { 146 | std::vector &val = vals[feas_idx]; 147 | feas_idx++; 148 | if (val.size() != _dim) { 149 | std::cerr << "ERROR fea size: " << val.size() << std::endl; 150 | return; 151 | } 152 | for (int i = slot_idx * _dim; i < (slot_idx + 1) * _dim; i++) { 153 | _val(line_id, i) += val[i - slot_idx * _dim]; 154 | } 155 | } 156 | } 157 | line_id++; 158 | } 159 | } 160 | 161 | void push(std::vector> &lines) override { 162 | if (!_trainable || !_has_grad) { 163 | return; 164 | } 165 | 166 | std::vector feas; 167 | std::vector> vals; 168 | 169 | int line_id = 0; 170 | for (auto &line : lines) { 171 | for (auto &slot_pair : line) { 172 | int slot_id = slot_pair.first; 173 | std::vector &feas_str = slot_pair.second; 174 | 175 | int slot_idx = _slot_id_to_idx_mp[slot_id]; 176 | 177 | for (auto fea_str : feas_str) { 178 | std::string slot_fea = boost::lexical_cast(slot_id) + ":" + fea_str; 179 | ull tmp_key = BKDRHash(slot_fea); 180 | feas.push_back(tmp_key); 181 | std::vector val; 182 | for (int i = slot_idx * _dim; i < (slot_idx + 1) * _dim; i++) { 183 | val.push_back(_grad(line_id, i)); 184 | } 185 | vals.emplace_back(val); 186 | } 187 | } 188 | line_id++; 189 | } 190 | 191 | _optimizer->push(feas, vals); 192 | } 193 | 194 | std::vector _slot_vec; 195 | int _dim; 196 | std::shared_ptr _optimizer; 197 | std::unordered_map _slot_id_to_idx_mp; 198 | }; 199 | 200 | class MatrixValueMap { 201 | public: 202 | std::shared_ptr get_matrix_value(std::string name) { 203 | return _matrix_value_map[name]; 204 | } 205 | 206 | void add_matrix_value(std::string name, std::shared_ptr ma) { 207 | _matrix_value_map[name] = ma; 208 | } 209 | 210 | std::unordered_map> _matrix_value_map; 211 | }; 212 | 213 | inline MatrixValueMap &global_matrix_value() { 214 | static MatrixValueMap mp; 215 | return mp; 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /include/network.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "all.h" 4 | #include "instance.h" 5 | #include "optimizer.h" 6 | #include "loss_func.h" 7 | #include "matrix_val.h" 8 | #include "dense_layer.h" 9 | #include "ioc.h" 10 | #include "tools.h" 11 | 12 | namespace YCDL { 13 | class Network { 14 | public: 15 | std::shared_ptr _optimizer; 16 | std::vector > _matrix_vec; 17 | std::vector> _layer_vec; 18 | std::shared_ptr _input; 19 | std::shared_ptr _label; 20 | 21 | void load_weight() { 22 | nlohmann::json &conf = global_conf(); 23 | 24 | auto input_params = conf["input_params"]; 25 | std::vector slot_vec_str; 26 | std::vector slot_vec; 27 | std::string slots_ids = input_params["slots"]; 28 | boost::split(slot_vec_str, slots_ids, boost::is_any_of(",")); 29 | for (auto &str : slot_vec_str) { 30 | slot_vec.push_back(boost::lexical_cast(str)); 31 | } 32 | int dim = input_params["dim"]; 33 | bool is_train = true; 34 | _input = std::make_shared(); 35 | _input->init(slot_vec, dim, _optimizer, is_train); 36 | global_matrix_value().add_matrix_value(input_params["input_name"], _input); 37 | _label = std::make_shared(); 38 | global_matrix_value().add_matrix_value(input_params["label_name"], _label); 39 | 40 | auto params = conf["params"]; 41 | for (int i = 0; i < params.size(); i++) { 42 | std::string key_str = params[i]["name"]; 43 | ull key = BKDRHash(key_str); 44 | int row = params[i]["row"]; 45 | int col = params[i]["col"]; 46 | bool need_gradient = params[i]["need_gradient"]; 47 | std::shared_ptr m_ptr = std::make_shared(); 48 | m_ptr->init(key_str, row, col, _optimizer, need_gradient); 49 | _matrix_vec.push_back(m_ptr); 50 | global_matrix_value().add_matrix_value(key_str, m_ptr); 51 | } 52 | } 53 | 54 | void load_layer() { 55 | nlohmann::json &conf = global_conf(); 56 | auto layers = conf["layers"]; 57 | for (int i = 0; i < layers.size(); i++) { 58 | std::string layer_name = layers[i]["name"]; 59 | std::shared_ptr tmp_layer = MakeLayer(layer_name); 60 | tmp_layer->load_conf(layers[i]); 61 | _layer_vec.push_back(tmp_layer); 62 | } 63 | } 64 | 65 | Network(std::shared_ptr opt) { 66 | _optimizer = opt; 67 | load_weight(); 68 | load_layer(); 69 | } 70 | 71 | void forward(std::vector &instances, bool is_train = true) { 72 | 73 | std::vector> feas_lines; 74 | std::vector labels; 75 | for (int j = 0; j < instances.size(); j++) { 76 | Instance &ins = instances[j]; 77 | feas_lines.push_back(ins.slot_feas); 78 | labels.push_back(ins.label); 79 | } 80 | _input->pull(feas_lines, is_train); 81 | _label->update_value(labels, labels.size(), 1); 82 | 83 | for (int i = 0; i < _matrix_vec.size(); i++) { 84 | auto &weight = _matrix_vec[i]; 85 | weight->pull(is_train); 86 | } 87 | 88 | for (int i = 0; i < _layer_vec.size(); i++) { 89 | auto &layer = _layer_vec[i]; 90 | layer->forward(); 91 | } 92 | } 93 | 94 | void backward(std::vector &instances) { 95 | for (int i = _layer_vec.size() - 1; i >= 0; i--) { 96 | auto &layer = _layer_vec[i]; 97 | layer->backward(); 98 | } 99 | 100 | for (int i = 0; i < _matrix_vec.size(); i++) { 101 | auto &weight = _matrix_vec[i]; 102 | weight->push(); 103 | } 104 | 105 | std::vector> feas_lines; 106 | for (int j = 0; j < instances.size(); j++) { 107 | Instance &ins = instances[j]; 108 | feas_lines.push_back(ins.slot_feas); 109 | } 110 | _input->push(feas_lines); 111 | } 112 | 113 | void stat(std::vector &instances) { 114 | 115 | auto label = global_matrix_value().get_matrix_value("label"); 116 | auto predict = global_matrix_value().get_matrix_value("predict"); 117 | forward(instances, false); 118 | 119 | double loss = 0.0; 120 | double pre_avg = 0.0; 121 | double label_avg = 0.0; 122 | std::vector > label_pre; 123 | for (int i = 0; i < label->_val.rows(); i++) { 124 | for (int j = 0; j < label->_val.cols(); j++) { 125 | double val = label->_val(i, j) - predict->_val(i, j); 126 | int l = 0; 127 | if (label->_val(i, j) > 0.5) l = 1; 128 | label_pre.push_back(std::make_pair(l, predict->_val(i, j))); 129 | loss += val * val / 2; 130 | label_avg += label->_val(i, j); 131 | pre_avg += predict->_val(i, j); 132 | } 133 | } 134 | loss /= label->_val.size(); 135 | label_avg /= label->_val.size(); 136 | pre_avg /= label->_val.size(); 137 | std::cout << "loss: " << loss << std::endl; 138 | std::vector metric = calc(label_pre); 139 | std::cout << "auc: " << calc_auc(label_pre) << ", loss: " << loss << ", acc: " << metric[0] 140 | << ", pre: " << metric[1] << ", recall: " << metric[2] << ", pre_avg: " << pre_avg << ", label_avg: " 141 | << label_avg << std::endl; 142 | } 143 | }; 144 | } 145 | -------------------------------------------------------------------------------- /include/optimizer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "all.h" 3 | #include "tools.h" 4 | 5 | namespace YCDL { 6 | 7 | class Optimizer : Layer { 8 | public: 9 | virtual void push(std::vector feas, std::vector> &vals) {}; 10 | 11 | virtual int pull(std::vector feas, std::vector> &vals, bool is_train = true, int dim = 1) {}; 12 | 13 | virtual void print_weight() {}; 14 | 15 | virtual void init(ull fea, int dim) {}; 16 | 17 | virtual void load_conf(const nlohmann::json &conf) {}; 18 | 19 | virtual void push_over(std::vector feas, std::vector> &vals) {}; 20 | }; 21 | 22 | class SGD : public Optimizer { 23 | public: 24 | SGD() { 25 | load_conf(global_conf()["optimizer"]); 26 | } 27 | 28 | virtual void push(std::vector feas, std::vector> &vals) override { 29 | for (int i = 0; i < feas.size(); i++) { 30 | ull fea = feas[i]; 31 | auto &arr = _mp[fea]; 32 | for (int j = 0; j < vals[i].size(); j++) { 33 | arr[j] -= _learning_rate * vals[i][j]; 34 | } 35 | } 36 | } 37 | 38 | virtual void push_over(std::vector feas, std::vector> &vals) override { 39 | for (int i = 0; i < feas.size(); i++) { 40 | ull fea = feas[i]; 41 | auto &arr = _mp[fea]; 42 | for (int j = 0; j < vals[i].size(); j++) { 43 | arr[j] = vals[i][j]; 44 | } 45 | } 46 | } 47 | 48 | virtual void init(ull fea, int dim) override { 49 | std::vector w; 50 | for (int i = 0; i < dim; i++) { 51 | w.push_back(get_random_double()); 52 | } 53 | _mp[fea] = move(w); 54 | } 55 | 56 | virtual int pull(std::vector feas, std::vector> &vals, bool is_train = true, int dim = 1) override { 57 | int not_hit = 0; 58 | for (int i = 0; i < feas.size(); i++) { 59 | ull fea = feas[i]; 60 | auto iter = _mp.find(fea); 61 | if (!is_train && iter == _mp.end()) { 62 | not_hit++; 63 | continue; 64 | } 65 | if (iter == _mp.end()) { 66 | init(fea, dim); 67 | } 68 | // deep cooy 69 | vals[i] = _mp[fea]; 70 | } 71 | return not_hit; 72 | } 73 | 74 | virtual void print_weight() override { 75 | for (auto x : _mp) { 76 | std::cout << x.first; 77 | for (auto y : x.second) { 78 | std::cout << " " << y; 79 | } 80 | puts(""); 81 | } 82 | } 83 | 84 | virtual void load_conf(const nlohmann::json &conf) override { 85 | if (conf.find("learning_rate") != conf.end()) { 86 | _learning_rate = conf["learning_rate"]; 87 | } else { 88 | _learning_rate = 0.1; 89 | } 90 | } 91 | 92 | double _learning_rate; 93 | std::unordered_map > _mp; 94 | }; 95 | 96 | class Adagrad : public SGD { 97 | public: 98 | 99 | Adagrad() { 100 | load_conf(global_conf()["optimizer"]); 101 | } 102 | 103 | virtual void push(std::vector feas, std::vector> &vals) override { 104 | for (int i = 0; i < feas.size(); i++) { 105 | ull fea = feas[i]; 106 | auto &arr = _mp[fea], &arr2 = _mp2[fea]; 107 | for (int j = 0; j < vals[i].size(); j++) { 108 | arr2[j] += vals[i][j] * vals[i][j]; 109 | arr[j] -= _learning_rate * vals[i][j] / (sqrt(arr2[j]) + _eps); 110 | } 111 | } 112 | } 113 | 114 | virtual void init(ull fea, int dim) override { 115 | std::vector w, w2; 116 | for (int i = 0; i < dim; i++) { 117 | w.push_back(get_random_double()); 118 | w2.push_back(0.0); 119 | } 120 | _mp[fea] = move(w); 121 | _mp2[fea] = move(w2); 122 | } 123 | 124 | virtual void load_conf(const nlohmann::json &conf) override { 125 | SGD::load_conf(conf); 126 | if (conf.find("eps") != conf.end()) { 127 | _eps = conf["eps"]; 128 | } else { 129 | _eps = 1e-7; 130 | } 131 | } 132 | 133 | double _eps; 134 | std::unordered_map > _mp2; 135 | }; 136 | } 137 | -------------------------------------------------------------------------------- /include/tools.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "all.h" 3 | 4 | namespace YCDL { 5 | template 6 | void print_vec(const std::string &comment, const std::vector segs) { 7 | std::cout << comment << ": "; 8 | for (int i = 0; i < segs.size(); i++) 9 | std::cout << segs[i] << " "; 10 | puts(""); 11 | } 12 | 13 | template 14 | void line_split(const std::string &line, const std::string &sep, std::vector &ans) { 15 | std::vector segs; 16 | boost::split(segs, line, boost::is_any_of(sep)); 17 | for (int i = 0; i < segs.size(); i++) { 18 | ans.emplace_back(boost::lexical_cast(segs[i].c_str())); 19 | } 20 | return; 21 | } 22 | 23 | struct RandomDouble { 24 | RandomDouble(double l = -1, double u = 1) { 25 | lower_bound = l; 26 | upper_bound = u; 27 | unif = std::uniform_real_distribution(lower_bound, upper_bound); 28 | } 29 | 30 | double get_random() { 31 | return unif(re); 32 | } 33 | 34 | double lower_bound; 35 | double upper_bound; 36 | std::uniform_real_distribution unif; 37 | std::default_random_engine re; 38 | }; 39 | 40 | 41 | inline double get_random_double() { 42 | static RandomDouble rd; 43 | return rd.get_random(); 44 | } 45 | 46 | int get_file_len(std::string path) { 47 | std::ifstream file; 48 | file.open(path); 49 | int cnt = 0; 50 | std::string s; 51 | while (getline(file, s)) { 52 | cnt += 1; 53 | } 54 | file.close(); 55 | return cnt; 56 | } 57 | 58 | double sigmoid(double x) { 59 | static double overflow = 20.0; 60 | if (x > overflow) x = overflow; 61 | if (x < -overflow) x = -overflow; 62 | return 1.0 / (1.0 + exp(-x)); 63 | } 64 | 65 | bool cmp(std::pair a, std::pair b) { 66 | return a.second < b.second; 67 | } 68 | 69 | double calc_auc(std::vector > label_pre) { 70 | sort(label_pre.begin(), label_pre.end(), cmp); 71 | std::unordered_map > val_mp; 72 | int p = 0, n = 0, sum_rank = 0; 73 | for (int i = 0; i < label_pre.size(); i++) { 74 | if (label_pre[i].first == 1) 75 | p++, sum_rank += (i + 1); 76 | else 77 | n++; 78 | } 79 | int N = n * p; 80 | if (N == 0)return 0.0; 81 | return (sum_rank - p * (p + 1) / 2.0) / 1.0 / N; 82 | } 83 | 84 | /* 85 | double calc_auc(vector > & label_pre) { 86 | const int bucket_size = 10000000; 87 | vector bucket1(bucket_size, 0.0), bucket2(bucket_size, 0.0); 88 | for (int i = 0; i < label_pre.size(); i++) { 89 | pair p = label_pre[i]; 90 | int bucket_id = min(bucket_size - 1, int(p.second * bucket_size)); 91 | bucket1[bucket_id] += p.second; 92 | bucket2[bucket_id] += (1 - p.second); 93 | } 94 | double tp = 0.0, fp = 0.0, area = 0.0; 95 | for (int i = bucket2.size() - 1; i >= 0; i--) { 96 | double newtp = tp + bucket1[i]; 97 | double newfp = fp + bucket2[i]; 98 | area += (newtp + tp) * (newfp - fp) / 2; 99 | tp = newtp; 100 | fp = newfp; 101 | } 102 | return area / tp / fp; 103 | } 104 | */ 105 | 106 | std::vector calc(std::vector > &label_pre, double threshord = 0.5) { 107 | double tp = 0.0, fp = 0.0, tn = 0.0, fn = 0.0; 108 | for (int i = 0; i < label_pre.size(); i++) { 109 | std::pair &p = label_pre[i]; 110 | // cout << p.first << " " << p.second << endl; 111 | if (p.first == 1 && p.second > threshord) { 112 | tp++; 113 | } 114 | if (p.first == 0 && p.second < threshord) { 115 | tn++; 116 | } 117 | if (p.first == 0 && p.second > threshord) { 118 | fp++; 119 | } 120 | if (p.first == 1 && p.second < threshord) { 121 | fn++; 122 | } 123 | } 124 | 125 | double acc = (tp + tn) / (tp + tn + fp + fn); 126 | double pre = 0.0, recall = 0.0; 127 | if (tp + fp)pre = tp / (tp + fp); 128 | if (tp + fn)recall = tp / (tp + fn); 129 | std::vector ans = {acc, pre, recall}; 130 | return ans; 131 | } 132 | } -------------------------------------------------------------------------------- /readme.txt: -------------------------------------------------------------------------------- 1 | 1.基于pslite 的DNN框架 2 | 3 | 2.依赖的库有： 4 | boost 5 | Eigen 6 | ps-lite 7 | generator：https://github.com/TheLartians/Generator 8 | json: https://github.com/nlohmann/json.git 9 | 10 | 3. 依赖库安装 11 | ./run_build.sh 12 | 13 | 4. 编译 14 | ./run.sh build 15 | 16 | 5. test 17 | ./run.sh test 18 | 19 | 6. 运行demo 20 | 单机LR ./bin/output/lr_uci 21 | 单机DNN ./bin/output/network 22 | 伪分布式LR sh script/local.sh 2 2 ./bin/output/lr_uci_dist 23 | 伪分布式DNN sh script/local.sh 2 2 ./bin/output/network_dist 24 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | set -o pipefail 3 | 4 | DIR=`pwd` 5 | export CPPFLAGS=-I${DIR}/thirdparty/ps-lite/deps/include 6 | export LDFLAGS=-L${DIR}/thirdparty/ps-lite/deps/lib 7 | export PATH=${DIR}/thirdparty/ps-lite/deps/bin:$PATH 8 | export LD_LIBRARY_PATH=${DIR}/thirdparty/ps-lite/deps/lib:$LD_LIBRARY_PATH 9 | 10 | function build_all() { 11 | cd ./thirdparty/ps-lite 12 | make 13 | cd - 14 | mkdir -p build 15 | cd ./build/ 16 | cmake .. 17 | make 18 | cd .. 19 | } 20 | 21 | function test_all() { 22 | echo "test begin" 23 | ls ./bin/test | while read file 24 | do 25 | ./bin/test/$file > /dev/null 26 | echo "test $file done" 27 | done 28 | echo "test all done" 29 | } 30 | 31 | function run_all() { 32 | build_all 33 | test_all 34 | } 35 | 36 | function main() { 37 | local cmd=${1:-help} 38 | case ${cmd} in 39 | all) 40 | run_all "$@" 41 | ;; 42 | build) 43 | build_all "$@" 44 | ;; 45 | test) 46 | test_all "$@" 47 | ;; 48 | help) 49 | echo "Usage: ${BASH_SOURCE} {test}" 50 | return 0; 51 | ;; 52 | *) 53 | echo "unsupport command [${cmd}]" 54 | echo "Usage: ${BASH_SOURCE} {test}" 55 | return 1; 56 | ;; 57 | esac 58 | 59 | } 60 | 61 | main "$@" 62 | -------------------------------------------------------------------------------- /run_build.sh: -------------------------------------------------------------------------------- 1 | set -eux 2 | DIR=`pwd` 3 | 4 | ## install protobuf zmq 5 | wget https://github.com/protocolbuffers/protobuf/releases/download/v3.5.1/protobuf-cpp-3.5.1.tar.gz 6 | wget https://raw.githubusercontent.com/mli/deps/master/build/zeromq-4.1.4.tar.gz 7 | 8 | ## install eigen boost json ps-lite 9 | mkdir -p ./thirdparty/ 10 | wget https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.tar.gz 11 | tar -zxvf ./eigen-3.3.7.tar.gz 12 | mv eigen-3.3.7 ./thirdparty/ 13 | 14 | wget https://dl.bintray.com/boostorg/release/1.64.0/source/boost_1_64_0.tar.gz 15 | tar zxvf boost_1_64_0.tar.gz 16 | cd boost_1_64_0 17 | ./bootstrap.sh --with-libraries=all --with-toolset=gcc 18 | ./b2 toolset=gcc 19 | ./b2 install --prefix=${DIR}/thirdparty/boost_configure 20 | cd - 21 | 22 | cd ./thirdparty/ 23 | git clone https://github.com/nlohmann/json.git 24 | cd - 25 | 26 | cd ./thirdparty/ 27 | git clone https://github.com/dmlc/ps-lite.git 28 | cd 29 | -------------------------------------------------------------------------------- /script/kill.sh: -------------------------------------------------------------------------------- 1 | ps -ef | grep lr_uci_dist | awk '{ print $2 }' | xargs kill -9 2 | ps -ef | grep network_dist | awk '{ print $2 }' | xargs kill -9 3 | 4 | -------------------------------------------------------------------------------- /script/local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # set -x 3 | if [ $# -lt 3 ]; then 4 | echo "usage: $0 num_servers num_workers bin [args..]" 5 | exit -1; 6 | fi 7 | 8 | export DMLC_NUM_SERVER=$1 9 | shift 10 | export DMLC_NUM_WORKER=$1 11 | shift 12 | bin=$1 13 | shift 14 | arg="$@" 15 | 16 | # start the scheduler 17 | export DMLC_PS_ROOT_URI='127.0.0.1' 18 | export DMLC_PS_ROOT_PORT=8000 19 | export DMLC_ROLE='scheduler' 20 | ${bin} ${arg} & 21 | 22 | 23 | # start servers 24 | export DMLC_ROLE='server' 25 | for ((i=0; i<${DMLC_NUM_SERVER}; ++i)); do 26 | export HEAPPROFILE=./S${i} 27 | ${bin} ${arg} & 28 | done 29 | 30 | # start workers 31 | export DMLC_ROLE='worker' 32 | for ((i=0; i<${DMLC_NUM_WORKER}; ++i)); do 33 | export HEAPPROFILE=./W${i} 34 | ${bin} ${arg} & 35 | done 36 | 37 | wait 38 | -------------------------------------------------------------------------------- /script/ps_local.sh: -------------------------------------------------------------------------------- 1 | ./script/local.sh 1 1 bin/output/lr_uci_dist 2 | 3 | -------------------------------------------------------------------------------- /src/lr.cpp: -------------------------------------------------------------------------------- 1 | #include "all.h" 2 | #include "tools.h" 3 | #include "dataload.h" 4 | #include "lr.h" 5 | #include "ioc.h" 6 | #include "init.h" 7 | 8 | int main() { 9 | YCDL::initialize(); 10 | 11 | nlohmann::json& conf = YCDL::global_conf(); 12 | std::ifstream ifs("conf/config.json"); 13 | ifs >> conf; 14 | 15 | int epoch = conf["epoch"]; 16 | std::string train_file = conf["train_file"]; 17 | std::string test_file = conf["test_file"]; 18 | int batch_size = conf["batch_size"]; 19 | int shuffle_num = conf["shuffle_num"]; 20 | std::string optimizer_name = conf["optimizer"]["name"]; 21 | std::string loss_func_name = conf["loss_func_name"]; 22 | double learning_rate = conf["optimizer"]["learning_rate"]; 23 | int dim = conf["optimizer"]["dim"]; 24 | 25 | std::shared_ptr optimizer = YCDL::MakeLayer(optimizer_name); 26 | std::shared_ptr loss_function = YCDL::MakeLayer(loss_func_name); 27 | YCDL::LRmodel lr = YCDL::LRmodel(optimizer, loss_function, dim); 28 | 29 | int total_num = YCDL::get_file_len(train_file); 30 | int epoch_batch_num = total_num / batch_size; 31 | int iter = 0; 32 | 33 | auto data_iter = YCDL::dataload(train_file, epoch, batch_size, true, shuffle_num); 34 | for (std::vector instances : data_iter) { 35 | iter++; 36 | 37 | lr.forward(instances); 38 | lr.backward(instances); 39 | // test auc 40 | std::vector train_ins, test_ins; 41 | if (iter % epoch_batch_num == 0) { 42 | int epoch_num = iter / epoch_batch_num; 43 | // learning_rate adjust 44 | 45 | std::cout << "epoch_num: " << epoch_num << std::endl; 46 | std::cout << "train:\n"; 47 | auto data_iter_test = YCDL::dataload(train_file, 1, 100, false); 48 | for (std::vector instances: data_iter_test) { 49 | if (train_ins.size() > 100000) break; 50 | for (auto x: instances) { 51 | train_ins.emplace_back(std::move(x)); 52 | } 53 | } 54 | lr.stat(train_ins); 55 | train_ins.clear(); 56 | } 57 | 58 | if (iter % 100000 == 0) { 59 | std::cout << "iter: " << iter << std::endl; 60 | std::cout << "test:\n"; 61 | auto data_iter_test = YCDL::dataload(test_file, 1, 100, false); 62 | for (std::vector instances: data_iter_test) { 63 | for (auto x: instances) { 64 | test_ins.emplace_back(std::move(x)); 65 | } 66 | } 67 | lr.stat(test_ins); 68 | test_ins.clear(); 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/lr_uci.cpp: -------------------------------------------------------------------------------- 1 | #include "all.h" 2 | #include "tools.h" 3 | #include "lr.h" 4 | #include "ioc.h" 5 | #include "init.h" 6 | #include "instance.h" 7 | #include 8 | 9 | YCDL::ull deal_num (std::string s, int delim, std::string head) { 10 | std::string t = boost::lexical_cast(boost::lexical_cast(s) / delim); 11 | return YCDL::BKDRHash(head+t); 12 | } 13 | 14 | YCDL::ull deal_category(std::string s, std::string head) { 15 | return YCDL::BKDRHash(head+s); 16 | } 17 | 18 | YCDL::ull deal_label(std::string s) { 19 | if (s == "\"yes\"") 20 | return 1; 21 | return 0; 22 | } 23 | 24 | YCDL::Generator > dataload(std::string path, int epoch, int batch_size, bool is_train, int shuffle_num = 1) { 25 | return YCDL::Generator >([=](YCDL::Yield > &yield) { 26 | std::string s; 27 | int cnt = 0, pool_sz = shuffle_num * batch_size; 28 | std::vector instances, ans; 29 | int nums = epoch; 30 | std::vector heads = {"age","job","marital","education","default","balance","housing","loan","contact","day","month", 31 | "duration","campaign","pdays","previous","poutcome","y"}; 32 | std::vector ways = {"num-10","cat","cat","cat","cat","num-10","cat","cat","cat","num-10","cat","num-10","num-10","num-100","num-1","cat","label"}; 33 | 34 | while(nums) { 35 | std::ifstream file; 36 | file.open(path); 37 | nums--; 38 | while (getline(file, s)) { 39 | cnt += 1; 40 | std::vector segs; 41 | YCDL::line_split(s, ";", segs); 42 | YCDL::Instance ins; 43 | int len = segs.size(); 44 | for (int i = 0; i < len - 1; i++) { 45 | YCDL::ull value; 46 | if (ways[i] == "cat") { 47 | value = deal_category(segs[i], heads[i]); 48 | } else if (ways[i].substr(0,3) == "num") { 49 | int delim = boost::lexical_cast(ways[i].substr(4)); 50 | // cout << delim << " delim\n"; 51 | value = deal_num(segs[i], delim, heads[i]); 52 | } 53 | ins.feas.push_back(value); 54 | ins.vals.push_back(1.0); 55 | //cout << heads[i] << " " << ways[i] << " " << i << " " << value << " " << segs[i] << endl; 56 | } 57 | ins.feas.push_back(0); 58 | ins.vals.push_back(1.0); 59 | ins.label = boost::lexical_cast(deal_label(segs[len-1])); 60 | //cout << deal_label(segs[len-1]) << endl; 61 | 62 | instances.emplace_back(std::move(ins)); 63 | if (cnt == pool_sz) { 64 | cnt = 0; 65 | if (is_train) { 66 | random_shuffle(instances.begin(), instances.end()); 67 | } 68 | for (int i = 0; i < pool_sz; i+=batch_size) { 69 | ans.assign(instances.begin() + i, instances.begin() + i + batch_size); 70 | yield(ans); 71 | } 72 | instances.clear(); 73 | } 74 | } 75 | file.close(); 76 | } 77 | if (cnt != pool_sz && instances.size() != 0) { 78 | if (is_train) { 79 | random_shuffle(instances.begin(), instances.end()); 80 | } 81 | for (int i = 0; i < cnt; i += batch_size) { 82 | if (i + batch_size < cnt) { 83 | ans.assign(instances.begin() + i, instances.begin() + i + batch_size); 84 | } else { 85 | ans.clear(); 86 | ans.assign(instances.begin() + i, instances.end()); 87 | } 88 | yield(ans); 89 | } 90 | instances.clear(); 91 | } 92 | }); 93 | } 94 | 95 | int main() { 96 | YCDL::initialize(); 97 | 98 | nlohmann::json& conf = YCDL::global_conf(); 99 | std::ifstream ifs("conf/config.json"); 100 | ifs >> conf; 101 | 102 | int epoch = conf["epoch"]; 103 | std::string train_file = "./data/uci_data/bank/train"; 104 | std::string test_file = "./data/uci_data/bank/test"; 105 | int batch_size = conf["batch_size"]; 106 | int shuffle_num = conf["shuffle_num"]; 107 | std::string optimizer_name = conf["optimizer"]["name"]; 108 | std::string loss_func_name = conf["loss_func_name"]; 109 | double learning_rate = conf["optimizer"]["learning_rate"]; 110 | int dim = conf["optimizer"]["dim"]; 111 | 112 | std::shared_ptr optimizer = YCDL::MakeLayer(optimizer_name); 113 | std::shared_ptr loss_function = YCDL::MakeLayer(loss_func_name); 114 | YCDL::LRmodel lr = YCDL::LRmodel(optimizer, loss_function, dim); 115 | 116 | int total_num = YCDL::get_file_len(train_file); 117 | int epoch_batch_num = total_num / batch_size; 118 | int iter = 0; 119 | 120 | auto data_iter = dataload(train_file, epoch, batch_size, true, shuffle_num); 121 | for (std::vector instances : data_iter) { 122 | iter++; 123 | //cout << "print weight\n"; 124 | lr.forward(instances); 125 | //optimizer->print_weight(); 126 | lr.backward(instances); 127 | // test auc 128 | std::vector train_ins, test_ins; 129 | int epoch_num = iter / epoch_batch_num; 130 | if (iter % epoch_batch_num == 0) { 131 | std::cout << "epoch_num: " << epoch_num << std::endl; 132 | std::cout << "train:\n"; 133 | auto data_iter_test = dataload(train_file, 1, 100, false); 134 | for (std::vector instances: data_iter_test) { 135 | if (train_ins.size() > 100000) break; 136 | for (auto x: instances) { 137 | train_ins.emplace_back(std::move(x)); 138 | } 139 | } 140 | lr.stat(train_ins); 141 | train_ins.clear(); 142 | 143 | // test 144 | if (true) { 145 | std::cout << "iter: " << iter << std::endl; 146 | std::cout << "test:\n"; 147 | auto data_iter_test = dataload(test_file, 1, 100, false); 148 | for (std::vector instances: data_iter_test) { 149 | for (auto x: instances) { 150 | test_ins.emplace_back(std::move(x)); 151 | } 152 | } 153 | lr.stat(test_ins); 154 | test_ins.clear(); 155 | } 156 | } 157 | } 158 | 159 | } 160 | -------------------------------------------------------------------------------- /src/lr_uci_dist.cpp: -------------------------------------------------------------------------------- 1 | #include "all.h" 2 | #include "tools.h" 3 | #include "lr.h" 4 | #include "ioc.h" 5 | #include "init.h" 6 | #include "instance.h" 7 | #include 8 | #include "ps/ps.h" 9 | #include "dist_optimizer.h" 10 | #include 11 | 12 | YCDL::ull deal_num (std::string s, int delim, std::string head) { 13 | std::string t = boost::lexical_cast(boost::lexical_cast(s) / delim); 14 | return YCDL::BKDRHash(head+t); 15 | } 16 | 17 | YCDL::ull deal_category(std::string s, std::string head) { 18 | return YCDL::BKDRHash(head+s); 19 | } 20 | 21 | YCDL::ull deal_label(std::string s) { 22 | if (s == "\"yes\"") 23 | return 1; 24 | return 0; 25 | } 26 | 27 | YCDL::Generator > dataload(std::string path, int epoch, int batch_size, bool is_train, int shuffle_num = 1) { 28 | return YCDL::Generator >([=](YCDL::Yield > &yield) { 29 | std::string s; 30 | int cnt = 0, pool_sz = shuffle_num * batch_size; 31 | std::vector instances, ans; 32 | int nums = epoch; 33 | std::vector heads = {"age","job","marital","education","default","balance","housing","loan","contact","day","month", 34 | "duration","campaign","pdays","previous","poutcome","y"}; 35 | std::vector ways = {"num-10","cat","cat","cat","cat","num-10","cat","cat","cat","num-10","cat","num-10","num-10","num-100","num-1","cat","label"}; 36 | 37 | while(nums) { 38 | std::ifstream file; 39 | file.open(path); 40 | nums--; 41 | while (getline(file, s)) { 42 | cnt += 1; 43 | std::vector segs; 44 | YCDL::line_split(s, ";", segs); 45 | YCDL::Instance ins; 46 | int len = segs.size(); 47 | for (int i = 0; i < len - 1; i++) { 48 | YCDL::ull value; 49 | if (ways[i] == "cat") { 50 | value = deal_category(segs[i], heads[i]); 51 | } else if (ways[i].substr(0,3) == "num") { 52 | int delim = boost::lexical_cast(ways[i].substr(4)); 53 | // cout << delim << " delim\n"; 54 | value = deal_num(segs[i], delim, heads[i]); 55 | } 56 | ins.feas.push_back(value); 57 | ins.vals.push_back(1.0); 58 | //cout << heads[i] << " " << ways[i] << " " << i << " " << value << " " << segs[i] << endl; 59 | } 60 | ins.feas.push_back(0); 61 | ins.vals.push_back(1.0); 62 | ins.label = boost::lexical_cast(deal_label(segs[len-1])); 63 | //cout << deal_label(segs[len-1]) << endl; 64 | 65 | instances.emplace_back(std::move(ins)); 66 | if (cnt == pool_sz) { 67 | cnt = 0; 68 | if (is_train) { 69 | random_shuffle(instances.begin(), instances.end()); 70 | } 71 | for (int i = 0; i < pool_sz; i+=batch_size) { 72 | ans.assign(instances.begin() + i, instances.begin() + i + batch_size); 73 | yield(ans); 74 | } 75 | instances.clear(); 76 | } 77 | } 78 | file.close(); 79 | } 80 | if (cnt != pool_sz && instances.size() != 0) { 81 | if (is_train) { 82 | random_shuffle(instances.begin(), instances.end()); 83 | } 84 | for (int i = 0; i < cnt; i += batch_size) { 85 | if (i + batch_size < cnt) { 86 | ans.assign(instances.begin() + i, instances.begin() + i + batch_size); 87 | } else { 88 | ans.clear(); 89 | ans.assign(instances.begin() + i, instances.end()); 90 | } 91 | yield(ans); 92 | } 93 | instances.clear(); 94 | } 95 | }); 96 | } 97 | 98 | void start_server() { 99 | 100 | auto & conf = YCDL::global_conf(); 101 | std::string optimizer_name = conf["optimizer"]["name"]; 102 | auto server = YCDL::MakeLayer2(optimizer_name); 103 | ps::RegisterExitCallback([server]() { delete server; }); 104 | } 105 | 106 | void start_work() { 107 | 108 | int rank = ps::MyRank(); 109 | auto & conf = YCDL::global_conf(); 110 | int epoch = conf["epoch"]; 111 | std::string train_file = "./data/uci_data/dist_bank/train_" + std::to_string(rank); 112 | std::string test_file = "./data/uci_data/bank/test"; 113 | int batch_size = conf["batch_size"]; 114 | int shuffle_num = conf["shuffle_num"]; 115 | std::string optimizer_name = "DistWorker"; 116 | std::string loss_func_name = conf["loss_func_name"]; 117 | int dim = conf["optimizer"]["dim"]; 118 | 119 | std::shared_ptr optimizer = YCDL::MakeLayer(optimizer_name); 120 | std::shared_ptr loss_function = YCDL::MakeLayer(loss_func_name); 121 | YCDL::LRmodel lr = YCDL::LRmodel(optimizer, loss_function, dim); 122 | 123 | int total_num = YCDL::get_file_len(train_file); 124 | int epoch_batch_num = total_num / batch_size; 125 | int iter = 0; 126 | 127 | auto data_iter = dataload(train_file, epoch, batch_size, true, shuffle_num); 128 | for (std::vector instances : data_iter) { 129 | iter++; 130 | //cout << "print weight\n"; 131 | lr.forward(instances); 132 | //optimizer->print_weight(); 133 | lr.backward(instances); 134 | // test auc 135 | std::vector train_ins, test_ins; 136 | int epoch_num = iter / epoch_batch_num; 137 | if (iter % epoch_batch_num == 0 && rank == 0) { 138 | 139 | std::cout << "epoch_num: " << epoch_num << std::endl; 140 | std::cout << "train:\n"; 141 | 142 | auto data_iter_test = dataload("./data/uci_data/bank/train", 1, 100, false); 143 | for (std::vector instances: data_iter_test) { 144 | if (train_ins.size() > 100000) break; 145 | for (auto x: instances) { 146 | train_ins.emplace_back(std::move(x)); 147 | } 148 | } 149 | lr.stat(train_ins); 150 | train_ins.clear(); 151 | 152 | // test 153 | if (true) { 154 | std::cout << "iter: " << iter << std::endl; 155 | std::cout << "test:\n"; 156 | auto data_iter_test = dataload(test_file, 1, 100, false); 157 | for (std::vector instances: data_iter_test) { 158 | for (auto x: instances) { 159 | test_ins.emplace_back(std::move(x)); 160 | } 161 | } 162 | lr.stat(test_ins); 163 | test_ins.clear(); 164 | } 165 | } 166 | } 167 | } 168 | 169 | void global_init() { 170 | YCDL::initialize(); 171 | 172 | nlohmann::json &conf = YCDL::global_conf(); 173 | std::ifstream ifs("conf/dist_config.json"); 174 | ifs >> conf; 175 | } 176 | 177 | int main() { 178 | 179 | ps::Start(0); 180 | 181 | global_init(); 182 | if (ps::IsServer()) { 183 | 184 | std::cout << "server:" << ps::MyRank() << " running\n"; 185 | start_server(); 186 | } else if (ps::IsWorker()){ 187 | std::cout << "worker:" << ps::MyRank() << " running\n"; 188 | start_work(); 189 | } 190 | 191 | ps::Finalize(0); 192 | } 193 | -------------------------------------------------------------------------------- /src/network.cpp: -------------------------------------------------------------------------------- 1 | #include "all.h" 2 | #include "tools.h" 3 | #include "network.h" 4 | #include "ioc.h" 5 | #include "init.h" 6 | #include "instance.h" 7 | #include 8 | using namespace std; 9 | 10 | YCDL::ull deal_num (std::string s, int delim, std::string head) { 11 | std::string t = boost::lexical_cast(boost::lexical_cast(s) / delim); 12 | return YCDL::BKDRHash(head+t); 13 | } 14 | 15 | YCDL::ull deal_category(std::string s, std::string head) { 16 | return YCDL::BKDRHash(head+s); 17 | } 18 | 19 | YCDL::ull deal_label(std::string s) { 20 | if (s == "\"yes\"") 21 | return 1; 22 | return 0; 23 | } 24 | 25 | YCDL::Generator > dataload(std::string path, int epoch, int batch_size, bool is_train, int shuffle_num = 1) { 26 | return YCDL::Generator >([=](YCDL::Yield > &yield) { 27 | std::string s; 28 | int cnt = 0, pool_sz = shuffle_num * batch_size; 29 | std::vector instances, ans; 30 | int nums = epoch; 31 | std::vector heads = {"age","job","marital","education","default","balance","housing","loan","contact","day","month", 32 | "duration","campaign","pdays","previous","poutcome","y"}; 33 | std::vector ways = {"num-10","cat","cat","cat","cat","num-10","cat","cat","cat","num-10","cat","num-10","num-10","num-100","num-1","cat","label"}; 34 | 35 | while(nums) { 36 | std::ifstream file; 37 | file.open(path); 38 | nums--; 39 | while (getline(file, s)) { 40 | cnt += 1; 41 | std::vector segs; 42 | YCDL::line_split(s, ";", segs); 43 | YCDL::Instance ins; 44 | int len = segs.size(); 45 | for (int i = 0; i < len - 1; i++) { 46 | YCDL::ull value; 47 | if (ways[i] == "cat") { 48 | value = deal_category(segs[i], heads[i]); 49 | } else if (ways[i].substr(0,3) == "num") { 50 | int delim = boost::lexical_cast(ways[i].substr(4)); 51 | value = deal_num(segs[i], delim, heads[i]); 52 | } 53 | ins.slot_feas.push_back({i, vector(1, boost::lexical_cast(value))}); 54 | //ins.feas.push_back(value); 55 | //ins.vals.push_back(1.0); 56 | } 57 | //ins.feas.push_back(0); 58 | //ins.vals.push_back(1.0); 59 | ins.label = boost::lexical_cast(deal_label(segs[len-1])); 60 | 61 | instances.emplace_back(std::move(ins)); 62 | if (cnt == pool_sz) { 63 | cnt = 0; 64 | if (is_train) { 65 | random_shuffle(instances.begin(), instances.end()); 66 | } 67 | for (int i = 0; i < pool_sz; i+=batch_size) { 68 | ans.assign(instances.begin() + i, instances.begin() + i + batch_size); 69 | yield(ans); 70 | } 71 | instances.clear(); 72 | } 73 | } 74 | file.close(); 75 | } 76 | if (cnt != pool_sz && instances.size() != 0) { 77 | if (is_train) { 78 | random_shuffle(instances.begin(), instances.end()); 79 | } 80 | for (int i = 0; i < cnt; i += batch_size) { 81 | if (i + batch_size < cnt) { 82 | ans.assign(instances.begin() + i, instances.begin() + i + batch_size); 83 | } else { 84 | ans.clear(); 85 | ans.assign(instances.begin() + i, instances.end()); 86 | } 87 | yield(ans); 88 | } 89 | instances.clear(); 90 | } 91 | }); 92 | } 93 | 94 | int main() { 95 | YCDL::initialize(); 96 | 97 | nlohmann::json& conf = YCDL::global_conf(); 98 | std::ifstream ifs("conf/nn_config.json"); 99 | ifs >> conf; 100 | 101 | int epoch = conf["epoch"]; 102 | std::string train_file = conf["train_file"]; 103 | std::string test_file = conf["test_file"]; 104 | int batch_size = conf["batch_size"]; 105 | int shuffle_num = conf["shuffle_num"]; 106 | std::string optimizer_name = conf["optimizer"]["name"]; 107 | std::shared_ptr optimizer = YCDL::MakeLayer(optimizer_name); 108 | YCDL::Network network = YCDL::Network(optimizer); 109 | 110 | int total_num = YCDL::get_file_len(train_file); 111 | int epoch_batch_num = total_num / batch_size; 112 | int iter = 0; 113 | 114 | auto data_iter = dataload(train_file, epoch, batch_size, true, shuffle_num); 115 | for (std::vector instances : data_iter) { 116 | iter++; 117 | //cout << "print weight\n"; 118 | network.forward(instances); 119 | //optimizer->print_weight(); 120 | network.backward(instances); 121 | // test auc 122 | std::vector train_ins, test_ins; 123 | int epoch_num = iter / epoch_batch_num; 124 | if (iter % epoch_batch_num == 0) { 125 | std::cout << "epoch_num: " << epoch_num << std::endl; 126 | std::cout << "train:\n"; 127 | auto data_iter_test = dataload(train_file, 1, 100, false); 128 | for (std::vector instances: data_iter_test) { 129 | if (train_ins.size() > 100000) break; 130 | for (auto x: instances) { 131 | train_ins.emplace_back(std::move(x)); 132 | } 133 | } 134 | network.stat(train_ins); 135 | train_ins.clear(); 136 | 137 | // test 138 | if (true) { 139 | std::cout << "iter: " << iter << std::endl; 140 | std::cout << "test:\n"; 141 | auto data_iter_test = dataload(test_file, 1, 100, false); 142 | for (vector instances: data_iter_test) { 143 | for (auto x: instances) { 144 | test_ins.emplace_back(std::move(x)); 145 | } 146 | } 147 | network.stat(test_ins); 148 | test_ins.clear(); 149 | } 150 | } 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/network_dist.cpp: -------------------------------------------------------------------------------- 1 | #include "all.h" 2 | #include "tools.h" 3 | #include "network.h" 4 | #include "ioc.h" 5 | #include "init.h" 6 | #include "instance.h" 7 | #include 8 | #include "dist_optimizer.h" 9 | 10 | YCDL::ull deal_num (std::string s, int delim, std::string head) { 11 | std::string t = boost::lexical_cast(boost::lexical_cast(s) / delim); 12 | return YCDL::BKDRHash(head+t); 13 | } 14 | 15 | YCDL::ull deal_category(std::string s, std::string head) { 16 | return YCDL::BKDRHash(head+s); 17 | } 18 | 19 | YCDL::ull deal_label(std::string s) { 20 | if (s == "\"yes\"") 21 | return 1; 22 | return 0; 23 | } 24 | 25 | YCDL::Generator > dataload(std::string path, int epoch, int batch_size, bool is_train, int shuffle_num = 1) { 26 | return YCDL::Generator >([=](YCDL::Yield > &yield) { 27 | std::string s; 28 | int cnt = 0, pool_sz = shuffle_num * batch_size; 29 | std::vector instances, ans; 30 | int nums = epoch; 31 | std::vector heads = {"age","job","marital","education","default","balance","housing","loan","contact","day","month", 32 | "duration","campaign","pdays","previous","poutcome","y"}; 33 | std::vector ways = {"num-10","cat","cat","cat","cat","num-10","cat","cat","cat","num-10","cat","num-10","num-10","num-100","num-1","cat","label"}; 34 | 35 | while(nums) { 36 | std::ifstream file; 37 | file.open(path); 38 | nums--; 39 | while (getline(file, s)) { 40 | cnt += 1; 41 | std::vector segs; 42 | YCDL::line_split(s, ";", segs); 43 | YCDL::Instance ins; 44 | int len = segs.size(); 45 | for (int i = 0; i < len - 1; i++) { 46 | YCDL::ull value; 47 | if (ways[i] == "cat") { 48 | value = deal_category(segs[i], heads[i]); 49 | } else if (ways[i].substr(0,3) == "num") { 50 | int delim = boost::lexical_cast(ways[i].substr(4)); 51 | value = deal_num(segs[i], delim, heads[i]); 52 | } 53 | ins.slot_feas.push_back({i, std::vector(1, boost::lexical_cast(value))}); 54 | //ins.feas.push_back(value); 55 | //ins.vals.push_back(1.0); 56 | } 57 | //ins.feas.push_back(0); 58 | //ins.vals.push_back(1.0); 59 | ins.label = boost::lexical_cast(deal_label(segs[len-1])); 60 | 61 | instances.emplace_back(std::move(ins)); 62 | if (cnt == pool_sz) { 63 | cnt = 0; 64 | if (is_train) { 65 | random_shuffle(instances.begin(), instances.end()); 66 | } 67 | for (int i = 0; i < pool_sz; i+=batch_size) { 68 | ans.assign(instances.begin() + i, instances.begin() + i + batch_size); 69 | yield(ans); 70 | } 71 | instances.clear(); 72 | } 73 | } 74 | file.close(); 75 | } 76 | if (cnt != pool_sz && instances.size() != 0) { 77 | if (is_train) { 78 | random_shuffle(instances.begin(), instances.end()); 79 | } 80 | for (int i = 0; i < cnt; i += batch_size) { 81 | if (i + batch_size < cnt) { 82 | ans.assign(instances.begin() + i, instances.begin() + i + batch_size); 83 | } else { 84 | ans.clear(); 85 | ans.assign(instances.begin() + i, instances.end()); 86 | } 87 | yield(ans); 88 | } 89 | instances.clear(); 90 | } 91 | }); 92 | } 93 | 94 | void global_init() { 95 | YCDL::initialize(); 96 | 97 | nlohmann::json &conf = YCDL::global_conf(); 98 | std::ifstream ifs("conf/dist_nn_config.json"); 99 | ifs >> conf; 100 | } 101 | 102 | void start_server() { 103 | 104 | auto & conf = YCDL::global_conf(); 105 | std::string optimizer_name = conf["optimizer"]["name"]; 106 | auto server = YCDL::MakeLayer2(optimizer_name); 107 | ps::RegisterExitCallback([server]() { delete server; }); 108 | } 109 | void start_work() { 110 | 111 | int rank = ps::MyRank(); 112 | auto & conf = YCDL::global_conf(); 113 | int epoch = conf["epoch"]; 114 | std::string train_file = "./data/uci_data/dist_bank/train_" + std::to_string(rank); 115 | std::string test_file = "./data/uci_data/bank/test"; 116 | int batch_size = conf["batch_size"]; 117 | int shuffle_num = conf["shuffle_num"]; 118 | std::string optimizer_name = "DistWorker"; 119 | 120 | std::shared_ptr optimizer = YCDL::MakeLayer(optimizer_name); 121 | YCDL::Network network = YCDL::Network(optimizer); 122 | 123 | int total_num = YCDL::get_file_len(train_file); 124 | int epoch_batch_num = total_num / batch_size; 125 | int iter = 0; 126 | 127 | auto data_iter = dataload(train_file, epoch, batch_size, true, shuffle_num); 128 | for (std::vector instances : data_iter) { 129 | iter++; 130 | //cout << "print weight\n"; 131 | network.forward(instances); 132 | //optimizer->print_weight(); 133 | network.backward(instances); 134 | // test auc 135 | std::vector train_ins, test_ins; 136 | int epoch_num = iter / epoch_batch_num; 137 | if (iter % epoch_batch_num == 0 && rank == 0) { 138 | 139 | std::cout << "epoch_num: " << epoch_num << std::endl; 140 | 141 | std::cout << "train:\n"; 142 | 143 | auto data_iter_test = dataload("./data/uci_data/bank/train", 1, 100, false); 144 | for (std::vector instances: data_iter_test) { 145 | if (train_ins.size() > 100000) break; 146 | for (auto x: instances) { 147 | train_ins.emplace_back(std::move(x)); 148 | } 149 | } 150 | network.stat(train_ins); 151 | train_ins.clear(); 152 | 153 | // test 154 | if (true) { 155 | std::cout << "iter: " << iter << std::endl; 156 | std::cout << "test:\n"; 157 | auto data_iter_test = dataload(test_file, 1, 100, false); 158 | for (std::vector instances: data_iter_test) { 159 | for (auto x: instances) { 160 | test_ins.emplace_back(std::move(x)); 161 | } 162 | } 163 | network.stat(test_ins); 164 | test_ins.clear(); 165 | } 166 | } 167 | } 168 | } 169 | 170 | int main() { 171 | 172 | ps::Start(0); 173 | 174 | global_init(); 175 | if (ps::IsServer()) { 176 | std::cout << "server:" << ps::MyRank() << " running\n"; 177 | start_server(); 178 | } else if (ps::IsWorker()){ 179 | std::cout << "worker:" << ps::MyRank() << " running\n"; 180 | start_work(); 181 | } 182 | 183 | ps::Finalize(0); 184 | return 0; 185 | } 186 | -------------------------------------------------------------------------------- /test/dataload.cpp: -------------------------------------------------------------------------------- 1 | #include "dataload.h" 2 | #include "tools.h" 3 | #include 4 | int main(){ 5 | int epoch = 10; 6 | std::string path = "./data/test0"; 7 | int batch_size = 9, line_len = YCDL::get_file_len(path); 8 | auto data_iter = YCDL::dataload(path, epoch, batch_size, false, 10); 9 | int cnt = 0; 10 | bool first = true; 11 | for (std::vector x : data_iter) { 12 | cnt += x.size(); 13 | if (first) { 14 | assert(cnt == batch_size); 15 | first = false; 16 | } 17 | for (auto y : x) 18 | YCDL::print_vec("", y.feas); 19 | } 20 | assert(cnt == line_len * 10); 21 | } -------------------------------------------------------------------------------- /test/eigen.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "eigen_func.h" 4 | int main() 5 | { 6 | Eigen::MatrixXf m(2,2), c(2,2); 7 | m(0,0) = 3; 8 | m(1,0) = 2.5; 9 | m(0,1) = -1; 10 | m(1,1) = m(1,0) + m(0,1); 11 | std::cout << m << std::endl; 12 | 13 | YCDL::matrix_multi(false, false, false, m, m, c); 14 | std::cout << c << std::endl; 15 | 16 | YCDL::matrix_multi_addition(false, false, false, m, m, c); 17 | std::cout << c << std::endl; 18 | } -------------------------------------------------------------------------------- /test/fibonacci.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | int main() { 5 | 6 | auto fibonacci_numbers = YCDL::Generator([=](YCDL::Yield &yield) { 7 | long a = 0, b = 1; 8 | yield(a); 9 | yield(b); 10 | while (true) { 11 | long t = a + b; 12 | yield(t); 13 | a = b; 14 | b = t; 15 | } 16 | }); 17 | 18 | for (auto i:fibonacci_numbers) { 19 | std::cout << i << std::endl; 20 | if (i > 10000) break; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /test/lr_test.cpp: -------------------------------------------------------------------------------- 1 | #include "all.h" 2 | #include "tools.h" 3 | #include "dataload.h" 4 | #include "lr.h" 5 | #include "optimizer.h" 6 | #include "ioc.h" 7 | #include "init.h" 8 | #include 9 | #include 10 | 11 | YCDL::Generator > dataload(std::vector& input, int epoch, int batch_size, bool is_train) { 12 | return YCDL::Generator >([=](YCDL::Yield > &yield) { 13 | int cnt = 0; 14 | std::vector instances; 15 | int nums = epoch; 16 | while(nums) { 17 | nums--; 18 | for (auto x : input) { 19 | cnt += 1; 20 | instances.emplace_back(std::move(x)); 21 | if (cnt == batch_size) { 22 | cnt = 0; 23 | if (is_train) { 24 | std::random_shuffle(instances.begin(), instances.end()); 25 | } 26 | yield(instances); 27 | instances.clear(); 28 | } 29 | } 30 | } 31 | if (cnt != batch_size && instances.size() != 0) 32 | if (is_train) { 33 | std::random_shuffle(instances.begin(), instances.end()); 34 | } 35 | yield(instances); 36 | instances.clear(); 37 | }); 38 | } 39 | 40 | int main() { 41 | YCDL::initialize(); 42 | 43 | nlohmann::json& conf = YCDL::global_conf(); 44 | std::ifstream ifs("conf/config.json"); 45 | ifs >> conf; 46 | 47 | 48 | std::string optimizer_name = conf["optimizer"]["name"]; 49 | std::string loss_func_name = conf["loss_func_name"]; 50 | int dim = conf["optimizer"]["dim"]; 51 | 52 | std::shared_ptr optimizer = YCDL::MakeLayer(optimizer_name); 53 | std::shared_ptr loss_function = YCDL::MakeLayer(loss_func_name); 54 | YCDL::LRmodel lr = YCDL::LRmodel(optimizer, loss_function, dim); 55 | 56 | std::vector train_data, test_data; 57 | srand((int)time(0)); 58 | for (int i = 0; i < 10000; i++) { 59 | double x =(rand() % 10) / 10.0; 60 | double y = x * 2 + (rand()%10 - 4) / 10.0 - 1; 61 | double bias = 1.0; 62 | YCDL::Instance ins; 63 | if (y - x*2 + 1> 0){ 64 | ins.label = 1; 65 | } else { 66 | ins.label = 0; 67 | } 68 | ins.feas.push_back(0); 69 | ins.feas.push_back(1); 70 | ins.feas.push_back(2); 71 | ins.vals.push_back(1.0); 72 | ins.vals.push_back(x); 73 | ins.vals.push_back(y); 74 | train_data.emplace_back(std::move(ins)); 75 | } 76 | 77 | for (int i = 0; i < 1000; i++) { 78 | double x =(rand() % 10) / 10.0; 79 | double y = x * 2 + (rand()%10 - 4) / 10.0 - 1; 80 | double bias = 1.0; 81 | YCDL::Instance ins; 82 | if (y - x*2 + 1> 0) { 83 | ins.label = 1; 84 | } else { 85 | ins.label = 0; 86 | } 87 | ins.feas.push_back(0); 88 | ins.feas.push_back(1); 89 | ins.feas.push_back(2); 90 | ins.vals.push_back(1.0); 91 | ins.vals.push_back(x); 92 | ins.vals.push_back(y); 93 | test_data.emplace_back(std::move(ins)); 94 | } 95 | 96 | int epoch = 10; 97 | int batch_size = 50; 98 | int total_num = 10000; 99 | int epoch_batch = total_num / batch_size; 100 | auto train_data_iter = dataload(train_data, epoch, batch_size, true); 101 | int iter = 0; 102 | for (std::vector instances : train_data_iter) { 103 | lr.forward(instances); 104 | lr.backward(instances); 105 | iter += 1; 106 | if (iter % epoch_batch == 0) { 107 | int epoch_num = iter / epoch_batch; 108 | std::cout << "epoch_num: " << epoch_num << std::endl; 109 | optimizer->print_weight(); 110 | lr.stat(test_data); 111 | } 112 | } 113 | } 114 | /* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ 115 | -------------------------------------------------------------------------------- /test/range.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | YCDL::Generator range(int max){ 5 | return YCDL::Generator([=](YCDL::Yield &yield){ 6 | for(int i = 0;i 3 | #include 4 | #include 5 | #include 6 | 7 | using namespace nlohmann; 8 | int main() { 9 | json j; 10 | j["number"] = 1; 11 | j["float"] = 1.5; 12 | j["string"] = "this is a string"; 13 | j["boolean"] = true; 14 | j["user"]["id"] = 10; 15 | j["user"]["name"] = "Nomango"; 16 | std::cout << j["user"]["name"] << std::endl; 17 | 18 | 19 | // 从文件读取 JSON 20 | std::ifstream ifs("conf/test.json"); 21 | json j2; 22 | ifs >> j2; 23 | 24 | std::cout << j2["global"]["epoch"] << std::endl; 25 | std::string s = j2["global"]["test_file"]; 26 | std::cout << s << std::endl; 27 | 28 | auto layers = j2["layers"]; 29 | 30 | // 使用迭代器遍历 31 | 32 | for (int i =0; i < layers.size(); i++) { 33 | auto &layer = layers[i]; 34 | for (auto iter = layer.begin(); iter != layer.end(); iter++) { 35 | std::string tmp = iter.key(); 36 | if (tmp == "name") 37 | std::cout << iter.key() << ":" << iter.value() << std::endl; 38 | } 39 | } 40 | 41 | } --------------------------------------------------------------------------------