├── .gitignore ├── Data.cpp ├── Data.hpp ├── LBLM.cpp ├── LBLM.hpp ├── Makefile ├── Matrix.hpp ├── NTF.cpp ├── NTF.hpp ├── README.md ├── Rand.cpp ├── Rand.hpp ├── SVO.hpp ├── SVOPN.hpp ├── Show.cpp ├── Sigmoid.hpp ├── Tanh.hpp ├── Test.cpp ├── Utils.hpp ├── Vocabulary.cpp ├── Vocabulary.hpp ├── main.cpp ├── objs └── dummy ├── test_data ├── SVO.txt ├── SVO_ave.txt ├── VO+S.txt ├── VO+S_ave.txt ├── VO+Sadd.txt ├── VO+Sadd_ave.txt ├── VO.txt └── VO_ave.txt └── train_data └── dummy /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | 19 | # Compiled Static libraries 20 | *.lai 21 | *.la 22 | *.a 23 | *.lib 24 | 25 | # Executables 26 | *.exe 27 | *.out 28 | *.app 29 | -------------------------------------------------------------------------------- /Data.cpp: -------------------------------------------------------------------------------- 1 | #include "Data.hpp" 2 | 3 | Rand Data::rndModel(Rand::r_.next()); 4 | Rand Data::rndData(Rand::r_.next()); 5 | -------------------------------------------------------------------------------- /Data.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include "Rand.hpp" 5 | 6 | class Vocabulary; 7 | 8 | class Data{ 9 | public: 10 | enum TYPE{ 11 | SVO_, 12 | SVOPN_, 13 | }; 14 | 15 | enum SET{ 16 | TRAIN, 17 | DEV, 18 | TEST, 19 | }; 20 | 21 | static Rand rndModel; 22 | static Rand rndData; 23 | 24 | Data::TYPE type; 25 | Data::SET set; 26 | }; 27 | -------------------------------------------------------------------------------- /LBLM.cpp: -------------------------------------------------------------------------------- 1 | #include "LBLM.hpp" 2 | #include 3 | 4 | #define ETA 1.0 5 | 6 | void LBLM::init(const int dim, Vocabulary& voc){ 7 | const double scale1 = 1.0/dim; 8 | const double scale2 = 1.0/(dim*dim); 9 | 10 | this->nounVector = MatD(dim, voc.nounStr.size()); 11 | Data::rndModel.gauss(this->nounVector, scale1); 12 | this->verbVector = MatD(dim, voc.verbStr.size()); 13 | Data::rndModel.gauss(this->verbVector, scale1); 14 | this->prepVector = MatD(dim, voc.prepStr.size()); 15 | Data::rndModel.gauss(this->prepVector, scale1); 16 | 17 | if (voc.nullIndex >= 0){ 18 | this->nounVector.col(voc.nullIndex).fill(0.0); 19 | } 20 | 21 | this->subjCompWeight = MatD::Ones(dim, 1); 22 | this->verbCompWeight = MatD::Ones(dim, 1); 23 | this->objCompWeight = MatD::Ones(dim, 1); 24 | 25 | this->subjWeight = MatD(dim, 1); 26 | Data::rndModel.gauss(this->subjWeight, scale2); 27 | this->verbWeight = MatD(dim, 1); 28 | Data::rndModel.gauss(this->verbWeight, scale2); 29 | this->objWeight = MatD(dim, 1); 30 | Data::rndModel.gauss(this->objWeight, scale2); 31 | 32 | this->prepWeight = MatD(dim, 1); 33 | Data::rndModel.gauss(this->prepWeight, scale2); 34 | this->pobjWeight = MatD(dim, 1); 35 | Data::rndModel.gauss(this->pobjWeight, scale2); 36 | this->svoWeight = MatD(dim, 1); 37 | Data::rndModel.gauss(this->svoWeight, scale2); 38 | 39 | this->nounScoreWeight = MatD::Zero(this->nounVector.cols(), this->nounVector.rows()); 40 | this->verbScoreWeight = MatD::Zero(this->verbVector.cols(), this->verbVector.rows()); 41 | this->prepScoreWeight = MatD::Zero(this->prepVector.cols(), this->prepVector.rows()); 42 | 43 | //gradient 44 | this->nounGrad = MatD::Zero(this->nounVector.rows(), this->nounVector.cols()); 45 | this->verbGrad = MatD::Zero(this->verbVector.rows(), this->verbVector.cols()); 46 | this->prepGrad = MatD::Zero(this->prepVector.rows(), this->prepVector.cols()); 47 | 48 | this->scwGrad = MatD::Zero(this->subjCompWeight.rows(), this->subjCompWeight.cols()); 49 | this->vcwGrad = MatD::Zero(this->verbCompWeight.rows(), this->verbCompWeight.cols()); 50 | this->ocwGrad = MatD::Zero(this->objCompWeight.rows(), this->objCompWeight.cols()); 51 | 52 | this->swGrad = MatD::Zero(this->subjWeight.rows(), this->subjWeight.cols()); 53 | this->vwGrad = MatD::Zero(this->verbWeight.rows(), this->verbWeight.cols()); 54 | this->owGrad = MatD::Zero(this->objWeight.rows(), this->objWeight.cols()); 55 | 56 | this->pwGrad = MatD::Zero(this->prepWeight.rows(), this->prepWeight.cols()); 57 | this->powGrad = MatD::Zero(this->pobjWeight.rows(), this->pobjWeight.cols()); 58 | this->svowGrad = MatD::Zero(this->svoWeight.rows(), this->svoWeight.cols()); 59 | 60 | this->nounScoreGrad = MatD::Zero(this->nounScoreWeight.rows(), this->nounScoreWeight.cols()); 61 | this->verbScoreGrad = MatD::Zero(this->verbScoreWeight.rows(), this->verbScoreWeight.cols()); 62 | this->prepScoreGrad = MatD::Zero(this->prepScoreWeight.rows(), this->prepScoreWeight.cols()); 63 | 64 | //AdaGrad 65 | this->nounGradHist = MatD::Ones(this->nounVector.rows(), this->nounVector.cols())*ETA; 66 | this->verbGradHist = MatD::Ones(this->verbVector.rows(), this->verbVector.cols())*ETA; 67 | this->prepGradHist = MatD::Ones(this->prepVector.rows(), this->prepVector.cols())*ETA; 68 | 69 | this->scwGradHist = MatD::Ones(this->subjCompWeight.rows(), this->subjCompWeight.cols())*ETA; 70 | this->vcwGradHist = MatD::Ones(this->verbCompWeight.rows(), this->verbCompWeight.cols())*ETA; 71 | this->ocwGradHist = MatD::Ones(this->objCompWeight.rows(), this->objCompWeight.cols())*ETA; 72 | 73 | this->swGradHist = MatD::Ones(this->subjWeight.rows(), this->subjWeight.cols())*ETA; 74 | this->vwGradHist = MatD::Ones(this->verbWeight.rows(), this->verbWeight.cols())*ETA; 75 | this->owGradHist = MatD::Ones(this->objWeight.rows(), this->objWeight.cols())*ETA; 76 | 77 | this->pwGradHist = MatD::Ones(this->prepWeight.rows(), this->prepWeight.cols())*ETA; 78 | this->powGradHist = MatD::Ones(this->pobjWeight.rows(), this->pobjWeight.cols())*ETA; 79 | this->svowGradHist = MatD::Ones(this->svoWeight.rows(), this->svoWeight.cols())*ETA; 80 | 81 | this->nounScoreGradHist = MatD::Ones(this->nounScoreWeight.rows(), this->nounScoreWeight.cols())*ETA; 82 | this->verbScoreGradHist = MatD::Ones(this->verbScoreWeight.rows(), this->verbScoreWeight.cols())*ETA; 83 | this->prepScoreGradHist = MatD::Ones(this->prepScoreWeight.rows(), this->prepScoreWeight.cols())*ETA; 84 | } 85 | 86 | void LBLM::train(std::vector& instance, std::vector& type, std::vector& dummy, Vocabulary& voc, const double learningRate, const int maxItr, const int miniBatchSize, const int numNeg){ 87 | const int prog = instance.size()/10; 88 | int counter = 0; 89 | 90 | for (int itr = 0; itr < maxItr; ++itr){ 91 | counter = 0; 92 | std::random_shuffle(instance.begin(), instance.end()); 93 | ::printf("LBLM: itr %2d/%2d, learning rate %.6f, dim %d\n", itr+1, maxItr, learningRate, (int)this->nounVector.rows()); 94 | 95 | for (int i = 0; i < (int)instance.size(); ++i){ 96 | if (instance[i]->type == Data::SVO_){ 97 | this->train((SVO*)instance[i], voc); 98 | //this->gradCheck((SVO::SVO*)instance[i]); 99 | } 100 | else { 101 | this->train((SVOPN*)instance[i], voc); 102 | } 103 | 104 | if ((i+1)%miniBatchSize == 0 || i == (int)instance.size()-1){ 105 | this->update(learningRate, voc.nullIndex); 106 | } 107 | 108 | if ((i+1)%prog == 0){ 109 | std::cout << (++counter)*10 << "% " << std::flush; 110 | } 111 | } 112 | 113 | std::cout << std::endl; 114 | 115 | MatD accDev = MatD::Zero(numNeg, 1); 116 | double devCount = 0.0; 117 | 118 | for (int j = 0; j < (int)type.size(); ++j){ 119 | if (type[j]->type == Data::SVO_){ 120 | SVO* sample = (SVO*)type[j]; 121 | 122 | if (sample->set == Data::DEV){ 123 | devCount += 1.0; 124 | } 125 | else { 126 | continue; 127 | } 128 | 129 | double vo2s = this->score(sample->s, sample->v, sample->o, LBLM::S); 130 | double sv2o = this->score(sample->s, sample->v, sample->o, LBLM::O); 131 | double so2v = this->score(sample->s, sample->v, sample->o, LBLM::V); 132 | 133 | for (int k = 0; k < numNeg; ++k){ 134 | double vo2s_ = this->score(sample->s_[k], sample->v, sample->o, LBLM::S); 135 | double sv2o_ = this->score(sample->s, sample->v, sample->o_[k], LBLM::O); 136 | double so2v_ = this->score(sample->s, sample->v_[k], sample->o, LBLM::V); 137 | 138 | if (vo2s > vo2s_ && sv2o > sv2o_ && so2v > so2v_){ 139 | accDev.coeffRef(k, 0) += 1.0; 140 | } 141 | } 142 | } 143 | else if (type[j]->type == Data::SVOPN_){ 144 | SVOPN* sample = (SVOPN*)type[j]; 145 | 146 | if (sample->set == Data::DEV){ 147 | devCount += 1.0; 148 | } 149 | else { 150 | continue; 151 | } 152 | 153 | double vo2s = this->score(sample->p, sample->n, sample->svo, LBLM::P); 154 | double sv2o = this->score(sample->p, sample->n, sample->svo, LBLM::N); 155 | 156 | for (int k = 0; k < numNeg; ++k){ 157 | double vo2s_ = this->score(sample->p_[k], sample->n, sample->svo, LBLM::P); 158 | double sv2o_ = this->score(sample->p, sample->n_[k], sample->svo, LBLM::N); 159 | 160 | if (vo2s > vo2s_ && sv2o > sv2o_){ 161 | accDev.coeffRef(k, 0) += 1.0; 162 | } 163 | } 164 | } 165 | } 166 | 167 | accDev /= devCount; 168 | accDev *= 100.0; 169 | printf("\tDev Acc: %f (%f)", accDev.array().sum()/accDev.rows(), Utils::stdDev(accDev)); 170 | std::cout << std::endl; 171 | } 172 | } 173 | 174 | void LBLM::train(SVO* svo, Vocabulary& voc){ 175 | static MatD predict; 176 | static MatD WVs, WVv, WVo; 177 | static double scorePos, scoreNeg; 178 | static MatD delta; 179 | 180 | WVs = this->subjWeight.array()*this->nounVector.col(svo->s).array(); 181 | WVv = this->verbWeight.array()*this->verbVector.col(svo->v).array(); 182 | WVo = this->objWeight.array()*this->nounVector.col(svo->o).array(); 183 | 184 | //V 185 | do { 186 | this->negV = voc.verbList[(Data::rndData.next() >> 16)%voc.verbList.size()]; 187 | } while (voc.exist(svo->s, this->negV, svo->o)); 188 | 189 | predict = WVs+WVo; 190 | Tanh::tanh(predict); 191 | scorePos = (this->verbScoreWeight.row(svo->v)*predict).coeff(0, 0); 192 | scoreNeg = (this->verbScoreWeight.row(this->negV)*predict).coeff(0, 0); 193 | 194 | if (scorePos <= scoreNeg+1.0){ 195 | this->verbScoreMap[this->negV] = 1; 196 | this->verbScoreMap[svo->v] = 1; 197 | this->verbScoreGrad.row(svo->v) -= predict.transpose(); 198 | this->verbScoreGrad.row(this->negV) += predict.transpose(); 199 | delta = Tanh::tanhPrime(predict).array()*(this->verbScoreWeight.row(this->negV)-this->verbScoreWeight.row(svo->v)).transpose().array(); 200 | this->swGrad.array() += delta.array()*this->nounVector.col(svo->s).array(); 201 | this->owGrad.array() += delta.array()*this->nounVector.col(svo->o).array(); 202 | this->nounGrad.col(svo->s).array() += delta.array()*this->subjWeight.array(); 203 | this->nounGrad.col(svo->o).array() += delta.array()*this->objWeight.array(); 204 | this->nounMap[svo->s] = 1; 205 | this->nounMap[svo->o] = 1; 206 | } 207 | 208 | //S 209 | do { 210 | this->negS = voc.vsubjList[(Data::rndData.next() >> 16)%voc.vsubjList.size()]; 211 | } while (voc.exist(this->negS, svo->v, svo->o)); 212 | 213 | predict = WVv+WVo; 214 | Tanh::tanh(predict); 215 | scorePos = (this->nounScoreWeight.row(svo->s)*predict).coeff(0, 0); 216 | scoreNeg = (this->nounScoreWeight.row(this->negS)*predict).coeff(0, 0); 217 | 218 | if (scorePos <= scoreNeg+1.0){ 219 | this->nounScoreMap[this->negS] = 1; 220 | this->nounScoreMap[svo->s] = 1; 221 | this->nounScoreGrad.row(svo->s) -= predict.transpose(); 222 | this->nounScoreGrad.row(this->negS) += predict.transpose(); 223 | delta = Tanh::tanhPrime(predict).array()*(this->nounScoreWeight.row(this->negS)-this->nounScoreWeight.row(svo->s)).transpose().array(); 224 | this->vwGrad.array() += delta.array()*this->verbVector.col(svo->v).array(); 225 | this->owGrad.array() += delta.array()*this->nounVector.col(svo->o).array(); 226 | this->verbGrad.col(svo->v).array() += delta.array()*this->verbWeight.array(); 227 | this->nounGrad.col(svo->o).array() += delta.array()*this->objWeight.array(); 228 | this->verbMap[svo->v] = 1; 229 | this->nounMap[svo->o] = 1; 230 | } 231 | 232 | //O 233 | do { 234 | this->negO = voc.vobjList[(Data::rndData.next() >> 16)%voc.vobjList.size()]; 235 | } while (voc.exist(svo->s, svo->v, this->negO) || voc.exist(this->negS, svo->v, this->negO)); 236 | 237 | predict = WVs+WVv; 238 | Tanh::tanh(predict); 239 | scorePos = (this->nounScoreWeight.row(svo->o)*predict).coeff(0, 0); 240 | scoreNeg = (this->nounScoreWeight.row(this->negO)*predict).coeff(0, 0); 241 | 242 | if (scorePos <= scoreNeg+1.0){ 243 | this->nounScoreMap[this->negO] = 1; 244 | this->nounScoreMap[svo->o] = 1; 245 | this->nounScoreGrad.row(svo->o) -= predict.transpose(); 246 | this->nounScoreGrad.row(this->negO) += predict.transpose(); 247 | delta = Tanh::tanhPrime(predict).array()*(this->nounScoreWeight.row(this->negO)-this->nounScoreWeight.row(svo->o)).transpose().array(); 248 | this->vwGrad.array() += delta.array()*this->verbVector.col(svo->v).array(); 249 | this->swGrad.array() += delta.array()*this->nounVector.col(svo->s).array(); 250 | this->verbGrad.col(svo->v).array() += delta.array()*this->verbWeight.array(); 251 | this->nounGrad.col(svo->s).array() += delta.array()*this->subjWeight.array(); 252 | this->verbMap[svo->v] = 1; 253 | this->nounMap[svo->s] = 1; 254 | } 255 | } 256 | 257 | void LBLM::train(SVOPN* svopn, Vocabulary& voc){ 258 | static MatD predict; 259 | static MatD svo, svoPrime; 260 | static MatD WVp, WVn, WVsvo; 261 | static double scorePos, scoreNeg; 262 | static MatD delta, deltaComp = MatD(this->nounVector.rows(), 1); 263 | static bool flg; 264 | 265 | this->compose(svo, svopn->svo); 266 | svoPrime = Tanh::tanhPrime(svo); 267 | svoPrime.array() *= this->svoWeight.array(); 268 | WVp = this->prepWeight.array()*this->prepVector.col(svopn->p).array(); 269 | WVn = this->pobjWeight.array()*this->nounVector.col(svopn->n).array(); 270 | WVsvo = this->svoWeight.array()*svo.array(); 271 | 272 | deltaComp.setZero(); 273 | flg = false; 274 | 275 | //P 276 | do { 277 | this->negP = voc.prepList[(Data::rndData.next() >> 16)%voc.prepList.size()]; 278 | } while (voc.exist(this->negP, svopn->n, svopn->svo)); 279 | 280 | predict = WVn+WVsvo; 281 | Tanh::tanh(predict); 282 | scorePos = (this->prepScoreWeight.row(svopn->p)*predict).coeff(0, 0); 283 | scoreNeg = (this->prepScoreWeight.row(this->negP)*predict).coeff(0, 0); 284 | 285 | if (scorePos <= scoreNeg+1.0){ 286 | this->prepScoreGrad.row(svopn->p) -= predict.transpose(); 287 | this->prepScoreGrad.row(this->negP) += predict.transpose(); 288 | this->prepScoreMap[this->negP] = 1; 289 | this->prepScoreMap[svopn->p] = 1; 290 | 291 | delta = Tanh::tanhPrime(predict).array()*(this->prepScoreWeight.row(this->negP)-this->prepScoreWeight.row(svopn->p)).transpose().array(); 292 | 293 | this->powGrad.array() += delta.array()*this->nounVector.col(svopn->n).array(); 294 | this->svowGrad.array() += delta.array()*svo.array(); 295 | 296 | this->nounGrad.col(svopn->n).array() += delta.array()*this->pobjWeight.array(); 297 | this->nounMap[svopn->n] = 1; 298 | 299 | deltaComp.array() += delta.array()*svoPrime.array(); 300 | flg = true; 301 | } 302 | 303 | //N 304 | do { 305 | this->negN = voc.pobjList[(Data::rndData.next() >> 16)%voc.pobjList.size()]; 306 | } while (voc.exist(svopn->p, this->negN, svopn->svo)); 307 | 308 | predict = WVp+WVsvo; 309 | Tanh::tanh(predict); 310 | scorePos = (this->nounScoreWeight.row(svopn->n)*predict).coeff(0, 0); 311 | scoreNeg = (this->nounScoreWeight.row(this->negN)*predict).coeff(0, 0); 312 | 313 | if (scorePos <= scoreNeg+1.0){ 314 | this->nounScoreGrad.row(svopn->n) -= predict.transpose(); 315 | this->nounScoreGrad.row(this->negN) += predict.transpose(); 316 | this->nounScoreMap[this->negN] = 1; 317 | this->nounScoreMap[svopn->n] = 1; 318 | 319 | delta = Tanh::tanhPrime(predict).array()*(this->nounScoreWeight.row(this->negN)-this->nounScoreWeight.row(svopn->n)).transpose().array(); 320 | 321 | this->pwGrad.array() += delta.array()*this->prepVector.col(svopn->p).array(); 322 | this->svowGrad.array() += delta.array()*svo.array(); 323 | 324 | this->prepGrad.col(svopn->p).array() += delta.array()*this->prepWeight.array(); 325 | this->prepMap[svopn->p] = 1; 326 | 327 | deltaComp.array() += delta.array()*svoPrime.array(); 328 | flg = true; 329 | } 330 | 331 | if (flg){ 332 | this->nounGrad.col(svopn->svo->s).array() += deltaComp.array()*this->subjCompWeight.array(); 333 | this->verbGrad.col(svopn->svo->v).array() += deltaComp.array()*this->verbCompWeight.array(); 334 | this->nounGrad.col(svopn->svo->o).array() += deltaComp.array()*this->objCompWeight.array(); 335 | this->scwGrad.array() += deltaComp.array()*this->nounVector.col(svopn->svo->s).array(); 336 | this->vcwGrad.array() += deltaComp.array()*this->verbVector.col(svopn->svo->v).array(); 337 | this->ocwGrad.array() += deltaComp.array()*this->nounVector.col(svopn->svo->o).array(); 338 | this->nounMap[svopn->svo->s] = 1; 339 | this->verbMap[svopn->svo->v] = 1; 340 | this->nounMap[svopn->svo->o] = 1; 341 | } 342 | } 343 | 344 | void LBLM::update(const double learningRate, const int exception){ 345 | for (auto it = this->nounMap.begin(); it != this->nounMap.end(); ++it){ 346 | if (it->first == exception){ 347 | this->nounGrad.col(it->first).setZero(); 348 | continue; 349 | } 350 | 351 | this->nounGradHist.col(it->first).array() += this->nounGrad.col(it->first).array().square(); 352 | this->nounGrad.col(it->first).array() /= this->nounGradHist.col(it->first).array().sqrt(); 353 | this->nounVector.col(it->first) -= learningRate*this->nounGrad.col(it->first); 354 | this->nounGrad.col(it->first).setZero(); 355 | } 356 | for (auto it = this->verbMap.begin(); it != this->verbMap.end(); ++it){ 357 | this->verbGradHist.col(it->first).array() += this->verbGrad.col(it->first).array().square(); 358 | this->verbGrad.col(it->first).array() /= this->verbGradHist.col(it->first).array().sqrt(); 359 | this->verbVector.col(it->first) -= learningRate*this->verbGrad.col(it->first); 360 | this->verbGrad.col(it->first).setZero(); 361 | } 362 | for (auto it = this->prepMap.begin(); it != this->prepMap.end(); ++it){ 363 | this->prepGradHist.col(it->first).array() += this->prepGrad.col(it->first).array().square(); 364 | this->prepGrad.col(it->first).array() /= this->prepGradHist.col(it->first).array().sqrt(); 365 | this->prepVector.col(it->first) -= learningRate*this->prepGrad.col(it->first); 366 | this->prepGrad.col(it->first).setZero(); 367 | } 368 | 369 | for (auto it = this->nounScoreMap.begin(); it != this->nounScoreMap.end(); ++it){ 370 | this->nounScoreGradHist.row(it->first).array() += this->nounScoreGrad.row(it->first).array().square(); 371 | this->nounScoreGrad.row(it->first).array() /= this->nounScoreGradHist.row(it->first).array().sqrt(); 372 | this->nounScoreWeight.row(it->first) -= learningRate*this->nounScoreGrad.row(it->first); 373 | this->nounScoreGrad.row(it->first).setZero(); 374 | } 375 | for (auto it = this->verbScoreMap.begin(); it != this->verbScoreMap.end(); ++it){ 376 | this->verbScoreGradHist.row(it->first).array() += this->verbScoreGrad.row(it->first).array().square(); 377 | this->verbScoreGrad.row(it->first).array() /= this->verbScoreGradHist.row(it->first).array().sqrt(); 378 | this->verbScoreWeight.row(it->first) -= learningRate*this->verbScoreGrad.row(it->first); 379 | this->verbScoreGrad.row(it->first).setZero(); 380 | } 381 | for (auto it = this->prepScoreMap.begin(); it != this->prepScoreMap.end(); ++it){ 382 | this->prepScoreGradHist.row(it->first).array() += this->prepScoreGrad.row(it->first).array().square(); 383 | this->prepScoreGrad.row(it->first).array() /= this->prepScoreGradHist.row(it->first).array().sqrt(); 384 | this->prepScoreWeight.row(it->first) -= learningRate*this->prepScoreGrad.row(it->first); 385 | this->prepScoreGrad.row(it->first).setZero(); 386 | } 387 | 388 | this->nounMap.clear(); 389 | this->verbMap.clear(); 390 | this->prepMap.clear(); 391 | this->nounScoreMap.clear(); 392 | this->verbScoreMap.clear(); 393 | this->prepScoreMap.clear(); 394 | 395 | this->scwGradHist.array() += this->scwGrad.array().square(); 396 | this->scwGrad.array() /= this->scwGradHist.array().sqrt(); 397 | this->subjCompWeight -= learningRate*this->scwGrad; 398 | 399 | this->vcwGradHist.array() += this->vcwGrad.array().square(); 400 | this->vcwGrad.array() /= this->vcwGradHist.array().sqrt(); 401 | this->verbCompWeight -= learningRate*this->vcwGrad; 402 | 403 | this->ocwGradHist.array() += this->ocwGrad.array().square(); 404 | this->ocwGrad.array() /= this->ocwGradHist.array().sqrt(); 405 | this->objCompWeight -= learningRate*this->ocwGrad; 406 | 407 | this->swGradHist.array() += this->swGrad.array().square(); 408 | this->swGrad.array() /= this->swGradHist.array().sqrt(); 409 | this->subjWeight -= learningRate*this->swGrad; 410 | 411 | this->vwGradHist.array() += this->vwGrad.array().square(); 412 | this->vwGrad.array() /= this->vwGradHist.array().sqrt(); 413 | this->verbWeight -= learningRate*this->vwGrad; 414 | 415 | this->owGradHist.array() += this->owGrad.array().square(); 416 | this->owGrad.array() /= this->owGradHist.array().sqrt(); 417 | this->objWeight -= learningRate*this->owGrad; 418 | 419 | this->pwGradHist.array() += this->pwGrad.array().square(); 420 | this->pwGrad.array() /= this->pwGradHist.array().sqrt(); 421 | this->prepWeight -= learningRate*this->pwGrad; 422 | 423 | this->powGradHist.array() += this->powGrad.array().square(); 424 | this->powGrad.array() /= this->powGradHist.array().sqrt(); 425 | this->pobjWeight -= learningRate*this->powGrad; 426 | 427 | this->svowGradHist.array() += this->svowGrad.array().square(); 428 | this->svowGrad.array() /= this->svowGradHist.array().sqrt(); 429 | this->svoWeight -= learningRate*this->svowGrad; 430 | 431 | this->scwGrad.setZero(); 432 | this->vcwGrad.setZero(); 433 | this->ocwGrad.setZero(); 434 | this->swGrad.setZero(); 435 | this->vwGrad.setZero(); 436 | this->owGrad.setZero(); 437 | this->pwGrad.setZero(); 438 | this->powGrad.setZero(); 439 | this->svowGrad.setZero(); 440 | } 441 | 442 | void LBLM::gradCheck(SVO* svo){ 443 | const double eps = 1.0e-04; 444 | double val, objPlus, objMinus; 445 | 446 | printf("\nchecking gradients ...\n"); 447 | 448 | for (auto it = this->nounMap.begin(); it != this->nounMap.end(); ++it){ 449 | printf("----------- noun %10d -------------\n", it->first); 450 | for (int i = 0; i < this->nounGrad.rows(); ++i){ 451 | val = this->nounVector.coeff(i, it->first); 452 | this->nounVector.coeffRef(i, it->first) = val+eps; 453 | objPlus = this->objective(svo); 454 | this->nounVector.coeffRef(i, it->first) = val-eps; 455 | objMinus = this->objective(svo); 456 | this->nounVector.coeffRef(i, it->first) = val; 457 | printf("backprop: %.8f\n", this->nounGrad.coeff(i, it->first)); 458 | printf("numerical: %.8f\n", (objPlus-objMinus)/(2.0*eps)); 459 | } 460 | } 461 | 462 | for (auto it = this->verbMap.begin(); it != this->verbMap.end(); ++it){ 463 | printf("----------- verb %10d -------------\n", it->first); 464 | for (int i = 0; i < this->verbGrad.rows(); ++i){ 465 | val = this->verbVector.coeff(i, it->first); 466 | this->verbVector.coeffRef(i, it->first) = val+eps; 467 | objPlus = this->objective(svo); 468 | this->verbVector.coeffRef(i, it->first) = val-eps; 469 | objMinus = this->objective(svo); 470 | this->verbVector.coeffRef(i, it->first) = val; 471 | printf("backprop: %.8f\n", this->verbGrad.coeff(i, it->first)); 472 | printf("numerical: %.8f\n", (objPlus-objMinus)/(2.0*eps)); 473 | } 474 | } 475 | 476 | for (auto it = this->nounScoreMap.begin(); it != this->nounScoreMap.end(); ++it){ 477 | printf("----------- noun score weight %10d -------------\n", it->first); 478 | for (int i = 0; i < this->nounScoreGrad.cols(); ++i){ 479 | val = this->nounScoreWeight.coeff(it->first, i); 480 | this->nounScoreWeight.coeffRef(it->first, i) = val+eps; 481 | objPlus = this->objective(svo); 482 | this->nounScoreWeight.coeffRef(it->first, i) = val-eps; 483 | objMinus = this->objective(svo); 484 | this->nounScoreWeight.coeffRef(it->first, i) = val; 485 | printf("backprop: %.8f\n", this->nounScoreGrad.coeff(it->first, i)); 486 | printf("numerical: %.8f\n", (objPlus-objMinus)/(2.0*eps)); 487 | } 488 | } 489 | 490 | for (auto it = this->verbScoreMap.begin(); it != this->verbScoreMap.end(); ++it){ 491 | printf("----------- verb score weight %10d -------------\n", it->first); 492 | for (int i = 0; i < this->verbScoreGrad.cols(); ++i){ 493 | val = this->verbScoreWeight.coeff(it->first, i); 494 | this->verbScoreWeight.coeffRef(it->first, i) = val+eps; 495 | objPlus = this->objective(svo); 496 | this->verbScoreWeight.coeffRef(it->first, i) = val-eps; 497 | objMinus = this->objective(svo); 498 | this->verbScoreWeight.coeffRef(it->first, i) = val; 499 | printf("backprop: %.8f\n", this->verbScoreGrad.coeff(it->first, i)); 500 | printf("numerical: %.8f\n", (objPlus-objMinus)/(2.0*eps)); 501 | } 502 | } 503 | 504 | printf("----------- subj weight -------------\n"); 505 | for (int i = 0; i < this->subjWeight.rows(); ++i){ 506 | for (int j = 0; j < this->subjWeight.cols(); ++j){ 507 | val = this->subjWeight.coeff(i, j); 508 | this->subjWeight.coeffRef(i, j) = val+eps; 509 | objPlus = this->objective(svo); 510 | this->subjWeight.coeffRef(i, j) = val-eps; 511 | objMinus = this->objective(svo); 512 | this->subjWeight.coeffRef(i, j) = val; 513 | printf("backprop: %.8f\n", this->swGrad.coeff(i, j)); 514 | printf("numerical: %.8f\n", (objPlus-objMinus)/(2.0*eps)); 515 | } 516 | } 517 | 518 | printf("----------- verb weight -------------\n"); 519 | for (int i = 0; i < this->verbWeight.rows(); ++i){ 520 | for (int j = 0; j < this->verbWeight.cols(); ++j){ 521 | val = this->verbWeight.coeff(i, j); 522 | this->verbWeight.coeffRef(i, j) = val+eps; 523 | objPlus = this->objective(svo); 524 | this->verbWeight.coeffRef(i, j) = val-eps; 525 | objMinus = this->objective(svo); 526 | this->verbWeight.coeffRef(i, j) = val; 527 | printf("backprop: %.8f\n", this->vwGrad.coeff(i, j)); 528 | printf("numerical: %.8f\n", (objPlus-objMinus)/(2.0*eps)); 529 | } 530 | } 531 | 532 | printf("----------- obj weight -------------\n"); 533 | for (int i = 0; i < this->objWeight.rows(); ++i){ 534 | for (int j = 0; j < this->objWeight.cols(); ++j){ 535 | val = this->objWeight.coeff(i, j); 536 | this->objWeight.coeffRef(i, j) = val+eps; 537 | objPlus = this->objective(svo); 538 | this->objWeight.coeffRef(i, j) = val-eps; 539 | objMinus = this->objective(svo); 540 | this->objWeight.coeffRef(i, j) = val; 541 | printf("backprop: %.8f\n", this->owGrad.coeff(i, j)); 542 | printf("numerical: %.8f\n", (objPlus-objMinus)/(2.0*eps)); 543 | } 544 | } 545 | } 546 | 547 | void LBLM::save(const std::string& file){ 548 | std::ofstream ofs(file.c_str(), std::ios::out|std::ios::binary); 549 | 550 | assert(ofs); 551 | Utils::save(ofs, this->nounVector); 552 | Utils::save(ofs, this->verbVector); 553 | Utils::save(ofs, this->prepVector); 554 | 555 | Utils::save(ofs, this->subjCompWeight); 556 | Utils::save(ofs, this->verbCompWeight); 557 | Utils::save(ofs, this->objCompWeight); 558 | 559 | Utils::save(ofs, this->subjWeight); 560 | Utils::save(ofs, this->verbWeight); 561 | Utils::save(ofs, this->objWeight); 562 | 563 | Utils::save(ofs, this->prepWeight); 564 | Utils::save(ofs, this->pobjWeight); 565 | Utils::save(ofs, this->svoWeight); 566 | 567 | Utils::save(ofs, this->nounScoreWeight); 568 | Utils::save(ofs, this->verbScoreWeight); 569 | Utils::save(ofs, this->prepScoreWeight); 570 | } 571 | 572 | void LBLM::load(const std::string& file){ 573 | std::ifstream ifs(file.c_str(), std::ios::out|std::ios::binary); 574 | 575 | assert(ifs); 576 | Utils::load(ifs, this->nounVector); 577 | Utils::load(ifs, this->verbVector); 578 | Utils::load(ifs, this->prepVector); 579 | 580 | Utils::load(ifs, this->subjCompWeight); 581 | Utils::load(ifs, this->verbCompWeight); 582 | Utils::load(ifs, this->objCompWeight); 583 | 584 | Utils::load(ifs, this->subjWeight); 585 | Utils::load(ifs, this->verbWeight); 586 | Utils::load(ifs, this->objWeight); 587 | 588 | Utils::load(ifs, this->prepWeight); 589 | Utils::load(ifs, this->pobjWeight); 590 | Utils::load(ifs, this->svoWeight); 591 | 592 | Utils::load(ifs, this->nounScoreWeight); 593 | Utils::load(ifs, this->verbScoreWeight); 594 | Utils::load(ifs, this->prepScoreWeight); 595 | } 596 | -------------------------------------------------------------------------------- /LBLM.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "SVO.hpp" 4 | #include "SVOPN.hpp" 5 | #include "Vocabulary.hpp" 6 | #include "Tanh.hpp" 7 | #include "Utils.hpp" 8 | 9 | class LBLM{ 10 | public: 11 | enum TARGET{ 12 | S, V, O, 13 | P, N, 14 | }; 15 | 16 | //word vectors 17 | MatD nounVector; 18 | MatD verbVector; 19 | MatD prepVector; 20 | 21 | //composition weight vectors for SVO 22 | MatD subjCompWeight; 23 | MatD verbCompWeight; 24 | MatD objCompWeight; 25 | 26 | //context weight vectors for SVO 27 | MatD subjWeight; 28 | MatD verbWeight; 29 | MatD objWeight; 30 | 31 | //context weight vectors for SVO-P-N 32 | MatD prepWeight; 33 | MatD pobjWeight; 34 | MatD svoWeight; 35 | 36 | //scoring weight vectors 37 | MatD nounScoreWeight; 38 | MatD verbScoreWeight; 39 | MatD prepScoreWeight; 40 | 41 | MatD nounGrad, nounGradHist; 42 | MatD verbGrad, verbGradHist; 43 | MatD prepGrad, prepGradHist; 44 | MatD scwGrad, vcwGrad, ocwGrad; 45 | MatD scwGradHist, vcwGradHist, ocwGradHist; 46 | MatD swGrad, vwGrad, owGrad; 47 | MatD swGradHist, vwGradHist, owGradHist; 48 | MatD pwGrad, powGrad, svowGrad; 49 | MatD pwGradHist, powGradHist, svowGradHist; 50 | MatD nounScoreGrad, nounScoreGradHist; 51 | MatD verbScoreGrad, verbScoreGradHist; 52 | MatD prepScoreGrad, prepScoreGradHist; 53 | 54 | std::unordered_map nounMap; 55 | std::unordered_map verbMap; 56 | std::unordered_map prepMap; 57 | std::unordered_map nounScoreMap; 58 | std::unordered_map verbScoreMap; 59 | std::unordered_map prepScoreMap; 60 | 61 | int negS, negV, negO; 62 | int negP, negN; 63 | 64 | void init(const int dim, Vocabulary& voc); 65 | 66 | void compose(MatD& comp, SVO* svo); 67 | void train(std::vector& instance, std::vector& type, std::vector& dummy, Vocabulary& voc, const double learningRate, const int maxItr, const int miniBatchSize, const int numNeg); 68 | void train(SVO* svo, Vocabulary& voc); 69 | void train(SVOPN* svopn, Vocabulary& voc); 70 | void update(const double learningRate, const int exception = -1); 71 | 72 | double score(const int s, const int v, const int o, const LBLM::TARGET target); 73 | double objective(SVO* svo); 74 | double score(const int p, const int n, SVO* svo, const LBLM::TARGET target); 75 | void gradCheck(SVO* svo); 76 | void gradCheck(SVOPN* svopn); 77 | 78 | double testSVO(Vocabulary& voc, const std::string& type, const bool ave = false); 79 | double testVO(Vocabulary& voc, const bool ave = false); 80 | void SVOdist(Vocabulary& voc); 81 | void SVOdist(Vocabulary& voc, int k); 82 | void VOdist(Vocabulary& voc, int k); 83 | 84 | void save(const std::string& file); 85 | void load(const std::string& file); 86 | }; 87 | 88 | inline double LBLM::score(const int s, const int v, const int o, const LBLM::TARGET target){ 89 | MatD f; 90 | 91 | if (target == LBLM::S){ 92 | f = this->objWeight.array()*this->nounVector.col(o).array()+this->verbWeight.array()*this->verbVector.col(v).array(); 93 | Tanh::tanh(f); 94 | return (this->nounScoreWeight.row(s)*f).coeff(0, 0); 95 | } 96 | else if (target == LBLM::V){ 97 | f = this->subjWeight.array()*this->nounVector.col(s).array()+this->objWeight.array()*this->nounVector.col(o).array(); 98 | Tanh::tanh(f); 99 | return (this->verbScoreWeight.row(v)*f).coeff(0, 0); 100 | } 101 | else if (target == LBLM::O){ 102 | f = this->subjWeight.array()*this->nounVector.col(s).array()+this->verbWeight.array()*this->verbVector.col(v).array(); 103 | Tanh::tanh(f); 104 | return (this->nounScoreWeight.row(o)*f).coeff(0, 0); 105 | } 106 | 107 | return -1.0e+10; 108 | } 109 | 110 | inline double LBLM::objective(SVO* svo){ 111 | return 112 | Utils::max(1.0+this->score(this->negS, svo->v, svo->o, LBLM::S)-this->score(svo->s, svo->v, svo->o, LBLM::S), 0.0)+ 113 | Utils::max(1.0+this->score(svo->s, this->negV, svo->o, LBLM::V)-this->score(svo->s, svo->v, svo->o, LBLM::V), 0.0)+ 114 | Utils::max(1.0+this->score(svo->s, svo->v, this->negO, LBLM::O)-this->score(svo->s, svo->v, svo->o, LBLM::O), 0.0); 115 | } 116 | 117 | inline double LBLM::score(const int p, const int n, SVO* svo, const LBLM::TARGET target){ 118 | MatD f; 119 | MatD svoVec; 120 | 121 | this->compose(svoVec, svo); 122 | 123 | if (target == LBLM::P){ 124 | f = this->pobjWeight.array()*this->nounVector.col(n).array()+this->svoWeight.array()*svoVec.array(); 125 | Tanh::tanh(f); 126 | return (this->prepScoreWeight.row(p)*f).coeff(0, 0); 127 | } 128 | else if (target == LBLM::N){ 129 | f = this->prepWeight.array()*this->prepVector.col(p).array()+this->svoWeight.array()*svoVec.array(); 130 | Tanh::tanh(f); 131 | return (this->nounScoreWeight.row(n)*f).coeff(0, 0); 132 | } 133 | 134 | return -1.0e+10; 135 | } 136 | 137 | inline void LBLM::compose(MatD& comp, SVO* svo){ 138 | if (svo->s == -1){ 139 | comp = 140 | this->verbCompWeight.array()*this->verbVector.col(svo->v).array()+ 141 | this->objCompWeight.array()*this->nounVector.col(svo->o).array(); 142 | } 143 | else { 144 | comp = 145 | this->subjCompWeight.array()*this->nounVector.col(svo->s).array()+ 146 | this->verbCompWeight.array()*this->verbVector.col(svo->v).array()+ 147 | this->objCompWeight.array()*this->nounVector.col(svo->o).array(); 148 | } 149 | 150 | Tanh::tanh(comp); 151 | } 152 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CXX=g++ 2 | 3 | EIGEN_LOCATION=XXX 4 | BUILD_DIR=objs 5 | 6 | CXXFLAGS =-Wall 7 | CXXFLAGS+=-O3 8 | CXXFLAGS+=-std=c++0x 9 | CXXFLAGS+=-lm 10 | CXXFLAGS+=-fomit-frame-pointer 11 | CXXFLAGS+=-fno-schedule-insns2 12 | CXXFLAGS+=-fexceptions 13 | CXXFLAGS+=-funroll-loops 14 | CXXFLAGS+=-march=native 15 | CXXFLAGS+=-mfpmath=sse 16 | CXXFLAGS+=-msse4.2 17 | CXXFLAGS+=-mmmx 18 | CXXFLAGS+=-fopenmp 19 | CXXFLAGS+=-m64 20 | CXXFLAGS+=-DEIGEN_DONT_PARALLELIZE 21 | CXXFLAGS+=-DEIGEN_NO_DEBUG 22 | CXXFLAGS+=-DEIGEN_NO_STATIC_ASSERT 23 | CXXFLAGS+=-I$(EIGEN_LOCATION) 24 | 25 | CXXLIBS=-fopenmp 26 | 27 | SRCS=$(shell ls *.cpp) 28 | OBJS=$(SRCS:.cpp=.o) 29 | 30 | PROGRAM=svo_embedding 31 | 32 | all : $(BUILD_DIR) $(patsubst %,$(BUILD_DIR)/%,$(PROGRAM)) 33 | 34 | $(BUILD_DIR)/%.o : %.cpp 35 | $(CXX) -c $(CXXFLAGS) -o $@ $< 36 | 37 | $(BUILD_DIR)/$(PROGRAM) : $(patsubst %,$(BUILD_DIR)/%,$(OBJS)) 38 | $(CXX) $(CXXFLAGS) $(CXXLIBS) -o $@ $^ 39 | mv $(BUILD_DIR)/$(PROGRAM) ./ 40 | rm -f ?*~ 41 | 42 | clean: 43 | rm -f $(BUILD_DIR)/* $(PROGRAM) ?*~ 44 | -------------------------------------------------------------------------------- /Matrix.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | typedef Eigen::MatrixXd MatD; 6 | typedef Eigen::MatrixXi MatI; 7 | -------------------------------------------------------------------------------- /NTF.cpp: -------------------------------------------------------------------------------- 1 | #include "NTF.hpp" 2 | #include "Sigmoid.hpp" 3 | #include "Utils.hpp" 4 | #include 5 | #include 6 | 7 | #define ETA 1.0 8 | 9 | void NTF::init(const int dim, Vocabulary& voc){ 10 | const double scale1 = 1.0/dim; 11 | const double scale2 = 1.0/(dim*dim); 12 | 13 | this->nounVector = MatD(dim, voc.nounStr.size()); 14 | Data::rndModel.gauss(this->nounVector, scale1); 15 | 16 | if (voc.nullIndex >= 0){ 17 | this->nounVector.col(voc.nullIndex).fill(1.0); 18 | } 19 | 20 | for (int i = 0; i < (int)voc.verbStr.size(); ++i){ 21 | this->verbMatrix.push_back(MatD(dim, dim)); 22 | Data::rndModel.gauss(this->verbMatrix.back(), scale2); 23 | } 24 | for (int i = 0; i < (int)voc.prepStr.size(); ++i){ 25 | this->prepMatrix.push_back(MatD(dim, dim)); 26 | Data::rndModel.gauss(this->prepMatrix.back(), scale2); 27 | } 28 | 29 | //gradient 30 | this->nounGrad = MatD::Zero(this->nounVector.rows(), this->nounVector.cols()); 31 | 32 | for (int i = 0; i < (int)this->verbMatrix.size(); ++i){ 33 | this->verbGrad.push_back(MatD::Zero(this->verbMatrix[i].rows(), this->verbMatrix[i].cols())); 34 | } 35 | for (int i = 0; i < (int)this->prepMatrix.size(); ++i){ 36 | this->prepGrad.push_back(MatD::Zero(this->prepMatrix[i].rows(), this->prepMatrix[i].cols())); 37 | } 38 | 39 | //AdaGrad 40 | this->nvG = MatD::Ones(this->nounVector.rows(), this->nounVector.cols())*ETA; 41 | 42 | for (int i = 0; i < (int)this->verbMatrix.size(); ++i){ 43 | this->vmG.push_back(MatD::Ones(this->verbMatrix[i].rows(), this->verbMatrix[i].cols())*ETA); 44 | } 45 | for (int i = 0; i < (int)this->prepMatrix.size(); ++i){ 46 | this->pmG.push_back(MatD::Ones(this->prepMatrix[i].rows(), this->prepMatrix[i].cols())*ETA); 47 | } 48 | } 49 | 50 | void NTF::train(std::vector& sample, std::vector& type, std::vector& dummy, Vocabulary& voc, const double learningRate, const int maxItr, const int miniBatchSize, const int numNeg){ 51 | const int prog = sample.size()/10; 52 | int counter = 0; 53 | 54 | for (int itr = 0; itr < maxItr; ++itr){ 55 | counter = 0; 56 | std::random_shuffle(sample.begin(), sample.end()); 57 | ::printf("### NTF: itr %2d/%2d, learning rate %.6f, dim %d\n", itr+1, maxItr, learningRate, (int)this->nounVector.rows()); 58 | 59 | for (int i = 0; i < (int)sample.size(); ++i){ 60 | if (sample[i]->type == Data::SVO_){ 61 | this->trainSVO((SVO*)sample[i], voc); 62 | //gradCheck((SVO::SVO*)sample[i]); 63 | } 64 | else { 65 | this->trainSVOPN((SVOPN*)sample[i], voc); 66 | //gradCheck((SVOPN*)sample[i]); 67 | } 68 | 69 | if ((i+1)%miniBatchSize == 0 || i == (int)sample.size()-1){ 70 | this->update(learningRate, voc.nullIndex); 71 | } 72 | 73 | if ((i+1)%prog == 0){ 74 | std::cout << (++counter)*10 << "% " << std::flush; 75 | } 76 | } 77 | 78 | std::cout << std::endl; 79 | 80 | MatD accDev = MatD::Zero(numNeg, 1); 81 | double devCount = 0.0; 82 | 83 | for (int j = 0; j < (int)type.size(); ++j){ 84 | if (type[j]->type == Data::SVO_){ 85 | SVO* sample = (SVO*)type[j]; 86 | 87 | if (sample->set == Data::DEV){ 88 | devCount += 1.0; 89 | } 90 | else { 91 | continue; 92 | } 93 | 94 | double svo = this->score(sample->s, sample->v, sample->o); 95 | 96 | for (int k = 0; k < accDev.rows(); ++k){ 97 | double s_vo = this->score(sample->s_[k], sample->v, sample->o); 98 | double svo_ = this->score(sample->s, sample->v, sample->o_[k]); 99 | double sv_o = this->score(sample->s, sample->v_[k], sample->o); 100 | 101 | if (svo > s_vo && svo > svo_ && svo > sv_o){ 102 | accDev.coeffRef(k, 0) += 1.0; 103 | } 104 | } 105 | } 106 | else if (type[j]->type == Data::SVOPN_){ 107 | SVOPN* sample = (SVOPN*)type[j]; 108 | 109 | if (sample->set == Data::DEV){ 110 | devCount += 1.0; 111 | } 112 | else { 113 | continue; 114 | } 115 | 116 | double svoPn = this->score(sample->p, sample->n, sample->svo); 117 | 118 | for (int k = 0; k < accDev.rows(); ++k){ 119 | double svo_Pn = this->score(sample->p, sample->n, sample->svo_[k]); 120 | double svoPn_ = this->score(sample->p, sample->n_[k], sample->svo); 121 | double svoP_n = this->score(sample->p_[k], sample->n, sample->svo); 122 | 123 | if (svoPn > svo_Pn && svoPn > svoPn_ && svoPn > svoP_n){ 124 | accDev.coeffRef(k, 0) += 1.0; 125 | } 126 | } 127 | } 128 | } 129 | 130 | accDev /= devCount; 131 | accDev *= 100.0; 132 | printf("\tDev Acc: %f (%f)", accDev.array().sum()/accDev.rows(), Utils::stdDev(accDev)); 133 | std::cout << std::endl; 134 | } 135 | } 136 | 137 | void NTF::trainSVO(SVO* svo, Vocabulary& voc){ 138 | static double deltaPos, deltaNeg; 139 | static MatD vo, _vo, so, vs; 140 | 141 | this->verbMap[svo->v] = 1; 142 | this->nounMap[svo->s] = 1; 143 | this->nounMap[svo->o] = 1; 144 | vo = this->verbMatrix[svo->v]*this->nounVector.col(svo->o); 145 | so = this->nounVector.col(svo->s)*this->nounVector.col(svo->o).transpose(); 146 | vs = this->verbMatrix[svo->v].transpose()*this->nounVector.col(svo->s); 147 | deltaPos = Sigmoid::sigmoid(this->nounVector.col(svo->s).transpose()*vo)-1.0; 148 | 149 | this->nounGrad.col(svo->s) += deltaPos*vo; 150 | this->nounGrad.col(svo->o) += deltaPos*vs; 151 | this->verbGrad[svo->v] += deltaPos*so; 152 | 153 | //V 154 | do { 155 | this->negV = voc.verbList[(Data::rndData.next() >> 16)%voc.verbList.size()]; 156 | } while (voc.exist(svo->s, this->negV, svo->o)); 157 | 158 | this->verbMap[this->negV] = 1; 159 | 160 | _vo = this->verbMatrix[this->negV]*this->nounVector.col(svo->o); 161 | deltaNeg = Sigmoid::sigmoid(this->nounVector.col(svo->s).transpose()*_vo); 162 | this->nounGrad.col(svo->s) += deltaNeg*_vo; 163 | this->nounGrad.col(svo->o) += deltaNeg*(this->verbMatrix[this->negV].transpose()*this->nounVector.col(svo->s)); 164 | this->verbGrad[this->negV] += deltaNeg*so; 165 | 166 | //S 167 | do { 168 | this->negS = voc.vsubjList[(Data::rndData.next() >> 16)%voc.vsubjList.size()]; 169 | } while (voc.exist(this->negS, svo->v, svo->o)); 170 | 171 | this->nounMap[this->negS] = 1; 172 | 173 | deltaNeg = Sigmoid::sigmoid(this->nounVector.col(this->negS).transpose()*vo); 174 | this->nounGrad.col(svo->o) += deltaNeg*(this->verbMatrix[svo->v].transpose()*this->nounVector.col(this->negS)); 175 | this->verbGrad[svo->v] += deltaNeg*(this->nounVector.col(this->negS)*this->nounVector.col(svo->o).transpose()); 176 | this->nounGrad.col(this->negS) += deltaNeg*vo; 177 | 178 | //O 179 | do { 180 | this->negO = voc.vobjList[(Data::rndData.next() >> 16)%voc.vobjList.size()]; 181 | } while (voc.exist(svo->s, svo->v, this->negO) || voc.exist(this->negS, svo->v, this->negO)); 182 | 183 | this->nounMap[this->negO] = 1; 184 | 185 | _vo = this->verbMatrix[svo->v]*this->nounVector.col(this->negO); 186 | deltaNeg = Sigmoid::sigmoid(this->nounVector.col(svo->s).transpose()*_vo); 187 | this->nounGrad.col(svo->s) += deltaNeg*_vo; 188 | this->verbGrad[svo->v] += deltaNeg*(this->nounVector.col(svo->s)*this->nounVector.col(this->negO).transpose()); 189 | this->nounGrad.col(this->negO) += deltaNeg*vs; 190 | } 191 | 192 | void NTF::trainSVOPN(SVOPN* svopn, Vocabulary& voc){ 193 | static double deltaPos, deltaNeg; 194 | static MatD pn, _pn, psvo, svon; 195 | static MatD svo, svo_, vo, vo_, svoDelta, svoDelta_; 196 | 197 | this->prepMap[svopn->p] = 1; 198 | this->nounMap[svopn->n] = 1; 199 | this->verbMap[svopn->svo->v] = 1; 200 | this->nounMap[svopn->svo->s] = 1; 201 | this->nounMap[svopn->svo->o] = 1; 202 | 203 | vo = this->verbMatrix[svopn->svo->v]*this->nounVector.col(svopn->svo->o); 204 | svo = this->nounVector.col(svopn->svo->s).array()*vo.array(); 205 | pn = this->prepMatrix[svopn->p]*this->nounVector.col(svopn->n); 206 | psvo = this->prepMatrix[svopn->p].transpose()*svo; 207 | svon = svo*this->nounVector.col(svopn->n).transpose(); 208 | deltaPos = Sigmoid::sigmoid(svo.transpose()*pn)-1.0; 209 | 210 | this->nounGrad.col(svopn->n) += deltaPos*psvo; 211 | this->prepGrad[svopn->p] += deltaPos*svon; 212 | 213 | svoDelta = deltaPos*pn; 214 | 215 | //P 216 | do { 217 | this->negP = voc.prepList[(Data::rndData.next() >> 16)%voc.prepList.size()]; 218 | } while (voc.exist(this->negP, svopn->n, svopn->svo)); 219 | 220 | this->prepMap[this->negP] = 1; 221 | 222 | _pn = this->prepMatrix[this->negP]*this->nounVector.col(svopn->n); 223 | deltaNeg = Sigmoid::sigmoid(svo.transpose()*_pn); 224 | this->nounGrad.col(svopn->n) += deltaNeg*(this->prepMatrix[this->negP].transpose()*svo); 225 | this->prepGrad[this->negP] += deltaNeg*svon; 226 | svoDelta += deltaNeg*_pn; 227 | 228 | //SVO 229 | do { 230 | this->negSVO = voc.svoList[(Data::rndData.next() >> 16)%voc.svoList.size()]; 231 | } while (voc.exist(svopn->p, svopn->n, this->negSVO)); 232 | 233 | this->verbMap[this->negSVO->v] = 1; 234 | this->nounMap[this->negSVO->s] = 1; 235 | this->nounMap[this->negSVO->o] = 1; 236 | 237 | vo_ = this->verbMatrix[this->negSVO->v]*this->nounVector.col(this->negSVO->o); 238 | svo_ = this->nounVector.col(this->negSVO->s).array()*vo_.array(); 239 | deltaNeg = Sigmoid::sigmoid(svo_.transpose()*pn); 240 | this->nounGrad.col(svopn->n) += deltaNeg*(this->prepMatrix[svopn->p].transpose()*svo_); 241 | this->prepGrad[svopn->p] += deltaNeg*(svo_*this->nounVector.col(svopn->n).transpose()); 242 | svoDelta_ = deltaNeg*pn; 243 | 244 | //N 245 | do { 246 | this->negN = voc.pobjList[(Data::rndData.next() >> 16)%voc.pobjList.size()]; 247 | } while (voc.exist(svopn->p, this->negN, svopn->svo)); 248 | 249 | this->nounMap[this->negN] = 1; 250 | 251 | _pn = this->prepMatrix[svopn->p]*this->nounVector.col(this->negN); 252 | deltaNeg = Sigmoid::sigmoid(svo.transpose()*_pn); 253 | this->nounGrad.col(this->negN) += deltaNeg*psvo; 254 | this->prepGrad[svopn->p] += deltaNeg*(svo*this->nounVector.col(this->negN).transpose()); 255 | svoDelta += deltaNeg*_pn; 256 | 257 | //pos 258 | this->nounGrad.col(svopn->svo->s).array() += svoDelta.array()*vo.array(); 259 | svoDelta.array() *= this->nounVector.col(svopn->svo->s).array(); 260 | this->verbGrad[svopn->svo->v] += svoDelta*this->nounVector.col(svopn->svo->o).transpose(); 261 | this->nounGrad.col(svopn->svo->o) += this->verbMatrix[svopn->svo->v].transpose()*svoDelta; 262 | //neg 263 | this->nounGrad.col(this->negSVO->s).array() += svoDelta_.array()*vo_.array(); 264 | svoDelta_.array() *= this->nounVector.col(this->negSVO->s).array(); 265 | this->verbGrad[this->negSVO->v] += svoDelta_*this->nounVector.col(this->negSVO->o).transpose(); 266 | this->nounGrad.col(this->negSVO->o) += this->verbMatrix[this->negSVO->v].transpose()*svoDelta_; 267 | } 268 | 269 | 270 | void NTF::update(const double learningRate, const int exception){ 271 | for (auto it = this->nounMap.begin(); it != this->nounMap.end(); ++it){ 272 | if (it->first == exception){ 273 | this->nounGrad.col(it->first).setZero(); 274 | continue; 275 | } 276 | 277 | //AdaGrad 278 | this->nvG.col(it->first).array() += this->nounGrad.col(it->first).array().square(); 279 | this->nounGrad.col(it->first).array() /= this->nvG.col(it->first).array().sqrt(); 280 | this->nounVector.col(it->first) -= learningRate*this->nounGrad.col(it->first); 281 | 282 | this->nounGrad.col(it->first).setZero(); 283 | } 284 | 285 | for (auto it = this->verbMap.begin(); it != this->verbMap.end(); ++it){ 286 | //AdaGrad 287 | this->vmG[it->first].array() += this->verbGrad[it->first].array().square(); 288 | this->verbGrad[it->first].array() /= this->vmG[it->first].array().sqrt(); 289 | this->verbMatrix[it->first] -= learningRate*this->verbGrad[it->first]; 290 | 291 | this->verbGrad[it->first].setZero(); 292 | } 293 | 294 | for (auto it = this->prepMap.begin(); it != this->prepMap.end(); ++it){ 295 | //AdaGrad 296 | this->pmG[it->first].array() += this->prepGrad[it->first].array().square(); 297 | this->prepGrad[it->first].array() /= this->pmG[it->first].array().sqrt(); 298 | this->prepMatrix[it->first] -= learningRate*this->prepGrad[it->first]; 299 | 300 | this->prepGrad[it->first].setZero(); 301 | } 302 | 303 | this->nounMap.clear(); 304 | this->verbMap.clear(); 305 | this->prepMap.clear(); 306 | } 307 | 308 | void NTF::gradCheck(SVO* svo){ 309 | const double eps = 1.0e-04; 310 | double val, objPlus, objMinus; 311 | 312 | printf("\nchecking gradients ...\n"); 313 | 314 | for (auto it = this->nounMap.begin(); it != this->nounMap.end(); ++it){ 315 | printf("----------- noun %10d -------------\n", it->first); 316 | 317 | for (int i = 0; i < this->nounGrad.rows(); ++i){ 318 | val = this->nounVector.coeff(i, it->first); 319 | this->nounVector.coeffRef(i, it->first) = val+eps; 320 | objPlus = this->objective(svo); 321 | this->nounVector.coeffRef(i, it->first) = val-eps; 322 | objMinus = this->objective(svo); 323 | this->nounVector.coeffRef(i, it->first) = val; 324 | printf("backprop: %.8f\n", this->nounGrad.coeff(i, it->first)); 325 | printf("numerical: %.8f\n", (objPlus-objMinus)/(2.0*eps)); 326 | } 327 | } 328 | 329 | for (auto it = this->verbMap.begin(); it != this->verbMap.end(); ++it){ 330 | printf("----------- verb %10d -------------\n", it->first); 331 | 332 | for (int i = 0; i < this->verbGrad[it->first].rows(); ++i){ 333 | for (int j = 0; j < this->verbGrad[it->first].cols(); ++j){ 334 | val = this->verbMatrix[it->first].coeff(i, j); 335 | this->verbMatrix[it->first].coeffRef(i, j) = val+eps; 336 | objPlus = this->objective(svo); 337 | this->verbMatrix[it->first].coeffRef(i, j) = val-eps; 338 | objMinus = this->objective(svo); 339 | this->verbMatrix[it->first].coeffRef(i, j) = val; 340 | printf("backprop: %.8f\n", this->verbGrad[it->first].coeff(i, j)); 341 | printf("numerical: %.8f\n", (objPlus-objMinus)/(2.0*eps)); 342 | } 343 | } 344 | } 345 | } 346 | 347 | void NTF::gradCheck(SVOPN* svopn){ 348 | const double eps = 1.0e-04; 349 | double val, objPlus, objMinus; 350 | 351 | printf("\nchecking gradients ...\n"); 352 | 353 | for (auto it = this->nounMap.begin(); it != this->nounMap.end(); ++it){ 354 | printf("----------- noun %10d -------------\n", it->first); 355 | 356 | for (int i = 0; i < this->nounGrad.rows(); ++i){ 357 | val = this->nounVector.coeff(i, it->first); 358 | this->nounVector.coeffRef(i, it->first) = val+eps; 359 | objPlus = this->objective(svopn); 360 | this->nounVector.coeffRef(i, it->first) = val-eps; 361 | objMinus = this->objective(svopn); 362 | this->nounVector.coeffRef(i, it->first) = val; 363 | printf("backprop: %.8f\n", this->nounGrad.coeff(i, it->first)); 364 | printf("numerical: %.8f\n", (objPlus-objMinus)/(2.0*eps)); 365 | } 366 | } 367 | 368 | for (auto it = this->verbMap.begin(); it != this->verbMap.end(); ++it){ 369 | printf("----------- verb %10d -------------\n", it->first); 370 | 371 | for (int i = 0; i < this->verbGrad[it->first].rows(); ++i){ 372 | for (int j = 0; j < this->verbGrad[it->first].cols(); ++j){ 373 | val = this->verbMatrix[it->first].coeff(i, j); 374 | this->verbMatrix[it->first].coeffRef(i, j) = val+eps; 375 | objPlus = this->objective(svopn); 376 | this->verbMatrix[it->first].coeffRef(i, j) = val-eps; 377 | objMinus = this->objective(svopn); 378 | this->verbMatrix[it->first].coeffRef(i, j) = val; 379 | printf("backprop: %.8f\n", this->verbGrad[it->first].coeff(i, j)); 380 | printf("numerical: %.8f\n", (objPlus-objMinus)/(2.0*eps)); 381 | } 382 | } 383 | } 384 | 385 | for (auto it = this->prepMap.begin(); it != this->prepMap.end(); ++it){ 386 | printf("----------- prep %10d -------------\n", it->first); 387 | 388 | for (int i = 0; i < this->prepGrad[it->first].rows(); ++i){ 389 | for (int j = 0; j < this->prepGrad[it->first].cols(); ++j){ 390 | val = this->prepMatrix[it->first].coeff(i, j); 391 | this->prepMatrix[it->first].coeffRef(i, j) = val+eps; 392 | objPlus = this->objective(svopn); 393 | this->prepMatrix[it->first].coeffRef(i, j) = val-eps; 394 | objMinus = this->objective(svopn); 395 | this->prepMatrix[it->first].coeffRef(i, j) = val; 396 | printf("backprop: %.8f\n", this->prepGrad[it->first].coeff(i, j)); 397 | printf("numerical: %.8f\n", (objPlus-objMinus)/(2.0*eps)); 398 | } 399 | } 400 | } 401 | } 402 | 403 | void NTF::save(const std::string& file){ 404 | std::ofstream ofs(file.c_str(), std::ios::out|std::ios::binary); 405 | 406 | assert(ofs); 407 | Utils::save(ofs, this->nounVector); 408 | 409 | for (int i = 0; i < (int)this->verbMatrix.size(); ++i){ 410 | Utils::save(ofs, this->verbMatrix[i]); 411 | } 412 | for (int i = 0; i < (int)this->prepMatrix.size(); ++i){ 413 | Utils::save(ofs, this->prepMatrix[i]); 414 | } 415 | } 416 | 417 | void NTF::load(const std::string& file){ 418 | std::ifstream ifs(file.c_str(), std::ios::out|std::ios::binary); 419 | 420 | assert(ifs); 421 | Utils::load(ifs, this->nounVector); 422 | 423 | for (int i = 0; i < (int)this->verbMatrix.size(); ++i){ 424 | Utils::load(ifs, this->verbMatrix[i]); 425 | } 426 | for (int i = 0; i < (int)this->prepMatrix.size(); ++i){ 427 | Utils::load(ifs, this->prepMatrix[i]); 428 | } 429 | } 430 | -------------------------------------------------------------------------------- /NTF.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "SVO.hpp" 4 | #include "SVOPN.hpp" 5 | #include "Vocabulary.hpp" 6 | #include "Utils.hpp" 7 | #include "Sigmoid.hpp" 8 | 9 | class NTF{ 10 | public: 11 | MatD nounVector, nvG; 12 | std::vector verbMatrix, vmG; 13 | std::vector prepMatrix, pmG; 14 | 15 | MatD nounGrad; 16 | std::vector verbGrad, prepGrad; 17 | std::unordered_map nounMap, verbMap, prepMap; 18 | 19 | int negS, negV, negO; //for gradient checking 20 | int negP, negN; SVO* negSVO; //for gradient checking 21 | 22 | void init(const int dim, Vocabulary& voc); 23 | 24 | double score(const int s, const int v, const int o); 25 | double objective(SVO* svo); 26 | double score(const int p, const int n, SVO* svo); 27 | double objective(SVOPN* svopn); 28 | 29 | void train(std::vector& sample, std::vector& type, std::vector& dummy, Vocabulary& voc, const double learningRate, const int maxItr, const int miniBatchSize, const int numNeg); 30 | void trainSVO(SVO* svo, Vocabulary& voc); 31 | void trainSVOPN(SVOPN* svopn, Vocabulary& voc); 32 | void update(const double learningRate, const int exception = -1); 33 | 34 | void gradCheck(SVO* svo); 35 | void gradCheck(SVOPN* svopn); 36 | 37 | void save(const std::string& file); 38 | void load(const std::string& file); 39 | 40 | double testSVO(Vocabulary& voc, const std::string& type, const bool ave = false); 41 | double testVO(Vocabulary& voc, const bool ave = false); 42 | void SVOdist(Vocabulary& voc); 43 | void SVOdist(Vocabulary& voc, int k); 44 | void VOdist(Vocabulary& voc, int k); 45 | void SVO_PNdist(Vocabulary& voc, int k); 46 | void vRowKnn(Vocabulary& voc, int k); 47 | void vColKnn(Vocabulary& voc, int k); 48 | }; 49 | 50 | inline double NTF::score(const int s, const int v, const int o){ 51 | return 52 | this->nounVector.col(s).transpose()* 53 | this->verbMatrix[v]* 54 | this->nounVector.col(o); 55 | } 56 | 57 | inline double NTF::objective(SVO* svo){ 58 | return 59 | -log(Sigmoid::sigmoid(this->score(svo->s, svo->v, svo->o)))- 60 | log(1.0-Sigmoid::sigmoid(this->score(this->negS, svo->v, svo->o)))- 61 | log(1.0-Sigmoid::sigmoid(this->score(svo->s, this->negV, svo->o)))- 62 | //log(1.0-Sigmoid::sigmoid(this->score(this->negS, svo->v, this->negO)))+ 63 | log(1.0-Sigmoid::sigmoid(this->score(svo->s, svo->v, this->negO))); 64 | } 65 | 66 | inline double NTF::score(const int p, const int n, SVO* svo){ 67 | MatD SVO = this->nounVector.col(svo->s).array()*(this->verbMatrix[svo->v]*this->nounVector.col(svo->o)).array(); 68 | 69 | return 70 | SVO.col(0).transpose()* 71 | this->prepMatrix[p]* 72 | this->nounVector.col(n); 73 | } 74 | 75 | inline double NTF::objective(SVOPN* svopn){ 76 | return 77 | -log(Sigmoid::sigmoid(this->score(svopn->p, svopn->n, svopn->svo)))- 78 | log(1.0-Sigmoid::sigmoid(this->score(this->negP, svopn->n, svopn->svo)))- 79 | log(1.0-Sigmoid::sigmoid(this->score(svopn->p, this->negN, svopn->svo)))- 80 | log(1.0-Sigmoid::sigmoid(this->score(svopn->p, svopn->n, this->negSVO))); 81 | } 82 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Learning embeddings for transitive verb phrases 2 | 3 | This repository includes the code to learn embeddings for transitive verb phrases using predicate-argument structures presented in our papers [1, 2]. 4 | You can reproduce the experimental results reported in our CVSC 2015 paper [2]. 5 | 6 | The training data used in our paper [2] is publicly available at our web site. 7 | You can put the training data in the train_data directory and run the code to train SVO embeddings. 8 | 9 | [1] Kazuma Hashimoto, Pontus Stenetorp, Makoto Miwa, and Yoshimasa Tsuruoka. 2014. Jointly Learning Word Representations and Composition Functions Using Predicate-Argument Structures. In Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing. pages 1544-1555. 10 | 11 | [2] Kazuma Hashimoto and Yoshimasa Tsuruoka. 2015. Learning Embeddings for Transitive Verb Disambiguation by Implicit Tensor Factorization. In Proceedings of the 3rd Workshop on Continuous Vector Space Models and their Compositionality. pages 1-11. 12 | 13 | @InProceedings{hashimoto-EtAl:2014:EMNLP2014, 14 | author = {Hashimoto, Kazuma and Stenetorp, Pontus and Miwa, Makoto and Tsuruoka, Yoshimasa}, 15 | title = {Jointly Learning Word Representations and Composition Functions Using Predicate-Argument Structures}, 16 | booktitle = {Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP)}, 17 | month = {October}, 18 | year = {2014}, 19 | address = {Doha, Qatar}, 20 | publisher = {Association for Computational Linguistics}, 21 | pages = {1544--1555}, 22 | url = {http://www.aclweb.org/anthology/D14-1163} 23 | } 24 | 25 | @InProceedings{hashimoto-tsuruoka:2015:CVSC, 26 | author = {Hashimoto, Kazuma and Tsuruoka, Yoshimasa}, 27 | title = {Learning Embeddings for Transitive Verb Disambiguation by Implicit Tensor Factorization}, 28 | booktitle = {Proceedings of the 3rd Workshop on Continuous Vector Space Models and their Compositionality (CVSC)}, 29 | month = {July}, 30 | year = {2015}, 31 | address = {Beijing, China}, 32 | publisher = {Association for Computational Linguistics}, 33 | pages = {1--11}, 34 | url = {http://www.aclweb.org/anthology/W15-4001} 35 | } 36 | -------------------------------------------------------------------------------- /Rand.cpp: -------------------------------------------------------------------------------- 1 | #include "Rand.hpp" 2 | 3 | Rand Rand::r_ = Rand(); 4 | -------------------------------------------------------------------------------- /Rand.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Matrix.hpp" 4 | 5 | class Rand{ 6 | public: 7 | static Rand r_; 8 | 9 | Rand(unsigned long w_ = 88675123): 10 | x(123456789), y(362436069), z(521288629), w(w_) {}; 11 | 12 | unsigned long next(); 13 | double zero2one(); 14 | double gauss(double sigma, double mu = 0.0); 15 | void gauss(MatD& mat, double sigma, double mu = 0.0); 16 | 17 | private: 18 | unsigned long x; 19 | unsigned long y; 20 | unsigned long z; 21 | unsigned long w; 22 | unsigned long t; //tmp 23 | }; 24 | 25 | inline unsigned long Rand::next(){ 26 | this->t=(this->x^(this->x<<11)); 27 | this->x=this->y; 28 | this->y=this->z; 29 | this->z=this->w; 30 | return (this->w=(this->w^(this->w>>19))^(this->t^(this->t>>8))); 31 | } 32 | 33 | inline double Rand::zero2one(){ 34 | return ((this->next()&0xFFFF)+1)/65536.0; 35 | } 36 | 37 | inline double Rand::gauss(double sigma, double mu){ 38 | return 39 | mu+ 40 | sigma* 41 | sqrt(-2.0*log(this->zero2one()))* 42 | sin(2.0*M_PI*this->zero2one()); 43 | } 44 | 45 | inline void Rand::gauss(MatD& mat, double sigma, double mu){ 46 | for (int i = 0; i < mat.rows(); ++i){ 47 | for (int j = 0; j < mat.cols(); ++j){ 48 | mat.coeffRef(i, j) = this->gauss(sigma, mu); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /SVO.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Data.hpp" 4 | 5 | class SVO : public Data{ 6 | public: 7 | int s, v, o; 8 | int *s_, *v_, *o_; 9 | }; 10 | -------------------------------------------------------------------------------- /SVOPN.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Data.hpp" 4 | #include "SVO.hpp" 5 | 6 | class SVOPN : public Data{ 7 | public: 8 | int p, n; 9 | SVO* svo; 10 | 11 | int *p_, *n_; 12 | SVO **svo_; 13 | }; 14 | -------------------------------------------------------------------------------- /Show.cpp: -------------------------------------------------------------------------------- 1 | #include "Vocabulary.hpp" 2 | #include "NTF.hpp" 3 | #include "LBLM.hpp" 4 | #include "Utils.hpp" 5 | #include "Sigmoid.hpp" 6 | #include 7 | #include 8 | 9 | void NTF::SVOdist(Vocabulary& voc, int k){ 10 | printf("kNN SVOs\n"); 11 | 12 | std::vector res; 13 | 14 | for (std::string line; std::getline(std::cin, line) && line != "q"; ){ 15 | Utils::split(line, res); 16 | 17 | if (res.size() != 3){ 18 | continue; 19 | } 20 | 21 | std::string s = res[0]; 22 | std::string v = res[1]; 23 | std::string o = res[2]; 24 | 25 | if (!voc.verbIndex.count(v) || (!voc.nounIndex.count(s) && s != voc.null) || !voc.nounIndex.count(o)){ 26 | printf("retry\n~\n"); 27 | continue; 28 | } 29 | 30 | MatD svo; 31 | 32 | if (s == voc.null){ 33 | svo = this->verbMatrix[voc.verbIndex.at(v)]*this->nounVector.col(voc.nounIndex.at(o)); 34 | } 35 | else { 36 | svo = this->nounVector.col(voc.nounIndex.at(s)).array()*(this->verbMatrix[voc.verbIndex.at(v)]*this->nounVector.col(voc.nounIndex.at(o))).array(); 37 | } 38 | 39 | MatD dist(voc.svoListUniq.size(), 1); 40 | 41 | #pragma omp parallel for num_threads(12) 42 | for (int i = 0; i < (int)voc.svoListUniq.size(); ++i){ 43 | MatD tmp = this->nounVector.col(voc.svoListUniq[i]->s).array()*(this->verbMatrix[voc.svoListUniq[i]->v]*this->nounVector.col(voc.svoListUniq[i]->o)).array(); 44 | 45 | dist.coeffRef(i, 0) = Utils::cosDis(svo, tmp); 46 | } 47 | 48 | std::vector > best, worst; 49 | 50 | for (int i = 0; i < k; ++i){ 51 | int row, col; 52 | std::string str; 53 | 54 | dist.maxCoeff(&row, &col); 55 | str = voc.nounStr[voc.svoListUniq[row]->s]+" "+voc.verbStr[voc.svoListUniq[row]->v]+" "+voc.nounStr[voc.svoListUniq[row]->o]; 56 | best.push_back(std::pair(str, dist.coeff(row, col))); 57 | dist.coeffRef(row, col) = 0.0; 58 | } 59 | 60 | for (int i = 0; i < k/2; ++i){ 61 | int row, col; 62 | std::string str; 63 | 64 | dist.minCoeff(&row, &col); 65 | str = voc.nounStr[voc.svoListUniq[row]->s]+" "+voc.verbStr[voc.svoListUniq[row]->v]+" "+voc.nounStr[voc.svoListUniq[row]->o]; 66 | worst.push_back(std::pair(str, dist.coeff(row, col))); 67 | dist.coeffRef(row, col) = 0.0; 68 | } 69 | 70 | for (int i = 0; i < (int)best.size(); ++i){ 71 | printf("%s:%f\n", best[i].first.c_str(), best[i].second); 72 | } 73 | for (int i = (int)worst.size()-1; i >= 0; --i){ 74 | printf("%s:%f\n", worst[i].first.c_str(), worst[i].second); 75 | } 76 | 77 | printf("~\n"); 78 | } 79 | } 80 | 81 | void NTF::VOdist(Vocabulary& voc, int k){ 82 | printf("kNN VOs\n"); 83 | 84 | std::vector res; 85 | 86 | for (std::string line; std::getline(std::cin, line) && line != "q"; ){ 87 | Utils::split(line, res); 88 | 89 | if (res.size() != 2){ 90 | continue; 91 | } 92 | 93 | std::string v = res[0]; 94 | std::string o = res[1]; 95 | 96 | if (!voc.verbIndex.count(v) || !voc.nounIndex.count(o)){ 97 | continue; 98 | } 99 | 100 | MatD vo = this->verbMatrix[voc.verbIndex.at(v)]*this->nounVector.col(voc.nounIndex.at(o)); 101 | MatD dist(voc.svoListUniq.size(), 1); 102 | 103 | for (int i = 0; i < (int)voc.svoListUniq.size(); ++i){ 104 | MatD tmp = this->verbMatrix[voc.svoListUniq[i]->v]*this->nounVector.col(voc.svoListUniq[i]->o); 105 | 106 | dist.coeffRef(i, 0) = Utils::cosDis(vo, tmp); 107 | } 108 | 109 | double prevScore = -1.0; 110 | 111 | for (int i = 0; i < k; ++i){ 112 | int row, col; 113 | std::string str; 114 | 115 | dist.maxCoeff(&row, &col); 116 | 117 | if (dist.coeff(row, col) == prevScore){ 118 | dist.coeffRef(row, col) = -1.0; 119 | --i; 120 | continue; 121 | } 122 | 123 | prevScore = dist.coeff(row, col); 124 | str = voc.verbStr[voc.svoListUniq[row]->v]+" "+voc.nounStr[voc.svoListUniq[row]->o]; 125 | printf("%2d: %s (%f)\n", i+1, str.c_str(), dist.coeff(row, col)); 126 | dist.coeffRef(row, col) = -1.0; 127 | } 128 | } 129 | } 130 | 131 | void NTF::SVO_PNdist(Vocabulary& voc, int k){ 132 | printf("kNN SVOs\n"); 133 | 134 | std::vector res; 135 | 136 | for (std::string line; std::getline(std::cin, line) && line != "q"; ){ 137 | Utils::split(line, res); 138 | 139 | if (res.size() != 5){ 140 | continue; 141 | } 142 | 143 | std::string s = res[0]; 144 | std::string v = res[1]; 145 | std::string o = res[2]; 146 | std::string p = res[3]; 147 | std::string n = res[4]; 148 | 149 | if (!voc.verbIndex.count(v) || !voc.nounIndex.count(s) || !voc.nounIndex.count(o) || !voc.prepIndex.count(p) || !voc.nounIndex.count(n)){ 150 | continue; 151 | } 152 | 153 | MatD svo = this->nounVector.col(voc.nounIndex.at(s)).array()*(this->verbMatrix[voc.verbIndex.at(v)]*this->nounVector.col(voc.nounIndex.at(o))).array(); 154 | MatD svo_pn = svo.array()*(this->prepMatrix[voc.prepIndex.at(p)]*this->nounVector.col(voc.nounIndex.at(n))).array(); 155 | MatD dist(voc.svopnListUniq.size(), 1); 156 | 157 | for (int i = 0; i < (int)voc.svopnListUniq.size(); ++i){ 158 | MatD tmp = this->nounVector.col(voc.svopnListUniq[i]->svo->s).array()*(this->verbMatrix[voc.svopnListUniq[i]->svo->v]*this->nounVector.col(voc.svopnListUniq[i]->svo->o)).array(); 159 | MatD tmp2 = tmp.array()*(this->prepMatrix[voc.svopnListUniq[i]->p]*this->nounVector.col(voc.svopnListUniq[i]->n)).array(); 160 | 161 | dist.coeffRef(i, 0) = Utils::cosDis(svo_pn, tmp2); 162 | } 163 | 164 | for (int i = 0; i < k; ++i){ 165 | int row, col; 166 | std::string str; 167 | 168 | dist.maxCoeff(&row, &col); 169 | str = voc.nounStr[voc.svopnListUniq[row]->svo->s]+" "+voc.verbStr[voc.svopnListUniq[row]->svo->v]+" "+voc.nounStr[voc.svopnListUniq[row]->svo->o]+" "+voc.prepStr[voc.svopnListUniq[row]->p]+" "+voc.nounStr[voc.svopnListUniq[row]->n]; 170 | printf("%2d: %s (%f)\n", i+1, str.c_str(), dist.coeff(row, col)); 171 | dist.coeffRef(row, col) = -1.0; 172 | } 173 | } 174 | } 175 | 176 | void NTF::SVOdist(Vocabulary& voc){ 177 | printf("Distance between SVOs\n"); 178 | 179 | std::vector res; 180 | 181 | for (std::string line; std::getline(std::cin, line) && line != "q"; ){ 182 | Utils::split(line, res); 183 | 184 | if (res.size() != 6){ 185 | continue; 186 | } 187 | 188 | std::string s1 = res[0]; 189 | std::string v1 = res[1]; 190 | std::string o1 = res[2]; 191 | std::string s2 = res[3]; 192 | std::string v2 = res[4]; 193 | std::string o2 = res[5]; 194 | 195 | if (!voc.verbIndex.count(v1) || !voc.verbIndex.count(v2) || (!voc.nounIndex.count(s1) && s1 != voc.null) || (!voc.nounIndex.count(s2) && s2 != voc.null) || !voc.nounIndex.count(o1) || !voc.nounIndex.count(o2)){ 196 | continue; 197 | } 198 | 199 | MatD svo1, svo2; 200 | 201 | if (s1 == voc.null){ 202 | svo1 = this->verbMatrix[voc.verbIndex.at(v1)]*this->nounVector.col(voc.nounIndex.at(o1)); 203 | } 204 | else { 205 | svo1 = this->nounVector.col(voc.nounIndex.at(s1)).array()*(this->verbMatrix[voc.verbIndex.at(v1)]*this->nounVector.col(voc.nounIndex.at(o1))).array(); 206 | } 207 | 208 | if (s2 == voc.null){ 209 | svo2 = this->verbMatrix[voc.verbIndex.at(v2)]*this->nounVector.col(voc.nounIndex.at(o2)); 210 | } 211 | else { 212 | svo2 = this->nounVector.col(voc.nounIndex.at(s2)).array()*(this->verbMatrix[voc.verbIndex.at(v2)]*this->nounVector.col(voc.nounIndex.at(o2))).array(); 213 | } 214 | 215 | printf("%f\n", Utils::cosDis(svo1, svo2)); 216 | if (s1 != voc.null && s2 != voc.null){ 217 | printf("%s, %s: %f\n", s1.c_str(), s2.c_str(), Utils::cosDis(this->nounVector.col(voc.nounIndex.at(s1)), this->nounVector.col(voc.nounIndex.at(s2)))); 218 | } 219 | printf("%s, %s: %f\n", v1.c_str(), v2.c_str(), Utils::cosDis(this->verbMatrix[voc.verbIndex.at(v1)], this->verbMatrix[voc.verbIndex.at(v2)])); 220 | printf("%s, %s: %f\n\n", o1.c_str(), o2.c_str(), Utils::cosDis(this->nounVector.col(voc.nounIndex.at(o1)), this->nounVector.col(voc.nounIndex.at(o2)))); 221 | } 222 | } 223 | 224 | void NTF::vRowKnn(Vocabulary& voc, int k){ 225 | printf("Verb Knn for rows\n"); 226 | 227 | std::vector res; 228 | 229 | for (std::string line; std::getline(std::cin, line) && line != "q"; ){ 230 | Utils::split(line, res); 231 | 232 | if (res.size() != 2){ 233 | continue; 234 | } 235 | 236 | std::string v = res[0]; 237 | int r = atoi(res[1].c_str()); 238 | 239 | if (!voc.verbIndex.count(v) || r >= this->verbMatrix[0].rows()){ 240 | continue; 241 | } 242 | 243 | MatD dist(1, this->verbMatrix.size()); 244 | 245 | if (r < 0){ 246 | MatD target = this->verbMatrix[voc.verbIndex.at(v)]; 247 | 248 | for (int i = 0; i < dist.cols(); ++i){ 249 | dist.coeffRef(0, i) = Utils::cosDis(target, this->verbMatrix[i]); 250 | } 251 | } 252 | else { 253 | MatD target = this->verbMatrix[voc.verbIndex.at(v)].row(r).transpose(); 254 | 255 | for (int i = 0; i < dist.cols(); ++i){ 256 | dist.coeffRef(0, i) = Utils::cosDis(target, this->verbMatrix[i].row(r).transpose()); 257 | } 258 | } 259 | 260 | for (int i = 0; i < k; ++i){ 261 | int row, col; 262 | 263 | dist.maxCoeff(&row, &col); 264 | printf("(%.5f) %s\n", dist.coeff(row, col), voc.verbStr[col].c_str()); 265 | dist.coeffRef(row, col) = -1.0; 266 | } 267 | 268 | printf("\n"); 269 | } 270 | } 271 | 272 | void NTF::vColKnn(Vocabulary& voc, int k){ 273 | printf("Verb Knn for cols\n"); 274 | 275 | std::vector res; 276 | 277 | for (std::string line; std::getline(std::cin, line) && line != "q"; ){ 278 | Utils::split(line, res); 279 | 280 | if (res.size() != 2){ 281 | continue; 282 | } 283 | 284 | std::string v = res[0]; 285 | int r = atoi(res[1].c_str()); 286 | 287 | if (!voc.verbIndex.count(v) || r >= this->verbMatrix[0].cols()){ 288 | continue; 289 | } 290 | 291 | MatD dist(1, this->verbMatrix.size()); 292 | 293 | if (r < 0){ 294 | MatD target = this->verbMatrix[voc.verbIndex.at(v)]; 295 | 296 | for (int i = 0; i < dist.cols(); ++i){ 297 | dist.coeffRef(0, i) = Utils::cosDis(target, this->verbMatrix[i]); 298 | } 299 | } 300 | else { 301 | MatD target = this->verbMatrix[voc.verbIndex.at(v)].col(r); 302 | 303 | for (int i = 0; i < dist.cols(); ++i){ 304 | dist.coeffRef(0, i) = Utils::cosDis(target, this->verbMatrix[i].col(r)); 305 | } 306 | } 307 | 308 | for (int i = 0; i < k; ++i){ 309 | int row, col; 310 | 311 | dist.maxCoeff(&row, &col); 312 | printf("(%.5f) %s\n", dist.coeff(row, col), voc.verbStr[col].c_str()); 313 | dist.coeffRef(row, col) = -1.0; 314 | } 315 | 316 | printf("\n"); 317 | } 318 | } 319 | 320 | void LBLM::SVOdist(Vocabulary& voc, int k){ 321 | printf("kNN SVOs\n"); 322 | 323 | std::vector res; 324 | 325 | for (std::string line; std::getline(std::cin, line) && line != "q"; ){ 326 | Utils::split(line, res); 327 | 328 | if (res.size() != 3){ 329 | continue; 330 | } 331 | 332 | std::string s = res[0]; 333 | std::string v = res[1]; 334 | std::string o = res[2]; 335 | 336 | if (!voc.verbIndex.count(v) || !voc.nounIndex.count(s) || !voc.nounIndex.count(o)){ 337 | continue; 338 | } 339 | 340 | SVO svoInst; 341 | MatD svo, svoTmp; 342 | 343 | svoInst.s = voc.nounIndex.at(s); svoInst.v = voc.verbIndex.at(v); svoInst.o = voc.nounIndex.at(o); 344 | this->compose(svo, &svoInst); 345 | 346 | MatD dist(voc.svoListUniq.size(), 1); 347 | 348 | for (int i = 0; i < (int)voc.svoListUniq.size(); ++i){ 349 | this->compose(svoTmp, voc.svoListUniq[i]); 350 | 351 | dist.coeffRef(i, 0) = Utils::cosDis(svo, svoTmp); 352 | } 353 | 354 | for (int i = 0; i < k; ++i){ 355 | int row, col; 356 | std::string str; 357 | 358 | dist.maxCoeff(&row, &col); 359 | str = voc.nounStr[voc.svoListUniq[row]->s]+" "+voc.verbStr[voc.svoListUniq[row]->v]+" "+voc.nounStr[voc.svoListUniq[row]->o]; 360 | printf("%2d: %s (%f)\n", i+1, str.c_str(), dist.coeff(row, col)); 361 | dist.coeffRef(row, col) = -1.0; 362 | } 363 | } 364 | } 365 | 366 | void LBLM::VOdist(Vocabulary& voc, int k){ 367 | printf("kNN VOs\n"); 368 | 369 | std::vector res; 370 | 371 | for (std::string line; std::getline(std::cin, line) && line != "q"; ){ 372 | Utils::split(line, res); 373 | 374 | if (res.size() != 2){ 375 | continue; 376 | } 377 | 378 | std::string v = res[0]; 379 | std::string o = res[1]; 380 | 381 | if (!voc.verbIndex.count(v) || !voc.nounIndex.count(o)){ 382 | continue; 383 | } 384 | 385 | SVO svoInst; 386 | MatD vo, voTmp; 387 | MatD dist(voc.svoListUniq.size(), 1); 388 | 389 | svoInst.s = voc.nullIndex; svoInst.v = voc.verbIndex.at(v); svoInst.o = voc.nounIndex.at(o); 390 | this->compose(vo, &svoInst); 391 | 392 | for (int i = 0; i < (int)voc.svoListUniq.size(); ++i){ 393 | svoInst.v = voc.svoListUniq[i]->v; svoInst.o = voc.svoListUniq[i]->o; 394 | this->compose(voTmp, &svoInst); 395 | dist.coeffRef(i, 0) = Utils::cosDis(vo, voTmp); 396 | } 397 | 398 | double prevScore = -1.0; 399 | 400 | for (int i = 0; i < k; ++i){ 401 | int row, col; 402 | std::string str; 403 | 404 | dist.maxCoeff(&row, &col); 405 | 406 | if (dist.coeff(row, col) == prevScore){ 407 | dist.coeffRef(row, col) = -1.0; 408 | --i; 409 | continue; 410 | } 411 | 412 | prevScore = dist.coeff(row, col); 413 | str = voc.verbStr[voc.svoListUniq[row]->v]+" "+voc.nounStr[voc.svoListUniq[row]->o]; 414 | printf("%2d: %s (%f)\n", i+1, str.c_str(), dist.coeff(row, col)); 415 | dist.coeffRef(row, col) = -1.0; 416 | } 417 | } 418 | } 419 | 420 | void LBLM::SVOdist(Vocabulary& voc){ 421 | printf("Distance between SVOs\n"); 422 | 423 | std::vector res; 424 | 425 | for (std::string line; std::getline(std::cin, line) && line != "q"; ){ 426 | Utils::split(line, res); 427 | 428 | if (res.size() != 6){ 429 | continue; 430 | } 431 | 432 | std::string s1 = res[0]; 433 | std::string v1 = res[1]; 434 | std::string o1 = res[2]; 435 | std::string s2 = res[3]; 436 | std::string v2 = res[4]; 437 | std::string o2 = res[5]; 438 | 439 | if (!voc.verbIndex.count(v1) || !voc.verbIndex.count(v2) || !voc.nounIndex.count(s1) || !voc.nounIndex.count(s2) || !voc.nounIndex.count(o1) || !voc.nounIndex.count(o2)){ 440 | continue; 441 | } 442 | 443 | SVO svo; 444 | MatD svo1, svo2; 445 | 446 | svo.s = voc.nounIndex.at(s1); svo.v = voc.verbIndex.at(v1); svo.o = voc.nounIndex.at(o1); 447 | this->compose(svo1, &svo); 448 | svo.s = voc.nounIndex.at(s2); svo.v = voc.verbIndex.at(v2); svo.o = voc.nounIndex.at(o2); 449 | this->compose(svo2, &svo); 450 | 451 | printf("%f\n", Utils::cosDis(svo1, svo2)); 452 | printf("%s, %s: %f\n", s1.c_str(), s2.c_str(), Utils::cosDis(this->nounVector.col(voc.nounIndex.at(s1)), this->nounVector.col(voc.nounIndex.at(s2)))); 453 | printf("%s, %s: %f\n", v1.c_str(), v2.c_str(), Utils::cosDis(this->verbVector.col(voc.verbIndex.at(v1)), this->verbVector.col(voc.verbIndex.at(v2)))); 454 | printf("%s, %s: %f\n\n", o1.c_str(), o2.c_str(), Utils::cosDis(this->nounVector.col(voc.nounIndex.at(o1)), this->nounVector.col(voc.nounIndex.at(o2)))); 455 | } 456 | } 457 | -------------------------------------------------------------------------------- /Sigmoid.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Matrix.hpp" 4 | 5 | class Sigmoid{ 6 | public: 7 | static double sigmoid(const double x); 8 | static double sigmoid(const MatD& x); 9 | static void sigmoid(MatD& x); 10 | }; 11 | 12 | inline double Sigmoid::sigmoid(const double x){ 13 | return 1.0/(1.0+::exp(-x)); 14 | } 15 | 16 | inline double Sigmoid::sigmoid(const MatD& x){ 17 | return 1.0/(1.0+::exp(-x.coeff(0, 0))); 18 | } 19 | 20 | inline void Sigmoid::sigmoid(MatD& x){ 21 | x = x.unaryExpr(std::ptr_fun((double (*)(const double))Sigmoid::sigmoid)); 22 | } 23 | -------------------------------------------------------------------------------- /Tanh.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Matrix.hpp" 4 | 5 | class Tanh{ 6 | public: 7 | static void tanh(MatD& x); 8 | static MatD tanhPrime(const MatD& x); 9 | }; 10 | 11 | //f(x) = tanh(x) 12 | inline void Tanh::tanh(MatD& x){ 13 | x = x.unaryExpr(std::ptr_fun(::tanh)); 14 | } 15 | 16 | //x must be a output of Tanh::tanh 17 | //f'(x) = 1-(f(x))^2 18 | inline MatD Tanh::tanhPrime(const MatD& x){ 19 | return 1.0-x.array().square(); 20 | } 21 | -------------------------------------------------------------------------------- /Test.cpp: -------------------------------------------------------------------------------- 1 | #include "NTF.hpp" 2 | #include "LBLM.hpp" 3 | 4 | struct sort_pred { 5 | bool operator()(const std::pair &left, const std::pair &right) { 6 | return left.second > right.second; 7 | } 8 | }; 9 | 10 | double calcRho(std::unordered_map& rankMap, std::unordered_map >& tieMap, std::vector >& output){ 11 | double rho = 0.0; 12 | 13 | std::sort(output.begin(), output.end(), sort_pred()); 14 | 15 | for (int i = 0; i < (int)output.size(); ++i){ 16 | double trueRank = 0.0; 17 | 18 | for (int j = 0; j < (int)tieMap.at(rankMap.at(output[i].first)).size(); ++j){ 19 | trueRank += tieMap.at(rankMap.at(output[i].first))[j]; 20 | } 21 | 22 | trueRank /= tieMap.at(rankMap.at(output[i].first)).size(); 23 | rho += (trueRank-i)*(trueRank-i); 24 | } 25 | 26 | rho *= 6.0; 27 | rho /= (output.size()*output.size()*output.size()-output.size()); 28 | rho = 1.0-rho; 29 | 30 | return rho; 31 | } 32 | 33 | double NTF::testSVO(Vocabulary& voc, const std::string& type, const bool ave){ 34 | std::string file; 35 | 36 | if (type == "grefen"){ 37 | file = (ave ? "./test_data/SVO_ave.txt" : "./test_data/SVO.txt"); 38 | } 39 | else if (type == "emnlp2013"){ 40 | file = (ave ? "./test_data/VO+S_ave.txt" : "./test_data/VO+S.txt"); 41 | } 42 | else if (type == "emnlp2013add"){ 43 | file = (ave ? "./test_data/VO+Sadd_ave.txt" : "./test_data/VO+Sadd.txt"); 44 | } 45 | else { 46 | assert(false); 47 | } 48 | 49 | std::ifstream ifs(file.c_str()); 50 | std::string line; 51 | int rank = 0; 52 | std::unordered_map rankMap; 53 | std::unordered_map > tieMap; 54 | std::vector > output; 55 | 56 | while(std::getline(ifs, line) && line != ""){ 57 | std::vector res; 58 | double goldScore; 59 | MatD s1, s2; 60 | 61 | Utils::split(line, res); 62 | 63 | if (type == "grefen"){ 64 | int s = voc.nounIndex.at(res[1]); 65 | int v = voc.verbIndex.at(res[0]); 66 | int o = voc.nounIndex.at(res[2]); 67 | int v_ = voc.verbIndex.at(res[3]); 68 | 69 | goldScore = atof(res[4].c_str()); 70 | 71 | s1 = this->nounVector.col(s).array()*(this->verbMatrix[v]*this->nounVector.col(o)).array(); 72 | s2 = this->nounVector.col(s).array()*(this->verbMatrix[v_]*this->nounVector.col(o)).array(); 73 | } 74 | else { 75 | int s = voc.nounIndex.at(res[0]); 76 | int v = voc.verbIndex.at(res[1]); 77 | int o = voc.nounIndex.at(res[2]); 78 | int s_ = voc.nounIndex.at(res[3]); 79 | int v_ = voc.verbIndex.at(res[4]); 80 | int o_ = voc.nounIndex.at(res[5]); 81 | 82 | goldScore = atof(res[6].c_str()); 83 | 84 | s1 = this->nounVector.col(s).array()*(this->verbMatrix[v]*this->nounVector.col(o)).array(); 85 | s2 = this->nounVector.col(s_).array()*(this->verbMatrix[v_]*this->nounVector.col(o_)).array(); 86 | } 87 | 88 | rankMap[rank] = goldScore; 89 | tieMap[goldScore].push_back(rank); 90 | output.push_back(std::pair(rank++, Utils::cosDis(s1, s2))); 91 | } 92 | 93 | return calcRho(rankMap, tieMap, output); 94 | } 95 | 96 | double NTF::testVO(Vocabulary& voc, const bool ave){ 97 | std::string file = (ave ? "./test_data/VO_ave.txt" : "./test_data/VO.txt"); 98 | std::ifstream ifs(file.c_str()); 99 | std::string line; 100 | int rank = 0; 101 | std::unordered_map rankMap; 102 | std::unordered_map > tieMap; 103 | std::vector > output; 104 | 105 | while(std::getline(ifs, line) && line != ""){ 106 | std::vector res; 107 | double goldScore; 108 | 109 | Utils::split(line, res); 110 | 111 | int v1 = voc.verbIndex.at(res[1]); 112 | int o1 = voc.nounIndex.at(res[0]); 113 | int v2 = voc.verbIndex.at(res[3]); 114 | int o2 = voc.nounIndex.at(res[2]); 115 | 116 | goldScore = atof(res[4].c_str()); 117 | 118 | rankMap[rank] = goldScore; 119 | tieMap[goldScore].push_back(rank); 120 | 121 | MatD s1 = this->verbMatrix[v1]*this->nounVector.col(o1); 122 | MatD s2 = this->verbMatrix[v2]*this->nounVector.col(o2); 123 | 124 | output.push_back(std::pair(rank++, Utils::cosDis(s1, s2))); 125 | } 126 | 127 | return calcRho(rankMap, tieMap, output); 128 | } 129 | 130 | double LBLM::testSVO(Vocabulary& voc, const std::string& type, const bool ave){ 131 | std::string file; 132 | 133 | if (type == "grefen"){ 134 | file = (ave ? "./test_data/SVO_ave.txt" : "./test_data/SVO.txt"); 135 | } 136 | else if (type == "emnlp2013"){ 137 | file = (ave ? "./test_data/VO+S_ave.txt" : "./test_data/VO+S.txt"); 138 | } 139 | else if (type == "emnlp2013add"){ 140 | file = (ave ? "./test_data/VO+Sadd_ave.txt" : "./test_data/VO+Sadd.txt"); 141 | } 142 | else { 143 | assert(false); 144 | } 145 | 146 | std::ifstream ifs(file.c_str()); 147 | std::string line; 148 | int rank = 0; 149 | std::unordered_map rankMap; 150 | std::unordered_map > tieMap; 151 | std::vector > output; 152 | 153 | while(std::getline(ifs, line) && line != ""){ 154 | std::vector res; 155 | double goldScore; 156 | MatD s1, s2; 157 | SVO svo; 158 | 159 | Utils::split(line, res); 160 | 161 | if (type == "grefen"){ 162 | int s = voc.nounIndex.at(res[1]); 163 | int v = voc.verbIndex.at(res[0]); 164 | int o = voc.nounIndex.at(res[2]); 165 | int v_ = voc.verbIndex.at(res[3]); 166 | 167 | goldScore = atof(res[4].c_str()); 168 | 169 | svo.s = s; svo.v = v; svo.o = o; 170 | this->compose(s1, &svo); 171 | svo.v = v_; 172 | this->compose(s2, &svo); 173 | } 174 | else { 175 | int s = voc.nounIndex.at(res[0]); 176 | int v = voc.verbIndex.at(res[1]); 177 | int o = voc.nounIndex.at(res[2]); 178 | int s_ = voc.nounIndex.at(res[3]); 179 | int v_ = voc.verbIndex.at(res[4]); 180 | int o_ = voc.nounIndex.at(res[5]); 181 | 182 | goldScore = atof(res[6].c_str()); 183 | 184 | svo.s = s; svo.v = v; svo.o = o; 185 | this->compose(s1, &svo); 186 | svo.s = s_; svo.v = v_; svo.o = o_; 187 | this->compose(s2, &svo); 188 | } 189 | 190 | rankMap[rank] = goldScore; 191 | tieMap[goldScore].push_back(rank); 192 | output.push_back(std::pair(rank++, Utils::cosDis(s1, s2))); 193 | } 194 | 195 | return calcRho(rankMap, tieMap, output); 196 | } 197 | 198 | double LBLM::testVO(Vocabulary& voc, const bool ave){ 199 | std::string file = (ave ? "./test_data/VO_ave.txt" : "./test_data/VO.txt"); 200 | std::ifstream ifs(file.c_str()); 201 | std::string line; 202 | int rank = 0; 203 | std::unordered_map rankMap; 204 | std::unordered_map > tieMap; 205 | std::vector > output; 206 | 207 | while(std::getline(ifs, line) && line != ""){ 208 | std::vector res; 209 | double goldScore; 210 | 211 | Utils::split(line, res); 212 | 213 | int v1 = voc.verbIndex.at(res[1]); 214 | int o1 = voc.nounIndex.at(res[0]); 215 | int v2 = voc.verbIndex.at(res[3]); 216 | int o2 = voc.nounIndex.at(res[2]); 217 | 218 | goldScore = atof(res[4].c_str()); 219 | 220 | rankMap[rank] = goldScore; 221 | tieMap[goldScore].push_back(rank); 222 | 223 | SVO svo; 224 | MatD s1, s2; 225 | 226 | svo.s = voc.nullIndex; svo.v = v1; svo.o = o1; 227 | this->compose(s1, &svo); 228 | svo.s = voc.nullIndex; svo.v = v2; svo.o = o2; 229 | this->compose(s2, &svo); 230 | 231 | output.push_back(std::pair(rank++, Utils::cosDis(s1, s2))); 232 | } 233 | 234 | return calcRho(rankMap, tieMap, output); 235 | } 236 | -------------------------------------------------------------------------------- /Utils.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Matrix.hpp" 4 | #include 5 | #include 6 | #include 7 | 8 | namespace Utils{ 9 | inline double max(const double& x, const double& y){ 10 | return x > y ? x : y; 11 | } 12 | 13 | inline bool isSpace(const char& c){ 14 | return (c == ' ' || c == '\t'); 15 | } 16 | 17 | inline void split(const std::string& str, std::vector& res){ 18 | bool tok = false; 19 | int beg = 0; 20 | 21 | res.clear(); 22 | 23 | for (int i = 0, len = str.length(); i < len; ++i){ 24 | if (!tok && !Utils::isSpace(str[i])){ 25 | beg = i; 26 | tok = true; 27 | } 28 | 29 | if (tok && (i == len-1 || Utils::isSpace(str[i]))){ 30 | tok = false; 31 | res.push_back((i == len-1) ? str.substr(beg, i-beg+1) : str.substr(beg, i-beg)); 32 | } 33 | } 34 | } 35 | 36 | inline double cosDis(const MatD& a, const MatD& b){ 37 | return (a.array()*b.array()).sum()/(a.norm()*b.norm()); 38 | //return a.col(0).dot(b.col(0))/(a.norm()*b.norm()); 39 | } 40 | 41 | inline void infNan(const double& x){ 42 | assert(!isnan(x) && !isinf(x)); 43 | } 44 | 45 | inline void save(std::ofstream& ofs, const MatD& params){ 46 | double val = 0.0; 47 | 48 | for (int i = 0; i < params.cols(); ++i){ 49 | for (int j = 0; j < params.rows(); ++j){ 50 | val = params.coeff(j, i); 51 | ofs.write((char*)&val, sizeof(double)); 52 | } 53 | } 54 | } 55 | 56 | inline void load(std::ifstream& ifs, MatD& params){ 57 | double val = 0.0; 58 | 59 | for (int i = 0; i < params.cols(); ++i){ 60 | for (int j = 0; j < params.rows(); ++j){ 61 | ifs.read((char*)&val, sizeof(double)); 62 | params.coeffRef(j, i) = val; 63 | } 64 | } 65 | } 66 | 67 | inline double stdDev(const Eigen::MatrixXd& input){ 68 | return ::sqrt(((Eigen::MatrixXd)((input.array()-input.sum()/input.rows()).pow(2.0))).sum()/(input.rows()-1)); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /Vocabulary.cpp: -------------------------------------------------------------------------------- 1 | #include "Vocabulary.hpp" 2 | #include "Utils.hpp" 3 | #include 4 | 5 | const std::string Vocabulary::null = "NULL"; 6 | 7 | void Vocabulary::split(const std::string& line, int& freq, std::string& pred, std::string& arg1, std::string& arg2){ 8 | for (int i = line.length()-1, space = 0, s1 = -1, s2 = -1, s3 = -1; ; --i){ 9 | if (line[i] == ' '){ 10 | if (space == 0){ 11 | s1 = i; 12 | ++space; 13 | arg2 = line.substr(s1+1); 14 | } 15 | else if (space == 1){ 16 | s2 = i; 17 | ++space; 18 | arg1 = line.substr(s2+1, s1-s2-1); 19 | } 20 | else if (space == 2){ 21 | s3 = i; 22 | ++space; 23 | pred = line.substr(s3+1, s2-s3-1); 24 | freq = atoi(line.substr(0, s3).c_str()); 25 | break; 26 | } 27 | } 28 | } 29 | 30 | assert(pred.find(" ") == std::string::npos && arg1.find(" ") == std::string::npos && arg2.find(" ") == std::string::npos); 31 | } 32 | 33 | void Vocabulary::readSVO(const std::string& fileName, std::vector& instance, std::vector& type){ 34 | std::ifstream ifs(fileName.c_str()); 35 | std::string subj, obj, verb; 36 | SVO* svo = 0; 37 | std::unordered_map::iterator it; 38 | int freq = -1; 39 | std::unordered_map subjCount, objCount, verbCount; 40 | 41 | for (std::string line; std::getline(ifs, line);){ 42 | Vocabulary::split(line, freq, verb, subj, obj); 43 | 44 | //only use SVO tuples 45 | if (verb == "be" || subj == Vocabulary::null || obj == Vocabulary::null){ 46 | continue; 47 | } 48 | 49 | svo = new SVO; 50 | svo->type = Data::SVO_; 51 | 52 | //object 53 | it = this->nounIndex.find(obj); 54 | if (it == this->nounIndex.end()){ 55 | this->nounStr.push_back(obj); 56 | this->nounIndex[obj] = this->nounStr.size()-1; 57 | } 58 | svo->o = this->nounIndex.at(obj); 59 | 60 | //subject 61 | it = this->nounIndex.find(subj); 62 | if (it == this->nounIndex.end()){ 63 | this->nounStr.push_back(subj); 64 | this->nounIndex[subj] = this->nounStr.size()-1; 65 | } 66 | svo->s = this->nounIndex.at(subj); 67 | 68 | //verb 69 | it = this->verbIndex.find(verb); 70 | if (it == this->verbIndex.end()){ 71 | this->verbStr.push_back(verb); 72 | this->verbIndex[verb] = this->verbStr.size()-1; 73 | this->svoMap.push_back(std::unordered_map >()); 74 | } 75 | svo->v = this->verbIndex.at(verb); 76 | 77 | //add this svo 78 | if (!this->svoMap[svo->v].count(svo->s)){ 79 | this->svoMap[svo->v][svo->s] = std::unordered_map(); 80 | } 81 | this->svoMap[svo->v].at(svo->s)[svo->o] = svo; 82 | 83 | type.push_back(svo); 84 | 85 | for (int j = 0; j < freq; ++j){ 86 | instance.push_back(svo); 87 | } 88 | 89 | if (subjCount.count(svo->s)){ 90 | subjCount.at(svo->s) += freq; 91 | } 92 | else { 93 | subjCount[svo->s] = freq; 94 | } 95 | if (objCount.count(svo->o)){ 96 | objCount.at(svo->o) += freq; 97 | } 98 | else { 99 | objCount[svo->o] = freq; 100 | } 101 | if (verbCount.count(svo->v)){ 102 | verbCount.at(svo->v) += freq; 103 | } 104 | else { 105 | verbCount[svo->v] = freq; 106 | } 107 | } 108 | 109 | for (auto it = subjCount.begin(); it != subjCount.end(); ++it){ 110 | for (int i = 0; i < (int)pow(it->second, 0.75); ++i){ 111 | this->vsubjList.push_back(it->first); 112 | } 113 | } 114 | for (auto it = objCount.begin(); it != objCount.end(); ++it){ 115 | for (int i = 0; i < (int)pow(it->second, 0.75); ++i){ 116 | this->vobjList.push_back(it->first); 117 | } 118 | } 119 | for (auto it = verbCount.begin(); it != verbCount.end(); ++it){ 120 | for (int i = 0; i < (int)pow(it->second, 0.75); ++i){ 121 | this->verbList.push_back(it->first); 122 | } 123 | } 124 | 125 | printf("# of SVO types: %zd\n", type.size()); 126 | printf("# of SVO instances: %zd\n", instance.size()); 127 | } 128 | 129 | void Vocabulary::readSVOPN(const std::string& fileName, std::vector& instance, std::vector& type){ 130 | std::ifstream ifs(fileName.c_str()); 131 | std::string prep, svo, n; 132 | std::string subj, verb, obj; 133 | int c1 = -1, c2 = -1; 134 | int s = -1, v = -1, o = -1; 135 | SVOPN* svopn = 0; 136 | std::unordered_map::iterator it; 137 | int freq = -1; 138 | std::unordered_map prepCount, pobjCount; 139 | std::unordered_map svoIndex; 140 | std::vector > svoCount; 141 | 142 | for (std::string line; std::getline(ifs, line);){ 143 | Vocabulary::split(line, freq, prep, svo, n); 144 | 145 | //check SVO 146 | c1 = svo.find(","); 147 | c2 = svo.rfind(","); 148 | verb = svo.substr(0, c1); 149 | subj = svo.substr(c1+1, c2-c1-1); 150 | obj = svo.substr(c2+1); 151 | 152 | if (obj == Vocabulary::null || verb == "be"){ 153 | continue; 154 | } 155 | 156 | svopn = new SVOPN; 157 | svopn->type = Data::SVOPN_; 158 | 159 | //n 160 | it = this->nounIndex.find(n); 161 | if (it == this->nounIndex.end()){ 162 | this->nounStr.push_back(n); 163 | this->nounIndex[n] = this->nounStr.size()-1; 164 | } 165 | svopn->n = this->nounIndex.at(n); 166 | 167 | //svo 168 | it = this->nounIndex.find(obj); 169 | if (it == this->nounIndex.end()){ 170 | this->nounStr.push_back(obj); 171 | this->nounIndex[obj] = this->nounStr.size()-1; 172 | } 173 | it = this->nounIndex.find(subj); 174 | if (it == this->nounIndex.end()){ 175 | this->nounStr.push_back(subj); 176 | this->nounIndex[subj] = this->nounStr.size()-1; 177 | } 178 | it = this->verbIndex.find(verb); 179 | if (it == this->verbIndex.end()){ 180 | this->verbStr.push_back(verb); 181 | this->verbIndex[verb] = this->verbStr.size()-1; 182 | this->svoMap.push_back(std::unordered_map >()); 183 | } 184 | s = this->nounIndex.at(subj); 185 | o = this->nounIndex.at(obj); 186 | v = this->verbIndex.at(verb); 187 | if (this->exist(s, v, o)){ 188 | svopn->svo = this->svoMap[v].at(s).at(o); 189 | } 190 | else { 191 | svopn->svo = new SVO; 192 | svopn->svo->type = Data::SVO_; 193 | svopn->svo->s = s; 194 | svopn->svo->v = v; 195 | svopn->svo->o = o; 196 | if (!this->svoMap[v].count(s)){ 197 | this->svoMap[v][s] = std::unordered_map(); 198 | } 199 | this->svoMap[v].at(s)[o] = svopn->svo; 200 | } 201 | 202 | //prep 203 | it = this->prepIndex.find(prep); 204 | if (it == this->prepIndex.end()){ 205 | this->prepStr.push_back(prep); 206 | this->prepIndex[prep] = this->prepStr.size()-1; 207 | this->svopnMap.push_back(std::unordered_map >()); 208 | } 209 | svopn->p = this->prepIndex.at(prep); 210 | 211 | //add this svo-pn 212 | if (!this->svopnMap[svopn->p].count(svopn->n)){ 213 | this->svopnMap[svopn->p][svopn->n] = std::unordered_map(); 214 | } 215 | 216 | this->svopnMap[svopn->p].at(svopn->n)[svopn->svo] = svopn; 217 | 218 | type.push_back(svopn); 219 | 220 | for (int j = 0; j < freq; ++j){ 221 | instance.push_back(svopn); 222 | } 223 | 224 | if (prepCount.count(svopn->p)){ 225 | prepCount.at(svopn->p) += freq; 226 | } 227 | else { 228 | prepCount[svopn->p] = freq; 229 | } 230 | if (pobjCount.count(svopn->n)){ 231 | pobjCount.at(svopn->n) += freq; 232 | } 233 | else { 234 | pobjCount[svopn->n] = freq; 235 | } 236 | 237 | if (svoIndex.count(svopn->svo)){ 238 | svoCount[svoIndex.at(svopn->svo)].second += freq; 239 | } 240 | else { 241 | svoIndex[svopn->svo] = svoCount.size(); 242 | svoCount.push_back(std::pair(svopn->svo, freq)); 243 | } 244 | } 245 | 246 | if (this->nounIndex.count(Vocabulary::null)){ 247 | this->nullIndex = this->nounIndex.at(Vocabulary::null); 248 | } 249 | 250 | for (auto it = prepCount.begin(); it != prepCount.end(); ++it){ 251 | for (int i = 0; i < (int)pow(it->second, 0.75); ++i){ 252 | this->prepList.push_back(it->first); 253 | } 254 | } 255 | for (auto it = pobjCount.begin(); it != pobjCount.end(); ++it){ 256 | for (int i = 0; i < (int)pow(it->second, 0.75); ++i){ 257 | this->pobjList.push_back(it->first); 258 | } 259 | } 260 | for (int i = 0; i < (int)svoCount.size(); ++i){ 261 | for (int j = 0; j < (int)pow(svoCount[i].second, 0.75); ++j){ 262 | this->svoList.push_back(svoCount[i].first); 263 | } 264 | } 265 | 266 | printf("# of SVO-P-N types: %zd\n", type.size()); 267 | printf("# of SVO-P-N instances: %zd\n", instance.size()); 268 | } 269 | -------------------------------------------------------------------------------- /Vocabulary.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Matrix.hpp" 4 | #include "Data.hpp" 5 | #include "SVO.hpp" 6 | #include "SVOPN.hpp" 7 | #include 8 | #include 9 | 10 | class Vocabulary{ 11 | public: 12 | static const std::string null; 13 | 14 | //verb 15 | std::unordered_map verbIndex; 16 | std::vector verbStr; 17 | std::vector verbList; 18 | 19 | //preposition 20 | std::unordered_map prepIndex; 21 | std::vector prepStr; 22 | std::vector prepList; 23 | 24 | //noun 25 | std::unordered_map nounIndex; 26 | std::vector nounStr; 27 | std::vector vsubjList; 28 | std::vector vobjList; 29 | std::vector pobjList; 30 | int nullIndex; 31 | 32 | //svo 33 | std::vector svoList, svoListUniq; 34 | 35 | std::vector svopnListUniq; 36 | 37 | //svo map 38 | std::vector > > svoMap; //v -> s -> o 39 | std::vector > > svopnMap; //p -> n -> svo 40 | 41 | Vocabulary(): nullIndex(-1) {}; 42 | 43 | void readSVO(const std::string& fileName, std::vector& instance, std::vector& type); 44 | void readSVOPN(const std::string& fileName, std::vector& instance, std::vector& type); 45 | 46 | bool exist(int s, int v, int o); 47 | bool exist(int p, int n, SVO* svo); 48 | 49 | private: 50 | static void split(const std::string& line, int& freq, std::string& pred, std::string& arg1, std::string& arg2); 51 | }; 52 | 53 | inline bool Vocabulary::exist(int s, int v, int o){ 54 | return this->svoMap[v].count(s) && this->svoMap[v].at(s).count(o); 55 | } 56 | 57 | inline bool Vocabulary::exist(int p, int n, SVO* svo){ 58 | return this->svopnMap[p].count(n) && this->svopnMap[p].at(n).count(svo); 59 | } 60 | -------------------------------------------------------------------------------- /main.cpp: -------------------------------------------------------------------------------- 1 | #include "Vocabulary.hpp" 2 | #include "Data.hpp" 3 | #include "SVO.hpp" 4 | #include "NTF.hpp" 5 | #include "LBLM.hpp" 6 | #include 7 | 8 | /* 9 | argv[1]: dim 10 | argv[2]: learning rate 11 | argv[3]: iteration 12 | argv[4]: ntf of lblm 13 | argv[5]: svopn ? 1 or 0 14 | argv[6]: bnc or wiki 15 | argv[7]: save (or load) file 16 | */ 17 | 18 | int main(int argc, char** argv){ 19 | const int dim = atoi(argv[1]); 20 | const double learningRate = atof(argv[2]); 21 | const int itr = atoi(argv[3]); 22 | const std::string model = (std::string)argv[4]; 23 | const bool svopn = (atoi(argv[5]) == 1); 24 | const int miniBatchSize = 100; 25 | const std::string corpus = (std::string)argv[6]; 26 | int numNeg = -1; 27 | Vocabulary voc; 28 | std::vector instance, instanceSVO, instanceSVOPN; 29 | std::vector typeSVO, typeSVOPN; 30 | std::vector type; 31 | NTF ntf; 32 | LBLM lblm; 33 | 34 | if (corpus == "bnc"){ 35 | numNeg = 50; 36 | voc.readSVO("./train_data/svo.bnc", instanceSVO, typeSVO); 37 | 38 | if (svopn){ 39 | voc.readSVOPN("./train_data/svo_pn.bnc", instanceSVOPN, typeSVOPN); 40 | } 41 | } 42 | else if (corpus == "wiki"){ 43 | numNeg = 10; 44 | voc.readSVO("./train_data/svo.wiki", instanceSVO, typeSVO); 45 | 46 | if (svopn){ 47 | voc.readSVOPN("./train_data/svo_pn.wiki", instanceSVOPN, typeSVOPN); 48 | } 49 | } 50 | else { 51 | assert(false); 52 | } 53 | 54 | for (int i = 0; i < (int)voc.svoMap.size(); ++i){ 55 | for (auto it1 = voc.svoMap[i].begin(); it1 != voc.svoMap[i].end(); ++it1){ 56 | for (auto it2 = it1->second.begin(); it2 != it1->second.end(); ++it2){ 57 | voc.svoListUniq.push_back(it2->second); 58 | } 59 | } 60 | } 61 | for (int i = 0; i < (int)voc.svopnMap.size(); ++i){ 62 | for (auto it1 = voc.svopnMap[i].begin(); it1 != voc.svopnMap[i].end(); ++it1){ 63 | for (auto it2 = it1->second.begin(); it2 != it1->second.end(); ++it2){ 64 | voc.svopnListUniq.push_back(it2->second); 65 | } 66 | } 67 | } 68 | 69 | //split 70 | for (int i = 0; i < (int)typeSVO.size(); ++i){ 71 | double rndTmp = Data::rndData.zero2one(); 72 | 73 | typeSVO[i]->set = (rndTmp <= 0.8 ? 74 | Data::TRAIN : 75 | (rndTmp <= 0.9 ? Data::DEV : Data::TEST)); 76 | type.push_back(typeSVO[i]); 77 | 78 | if (typeSVO[i]->set == Data::TRAIN){ 79 | continue; 80 | } 81 | 82 | ((SVO*)typeSVO[i])->s_ = new int[numNeg]; 83 | ((SVO*)typeSVO[i])->v_ = new int[numNeg]; 84 | ((SVO*)typeSVO[i])->o_ = new int[numNeg]; 85 | 86 | for (int j = 0; j < numNeg; ++j){ 87 | do { 88 | ((SVO*)typeSVO[i])->s_[j] = voc.vsubjList[(Data::rndData.next() >> 16)%voc.vsubjList.size()]; 89 | } while (voc.exist(((SVO*)typeSVO[i])->s_[j], ((SVO*)typeSVO[i])->v, ((SVO*)typeSVO[i])->o)); 90 | do { 91 | ((SVO*)typeSVO[i])->v_[j] = voc.verbList[(Data::rndData.next() >> 16)%voc.verbList.size()]; 92 | } while (voc.exist(((SVO*)typeSVO[i])->s, ((SVO*)typeSVO[i])->v_[j], ((SVO*)typeSVO[i])->o)); 93 | do { 94 | ((SVO*)typeSVO[i])->o_[j] = voc.vobjList[(Data::rndData.next() >> 16)%voc.vobjList.size()]; 95 | } while (voc.exist(((SVO*)typeSVO[i])->s, ((SVO*)typeSVO[i])->v, ((SVO*)typeSVO[i])->o_[j])); 96 | } 97 | } 98 | for (int i = 0; i < (int)typeSVOPN.size(); ++i){ 99 | double rndTmp = Data::rndData.zero2one(); 100 | 101 | typeSVOPN[i]->set = (rndTmp <= 0.8 ? 102 | Data::TRAIN : 103 | (rndTmp <= 0.9 ? Data::DEV : Data::TEST)); 104 | type.push_back(typeSVOPN[i]); 105 | 106 | if (typeSVOPN[i]->set == Data::TRAIN){ 107 | continue; 108 | } 109 | 110 | ((SVOPN*)typeSVOPN[i])->p_ = new int[numNeg]; 111 | ((SVOPN*)typeSVOPN[i])->n_ = new int[numNeg]; 112 | ((SVOPN*)typeSVOPN[i])->svo_ = new SVO*[numNeg]; 113 | 114 | for (int j = 0; j < numNeg; ++j){ 115 | do { 116 | ((SVOPN*)typeSVOPN[i])->p_[j] = voc.prepList[(Data::rndData.next() >> 16)%voc.prepList.size()]; 117 | } while (voc.exist(((SVOPN*)typeSVOPN[i])->p_[j], ((SVOPN*)typeSVOPN[i])->n, ((SVOPN*)typeSVOPN[i])->svo)); 118 | do { 119 | ((SVOPN*)typeSVOPN[i])->n_[j] = voc.pobjList[(Data::rndData.next() >> 16)%voc.pobjList.size()]; 120 | } while (voc.exist(((SVOPN*)typeSVOPN[i])->p, ((SVOPN*)typeSVOPN[i])->n_[j], ((SVOPN*)typeSVOPN[i])->svo)); 121 | do { 122 | ((SVOPN*)typeSVOPN[i])->svo_[j] = voc.svoList[(Data::rndData.next() >> 16)%voc.svoList.size()]; 123 | } while (voc.exist(((SVOPN*)typeSVOPN[i])->p, ((SVOPN*)typeSVOPN[i])->n, ((SVOPN*)typeSVOPN[i])->svo_[j])); 124 | } 125 | } 126 | 127 | for (int i = 0; i < (int)instanceSVO.size(); ++i){ 128 | if (instanceSVO[i]->set == Data::TRAIN){ 129 | instance.push_back(instanceSVO[i]); 130 | } 131 | } 132 | for (int i = 0; i < (int)instanceSVOPN.size(); ++i){ 133 | if (instanceSVOPN[i]->set == Data::TRAIN){ 134 | instance.push_back(instanceSVOPN[i]); 135 | } 136 | } 137 | 138 | //neural tensor factorization 139 | if (model == "ntf"){ 140 | ntf.init(dim, voc); 141 | ntf.train(instance, type, type, voc, learningRate, itr, miniBatchSize, numNeg); 142 | ntf.save((std::string)argv[7]); 143 | //ntf.load((std::string)argv[7]); 144 | 145 | //for test 146 | /* 147 | printf("\tGS'11: %g (%g)\n", ntf.testSVO(voc, "grefen"), ntf.testSVO(voc, "grefen", true)); 148 | printf("\tKS'13: %g (%g)\n", ntf.testSVO(voc, "emnlp2013"), ntf.testSVO(voc, "emnlp2013", true)); 149 | printf("\tKS'14: %g (%g)\n", ntf.testSVO(voc, "emnlp2013add"), ntf.testSVO(voc, "emnlp2013add", true)); 150 | printf("\tML'10: %g (%g)\n", ntf.testVO(voc), ntf.testVO(voc, true)); 151 | */ 152 | //ntf.VOdist(voc, 20); 153 | ntf.SVOdist(voc, 20); 154 | //ntf.SVOdist(voc); 155 | //ntf.vRowKnn(voc, 20); 156 | //ntf.vColKnn(voc, 20); 157 | } 158 | else if (model == "lblm"){ 159 | lblm.init(dim, voc); 160 | lblm.train(instance, type, type, voc, learningRate, itr, miniBatchSize, numNeg); 161 | lblm.save((std::string)argv[7]); 162 | //lblm.load((std::string)argv[7]); 163 | 164 | //for test 165 | /* 166 | printf("\tGS'11: %g (%g)\n", lblm.testSVO(voc, "grefen"), lblm.testSVO(voc, "grefen", true)); 167 | printf("\tKS'13: %g (%g)\n", lblm.testSVO(voc, "emnlp2013"), lblm.testSVO(voc, "emnlp2013", true)); 168 | printf("\tKS'14: %g (%g)\n", lblm.testSVO(voc, "emnlp2013add"), lblm.testSVO(voc, "emnlp2013add", true)); 169 | printf("\tML'10: %g (%g)\n", lblm.testVO(voc), lblm.testVO(voc, true)); 170 | */ 171 | lblm.VOdist(voc, 20); 172 | } 173 | else { 174 | assert(false); 175 | } 176 | 177 | return 0; 178 | } 179 | -------------------------------------------------------------------------------- /objs/dummy: -------------------------------------------------------------------------------- 1 | dummy 2 | -------------------------------------------------------------------------------- /test_data/SVO_ave.txt: -------------------------------------------------------------------------------- 1 | buy investor share purchase 7 2 | buy employee property purchase 7 3 | buy supporter ticket purchase 6.92308 4 | buy people product purchase 6.92308 5 | meet system criterion satisfy 6.84615 6 | buy people house purchase 6.84615 7 | provide study evidence supply 6.83333 8 | meet student requirement satisfy 6.76923 9 | meet service need satisfy 6.76923 10 | write man song spell 6.69231 11 | provide man money supply 6.61538 12 | provide employer training supply 6.61538 13 | run people company operate 6.53846 14 | provide government cash supply 6.53846 15 | meet beach standard satisfy 6.53846 16 | provide system facility supply 6.5 17 | provide company service supply 6.5 18 | draw report attention attract 6.5 19 | provide study information supply 6.41667 20 | draw proposal criticism attract 6.41667 21 | meet system requirement satisfy 6.38462 22 | run family hotel operate 6.30769 23 | show child interest express 6.25 24 | buy firm politician bribe 6.25 25 | provide school education supply 6.16667 26 | buy people security purchase 6.16667 27 | provide family home supply 5.92308 28 | meet representative official visit 5.84615 29 | try people program test 5.76923 30 | show map area picture 5.58333 31 | draw patient figure depict 5.58333 32 | try people door test 5.53846 33 | say spokesman decision state 5.5 34 | say man sentence state 5.41667 35 | draw table eye attract 5.41667 36 | run user application operate 5.30769 37 | accept management responsibility bear 5.25 38 | try court case judge 5.23077 39 | accept magazine liability bear 5.16667 40 | provide mother baby supply 5.15385 41 | show table result express 5.08333 42 | draw child picture depict 5.08333 43 | accept member share receive 5.08333 44 | run system application operate 5.07692 45 | write scholar book publish 5 46 | try justice king judge 5 47 | try demonstrator building test 4.92308 48 | try court offence judge 4.92308 49 | run machine program operate 4.92308 50 | draw woman hair depict 4.91667 51 | draw male female depict 4.91667 52 | accept user responsibility bear 4.91667 53 | accept society charge bear 4.91667 54 | try tribunal crime judge 4.84615 55 | show poll support express 4.83333 56 | write student name spell 4.76923 57 | write specialist papers publish 4.76923 58 | write child word spell 4.69231 59 | say speaker word state 4.66667 60 | buy model behaviour purchase 4.66667 61 | show map location picture 4.58333 62 | say man success state 4.58333 63 | buy politician election purchase 4.58333 64 | show figure increase express 4.5 65 | say people success state 4.5 66 | meet boy girl visit 4.46154 67 | say survey recovery state 4.41667 68 | write teacher book publish 4.38462 69 | try man experiment test 4.38462 70 | show study correlation express 4.33333 71 | buy firm politician purchase 4.33333 72 | say producer favour state 4.16667 73 | draw male female attract 4.16667 74 | write user word spell 4.15385 75 | try people sniffing test 4.15385 76 | show figure increase picture 4.08333 77 | say people prayer state 4.08333 78 | provide man money leave 4.07692 79 | write writer book publish 4 80 | buy politician election bribe 4 81 | accept patient treatment receive 4 82 | write user software publish 3.92308 83 | write man song publish 3.84615 84 | accept patient treatment bear 3.75 85 | write people letter publish 3.61538 86 | try court case test 3.61538 87 | say survey recovery allege 3.58333 88 | buy weapon chance purchase 3.58333 89 | accept shareholder offer receive 3.58333 90 | say priest prayer state 3.5 91 | buy people security bribe 3.5 92 | accept government recommendation bear 3.5 93 | show table result picture 3.41667 94 | say man sentence allege 3.41667 95 | run people round move 3.38462 96 | show cinema film picture 3.33333 97 | say man success allege 3.33333 98 | accept lawyer conviction bear 3.33333 99 | accept government recommendation receive 3.33333 100 | try judge action test 3.30769 101 | say priest mass state 3.25 102 | say people success allege 3.25 103 | draw man sword depict 3.25 104 | try judge action judge 3.23077 105 | draw parent line depict 3.16667 106 | accept shareholder offer bear 3.08333 107 | try justice king test 3.07692 108 | try tribunal crime test 3 109 | try people program judge 3 110 | show map location express 3 111 | say spokesman decision allege 3 112 | show map area express 2.91667 113 | write user word publish 2.84615 114 | write child word publish 2.84615 115 | accept user responsibility receive 2.83333 116 | accept society charge receive 2.83333 117 | meet child house visit 2.76923 118 | accept magazine liability receive 2.75 119 | write student name publish 2.69231 120 | run people round operate 2.69231 121 | say producer favour allege 2.66667 122 | accept government proposal bear 2.66667 123 | try court offence test 2.61538 124 | run shiver spine move 2.61538 125 | show study correlation picture 2.58333 126 | accept member share bear 2.58333 127 | accept management responsibility receive 2.58333 128 | accept lawyer conviction receive 2.58333 129 | accept government proposal receive 2.58333 130 | run runway parallel operate 2.53846 131 | meet sight eye satisfy 2.53846 132 | show poll support picture 2.5 133 | show child sign picture 2.5 134 | show child sign express 2.5 135 | show artist work picture 2.5 136 | show artist work express 2.5 137 | write scholar book spell 2.46154 138 | say speaker word allege 2.41667 139 | provide study information leave 2.41667 140 | provide mother baby leave 2.30769 141 | meet river sea visit 2.23077 142 | meet boy girl satisfy 2.23077 143 | draw table eye depict 2.16667 144 | write specialist papers spell 2.15385 145 | run runway parallel move 2.15385 146 | draw claimant benefit attract 2.08333 147 | try man experiment judge 2.07692 148 | run road parallel move 2.07692 149 | provide government cash leave 2.07692 150 | write writer book spell 2 151 | write people letter spell 2 152 | provide study evidence leave 2 153 | meet representative official satisfy 2 154 | draw patient figure attract 2 155 | try demonstrator building judge 1.92308 156 | show cinema film express 1.91667 157 | say priest prayer allege 1.91667 158 | provide family home leave 1.84615 159 | meet river sea satisfy 1.84615 160 | say people prayer allege 1.83333 161 | draw proposal criticism depict 1.83333 162 | draw parent line attract 1.83333 163 | write teacher book spell 1.76923 164 | run sweat face move 1.76923 165 | run road parallel operate 1.76923 166 | meet service need visit 1.76923 167 | draw report attention depict 1.75 168 | buy weapon chance bribe 1.75 169 | try people sniffing judge 1.69231 170 | run shiver spine operate 1.69231 171 | provide system facility leave 1.66667 172 | draw woman hair attract 1.66667 173 | buy model behaviour bribe 1.66667 174 | run machine program move 1.61538 175 | meet system criterion visit 1.61538 176 | meet child house satisfy 1.61538 177 | meet beach standard visit 1.61538 178 | say priest mass allege 1.58333 179 | try people door judge 1.53846 180 | show child interest picture 1.5 181 | provide school education leave 1.5 182 | provide company service leave 1.5 183 | draw claimant benefit depict 1.5 184 | draw child picture attract 1.5 185 | meet system requirement visit 1.46154 186 | meet student requirement visit 1.46154 187 | buy people product bribe 1.46154 188 | write user software spell 1.38462 189 | run people company move 1.38462 190 | provide employer training leave 1.38462 191 | buy people house bribe 1.38462 192 | draw man sword attract 1.33333 193 | run user application move 1.30769 194 | buy investor share bribe 1.30769 195 | run family hotel move 1.23077 196 | buy supporter ticket bribe 1.23077 197 | run sweat face operate 1.15385 198 | buy employee property bribe 1.15385 199 | run system application move 1.07692 200 | -------------------------------------------------------------------------------- /test_data/VO+S_ave.txt: -------------------------------------------------------------------------------- 1 | design reduce amount company cut cost 6.55556 2 | programme offer support service provide help 6.33333 3 | report stress importance study emphasise need 6.22222 4 | case require attention patient need treatment 6.16667 5 | product satisfy demand commodity meet requirement 6.11111 6 | government use power authority exercise influence 6 7 | people share interest member express view 5.72222 8 | employee start work graduate begin career 5.66667 9 | medication achieve result student reach level 5.61111 10 | medication achieve result drug produce effect 5.5 11 | pupil achieve end student reach level 5.22222 12 | pupil use knowledge authority exercise influence 5.11111 13 | superpower fight war force win battle 4.88889 14 | committee consider matter panel discuss issue 4.88889 15 | panel discuss issue project present problem 4.83333 16 | paper address question study pose problem 4.72222 17 | service provide help product satisfy demand 4.55556 18 | paper address question project present problem 4.55556 19 | researcher develop technique system use method 4.44444 20 | pupil use knowledge student acquire skill 4.38889 21 | team win match people play game 4.22222 22 | family receive letter survey collect information 4.16667 23 | force win battle people play game 4.05556 24 | study pose problem programme face difficulty 4 25 | member attend conference group hold meeting 4 26 | father stretch arm man wave hand 3.94444 27 | superpower fight war team win match 3.88889 28 | researcher develop technique government use power 3.83333 29 | charity help people school encourage child 3.77778 30 | survey collect information page provide datum 3.66667 31 | parent set example company provide system 3.61111 32 | project present problem programme face difficulty 3.55556 33 | writer read word family receive letter 3.5 34 | force win battle commodity meet requirement 3.5 35 | student acquire skill doctor use test 3.44444 36 | man lift hand question raise head 3.44444 37 | committee consider matter study pose problem 3.44444 38 | service provide help researcher develop technique 3.38889 39 | people play game worker join party 3.38889 40 | pupil use knowledge company provide system 3.33333 41 | charity help people panel discuss issue 3.33333 42 | people share interest authority exercise influence 3.27778 43 | member express view pupil achieve end 3.27778 44 | drug produce effect employee start work 3.22222 45 | woman ask man user send message 3.16667 46 | man hear word people remember name 3.16667 47 | programme face difficulty parent set example 3.05556 48 | member attend conference people share interest 3.05556 49 | question raise head paper address question 2.94444 50 | wife pour tea worker join party 2.83333 51 | author write book writer read word 2.77778 52 | agent sell property family buy home 2.77778 53 | wife pour tea woman drink water 2.72222 54 | product satisfy demand study emphasise need 2.72222 55 | commodity meet requirement charity help people 2.72222 56 | road cross line route follow road 2.66667 57 | delegate buy land agent sell property 2.61111 58 | page provide datum student reach level 2.55556 59 | writer read word people remember name 2.5 60 | mother leave house family buy home 2.5 61 | customer pay price company cut cost 2.5 62 | company provide system government use power 2.5 63 | man shut door gentleman close eye 2.44444 64 | user send message man hear word 2.38889 65 | route follow road parent set example 2.38889 66 | drug produce effect committee consider matter 2.33333 67 | plan increase number design reduce amount 2.22222 68 | patient need treatment system use method 2.16667 69 | company suffer loss firm cause injury 2.16667 70 | user send message family receive letter 2.11111 71 | member attend conference case require attention 2.11111 72 | case require attention customer pay price 2.11111 73 | worker join party employee leave company 2.05556 74 | author write book man hear word 2.05556 75 | woman drink water system use method 2 76 | firm cause injury question raise head 1.94444 77 | gentleman close eye father stretch arm 1.88889 78 | doctor use test physician pass time 1.88889 79 | pupil achieve end gentleman close eye 1.83333 80 | member express view company suffer loss 1.83333 81 | man wave hand employee leave company 1.83333 82 | programme offer support road cross line 1.77778 83 | patient need treatment programme offer support 1.77778 84 | girl like people woman ask man 1.77778 85 | delegate buy land mother leave house 1.77778 86 | student acquire skill family buy home 1.72222 87 | group hold meeting man lift hand 1.72222 88 | customer pay price graduate begin career 1.72222 89 | physician pass time road cross line 1.66667 90 | man wave hand employee start work 1.66667 91 | agent sell property group hold meeting 1.66667 92 | girl like people plan increase number 1.61111 93 | report stress importance company cut cost 1.55556 94 | medication achieve result man lift hand 1.55556 95 | man shut door route follow road 1.55556 96 | girl like people survey collect information 1.55556 97 | superpower fight war plan increase number 1.5 98 | report stress importance firm cause injury 1.5 99 | people remember name physician pass time 1.5 100 | graduate begin career company suffer loss 1.5 101 | author write book delegate buy land 1.5 102 | woman drink water doctor use test 1.44444 103 | woman ask man father stretch arm 1.44444 104 | team win match design reduce amount 1.38889 105 | wife pour tea study emphasise need 1.33333 106 | mother leave house school encourage child 1.33333 107 | school encourage child employee leave company 1.27778 108 | man shut door page provide datum 1.05556 109 | -------------------------------------------------------------------------------- /test_data/VO+Sadd_ave.txt: -------------------------------------------------------------------------------- 1 | medication achieve result drug produce effect 6.16 2 | pupil achieve end student reach level 6.08696 3 | government use power authority exercise influence 5.8 4 | committee consider matter panel discuss issue 5.8 5 | programme offer support service provide help 5.64 6 | product satisfy demand commodity meet requirement 5.28 7 | report stress importance study emphasise need 5.16 8 | employee start work graduate begin career 4.95238 9 | pupil use knowledge student acquire skill 4.84 10 | project present problem programme face difficulty 4.8 11 | member attend conference group hold meeting 4.625 12 | paper address question project present problem 4.36364 13 | superpower fight war force win battle 4.31818 14 | design reduce amount company cut cost 4.26087 15 | case require attention patient need treatment 4.24 16 | father stretch arm man wave hand 4.08 17 | team win match people play game 4.04167 18 | study pose problem programme face difficulty 3.95833 19 | panel discuss issue project present problem 3.73913 20 | paper address question study pose problem 3.64 21 | people share interest member express view 3.625 22 | delegate buy land agent sell property 3.36 23 | road cross line route follow road 3.31818 24 | researcher develop technique system use method 3.28 25 | survey collect information page provide datum 3.26087 26 | user send message family receive letter 3.20833 27 | service provide help product satisfy demand 3.16667 28 | committee consider matter study pose problem 3.16667 29 | author write book writer read word 3.16667 30 | company suffer loss firm cause injury 3.13043 31 | agent sell property family buy home 3.125 32 | member attend conference people share interest 2.95455 33 | user send message man hear word 2.84 34 | man lift hand question raise head 2.8 35 | wife pour tea woman drink water 2.69565 36 | man hear word people remember name 2.68 37 | question raise head paper address question 2.66667 38 | people play game worker join party 2.5 39 | charity help people school encourage child 2.43478 40 | woman ask man user send message 2.21739 41 | superpower fight war team win match 2.20833 42 | doctor use test physician pass time 2.16667 43 | writer read word family receive letter 2.09091 44 | service provide help researcher develop technique 2.08696 45 | patient need treatment programme offer support 2.08696 46 | pupil use knowledge authority exercise influence 1.92 47 | customer pay price company cut cost 1.91667 48 | route follow road parent set example 1.91304 49 | company provide system government use power 1.88 50 | parent set example company provide system 1.875 51 | medication achieve result student reach level 1.84 52 | man shut door gentleman close eye 1.8 53 | member express view pupil achieve end 1.76 54 | family receive letter survey collect information 1.68 55 | author write book man hear word 1.64 56 | girl like people woman ask man 1.63636 57 | force win battle people play game 1.6 58 | mother leave house family buy home 1.54545 59 | product satisfy demand study emphasise need 1.52 60 | commodity meet requirement charity help people 1.45833 61 | plan increase number design reduce amount 1.45455 62 | writer read word people remember name 1.44 63 | worker join party employee leave company 1.4 64 | researcher develop technique government use power 1.4 65 | charity help people panel discuss issue 1.38095 66 | report stress importance company cut cost 1.375 67 | gentleman close eye father stretch arm 1.36 68 | force win battle commodity meet requirement 1.36 69 | student acquire skill family buy home 1.32 70 | pupil use knowledge company provide system 1.32 71 | people share interest authority exercise influence 1.32 72 | member attend conference case require attention 1.29167 73 | pupil achieve end gentleman close eye 1.28 74 | page provide datum student reach level 1.28 75 | patient need treatment system use method 1.24 76 | man wave hand employee leave company 1.24 77 | student acquire skill doctor use test 1.20833 78 | group hold meeting man lift hand 1.20833 79 | delegate buy land mother leave house 1.2 80 | drug produce effect committee consider matter 1.18182 81 | agent sell property group hold meeting 1.16667 82 | author write book delegate buy land 1.13043 83 | woman drink water doctor use test 1.125 84 | man wave hand employee start work 1.12 85 | firm cause injury question raise head 1.12 86 | case require attention customer pay price 1.12 87 | graduate begin career company suffer loss 1.08696 88 | woman drink water system use method 1.08333 89 | wife pour tea worker join party 1.08333 90 | report stress importance firm cause injury 1.08 91 | programme face difficulty parent set example 1.08 92 | member express view company suffer loss 1.08 93 | medication achieve result man lift hand 1.04545 94 | customer pay price graduate begin career 1.04545 95 | drug produce effect employee start work 1.04167 96 | woman ask man father stretch arm 1.04 97 | superpower fight war plan increase number 1.04 98 | school encourage child employee leave company 1.04 99 | physician pass time road cross line 1.04 100 | mother leave house school encourage child 1.04 101 | girl like people survey collect information 1.04 102 | wife pour tea study emphasise need 1 103 | team win match design reduce amount 1 104 | programme offer support road cross line 1 105 | people remember name physician pass time 1 106 | man shut door route follow road 1 107 | man shut door page provide datum 1 108 | girl like people plan increase number 1 109 | -------------------------------------------------------------------------------- /test_data/VO.txt: -------------------------------------------------------------------------------- 1 | work start career begin 7 2 | work start career begin 7 3 | work start career begin 7 4 | work start career begin 7 5 | work start career begin 7 6 | work start career begin 7 7 | word read letter receive 7 8 | water drink method use 7 9 | war fight match win 7 10 | war fight battle win 7 11 | war fight battle win 7 12 | war fight battle win 7 13 | view express end achieve 7 14 | technique develop method use 7 15 | technique develop method use 7 16 | support offer help provide 7 17 | support offer help provide 7 18 | support offer help provide 7 19 | support offer help provide 7 20 | support offer help provide 7 21 | support offer help provide 7 22 | support offer help provide 7 23 | support offer help provide 7 24 | support offer help provide 7 25 | support offer help provide 7 26 | support offer help provide 7 27 | skill acquire test use 7 28 | skill acquire test use 7 29 | road follow example set 7 30 | result achieve level reach 7 31 | result achieve level reach 7 32 | result achieve level reach 7 33 | result achieve level reach 7 34 | result achieve level reach 7 35 | result achieve level reach 7 36 | result achieve effect produce 7 37 | result achieve effect produce 7 38 | result achieve effect produce 7 39 | result achieve effect produce 7 40 | result achieve effect produce 7 41 | result achieve effect produce 7 42 | question address problem pose 7 43 | question address problem pose 7 44 | question address problem pose 7 45 | problem present difficulty face 7 46 | problem present difficulty face 7 47 | problem pose difficulty face 7 48 | power use influence exercise 7 49 | power use influence exercise 7 50 | power use influence exercise 7 51 | power use influence exercise 7 52 | power use influence exercise 7 53 | power use influence exercise 7 54 | power use influence exercise 7 55 | power use influence exercise 7 56 | power use influence exercise 7 57 | power use influence exercise 7 58 | people like number increase 7 59 | people help child encourage 7 60 | matter consider problem pose 7 61 | matter consider issue discuss 7 62 | matter consider issue discuss 7 63 | matter consider issue discuss 7 64 | match win game play 7 65 | man ask message send 7 66 | letter receive information collect 7 67 | letter receive information collect 7 68 | land buy property sell 7 69 | knowledge use skill acquire 7 70 | knowledge use skill acquire 7 71 | knowledge use skill acquire 7 72 | knowledge use influence exercise 7 73 | knowledge use influence exercise 7 74 | issue discuss problem present 7 75 | issue discuss problem present 7 76 | interest share view express 7 77 | interest share view express 7 78 | interest share view express 7 79 | interest share view express 7 80 | interest share view express 7 81 | interest share view express 7 82 | information collect datum provide 7 83 | information collect datum provide 7 84 | information collect datum provide 7 85 | importance stress need emphasise 7 86 | importance stress need emphasise 7 87 | importance stress need emphasise 7 88 | importance stress need emphasise 7 89 | importance stress need emphasise 7 90 | importance stress need emphasise 7 91 | importance stress need emphasise 7 92 | importance stress need emphasise 7 93 | importance stress need emphasise 7 94 | importance stress need emphasise 7 95 | house leave home buy 7 96 | help provide technique develop 7 97 | help provide demand satisfy 7 98 | help provide demand satisfy 7 99 | head raise question address 7 100 | head raise question address 7 101 | hand wave company leave 7 102 | hand lift head raise 7 103 | hand lift head raise 7 104 | example set system provide 7 105 | end achieve level reach 7 106 | end achieve level reach 7 107 | effect produce work start 7 108 | door shut eye close 7 109 | door shut eye close 7 110 | demand satisfy requirement meet 7 111 | demand satisfy requirement meet 7 112 | demand satisfy requirement meet 7 113 | demand satisfy requirement meet 7 114 | demand satisfy requirement meet 7 115 | demand satisfy requirement meet 7 116 | demand satisfy requirement meet 7 117 | demand satisfy requirement meet 7 118 | book write word read 7 119 | battle win requirement meet 7 120 | battle win game play 7 121 | battle win game play 7 122 | attention require treatment need 7 123 | attention require treatment need 7 124 | attention require treatment need 7 125 | attention require treatment need 7 126 | attention require treatment need 7 127 | attention require treatment need 7 128 | attention require treatment need 7 129 | attention require treatment need 7 130 | attention require treatment need 7 131 | attention require treatment need 7 132 | amount reduce cost cut 7 133 | amount reduce cost cut 7 134 | amount reduce cost cut 7 135 | amount reduce cost cut 7 136 | amount reduce cost cut 7 137 | amount reduce cost cut 7 138 | amount reduce cost cut 7 139 | amount reduce cost cut 7 140 | amount reduce cost cut 7 141 | amount reduce cost cut 7 142 | amount reduce cost cut 7 143 | work start career begin 6 144 | work start career begin 6 145 | work start career begin 6 146 | work start career begin 6 147 | work start career begin 6 148 | work start career begin 6 149 | word read name remember 6 150 | word read letter receive 6 151 | word read letter receive 6 152 | word hear name remember 6 153 | word hear name remember 6 154 | war fight match win 6 155 | war fight match win 6 156 | war fight battle win 6 157 | war fight battle win 6 158 | technique develop power use 6 159 | technique develop power use 6 160 | technique develop method use 6 161 | technique develop method use 6 162 | technique develop method use 6 163 | tea pour water drink 6 164 | tea pour party join 6 165 | system provide power use 6 166 | support offer line cross 6 167 | support offer help provide 6 168 | support offer help provide 6 169 | support offer help provide 6 170 | support offer help provide 6 171 | result achieve level reach 6 172 | result achieve level reach 6 173 | result achieve level reach 6 174 | result achieve level reach 6 175 | result achieve effect produce 6 176 | result achieve effect produce 6 177 | result achieve effect produce 6 178 | result achieve effect produce 6 179 | question address problem present 6 180 | question address problem present 6 181 | question address problem present 6 182 | question address problem present 6 183 | question address problem present 6 184 | question address problem present 6 185 | question address problem present 6 186 | question address problem pose 6 187 | question address problem pose 6 188 | question address problem pose 6 189 | property sell home buy 6 190 | problem pose difficulty face 6 191 | problem pose difficulty face 6 192 | problem pose difficulty face 6 193 | price pay cost cut 6 194 | price pay cost cut 6 195 | power use influence exercise 6 196 | power use influence exercise 6 197 | power use influence exercise 6 198 | people like man ask 6 199 | people help issue discuss 6 200 | people help issue discuss 6 201 | people help child encourage 6 202 | people help child encourage 6 203 | people help child encourage 6 204 | number increase amount reduce 6 205 | matter consider problem pose 6 206 | matter consider issue discuss 6 207 | matter consider issue discuss 6 208 | matter consider issue discuss 6 209 | match win game play 6 210 | match win game play 6 211 | match win game play 6 212 | man ask message send 6 213 | man ask message send 6 214 | line cross road follow 6 215 | line cross road follow 6 216 | letter receive information collect 6 217 | knowledge use system provide 6 218 | knowledge use influence exercise 6 219 | knowledge use influence exercise 6 220 | knowledge use influence exercise 6 221 | knowledge use influence exercise 6 222 | knowledge use influence exercise 6 223 | knowledge use influence exercise 6 224 | issue discuss problem present 6 225 | issue discuss problem present 6 226 | issue discuss problem present 6 227 | issue discuss problem present 6 228 | issue discuss problem present 6 229 | issue discuss problem present 6 230 | interest share view express 6 231 | interest share view express 6 232 | interest share view express 6 233 | interest share view express 6 234 | interest share influence exercise 6 235 | interest share influence exercise 6 236 | importance stress need emphasise 6 237 | importance stress need emphasise 6 238 | importance stress need emphasise 6 239 | importance stress need emphasise 6 240 | help provide technique develop 6 241 | help provide technique develop 6 242 | help provide demand satisfy 6 243 | help provide demand satisfy 6 244 | help provide demand satisfy 6 245 | help provide demand satisfy 6 246 | help provide demand satisfy 6 247 | head raise question address 6 248 | game play party join 6 249 | example set system provide 6 250 | example set system provide 6 251 | example set system provide 6 252 | end achieve level reach 6 253 | end achieve level reach 6 254 | end achieve level reach 6 255 | end achieve level reach 6 256 | effect produce matter consider 6 257 | demand satisfy requirement meet 6 258 | demand satisfy requirement meet 6 259 | demand satisfy requirement meet 6 260 | demand satisfy requirement meet 6 261 | demand satisfy requirement meet 6 262 | demand satisfy requirement meet 6 263 | demand satisfy requirement meet 6 264 | demand satisfy need emphasise 6 265 | conference attend meeting hold 6 266 | conference attend meeting hold 6 267 | conference attend meeting hold 6 268 | conference attend meeting hold 6 269 | conference attend meeting hold 6 270 | conference attend interest share 6 271 | book write word hear 6 272 | book write land buy 6 273 | attention require treatment need 6 274 | attention require treatment need 6 275 | attention require treatment need 6 276 | attention require treatment need 6 277 | attention require treatment need 6 278 | arm stretch hand wave 6 279 | arm stretch hand wave 6 280 | amount reduce cost cut 6 281 | amount reduce cost cut 6 282 | amount reduce cost cut 6 283 | amount reduce cost cut 6 284 | amount reduce cost cut 6 285 | amount reduce cost cut 6 286 | work start career begin 5 287 | work start career begin 5 288 | work start career begin 5 289 | word read name remember 5 290 | word read letter receive 5 291 | word read letter receive 5 292 | word hear name remember 5 293 | word hear name remember 5 294 | word hear name remember 5 295 | war fight match win 5 296 | war fight match win 5 297 | war fight match win 5 298 | war fight match win 5 299 | war fight battle win 5 300 | war fight battle win 5 301 | war fight battle win 5 302 | war fight battle win 5 303 | war fight battle win 5 304 | war fight battle win 5 305 | war fight battle win 5 306 | view express end achieve 5 307 | view express end achieve 5 308 | view express end achieve 5 309 | treatment need method use 5 310 | technique develop power use 5 311 | technique develop power use 5 312 | technique develop power use 5 313 | technique develop power use 5 314 | technique develop power use 5 315 | technique develop method use 5 316 | technique develop method use 5 317 | technique develop method use 5 318 | technique develop method use 5 319 | technique develop method use 5 320 | tea pour water drink 5 321 | tea pour water drink 5 322 | tea pour party join 5 323 | tea pour party join 5 324 | tea pour party join 5 325 | system provide power use 5 326 | system provide power use 5 327 | support offer help provide 5 328 | skill acquire test use 5 329 | skill acquire test use 5 330 | road follow example set 5 331 | road follow example set 5 332 | result achieve level reach 5 333 | result achieve level reach 5 334 | result achieve level reach 5 335 | result achieve level reach 5 336 | result achieve level reach 5 337 | result achieve level reach 5 338 | result achieve effect produce 5 339 | result achieve effect produce 5 340 | result achieve effect produce 5 341 | result achieve effect produce 5 342 | result achieve effect produce 5 343 | requirement meet people help 5 344 | requirement meet people help 5 345 | question address problem present 5 346 | question address problem present 5 347 | question address problem present 5 348 | question address problem pose 5 349 | question address problem pose 5 350 | question address problem pose 5 351 | question address problem pose 5 352 | question address problem pose 5 353 | property sell home buy 5 354 | problem present difficulty face 5 355 | problem present difficulty face 5 356 | problem present difficulty face 5 357 | problem present difficulty face 5 358 | problem present difficulty face 5 359 | problem pose difficulty face 5 360 | problem pose difficulty face 5 361 | price pay cost cut 5 362 | power use influence exercise 5 363 | power use influence exercise 5 364 | people help issue discuss 5 365 | people help child encourage 5 366 | people help child encourage 5 367 | message send word hear 5 368 | message send word hear 5 369 | message send letter receive 5 370 | meeting hold hand lift 5 371 | matter consider problem pose 5 372 | matter consider problem pose 5 373 | matter consider problem pose 5 374 | matter consider issue discuss 5 375 | matter consider issue discuss 5 376 | matter consider issue discuss 5 377 | matter consider issue discuss 5 378 | matter consider issue discuss 5 379 | matter consider issue discuss 5 380 | matter consider issue discuss 5 381 | match win game play 5 382 | match win game play 5 383 | match win game play 5 384 | match win game play 5 385 | match win game play 5 386 | match win game play 5 387 | man ask message send 5 388 | loss suffer injury cause 5 389 | line cross road follow 5 390 | letter receive information collect 5 391 | letter receive information collect 5 392 | letter receive information collect 5 393 | land buy property sell 5 394 | knowledge use system provide 5 395 | knowledge use system provide 5 396 | knowledge use system provide 5 397 | knowledge use system provide 5 398 | knowledge use skill acquire 5 399 | knowledge use skill acquire 5 400 | knowledge use skill acquire 5 401 | knowledge use skill acquire 5 402 | knowledge use skill acquire 5 403 | knowledge use skill acquire 5 404 | knowledge use skill acquire 5 405 | knowledge use influence exercise 5 406 | knowledge use influence exercise 5 407 | knowledge use influence exercise 5 408 | knowledge use influence exercise 5 409 | knowledge use influence exercise 5 410 | knowledge use influence exercise 5 411 | issue discuss problem present 5 412 | issue discuss problem present 5 413 | interest share view express 5 414 | interest share view express 5 415 | interest share view express 5 416 | interest share view express 5 417 | interest share view express 5 418 | interest share view express 5 419 | interest share influence exercise 5 420 | interest share influence exercise 5 421 | interest share influence exercise 5 422 | information collect datum provide 5 423 | information collect datum provide 5 424 | information collect datum provide 5 425 | importance stress need emphasise 5 426 | importance stress need emphasise 5 427 | importance stress need emphasise 5 428 | house leave home buy 5 429 | help provide technique develop 5 430 | help provide technique develop 5 431 | help provide demand satisfy 5 432 | help provide demand satisfy 5 433 | help provide demand satisfy 5 434 | head raise question address 5 435 | hand lift head raise 5 436 | game play party join 5 437 | game play party join 5 438 | game play party join 5 439 | game play party join 5 440 | eye close arm stretch 5 441 | eye close arm stretch 5 442 | example set system provide 5 443 | example set system provide 5 444 | end achieve level reach 5 445 | end achieve level reach 5 446 | end achieve level reach 5 447 | end achieve level reach 5 448 | end achieve level reach 5 449 | end achieve level reach 5 450 | end achieve level reach 5 451 | end achieve level reach 5 452 | end achieve level reach 5 453 | effect produce work start 5 454 | effect produce work start 5 455 | effect produce work start 5 456 | door shut eye close 5 457 | door shut eye close 5 458 | difficulty face example set 5 459 | difficulty face example set 5 460 | difficulty face example set 5 461 | demand satisfy requirement meet 5 462 | demand satisfy requirement meet 5 463 | demand satisfy need emphasise 5 464 | datum provide level reach 5 465 | conference attend meeting hold 5 466 | conference attend meeting hold 5 467 | conference attend meeting hold 5 468 | conference attend meeting hold 5 469 | conference attend interest share 5 470 | conference attend interest share 5 471 | conference attend attention require 5 472 | book write word read 5 473 | battle win requirement meet 5 474 | battle win requirement meet 5 475 | battle win requirement meet 5 476 | battle win game play 5 477 | battle win game play 5 478 | battle win game play 5 479 | battle win game play 5 480 | battle win game play 5 481 | attention require price pay 5 482 | attention require price pay 5 483 | arm stretch hand wave 5 484 | arm stretch hand wave 5 485 | arm stretch hand wave 5 486 | arm stretch hand wave 5 487 | arm stretch hand wave 5 488 | arm stretch hand wave 5 489 | amount reduce cost cut 5 490 | work start career begin 4 491 | word read name remember 4 492 | word read letter receive 4 493 | word read letter receive 4 494 | word read letter receive 4 495 | word hear name remember 4 496 | word hear name remember 4 497 | water drink method use 4 498 | war fight number increase 4 499 | war fight match win 4 500 | war fight match win 4 501 | war fight match win 4 502 | war fight match win 4 503 | war fight battle win 4 504 | war fight battle win 4 505 | war fight battle win 4 506 | view express loss suffer 4 507 | view express end achieve 4 508 | view express end achieve 4 509 | view express end achieve 4 510 | view express end achieve 4 511 | treatment need support offer 4 512 | treatment need method use 4 513 | test use time pass 4 514 | technique develop power use 4 515 | technique develop power use 4 516 | technique develop power use 4 517 | technique develop method use 4 518 | technique develop method use 4 519 | technique develop method use 4 520 | tea pour water drink 4 521 | tea pour water drink 4 522 | tea pour water drink 4 523 | tea pour party join 4 524 | tea pour party join 4 525 | system provide power use 4 526 | system provide power use 4 527 | support offer help provide 4 528 | support offer help provide 4 529 | skill acquire test use 4 530 | skill acquire test use 4 531 | skill acquire test use 4 532 | skill acquire test use 4 533 | skill acquire home buy 4 534 | result achieve effect produce 4 535 | requirement meet people help 4 536 | requirement meet people help 4 537 | requirement meet people help 4 538 | question address problem present 4 539 | question address problem present 4 540 | question address problem present 4 541 | question address problem pose 4 542 | question address problem pose 4 543 | question address problem pose 4 544 | property sell meeting hold 4 545 | property sell home buy 4 546 | property sell home buy 4 547 | property sell home buy 4 548 | property sell home buy 4 549 | property sell home buy 4 550 | problem present difficulty face 4 551 | problem pose difficulty face 4 552 | problem pose difficulty face 4 553 | problem pose difficulty face 4 554 | problem pose difficulty face 4 555 | problem pose difficulty face 4 556 | problem pose difficulty face 4 557 | price pay cost cut 4 558 | price pay career begin 4 559 | power use influence exercise 4 560 | power use influence exercise 4 561 | people like number increase 4 562 | people like man ask 4 563 | people like information collect 4 564 | people help issue discuss 4 565 | people help issue discuss 4 566 | people help issue discuss 4 567 | people help issue discuss 4 568 | people help issue discuss 4 569 | people help issue discuss 4 570 | people help child encourage 4 571 | people help child encourage 4 572 | people help child encourage 4 573 | people help child encourage 4 574 | party join company leave 4 575 | party join company leave 4 576 | number increase amount reduce 4 577 | number increase amount reduce 4 578 | number increase amount reduce 4 579 | message send word hear 4 580 | message send word hear 4 581 | message send word hear 4 582 | message send letter receive 4 583 | message send letter receive 4 584 | matter consider problem pose 4 585 | matter consider problem pose 4 586 | matter consider problem pose 4 587 | matter consider problem pose 4 588 | matter consider issue discuss 4 589 | matter consider issue discuss 4 590 | match win game play 4 591 | match win game play 4 592 | match win game play 4 593 | man ask message send 4 594 | man ask message send 4 595 | man ask message send 4 596 | man ask arm stretch 4 597 | loss suffer injury cause 4 598 | loss suffer injury cause 4 599 | line cross road follow 4 600 | letter receive information collect 4 601 | letter receive information collect 4 602 | letter receive information collect 4 603 | letter receive information collect 4 604 | letter receive information collect 4 605 | land buy property sell 4 606 | land buy property sell 4 607 | land buy property sell 4 608 | land buy property sell 4 609 | land buy house leave 4 610 | land buy house leave 4 611 | knowledge use system provide 4 612 | knowledge use system provide 4 613 | knowledge use skill acquire 4 614 | knowledge use skill acquire 4 615 | knowledge use skill acquire 4 616 | knowledge use influence exercise 4 617 | issue discuss problem present 4 618 | issue discuss problem present 4 619 | issue discuss problem present 4 620 | issue discuss problem present 4 621 | interest share view express 4 622 | interest share influence exercise 4 623 | interest share influence exercise 4 624 | injury cause head raise 4 625 | injury cause head raise 4 626 | information collect datum provide 4 627 | information collect datum provide 4 628 | importance stress cost cut 4 629 | help provide technique develop 4 630 | help provide technique develop 4 631 | help provide demand satisfy 4 632 | help provide demand satisfy 4 633 | help provide demand satisfy 4 634 | head raise question address 4 635 | head raise question address 4 636 | hand wave company leave 4 637 | hand lift head raise 4 638 | hand lift head raise 4 639 | hand lift head raise 4 640 | hand lift head raise 4 641 | hand lift head raise 4 642 | game play party join 4 643 | game play party join 4 644 | game play party join 4 645 | game play party join 4 646 | eye close arm stretch 4 647 | example set system provide 4 648 | end achieve level reach 4 649 | end achieve level reach 4 650 | end achieve eye close 4 651 | effect produce work start 4 652 | effect produce work start 4 653 | effect produce work start 4 654 | effect produce work start 4 655 | effect produce matter consider 4 656 | effect produce matter consider 4 657 | difficulty face example set 4 658 | difficulty face example set 4 659 | difficulty face example set 4 660 | difficulty face example set 4 661 | difficulty face example set 4 662 | demand satisfy need emphasise 4 663 | demand satisfy need emphasise 4 664 | demand satisfy need emphasise 4 665 | datum provide level reach 4 666 | datum provide level reach 4 667 | datum provide level reach 4 668 | datum provide level reach 4 669 | datum provide level reach 4 670 | conference attend meeting hold 4 671 | conference attend interest share 4 672 | conference attend interest share 4 673 | conference attend interest share 4 674 | conference attend interest share 4 675 | conference attend attention require 4 676 | book write word read 4 677 | book write word read 4 678 | book write word read 4 679 | book write word read 4 680 | book write word read 4 681 | battle win requirement meet 4 682 | battle win requirement meet 4 683 | battle win requirement meet 4 684 | battle win requirement meet 4 685 | battle win requirement meet 4 686 | battle win game play 4 687 | battle win game play 4 688 | battle win game play 4 689 | battle win game play 4 690 | battle win game play 4 691 | attention require treatment need 4 692 | attention require treatment need 4 693 | attention require price pay 4 694 | arm stretch hand wave 4 695 | arm stretch hand wave 4 696 | arm stretch hand wave 4 697 | work start career begin 3 698 | word read name remember 3 699 | word read name remember 3 700 | word read name remember 3 701 | word read letter receive 3 702 | word read letter receive 3 703 | word read letter receive 3 704 | word hear name remember 3 705 | word hear name remember 3 706 | word hear name remember 3 707 | word hear name remember 3 708 | word hear name remember 3 709 | water drink test use 3 710 | water drink method use 3 711 | war fight number increase 3 712 | war fight number increase 3 713 | war fight number increase 3 714 | war fight match win 3 715 | war fight match win 3 716 | war fight battle win 3 717 | war fight battle win 3 718 | view express loss suffer 3 719 | view express loss suffer 3 720 | view express end achieve 3 721 | view express end achieve 3 722 | treatment need support offer 3 723 | treatment need support offer 3 724 | treatment need support offer 3 725 | treatment need method use 3 726 | treatment need method use 3 727 | treatment need method use 3 728 | treatment need method use 3 729 | time pass line cross 3 730 | time pass line cross 3 731 | time pass line cross 3 732 | time pass line cross 3 733 | time pass line cross 3 734 | test use time pass 3 735 | test use time pass 3 736 | test use time pass 3 737 | test use time pass 3 738 | technique develop power use 3 739 | technique develop power use 3 740 | technique develop power use 3 741 | technique develop power use 3 742 | technique develop method use 3 743 | technique develop method use 3 744 | technique develop method use 3 745 | tea pour water drink 3 746 | tea pour water drink 3 747 | tea pour water drink 3 748 | tea pour party join 3 749 | tea pour party join 3 750 | tea pour need emphasise 3 751 | system provide power use 3 752 | support offer line cross 3 753 | support offer line cross 3 754 | skill acquire test use 3 755 | skill acquire test use 3 756 | skill acquire test use 3 757 | skill acquire test use 3 758 | skill acquire test use 3 759 | skill acquire home buy 3 760 | skill acquire home buy 3 761 | skill acquire home buy 3 762 | road follow example set 3 763 | road follow example set 3 764 | road follow example set 3 765 | result achieve level reach 3 766 | result achieve hand lift 3 767 | requirement meet people help 3 768 | requirement meet people help 3 769 | requirement meet people help 3 770 | requirement meet people help 3 771 | question address problem present 3 772 | question address problem present 3 773 | question address problem present 3 774 | question address problem present 3 775 | question address problem pose 3 776 | question address problem pose 3 777 | property sell meeting hold 3 778 | property sell meeting hold 3 779 | property sell home buy 3 780 | property sell home buy 3 781 | property sell home buy 3 782 | property sell home buy 3 783 | problem present difficulty face 3 784 | problem present difficulty face 3 785 | problem present difficulty face 3 786 | problem present difficulty face 3 787 | problem pose difficulty face 3 788 | problem pose difficulty face 3 789 | price pay cost cut 3 790 | price pay cost cut 3 791 | price pay cost cut 3 792 | price pay career begin 3 793 | price pay career begin 3 794 | people like information collect 3 795 | people like information collect 3 796 | people help issue discuss 3 797 | people help issue discuss 3 798 | people help issue discuss 3 799 | people help issue discuss 3 800 | people help issue discuss 3 801 | people help child encourage 3 802 | people help child encourage 3 803 | people help child encourage 3 804 | party join company leave 3 805 | party join company leave 3 806 | party join company leave 3 807 | party join company leave 3 808 | number increase amount reduce 3 809 | number increase amount reduce 3 810 | name remember time pass 3 811 | name remember time pass 3 812 | name remember time pass 3 813 | message send word hear 3 814 | message send word hear 3 815 | message send letter receive 3 816 | message send letter receive 3 817 | message send letter receive 3 818 | meeting hold hand lift 3 819 | matter consider problem pose 3 820 | matter consider problem pose 3 821 | matter consider problem pose 3 822 | matter consider problem pose 3 823 | matter consider issue discuss 3 824 | match win game play 3 825 | match win game play 3 826 | match win amount reduce 3 827 | man ask message send 3 828 | man ask message send 3 829 | man ask message send 3 830 | man ask message send 3 831 | man ask arm stretch 3 832 | loss suffer injury cause 3 833 | loss suffer injury cause 3 834 | line cross road follow 3 835 | line cross road follow 3 836 | line cross road follow 3 837 | line cross road follow 3 838 | line cross road follow 3 839 | line cross road follow 3 840 | letter receive information collect 3 841 | letter receive information collect 3 842 | letter receive information collect 3 843 | letter receive information collect 3 844 | letter receive information collect 3 845 | letter receive information collect 3 846 | land buy property sell 3 847 | land buy property sell 3 848 | land buy property sell 3 849 | land buy house leave 3 850 | land buy house leave 3 851 | land buy house leave 3 852 | knowledge use system provide 3 853 | knowledge use system provide 3 854 | knowledge use system provide 3 855 | knowledge use system provide 3 856 | knowledge use system provide 3 857 | knowledge use system provide 3 858 | knowledge use system provide 3 859 | knowledge use skill acquire 3 860 | knowledge use skill acquire 3 861 | knowledge use skill acquire 3 862 | knowledge use influence exercise 3 863 | knowledge use influence exercise 3 864 | issue discuss problem present 3 865 | issue discuss problem present 3 866 | issue discuss problem present 3 867 | interest share view express 3 868 | interest share influence exercise 3 869 | interest share influence exercise 3 870 | interest share influence exercise 3 871 | interest share influence exercise 3 872 | interest share influence exercise 3 873 | interest share influence exercise 3 874 | injury cause head raise 3 875 | injury cause head raise 3 876 | injury cause head raise 3 877 | information collect datum provide 3 878 | information collect datum provide 3 879 | information collect datum provide 3 880 | importance stress need emphasise 3 881 | importance stress injury cause 3 882 | importance stress injury cause 3 883 | importance stress injury cause 3 884 | importance stress cost cut 3 885 | importance stress cost cut 3 886 | house leave home buy 3 887 | house leave home buy 3 888 | house leave home buy 3 889 | house leave home buy 3 890 | house leave home buy 3 891 | house leave child encourage 3 892 | house leave child encourage 3 893 | house leave child encourage 3 894 | help provide technique develop 3 895 | help provide technique develop 3 896 | help provide technique develop 3 897 | help provide technique develop 3 898 | help provide demand satisfy 3 899 | help provide demand satisfy 3 900 | help provide demand satisfy 3 901 | head raise question address 3 902 | hand wave work start 3 903 | hand wave work start 3 904 | hand wave work start 3 905 | hand wave work start 3 906 | hand wave company leave 3 907 | hand lift head raise 3 908 | hand lift head raise 3 909 | hand lift head raise 3 910 | hand lift head raise 3 911 | game play party join 3 912 | game play party join 3 913 | game play party join 3 914 | eye close arm stretch 3 915 | example set system provide 3 916 | example set system provide 3 917 | example set system provide 3 918 | example set system provide 3 919 | example set system provide 3 920 | example set system provide 3 921 | example set system provide 3 922 | end achieve level reach 3 923 | end achieve eye close 3 924 | end achieve eye close 3 925 | end achieve eye close 3 926 | effect produce work start 3 927 | effect produce work start 3 928 | effect produce work start 3 929 | effect produce matter consider 3 930 | effect produce matter consider 3 931 | effect produce matter consider 3 932 | effect produce matter consider 3 933 | door shut road follow 3 934 | door shut road follow 3 935 | door shut road follow 3 936 | door shut road follow 3 937 | difficulty face example set 3 938 | difficulty face example set 3 939 | difficulty face example set 3 940 | difficulty face example set 3 941 | demand satisfy need emphasise 3 942 | demand satisfy need emphasise 3 943 | demand satisfy need emphasise 3 944 | demand satisfy need emphasise 3 945 | datum provide level reach 3 946 | datum provide level reach 3 947 | conference attend meeting hold 3 948 | conference attend meeting hold 3 949 | conference attend meeting hold 3 950 | conference attend interest share 3 951 | conference attend interest share 3 952 | conference attend interest share 3 953 | conference attend interest share 3 954 | conference attend attention require 3 955 | conference attend attention require 3 956 | career begin loss suffer 3 957 | book write word read 3 958 | book write word read 3 959 | book write word read 3 960 | book write word hear 3 961 | book write word hear 3 962 | book write word hear 3 963 | book write word hear 3 964 | battle win requirement meet 3 965 | battle win requirement meet 3 966 | battle win requirement meet 3 967 | battle win requirement meet 3 968 | battle win requirement meet 3 969 | battle win game play 3 970 | battle win game play 3 971 | battle win game play 3 972 | attention require treatment need 3 973 | attention require price pay 3 974 | attention require price pay 3 975 | arm stretch hand wave 3 976 | arm stretch hand wave 3 977 | arm stretch hand wave 3 978 | work start career begin 2 979 | word read name remember 2 980 | word read name remember 2 981 | word read name remember 2 982 | word read name remember 2 983 | word read name remember 2 984 | word read name remember 2 985 | word read name remember 2 986 | word read name remember 2 987 | word read name remember 2 988 | word read letter receive 2 989 | word read letter receive 2 990 | word read letter receive 2 991 | word read letter receive 2 992 | word read letter receive 2 993 | word read letter receive 2 994 | word hear name remember 2 995 | water drink test use 2 996 | water drink test use 2 997 | water drink test use 2 998 | water drink test use 2 999 | water drink test use 2 1000 | water drink test use 2 1001 | water drink method use 2 1002 | water drink method use 2 1003 | water drink method use 2 1004 | water drink method use 2 1005 | water drink method use 2 1006 | water drink method use 2 1007 | water drink method use 2 1008 | war fight match win 2 1009 | war fight match win 2 1010 | war fight match win 2 1011 | war fight match win 2 1012 | war fight battle win 2 1013 | view express loss suffer 2 1014 | view express loss suffer 2 1015 | view express loss suffer 2 1016 | view express loss suffer 2 1017 | view express loss suffer 2 1018 | view express loss suffer 2 1019 | view express loss suffer 2 1020 | view express loss suffer 2 1021 | view express end achieve 2 1022 | view express end achieve 2 1023 | view express end achieve 2 1024 | view express end achieve 2 1025 | view express end achieve 2 1026 | view express end achieve 2 1027 | view express end achieve 2 1028 | treatment need support offer 2 1029 | treatment need support offer 2 1030 | treatment need support offer 2 1031 | treatment need support offer 2 1032 | treatment need support offer 2 1033 | treatment need method use 2 1034 | treatment need method use 2 1035 | treatment need method use 2 1036 | treatment need method use 2 1037 | treatment need method use 2 1038 | treatment need method use 2 1039 | time pass line cross 2 1040 | time pass line cross 2 1041 | test use time pass 2 1042 | test use time pass 2 1043 | test use time pass 2 1044 | test use time pass 2 1045 | test use time pass 2 1046 | technique develop power use 2 1047 | technique develop power use 2 1048 | technique develop power use 2 1049 | technique develop power use 2 1050 | tea pour water drink 2 1051 | tea pour water drink 2 1052 | tea pour water drink 2 1053 | tea pour party join 2 1054 | tea pour party join 2 1055 | tea pour party join 2 1056 | tea pour party join 2 1057 | tea pour party join 2 1058 | tea pour party join 2 1059 | tea pour need emphasise 2 1060 | tea pour need emphasise 2 1061 | tea pour need emphasise 2 1062 | tea pour need emphasise 2 1063 | system provide power use 2 1064 | system provide power use 2 1065 | system provide power use 2 1066 | system provide power use 2 1067 | system provide power use 2 1068 | system provide power use 2 1069 | support offer line cross 2 1070 | support offer line cross 2 1071 | support offer line cross 2 1072 | support offer line cross 2 1073 | support offer line cross 2 1074 | skill acquire test use 2 1075 | skill acquire test use 2 1076 | skill acquire home buy 2 1077 | skill acquire home buy 2 1078 | skill acquire home buy 2 1079 | skill acquire home buy 2 1080 | road follow example set 2 1081 | road follow example set 2 1082 | road follow example set 2 1083 | road follow example set 2 1084 | road follow example set 2 1085 | result achieve level reach 2 1086 | result achieve hand lift 2 1087 | result achieve hand lift 2 1088 | result achieve hand lift 2 1089 | result achieve hand lift 2 1090 | result achieve hand lift 2 1091 | result achieve hand lift 2 1092 | result achieve hand lift 2 1093 | result achieve hand lift 2 1094 | result achieve effect produce 2 1095 | result achieve effect produce 2 1096 | requirement meet people help 2 1097 | requirement meet people help 2 1098 | requirement meet people help 2 1099 | requirement meet people help 2 1100 | requirement meet people help 2 1101 | requirement meet people help 2 1102 | question address problem pose 2 1103 | property sell meeting hold 2 1104 | property sell meeting hold 2 1105 | property sell meeting hold 2 1106 | property sell meeting hold 2 1107 | property sell meeting hold 2 1108 | problem present difficulty face 2 1109 | problem present difficulty face 2 1110 | problem present difficulty face 2 1111 | problem pose difficulty face 2 1112 | problem pose difficulty face 2 1113 | problem pose difficulty face 2 1114 | price pay cost cut 2 1115 | price pay cost cut 2 1116 | price pay cost cut 2 1117 | price pay cost cut 2 1118 | price pay career begin 2 1119 | price pay career begin 2 1120 | price pay career begin 2 1121 | price pay career begin 2 1122 | price pay career begin 2 1123 | price pay career begin 2 1124 | power use influence exercise 2 1125 | people like number increase 2 1126 | people like number increase 2 1127 | people like man ask 2 1128 | people like man ask 2 1129 | people like man ask 2 1130 | people like man ask 2 1131 | people like man ask 2 1132 | people like man ask 2 1133 | people like information collect 2 1134 | people like information collect 2 1135 | people like information collect 2 1136 | people help child encourage 2 1137 | people help child encourage 2 1138 | people help child encourage 2 1139 | party join company leave 2 1140 | party join company leave 2 1141 | party join company leave 2 1142 | party join company leave 2 1143 | party join company leave 2 1144 | number increase amount reduce 2 1145 | number increase amount reduce 2 1146 | number increase amount reduce 2 1147 | number increase amount reduce 2 1148 | name remember time pass 2 1149 | name remember time pass 2 1150 | name remember time pass 2 1151 | message send word hear 2 1152 | message send word hear 2 1153 | message send word hear 2 1154 | message send word hear 2 1155 | message send letter receive 2 1156 | message send letter receive 2 1157 | message send letter receive 2 1158 | message send letter receive 2 1159 | meeting hold hand lift 2 1160 | meeting hold hand lift 2 1161 | meeting hold hand lift 2 1162 | meeting hold hand lift 2 1163 | meeting hold hand lift 2 1164 | meeting hold hand lift 2 1165 | meeting hold hand lift 2 1166 | matter consider problem pose 2 1167 | matter consider issue discuss 2 1168 | match win amount reduce 2 1169 | match win amount reduce 2 1170 | match win amount reduce 2 1171 | match win amount reduce 2 1172 | match win amount reduce 2 1173 | man ask message send 2 1174 | man ask message send 2 1175 | man ask arm stretch 2 1176 | man ask arm stretch 2 1177 | man ask arm stretch 2 1178 | loss suffer injury cause 2 1179 | loss suffer injury cause 2 1180 | loss suffer injury cause 2 1181 | loss suffer injury cause 2 1182 | loss suffer injury cause 2 1183 | loss suffer injury cause 2 1184 | loss suffer injury cause 2 1185 | line cross road follow 2 1186 | letter receive information collect 2 1187 | land buy property sell 2 1188 | land buy house leave 2 1189 | land buy house leave 2 1190 | knowledge use system provide 2 1191 | knowledge use influence exercise 2 1192 | issue discuss problem present 2 1193 | interest share influence exercise 2 1194 | injury cause head raise 2 1195 | injury cause head raise 2 1196 | injury cause head raise 2 1197 | injury cause head raise 2 1198 | injury cause head raise 2 1199 | information collect datum provide 2 1200 | information collect datum provide 2 1201 | information collect datum provide 2 1202 | information collect datum provide 2 1203 | information collect datum provide 2 1204 | information collect datum provide 2 1205 | importance stress injury cause 2 1206 | importance stress injury cause 2 1207 | importance stress injury cause 2 1208 | importance stress cost cut 2 1209 | importance stress cost cut 2 1210 | importance stress cost cut 2 1211 | house leave home buy 2 1212 | house leave home buy 2 1213 | house leave home buy 2 1214 | house leave home buy 2 1215 | house leave home buy 2 1216 | house leave home buy 2 1217 | house leave home buy 2 1218 | help provide technique develop 2 1219 | help provide technique develop 2 1220 | help provide technique develop 2 1221 | help provide technique develop 2 1222 | help provide technique develop 2 1223 | head raise question address 2 1224 | head raise question address 2 1225 | head raise question address 2 1226 | head raise question address 2 1227 | head raise question address 2 1228 | head raise question address 2 1229 | hand wave work start 2 1230 | hand wave work start 2 1231 | hand wave work start 2 1232 | hand wave work start 2 1233 | hand wave company leave 2 1234 | hand wave company leave 2 1235 | hand wave company leave 2 1236 | hand wave company leave 2 1237 | hand lift head raise 2 1238 | hand lift head raise 2 1239 | hand lift head raise 2 1240 | hand lift head raise 2 1241 | hand lift head raise 2 1242 | game play party join 2 1243 | game play party join 2 1244 | game play party join 2 1245 | game play party join 2 1246 | eye close arm stretch 2 1247 | eye close arm stretch 2 1248 | eye close arm stretch 2 1249 | example set system provide 2 1250 | end achieve eye close 2 1251 | end achieve eye close 2 1252 | end achieve eye close 2 1253 | end achieve eye close 2 1254 | end achieve eye close 2 1255 | end achieve eye close 2 1256 | effect produce work start 2 1257 | effect produce work start 2 1258 | effect produce work start 2 1259 | effect produce work start 2 1260 | effect produce matter consider 2 1261 | effect produce matter consider 2 1262 | effect produce matter consider 2 1263 | effect produce matter consider 2 1264 | effect produce matter consider 2 1265 | door shut road follow 2 1266 | door shut road follow 2 1267 | door shut eye close 2 1268 | door shut eye close 2 1269 | door shut eye close 2 1270 | door shut eye close 2 1271 | door shut eye close 2 1272 | door shut eye close 2 1273 | door shut datum provide 2 1274 | difficulty face example set 2 1275 | difficulty face example set 2 1276 | demand satisfy requirement meet 2 1277 | demand satisfy need emphasise 2 1278 | demand satisfy need emphasise 2 1279 | demand satisfy need emphasise 2 1280 | demand satisfy need emphasise 2 1281 | demand satisfy need emphasise 2 1282 | datum provide level reach 2 1283 | datum provide level reach 2 1284 | datum provide level reach 2 1285 | datum provide level reach 2 1286 | datum provide level reach 2 1287 | conference attend meeting hold 2 1288 | conference attend meeting hold 2 1289 | conference attend meeting hold 2 1290 | conference attend meeting hold 2 1291 | conference attend interest share 2 1292 | conference attend interest share 2 1293 | conference attend interest share 2 1294 | conference attend interest share 2 1295 | conference attend attention require 2 1296 | conference attend attention require 2 1297 | conference attend attention require 2 1298 | conference attend attention require 2 1299 | conference attend attention require 2 1300 | conference attend attention require 2 1301 | conference attend attention require 2 1302 | conference attend attention require 2 1303 | conference attend attention require 2 1304 | child encourage company leave 2 1305 | child encourage company leave 2 1306 | child encourage company leave 2 1307 | child encourage company leave 2 1308 | child encourage company leave 2 1309 | career begin loss suffer 2 1310 | career begin loss suffer 2 1311 | career begin loss suffer 2 1312 | career begin loss suffer 2 1313 | career begin loss suffer 2 1314 | career begin loss suffer 2 1315 | career begin loss suffer 2 1316 | book write word read 2 1317 | book write word hear 2 1318 | book write word hear 2 1319 | book write word hear 2 1320 | book write word hear 2 1321 | book write word hear 2 1322 | book write word hear 2 1323 | book write land buy 2 1324 | book write land buy 2 1325 | book write land buy 2 1326 | book write land buy 2 1327 | battle win requirement meet 2 1328 | battle win requirement meet 2 1329 | battle win game play 2 1330 | battle win game play 2 1331 | attention require price pay 2 1332 | attention require price pay 2 1333 | attention require price pay 2 1334 | attention require price pay 2 1335 | attention require price pay 2 1336 | arm stretch hand wave 2 1337 | arm stretch hand wave 2 1338 | arm stretch hand wave 2 1339 | arm stretch hand wave 2 1340 | word read name remember 1 1341 | word read name remember 1 1342 | word read name remember 1 1343 | word read letter receive 1 1344 | word hear name remember 1 1345 | word hear name remember 1 1346 | word hear name remember 1 1347 | word hear name remember 1 1348 | word hear name remember 1 1349 | water drink test use 1 1350 | water drink test use 1 1351 | water drink test use 1 1352 | water drink test use 1 1353 | water drink test use 1 1354 | water drink test use 1 1355 | water drink test use 1 1356 | water drink test use 1 1357 | water drink test use 1 1358 | water drink test use 1 1359 | water drink test use 1 1360 | water drink method use 1 1361 | water drink method use 1 1362 | water drink method use 1 1363 | water drink method use 1 1364 | water drink method use 1 1365 | water drink method use 1 1366 | water drink method use 1 1367 | water drink method use 1 1368 | war fight number increase 1 1369 | war fight number increase 1 1370 | war fight number increase 1 1371 | war fight number increase 1 1372 | war fight number increase 1 1373 | war fight number increase 1 1374 | war fight number increase 1 1375 | war fight number increase 1 1376 | war fight number increase 1 1377 | war fight number increase 1 1378 | war fight number increase 1 1379 | war fight number increase 1 1380 | war fight number increase 1 1381 | war fight number increase 1 1382 | war fight match win 1 1383 | view express loss suffer 1 1384 | view express loss suffer 1 1385 | view express loss suffer 1 1386 | view express loss suffer 1 1387 | view express loss suffer 1 1388 | view express loss suffer 1 1389 | view express loss suffer 1 1390 | view express end achieve 1 1391 | treatment need support offer 1 1392 | treatment need support offer 1 1393 | treatment need support offer 1 1394 | treatment need support offer 1 1395 | treatment need support offer 1 1396 | treatment need support offer 1 1397 | treatment need support offer 1 1398 | treatment need support offer 1 1399 | treatment need support offer 1 1400 | treatment need method use 1 1401 | treatment need method use 1 1402 | treatment need method use 1 1403 | treatment need method use 1 1404 | treatment need method use 1 1405 | treatment need method use 1 1406 | time pass line cross 1 1407 | time pass line cross 1 1408 | time pass line cross 1 1409 | time pass line cross 1 1410 | time pass line cross 1 1411 | time pass line cross 1 1412 | time pass line cross 1 1413 | time pass line cross 1 1414 | time pass line cross 1 1415 | time pass line cross 1 1416 | time pass line cross 1 1417 | test use time pass 1 1418 | test use time pass 1 1419 | test use time pass 1 1420 | test use time pass 1 1421 | test use time pass 1 1422 | test use time pass 1 1423 | test use time pass 1 1424 | test use time pass 1 1425 | technique develop method use 1 1426 | technique develop method use 1 1427 | tea pour water drink 1 1428 | tea pour water drink 1 1429 | tea pour water drink 1 1430 | tea pour water drink 1 1431 | tea pour water drink 1 1432 | tea pour water drink 1 1433 | tea pour party join 1 1434 | tea pour party join 1 1435 | tea pour party join 1 1436 | tea pour party join 1 1437 | tea pour need emphasise 1 1438 | tea pour need emphasise 1 1439 | tea pour need emphasise 1 1440 | tea pour need emphasise 1 1441 | tea pour need emphasise 1 1442 | tea pour need emphasise 1 1443 | tea pour need emphasise 1 1444 | tea pour need emphasise 1 1445 | tea pour need emphasise 1 1446 | tea pour need emphasise 1 1447 | tea pour need emphasise 1 1448 | tea pour need emphasise 1 1449 | tea pour need emphasise 1 1450 | system provide power use 1 1451 | system provide power use 1 1452 | system provide power use 1 1453 | system provide power use 1 1454 | system provide power use 1 1455 | system provide power use 1 1456 | support offer line cross 1 1457 | support offer line cross 1 1458 | support offer line cross 1 1459 | support offer line cross 1 1460 | support offer line cross 1 1461 | support offer line cross 1 1462 | support offer line cross 1 1463 | support offer line cross 1 1464 | support offer line cross 1 1465 | support offer line cross 1 1466 | skill acquire test use 1 1467 | skill acquire test use 1 1468 | skill acquire test use 1 1469 | skill acquire home buy 1 1470 | skill acquire home buy 1 1471 | skill acquire home buy 1 1472 | skill acquire home buy 1 1473 | skill acquire home buy 1 1474 | skill acquire home buy 1 1475 | skill acquire home buy 1 1476 | skill acquire home buy 1 1477 | skill acquire home buy 1 1478 | skill acquire home buy 1 1479 | road follow example set 1 1480 | road follow example set 1 1481 | road follow example set 1 1482 | road follow example set 1 1483 | road follow example set 1 1484 | road follow example set 1 1485 | road follow example set 1 1486 | result achieve hand lift 1 1487 | result achieve hand lift 1 1488 | result achieve hand lift 1 1489 | result achieve hand lift 1 1490 | result achieve hand lift 1 1491 | result achieve hand lift 1 1492 | result achieve hand lift 1 1493 | result achieve hand lift 1 1494 | result achieve hand lift 1 1495 | requirement meet people help 1 1496 | requirement meet people help 1 1497 | requirement meet people help 1 1498 | question address problem present 1 1499 | question address problem pose 1 1500 | property sell meeting hold 1 1501 | property sell meeting hold 1 1502 | property sell meeting hold 1 1503 | property sell meeting hold 1 1504 | property sell meeting hold 1 1505 | property sell meeting hold 1 1506 | property sell meeting hold 1 1507 | property sell meeting hold 1 1508 | property sell meeting hold 1 1509 | property sell meeting hold 1 1510 | property sell home buy 1 1511 | property sell home buy 1 1512 | property sell home buy 1 1513 | property sell home buy 1 1514 | property sell home buy 1 1515 | property sell home buy 1 1516 | property sell home buy 1 1517 | problem present difficulty face 1 1518 | problem present difficulty face 1 1519 | problem present difficulty face 1 1520 | problem pose difficulty face 1 1521 | price pay cost cut 1 1522 | price pay cost cut 1 1523 | price pay cost cut 1 1524 | price pay cost cut 1 1525 | price pay cost cut 1 1526 | price pay cost cut 1 1527 | price pay cost cut 1 1528 | price pay career begin 1 1529 | price pay career begin 1 1530 | price pay career begin 1 1531 | price pay career begin 1 1532 | price pay career begin 1 1533 | price pay career begin 1 1534 | price pay career begin 1 1535 | price pay career begin 1 1536 | price pay career begin 1 1537 | people like number increase 1 1538 | people like number increase 1 1539 | people like number increase 1 1540 | people like number increase 1 1541 | people like number increase 1 1542 | people like number increase 1 1543 | people like number increase 1 1544 | people like number increase 1 1545 | people like number increase 1 1546 | people like number increase 1 1547 | people like number increase 1 1548 | people like number increase 1 1549 | people like number increase 1 1550 | people like number increase 1 1551 | people like man ask 1 1552 | people like man ask 1 1553 | people like man ask 1 1554 | people like man ask 1 1555 | people like man ask 1 1556 | people like man ask 1 1557 | people like man ask 1 1558 | people like man ask 1 1559 | people like man ask 1 1560 | people like man ask 1 1561 | people like information collect 1 1562 | people like information collect 1 1563 | people like information collect 1 1564 | people like information collect 1 1565 | people like information collect 1 1566 | people like information collect 1 1567 | people like information collect 1 1568 | people like information collect 1 1569 | people like information collect 1 1570 | people like information collect 1 1571 | people like information collect 1 1572 | people like information collect 1 1573 | people help issue discuss 1 1574 | people help issue discuss 1 1575 | people help issue discuss 1 1576 | people help issue discuss 1 1577 | people help child encourage 1 1578 | people help child encourage 1 1579 | party join company leave 1 1580 | party join company leave 1 1581 | party join company leave 1 1582 | party join company leave 1 1583 | party join company leave 1 1584 | party join company leave 1 1585 | party join company leave 1 1586 | number increase amount reduce 1 1587 | number increase amount reduce 1 1588 | number increase amount reduce 1 1589 | number increase amount reduce 1 1590 | number increase amount reduce 1 1591 | number increase amount reduce 1 1592 | number increase amount reduce 1 1593 | number increase amount reduce 1 1594 | name remember time pass 1 1595 | name remember time pass 1 1596 | name remember time pass 1 1597 | name remember time pass 1 1598 | name remember time pass 1 1599 | name remember time pass 1 1600 | name remember time pass 1 1601 | name remember time pass 1 1602 | name remember time pass 1 1603 | name remember time pass 1 1604 | name remember time pass 1 1605 | name remember time pass 1 1606 | message send word hear 1 1607 | message send word hear 1 1608 | message send word hear 1 1609 | message send word hear 1 1610 | message send word hear 1 1611 | message send word hear 1 1612 | message send word hear 1 1613 | message send letter receive 1 1614 | message send letter receive 1 1615 | message send letter receive 1 1616 | message send letter receive 1 1617 | message send letter receive 1 1618 | message send letter receive 1 1619 | message send letter receive 1 1620 | message send letter receive 1 1621 | meeting hold hand lift 1 1622 | meeting hold hand lift 1 1623 | meeting hold hand lift 1 1624 | meeting hold hand lift 1 1625 | meeting hold hand lift 1 1626 | meeting hold hand lift 1 1627 | meeting hold hand lift 1 1628 | meeting hold hand lift 1 1629 | meeting hold hand lift 1 1630 | matter consider problem pose 1 1631 | matter consider problem pose 1 1632 | matter consider problem pose 1 1633 | matter consider problem pose 1 1634 | matter consider issue discuss 1 1635 | match win game play 1 1636 | match win game play 1 1637 | match win game play 1 1638 | match win amount reduce 1 1639 | match win amount reduce 1 1640 | match win amount reduce 1 1641 | match win amount reduce 1 1642 | match win amount reduce 1 1643 | match win amount reduce 1 1644 | match win amount reduce 1 1645 | match win amount reduce 1 1646 | match win amount reduce 1 1647 | match win amount reduce 1 1648 | match win amount reduce 1 1649 | match win amount reduce 1 1650 | man ask message send 1 1651 | man ask message send 1 1652 | man ask message send 1 1653 | man ask message send 1 1654 | man ask message send 1 1655 | man ask arm stretch 1 1656 | man ask arm stretch 1 1657 | man ask arm stretch 1 1658 | man ask arm stretch 1 1659 | man ask arm stretch 1 1660 | man ask arm stretch 1 1661 | man ask arm stretch 1 1662 | man ask arm stretch 1 1663 | man ask arm stretch 1 1664 | man ask arm stretch 1 1665 | man ask arm stretch 1 1666 | man ask arm stretch 1 1667 | man ask arm stretch 1 1668 | loss suffer injury cause 1 1669 | loss suffer injury cause 1 1670 | loss suffer injury cause 1 1671 | loss suffer injury cause 1 1672 | loss suffer injury cause 1 1673 | loss suffer injury cause 1 1674 | line cross road follow 1 1675 | line cross road follow 1 1676 | line cross road follow 1 1677 | line cross road follow 1 1678 | line cross road follow 1 1679 | line cross road follow 1 1680 | line cross road follow 1 1681 | land buy property sell 1 1682 | land buy property sell 1 1683 | land buy property sell 1 1684 | land buy property sell 1 1685 | land buy property sell 1 1686 | land buy property sell 1 1687 | land buy property sell 1 1688 | land buy property sell 1 1689 | land buy house leave 1 1690 | land buy house leave 1 1691 | land buy house leave 1 1692 | land buy house leave 1 1693 | land buy house leave 1 1694 | land buy house leave 1 1695 | land buy house leave 1 1696 | land buy house leave 1 1697 | land buy house leave 1 1698 | land buy house leave 1 1699 | land buy house leave 1 1700 | knowledge use system provide 1 1701 | knowledge use system provide 1 1702 | knowledge use system provide 1 1703 | knowledge use skill acquire 1 1704 | knowledge use skill acquire 1 1705 | interest share influence exercise 1 1706 | interest share influence exercise 1 1707 | interest share influence exercise 1 1708 | interest share influence exercise 1 1709 | injury cause head raise 1 1710 | injury cause head raise 1 1711 | injury cause head raise 1 1712 | injury cause head raise 1 1713 | injury cause head raise 1 1714 | injury cause head raise 1 1715 | injury cause head raise 1 1716 | injury cause head raise 1 1717 | information collect datum provide 1 1718 | importance stress injury cause 1 1719 | importance stress injury cause 1 1720 | importance stress injury cause 1 1721 | importance stress injury cause 1 1722 | importance stress injury cause 1 1723 | importance stress injury cause 1 1724 | importance stress injury cause 1 1725 | importance stress injury cause 1 1726 | importance stress injury cause 1 1727 | importance stress injury cause 1 1728 | importance stress injury cause 1 1729 | importance stress injury cause 1 1730 | importance stress cost cut 1 1731 | importance stress cost cut 1 1732 | importance stress cost cut 1 1733 | importance stress cost cut 1 1734 | importance stress cost cut 1 1735 | importance stress cost cut 1 1736 | importance stress cost cut 1 1737 | importance stress cost cut 1 1738 | importance stress cost cut 1 1739 | importance stress cost cut 1 1740 | importance stress cost cut 1 1741 | importance stress cost cut 1 1742 | house leave home buy 1 1743 | house leave home buy 1 1744 | house leave home buy 1 1745 | house leave home buy 1 1746 | house leave child encourage 1 1747 | house leave child encourage 1 1748 | house leave child encourage 1 1749 | house leave child encourage 1 1750 | house leave child encourage 1 1751 | house leave child encourage 1 1752 | house leave child encourage 1 1753 | house leave child encourage 1 1754 | house leave child encourage 1 1755 | house leave child encourage 1 1756 | house leave child encourage 1 1757 | house leave child encourage 1 1758 | house leave child encourage 1 1759 | house leave child encourage 1 1760 | house leave child encourage 1 1761 | help provide technique develop 1 1762 | help provide technique develop 1 1763 | help provide demand satisfy 1 1764 | help provide demand satisfy 1 1765 | head raise question address 1 1766 | head raise question address 1 1767 | head raise question address 1 1768 | head raise question address 1 1769 | head raise question address 1 1770 | hand wave work start 1 1771 | hand wave work start 1 1772 | hand wave work start 1 1773 | hand wave work start 1 1774 | hand wave work start 1 1775 | hand wave work start 1 1776 | hand wave work start 1 1777 | hand wave work start 1 1778 | hand wave work start 1 1779 | hand wave work start 1 1780 | hand wave company leave 1 1781 | hand wave company leave 1 1782 | hand wave company leave 1 1783 | hand wave company leave 1 1784 | hand wave company leave 1 1785 | hand wave company leave 1 1786 | hand wave company leave 1 1787 | hand wave company leave 1 1788 | hand wave company leave 1 1789 | hand wave company leave 1 1790 | hand wave company leave 1 1791 | hand lift head raise 1 1792 | game play party join 1 1793 | game play party join 1 1794 | eye close arm stretch 1 1795 | eye close arm stretch 1 1796 | eye close arm stretch 1 1797 | eye close arm stretch 1 1798 | eye close arm stretch 1 1799 | eye close arm stretch 1 1800 | eye close arm stretch 1 1801 | eye close arm stretch 1 1802 | eye close arm stretch 1 1803 | eye close arm stretch 1 1804 | eye close arm stretch 1 1805 | example set system provide 1 1806 | example set system provide 1 1807 | example set system provide 1 1808 | end achieve eye close 1 1809 | end achieve eye close 1 1810 | end achieve eye close 1 1811 | end achieve eye close 1 1812 | end achieve eye close 1 1813 | end achieve eye close 1 1814 | end achieve eye close 1 1815 | end achieve eye close 1 1816 | effect produce work start 1 1817 | effect produce work start 1 1818 | effect produce work start 1 1819 | effect produce matter consider 1 1820 | effect produce matter consider 1 1821 | effect produce matter consider 1 1822 | effect produce matter consider 1 1823 | effect produce matter consider 1 1824 | effect produce matter consider 1 1825 | door shut road follow 1 1826 | door shut road follow 1 1827 | door shut road follow 1 1828 | door shut road follow 1 1829 | door shut road follow 1 1830 | door shut road follow 1 1831 | door shut road follow 1 1832 | door shut road follow 1 1833 | door shut road follow 1 1834 | door shut road follow 1 1835 | door shut road follow 1 1836 | door shut road follow 1 1837 | door shut eye close 1 1838 | door shut eye close 1 1839 | door shut eye close 1 1840 | door shut eye close 1 1841 | door shut eye close 1 1842 | door shut eye close 1 1843 | door shut eye close 1 1844 | door shut eye close 1 1845 | door shut datum provide 1 1846 | door shut datum provide 1 1847 | door shut datum provide 1 1848 | door shut datum provide 1 1849 | door shut datum provide 1 1850 | door shut datum provide 1 1851 | door shut datum provide 1 1852 | door shut datum provide 1 1853 | door shut datum provide 1 1854 | door shut datum provide 1 1855 | door shut datum provide 1 1856 | door shut datum provide 1 1857 | door shut datum provide 1 1858 | door shut datum provide 1 1859 | door shut datum provide 1 1860 | door shut datum provide 1 1861 | door shut datum provide 1 1862 | difficulty face example set 1 1863 | difficulty face example set 1 1864 | difficulty face example set 1 1865 | difficulty face example set 1 1866 | demand satisfy need emphasise 1 1867 | demand satisfy need emphasise 1 1868 | demand satisfy need emphasise 1 1869 | demand satisfy need emphasise 1 1870 | datum provide level reach 1 1871 | datum provide level reach 1 1872 | datum provide level reach 1 1873 | datum provide level reach 1 1874 | datum provide level reach 1 1875 | conference attend meeting hold 1 1876 | conference attend interest share 1 1877 | conference attend interest share 1 1878 | conference attend interest share 1 1879 | conference attend attention require 1 1880 | conference attend attention require 1 1881 | conference attend attention require 1 1882 | conference attend attention require 1 1883 | conference attend attention require 1 1884 | child encourage company leave 1 1885 | child encourage company leave 1 1886 | child encourage company leave 1 1887 | child encourage company leave 1 1888 | child encourage company leave 1 1889 | child encourage company leave 1 1890 | child encourage company leave 1 1891 | child encourage company leave 1 1892 | child encourage company leave 1 1893 | child encourage company leave 1 1894 | child encourage company leave 1 1895 | child encourage company leave 1 1896 | child encourage company leave 1 1897 | career begin loss suffer 1 1898 | career begin loss suffer 1 1899 | career begin loss suffer 1 1900 | career begin loss suffer 1 1901 | career begin loss suffer 1 1902 | career begin loss suffer 1 1903 | career begin loss suffer 1 1904 | career begin loss suffer 1 1905 | career begin loss suffer 1 1906 | career begin loss suffer 1 1907 | book write word read 1 1908 | book write word read 1 1909 | book write word read 1 1910 | book write word read 1 1911 | book write word read 1 1912 | book write word read 1 1913 | book write word read 1 1914 | book write word hear 1 1915 | book write word hear 1 1916 | book write word hear 1 1917 | book write word hear 1 1918 | book write word hear 1 1919 | book write word hear 1 1920 | book write word hear 1 1921 | book write land buy 1 1922 | book write land buy 1 1923 | book write land buy 1 1924 | book write land buy 1 1925 | book write land buy 1 1926 | book write land buy 1 1927 | book write land buy 1 1928 | book write land buy 1 1929 | book write land buy 1 1930 | book write land buy 1 1931 | book write land buy 1 1932 | book write land buy 1 1933 | book write land buy 1 1934 | battle win requirement meet 1 1935 | battle win requirement meet 1 1936 | battle win game play 1 1937 | attention require price pay 1 1938 | attention require price pay 1 1939 | attention require price pay 1 1940 | attention require price pay 1 1941 | attention require price pay 1 1942 | attention require price pay 1 1943 | attention require price pay 1 1944 | attention require price pay 1 1945 | -------------------------------------------------------------------------------- /test_data/VO_ave.txt: -------------------------------------------------------------------------------- 1 | amount reduce cost cut 6.55556 2 | support offer help provide 6.33333 3 | importance stress need emphasise 6.22222 4 | attention require treatment need 6.16667 5 | demand satisfy requirement meet 6.11111 6 | power use influence exercise 6 7 | interest share view express 5.72222 8 | work start career begin 5.66667 9 | result achieve level reach 5.61111 10 | result achieve effect produce 5.5 11 | end achieve level reach 5.22222 12 | knowledge use influence exercise 5.11111 13 | war fight battle win 4.88889 14 | matter consider issue discuss 4.88889 15 | issue discuss problem present 4.83333 16 | question address problem pose 4.72222 17 | question address problem present 4.55556 18 | help provide demand satisfy 4.55556 19 | technique develop method use 4.44444 20 | knowledge use skill acquire 4.38889 21 | match win game play 4.22222 22 | letter receive information collect 4.16667 23 | battle win game play 4.05556 24 | problem pose difficulty face 4 25 | conference attend meeting hold 4 26 | arm stretch hand wave 3.94444 27 | war fight match win 3.88889 28 | technique develop power use 3.83333 29 | people help child encourage 3.77778 30 | information collect datum provide 3.66667 31 | example set system provide 3.61111 32 | problem present difficulty face 3.55556 33 | word read letter receive 3.5 34 | battle win requirement meet 3.5 35 | skill acquire test use 3.44444 36 | matter consider problem pose 3.44444 37 | hand lift head raise 3.44444 38 | help provide technique develop 3.38889 39 | game play party join 3.38889 40 | people help issue discuss 3.33333 41 | knowledge use system provide 3.33333 42 | view express end achieve 3.27778 43 | interest share influence exercise 3.27778 44 | effect produce work start 3.22222 45 | word hear name remember 3.16667 46 | man ask message send 3.16667 47 | difficulty face example set 3.05556 48 | conference attend interest share 3.05556 49 | head raise question address 2.94444 50 | tea pour party join 2.83333 51 | property sell home buy 2.77778 52 | book write word read 2.77778 53 | tea pour water drink 2.72222 54 | requirement meet people help 2.72222 55 | demand satisfy need emphasise 2.72222 56 | line cross road follow 2.66667 57 | land buy property sell 2.61111 58 | datum provide level reach 2.55556 59 | word read name remember 2.5 60 | system provide power use 2.5 61 | price pay cost cut 2.5 62 | house leave home buy 2.5 63 | door shut eye close 2.44444 64 | road follow example set 2.38889 65 | message send word hear 2.38889 66 | effect produce matter consider 2.33333 67 | number increase amount reduce 2.22222 68 | treatment need method use 2.16667 69 | loss suffer injury cause 2.16667 70 | message send letter receive 2.11111 71 | conference attend attention require 2.11111 72 | attention require price pay 2.11111 73 | party join company leave 2.05556 74 | book write word hear 2.05556 75 | water drink method use 2 76 | injury cause head raise 1.94444 77 | test use time pass 1.88889 78 | eye close arm stretch 1.88889 79 | view express loss suffer 1.83333 80 | hand wave company leave 1.83333 81 | end achieve eye close 1.83333 82 | treatment need support offer 1.77778 83 | support offer line cross 1.77778 84 | people like man ask 1.77778 85 | land buy house leave 1.77778 86 | skill acquire home buy 1.72222 87 | price pay career begin 1.72222 88 | meeting hold hand lift 1.72222 89 | time pass line cross 1.66667 90 | property sell meeting hold 1.66667 91 | hand wave work start 1.66667 92 | people like number increase 1.61111 93 | result achieve hand lift 1.55556 94 | people like information collect 1.55556 95 | importance stress cost cut 1.55556 96 | door shut road follow 1.55556 97 | war fight number increase 1.5 98 | name remember time pass 1.5 99 | importance stress injury cause 1.5 100 | career begin loss suffer 1.5 101 | book write land buy 1.5 102 | water drink test use 1.44444 103 | man ask arm stretch 1.44444 104 | match win amount reduce 1.38889 105 | tea pour need emphasise 1.33333 106 | house leave child encourage 1.33333 107 | child encourage company leave 1.27778 108 | door shut datum provide 1.05556 109 | -------------------------------------------------------------------------------- /train_data/dummy: -------------------------------------------------------------------------------- 1 | dummy 2 | --------------------------------------------------------------------------------