├── classifier ├── Reddit_vs_Holmes │ ├── ngram │ │ ├── 1gram.acc │ │ ├── 2gram.acc │ │ ├── 3gram.acc │ │ ├── 4gram.acc │ │ ├── 1gram.p │ │ ├── 2gram.p │ │ ├── 3gram.p │ │ └── 4gram.p │ └── neural │ │ ├── model.h5 │ │ ├── args.json │ │ ├── train.tsv │ │ └── train.log └── Reddit_vs_arxiv │ ├── ngram │ ├── 1gram.acc │ ├── 2gram.acc │ ├── 3gram.acc │ ├── 4gram.acc │ ├── 1gram.p │ ├── 2gram.p │ ├── 3gram.p │ └── 4gram.p │ └── neural │ ├── model.h5 │ ├── args.json │ ├── train.tsv │ └── train.log ├── .gitignore ├── .gitattributes ├── EMNLP paper.pdf ├── EMNLP poster.pdf ├── fig └── intro_fig.png ├── src ├── infer_args.csv ├── tf_lib.py ├── shared.py ├── main.py ├── dataset.py ├── vis.py ├── classifier.py ├── decode.py └── model.py ├── data ├── arXiv │ ├── README.md │ └── arxiv.py ├── toy │ ├── vocab.txt │ ├── base_conv_vali.num │ ├── base_conv_test.num │ └── base_conv_vali.txt ├── Holmes │ └── README.md └── README.md └── README.md /classifier/Reddit_vs_Holmes/ngram/1gram.acc: -------------------------------------------------------------------------------- 1 | 0.782 -------------------------------------------------------------------------------- /classifier/Reddit_vs_Holmes/ngram/2gram.acc: -------------------------------------------------------------------------------- 1 | 0.751 -------------------------------------------------------------------------------- /classifier/Reddit_vs_Holmes/ngram/3gram.acc: -------------------------------------------------------------------------------- 1 | 0.583 -------------------------------------------------------------------------------- /classifier/Reddit_vs_Holmes/ngram/4gram.acc: -------------------------------------------------------------------------------- 1 | 0.495 -------------------------------------------------------------------------------- /classifier/Reddit_vs_arxiv/ngram/1gram.acc: -------------------------------------------------------------------------------- 1 | 0.983 -------------------------------------------------------------------------------- /classifier/Reddit_vs_arxiv/ngram/2gram.acc: -------------------------------------------------------------------------------- 1 | 0.975 -------------------------------------------------------------------------------- /classifier/Reddit_vs_arxiv/ngram/3gram.acc: -------------------------------------------------------------------------------- 1 | 0.925 -------------------------------------------------------------------------------- /classifier/Reddit_vs_arxiv/ngram/4gram.acc: -------------------------------------------------------------------------------- 1 | 0.815 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *.pyc 3 | models/ 4 | out/ 5 | temp.py 6 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /EMNLP paper.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golsun/StyleFusion/HEAD/EMNLP paper.pdf -------------------------------------------------------------------------------- /EMNLP poster.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golsun/StyleFusion/HEAD/EMNLP poster.pdf -------------------------------------------------------------------------------- /fig/intro_fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golsun/StyleFusion/HEAD/fig/intro_fig.png -------------------------------------------------------------------------------- /classifier/Reddit_vs_Holmes/ngram/1gram.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golsun/StyleFusion/HEAD/classifier/Reddit_vs_Holmes/ngram/1gram.p -------------------------------------------------------------------------------- /classifier/Reddit_vs_Holmes/ngram/2gram.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golsun/StyleFusion/HEAD/classifier/Reddit_vs_Holmes/ngram/2gram.p -------------------------------------------------------------------------------- /classifier/Reddit_vs_Holmes/ngram/3gram.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golsun/StyleFusion/HEAD/classifier/Reddit_vs_Holmes/ngram/3gram.p -------------------------------------------------------------------------------- /classifier/Reddit_vs_Holmes/ngram/4gram.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golsun/StyleFusion/HEAD/classifier/Reddit_vs_Holmes/ngram/4gram.p -------------------------------------------------------------------------------- /classifier/Reddit_vs_arxiv/neural/model.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golsun/StyleFusion/HEAD/classifier/Reddit_vs_arxiv/neural/model.h5 -------------------------------------------------------------------------------- /classifier/Reddit_vs_arxiv/ngram/1gram.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golsun/StyleFusion/HEAD/classifier/Reddit_vs_arxiv/ngram/1gram.p -------------------------------------------------------------------------------- /classifier/Reddit_vs_arxiv/ngram/2gram.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golsun/StyleFusion/HEAD/classifier/Reddit_vs_arxiv/ngram/2gram.p -------------------------------------------------------------------------------- /classifier/Reddit_vs_arxiv/ngram/3gram.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golsun/StyleFusion/HEAD/classifier/Reddit_vs_arxiv/ngram/3gram.p -------------------------------------------------------------------------------- /classifier/Reddit_vs_arxiv/ngram/4gram.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golsun/StyleFusion/HEAD/classifier/Reddit_vs_arxiv/ngram/4gram.p -------------------------------------------------------------------------------- /classifier/Reddit_vs_Holmes/neural/model.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golsun/StyleFusion/HEAD/classifier/Reddit_vs_Holmes/neural/model.h5 -------------------------------------------------------------------------------- /classifier/Reddit_vs_arxiv/neural/args.json: -------------------------------------------------------------------------------- 1 | {"mode": "train", "encoder_depth": 2, "rnn_units": 32, "mlp_depth": 2, "mlp_units": 32, "lr": 0.0001, "dropout": 0.0, "tgt_only": true, "data_name": "base10m_vs_arxiv1m", "score_path": "", "restore": "", "n_max": -1, "include_punc": false} -------------------------------------------------------------------------------- /src/infer_args.csv: -------------------------------------------------------------------------------- 1 | method,latent 2 | beam_width,100 3 | n_rand,10 4 | r_rand,3 5 | softmax_temperature,1 6 | min_logP,-1000. 7 | wt_center,0. 8 | wt_rep,0. 9 | wt_len,0. 10 | wt_base,0. 11 | wt_clf,1. 12 | crit_logP,-2.5 13 | crit_rep,-0.3 14 | rep_allow,0.9 15 | lm_wt,0.95 -------------------------------------------------------------------------------- /classifier/Reddit_vs_Holmes/neural/args.json: -------------------------------------------------------------------------------- 1 | {"mode": "train", "encoder_depth": 2, "rnn_units": 32, "mlp_depth": 2, "mlp_units": 32, "lr": 0.0001, "dropout": 0.0, "tgt_only": true, "data_name": "base10m_Vs_holmes38k", "score_path": "", "restore": "", "n_max": -1, "include_punc": false} -------------------------------------------------------------------------------- /data/arXiv/README.md: -------------------------------------------------------------------------------- 1 | To build the arXiv-style dataset, please 2 | * first download the raw LaTeX source files `hep-th XXXX` from [this webiste](http://www.cs.cornell.edu/projects/kddcup/datasets.html). In the paper, we used the data from 1998 to 2002. 3 | * unzip the downloaded `hep-th XXXX.tar.gz` files 4 | * process the files with `python arxiv.py` -------------------------------------------------------------------------------- /data/toy/vocab.txt: -------------------------------------------------------------------------------- 1 | _SOS_ 2 | _EOS_ 3 | _UNK_ 4 | you 5 | ? 6 | . 7 | this 8 | so 9 | like 10 | how 11 | hi 12 | game 13 | do 14 | are 15 | , 16 | ! 17 | not 18 | it 19 | yes 20 | well 21 | the 22 | one 23 | much 24 | just 25 | is 26 | i 27 | good 28 | fine 29 | boring 30 | best 31 | bad 32 | 's 33 | 'm 34 | a 35 | assumption 36 | reasonable 37 | valid 38 | to 39 | be -------------------------------------------------------------------------------- /src/tf_lib.py: -------------------------------------------------------------------------------- 1 | 2 | from keras.models import Model, load_model, model_from_yaml 3 | from keras.layers import Input, GRU, Dense, Embedding, Dropout, Concatenate, Lambda, Add, Subtract, Multiply, GaussianNoise 4 | from keras.utils import plot_model 5 | from keras.callbacks import ModelCheckpoint 6 | from keras.optimizers import Adam, RMSprop 7 | from keras.callbacks import Callback 8 | from keras import backend as K 9 | import tensorflow as tf 10 | from keras.activations import hard_sigmoid 11 | import keras 12 | -------------------------------------------------------------------------------- /data/Holmes/README.md: -------------------------------------------------------------------------------- 1 | This folder contains the stylized dataset built upon the following Sherlock Holmes books by Arthur Conan Doyle: 2 | * A Study in Scarlet 3 | * His Last Bow 4 | * The Adventures of Sherlock Holmes 5 | * The Hound of the Baskervilles 6 | * The Memoirs of Sherlock Holmes 7 | * The Return of Sherlock Holmes 8 | * The Sign of the Four 9 | * The Valley of Fear 10 | 11 | The raw data is the books downloaded from [Gutenberg](https://www.gutenberg.org/wiki/Main_Page), a collection of books for which U.S. copyright has expired. If you are outside of the United States, please check the copyright terms before using this data. 12 | -------------------------------------------------------------------------------- /classifier/Reddit_vs_Holmes/neural/train.tsv: -------------------------------------------------------------------------------- 1 | 0.01 8 0.6344 0.7235 0.4910 2 | 0.03 5 0.4045 1.1514 0.4910 3 | 0.04 5 0.3360 1.1246 0.4910 4 | 0.05 5 0.3240 1.0559 0.4910 5 | 0.06 5 0.2792 1.0022 0.5020 6 | 0.08 5 0.2594 0.8556 0.5690 7 | 0.09 5 0.2195 0.6895 0.6870 8 | 0.10 5 0.1928 0.5804 0.7410 9 | 0.12 5 0.1737 0.5683 0.7690 10 | 0.13 5 0.1599 0.4971 0.7890 11 | 0.14 5 0.1487 0.5141 0.7890 12 | 0.15 5 0.1464 0.5520 0.7670 13 | 0.17 5 0.1415 0.5008 0.7920 14 | 0.18 5 0.1358 0.4912 0.7860 15 | 0.19 5 0.1246 0.5099 0.7830 16 | 0.20 5 0.1308 0.4687 0.7970 17 | 0.22 4 0.1280 0.4688 0.8000 18 | 0.23 5 0.1244 0.4730 0.8020 19 | 0.24 5 0.1283 0.4422 0.8120 20 | 0.26 5 0.1234 0.4532 0.8020 21 | 0.27 4 0.1204 0.4438 0.8120 22 | 0.28 4 0.1203 0.4519 0.8060 23 | 0.29 4 0.1230 0.4629 0.8080 24 | 0.31 5 0.1107 0.4673 0.8040 25 | 0.32 4 0.1111 0.4503 0.8090 26 | 0.33 5 0.1156 0.4594 0.8080 27 | -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## Format 3 | The model needs base conversational corpus (`base`, e.g. **Reddit**) as well as a stylized corpus (`bias`, e.g. **arXiv**). 4 | The `base` corpus should be conversational (so `base_conv`), and the `bias` corpus doesn't have to be. so we only need `bias_nonc` (`nonc` means non-conversational) 5 | 6 | To sum up, at least these files are required: `base_conv_XXX.num`, `bias_nonc_XXX.num` and `vocab.txt`, where `XXX` is `train`, `vali`, or `test`. 7 | See more discusion [here](https://github.com/golsun/StyleFusion/issues/3) 8 | 9 | 10 | * `vocab.txt` is the vocab list of tokens. 11 | * The first three token must be `_SOS_`, `_EOS_` and `_UNK_`, which represent "start of sentence", "end of sentence", and "unknown token". 12 | * The line ID (starts from 1, 0 is reserved for padding) of `vocab.txt` is the token index used in `*.num` files. For examples, unknown tokens will be represented by `3` which is the token index of `_UNK_`. 13 | 14 | * `*.num` files are sentences (in form of seq of token index), 15 | * for `conv`, each line is `src \t tgt`, where `\t` is the tab delimiter 16 | * for `nonc`, each line is a sentence. 17 | 18 | You may build a vocab using the [build_vocab](https://github.com/golsun/NLP-tools/blob/master/data_prepare.py#L266) function to generate `vocab.txt`, 19 | and then convert a raw text files to `*.num` 20 | (e.g. [train.txt](https://github.com/golsun/SpaceFusion/blob/master/data/toy/train.txt) to [train.num](https://github.com/golsun/SpaceFusion/blob/master/data/toy/train.num)) 21 | by the [text2num](https://github.com/golsun/NLP-tools/blob/master/data_prepare.py#L381) function 22 | 23 | 24 | ## Dataset 25 | In our paper, we trained the model using the following three datasets. 26 | * **Reddit**: the conversational dataset (`base_conv`), can be generated using this [script](https://github.com/golsun/SpaceFusion/tree/master/data#multi-ref-reddit). 27 | * **Sherlock Holmes**, one of style dataset (`bias_nonc`), avaialble [here](https://github.com/golsun/StyleFusion/tree/master/data/Holmes) 28 | * **arXiv**, another style corpus (`bias_nonc`), can be obtained following instructions [here](https://github.com/golsun/StyleFusion/tree/master/data/arXiv) 29 | * A [toy dataset](https://github.com/golsun/StyleFusion/tree/master/data/toy) is provied as an example following the format described above. 30 | 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # StyleFusion 2 | code/data for EMNLP'19 paper [Structuring Latent Spaces for Stylized Response Generation](https://arxiv.org/abs/1909.05361). 3 | 4 | Designed to build a **stylized** dialogue response generator, StyleFusion jointly learns from a conversational dataset and other formats of text (e.g., non-parallel, non-conversational stylized text dataset). In our EMNLP 2019 paper, we demonstrated its use to generate response in style of **Sherlock Holmes** and **arXiv**. StyleFusion is a generalized version of our previous work [SpaceFusion](https://github.com/golsun/SpaceFusion). 5 | 6 | More documents: 7 | * our EMNLP'19 [paper](https://arxiv.org/abs/1909.05361) and [poster](https://github.com/golsun/StyleFusion/blob/master/EMNLP%20poster.pdf). 8 | * A nice [introduction](https://mp.weixin.qq.com/s/rtAra15Qqnz9bLadSUSAlg) of our work (not official, by Shannon.AI, in Chinese) 9 | 10 | ## Dataset 11 | In our paper, we trained the model using the following three datasets. 12 | * **Reddit**: the conversational dataset (`base_conv`), can be generated using this [script](https://github.com/golsun/SpaceFusion/tree/master/data#multi-ref-reddit). 13 | * **Sherlock Holmes**, one of style dataset (`bias_nonc`), avaialble [here](https://github.com/golsun/StyleFusion/tree/master/data/Holmes) 14 | * **arXiv**, another style corpus (`bias_nonc`), can be obtained following instructions [here](https://github.com/golsun/StyleFusion/tree/master/data/arXiv) 15 | * A [toy dataset](https://github.com/golsun/StyleFusion/tree/master/data/toy) is provied as an example following the format described above. 16 | * We also provided the **test data** [here](https://github.com/golsun/StyleFusion/tree/master/data/test) 17 | 18 | See [here](https://github.com/golsun/StyleFusion/blob/master/data/README.md) for more details and instructions. 19 | 20 | ## Usage 21 | * to train a model `python src/main.py train` 22 | * to interact with a trained model `python src/main.py cmd --restore=[path_to_model_file]` 23 | * using the [provided style classifiers](https://github.com/golsun/StyleFusion/tree/master/classifier) 24 | * interactive demo: `python src/classifier.py [fld_clf]`, where `[fld_clf]` is the folder where the classifier model exists, e.g., `classifier/Reddit_vs_arXiv/neural` 25 | * evaluate a tsv file: `python src/classifier.py [fld_clf] [path_to_be_evaluated]`. 26 | 27 | ## Citation 28 | Please cite our EMNLP paper if this repo is useful to your work :) 29 | ``` 30 | @article{gao2019stylefusion, 31 | title={Structuring Latent Spaces for Stylized Response Generation}, 32 | author={Gao, Xiang and Zhang, Yizhe and Lee, Sungjin and Galley, Michel and Brockett, Chris and Gao, Jianfeng and Dolan, Bill}, 33 | journal={EMNLP 2019}, 34 | year={2019} 35 | } 36 | ``` 37 | 38 | ![](https://github.com/golsun/StyleFusion/blob/master/fig/intro_fig.png) 39 | 40 | 41 | -------------------------------------------------------------------------------- /classifier/Reddit_vs_arxiv/neural/train.tsv: -------------------------------------------------------------------------------- 1 | 0.01 8 0.6876 0.6743 0.6150 2 | 0.03 5 0.5827 0.3854 0.9180 3 | 0.04 5 0.2182 0.1507 0.9540 4 | 0.05 5 0.1291 0.1179 0.9650 5 | 0.06 4 0.1128 0.0960 0.9710 6 | 0.08 4 0.0952 0.0815 0.9750 7 | 0.09 4 0.0815 0.0703 0.9820 8 | 0.10 4 0.0746 0.0650 0.9800 9 | 0.12 4 0.0703 0.0595 0.9840 10 | 0.13 4 0.0688 0.0556 0.9850 11 | 0.14 4 0.0599 0.0532 0.9860 12 | 0.15 4 0.0594 0.0513 0.9850 13 | 0.17 5 0.0556 0.0484 0.9860 14 | 0.18 5 0.0537 0.0474 0.9870 15 | 0.19 5 0.0516 0.0482 0.9880 16 | 0.20 5 0.0511 0.0436 0.9890 17 | 0.22 5 0.0490 0.0422 0.9890 18 | 0.23 11 0.0446 0.0411 0.9910 19 | 0.24 5 0.0453 0.0405 0.9920 20 | 0.26 5 0.0467 0.0397 0.9920 21 | 0.27 5 0.0429 0.0382 0.9920 22 | 0.28 5 0.0478 0.0377 0.9920 23 | 0.29 5 0.0463 0.0358 0.9920 24 | 0.31 5 0.0422 0.0367 0.9900 25 | 0.32 5 0.0360 0.0368 0.9910 26 | 0.33 5 0.0410 0.0351 0.9930 27 | 0.35 5 0.0407 0.0368 0.9910 28 | 0.36 5 0.0368 0.0341 0.9930 29 | 0.37 5 0.0395 0.0339 0.9920 30 | 0.38 4 0.0376 0.0327 0.9940 31 | 0.40 4 0.0405 0.0326 0.9920 32 | 0.41 5 0.0389 0.0336 0.9910 33 | 0.42 4 0.0366 0.0322 0.9910 34 | 0.44 4 0.0396 0.0337 0.9920 35 | 0.45 5 0.0397 0.0304 0.9920 36 | 0.46 4 0.0369 0.0297 0.9930 37 | 0.47 5 0.0355 0.0301 0.9940 38 | 0.49 5 0.0360 0.0284 0.9930 39 | 0.50 5 0.0344 0.0288 0.9910 40 | 0.51 5 0.0336 0.0277 0.9930 41 | 0.52 5 0.0342 0.0266 0.9940 42 | 0.54 5 0.0326 0.0267 0.9950 43 | 0.55 5 0.0299 0.0263 0.9950 44 | 0.56 4 0.0323 0.0265 0.9930 45 | 0.58 5 0.0326 0.0255 0.9940 46 | 0.59 5 0.0352 0.0248 0.9950 47 | 0.60 4 0.0331 0.0275 0.9930 48 | 0.61 4 0.0326 0.0266 0.9930 49 | 0.63 5 0.0289 0.0251 0.9940 50 | 0.64 4 0.0308 0.0236 0.9950 51 | 0.65 4 0.0354 0.0246 0.9940 52 | 0.67 5 0.0339 0.0248 0.9930 53 | 0.68 5 0.0288 0.0255 0.9930 54 | 0.69 4 0.0295 0.0243 0.9940 55 | 0.70 4 0.0292 0.0243 0.9940 56 | 0.72 5 0.0272 0.0243 0.9940 57 | 0.73 5 0.0368 0.0236 0.9950 58 | 0.74 4 0.0292 0.0245 0.9950 59 | 0.76 5 0.0284 0.0257 0.9930 60 | 0.77 5 0.0298 0.0235 0.9940 61 | 0.78 5 0.0284 0.0250 0.9940 62 | 0.79 5 0.0277 0.0238 0.9940 63 | 0.81 5 0.0269 0.0247 0.9930 64 | 0.82 5 0.0300 0.0235 0.9950 65 | 0.83 5 0.0279 0.0249 0.9930 66 | 0.84 4 0.0297 0.0264 0.9940 67 | 0.86 4 0.0336 0.0248 0.9940 68 | 0.87 5 0.0271 0.0245 0.9930 69 | 0.88 5 0.0288 0.0234 0.9940 70 | 0.90 5 0.0283 0.0249 0.9930 71 | 0.91 5 0.0276 0.0231 0.9940 72 | 0.92 4 0.0289 0.0224 0.9940 73 | 0.93 4 0.0309 0.0218 0.9940 74 | 0.95 4 0.0246 0.0218 0.9940 75 | 0.96 4 0.0262 0.0217 0.9930 76 | 0.97 4 0.0274 0.0222 0.9930 77 | 0.99 5 0.0291 0.0225 0.9930 78 | 1.00 5 0.0290 0.0225 0.9930 79 | 1.01 5 0.0272 0.0223 0.9930 80 | 1.02 4 0.0297 0.0236 0.9930 81 | 1.04 4 0.0255 0.0234 0.9930 82 | 1.05 4 0.0244 0.0216 0.9930 83 | 1.06 4 0.0266 0.0215 0.9930 84 | 1.08 4 0.0302 0.0216 0.9930 85 | 1.09 4 0.0257 0.0217 0.9930 86 | 1.10 4 0.0310 0.0219 0.9940 87 | 1.11 4 0.0302 0.0230 0.9930 88 | 1.13 4 0.0277 0.0209 0.9960 89 | 1.14 4 0.0247 0.0203 0.9950 90 | 1.15 4 0.0270 0.0200 0.9960 91 | 1.16 5 0.0279 0.0205 0.9950 92 | 1.18 4 0.0253 0.0213 0.9940 93 | 1.19 4 0.0306 0.0196 0.9990 94 | 1.20 5 0.0288 0.0198 0.9960 95 | 1.22 5 0.0268 0.0203 0.9960 96 | 1.23 4 0.0247 0.0193 0.9990 97 | 1.24 5 0.0279 0.0196 0.9990 98 | 1.25 5 0.0295 0.0199 0.9950 99 | 1.27 5 0.0266 0.0198 0.9970 100 | 1.28 5 0.0227 0.0191 0.9980 101 | 1.29 6 0.0265 0.0190 0.9960 102 | -------------------------------------------------------------------------------- /src/shared.py: -------------------------------------------------------------------------------- 1 | import os, random, sys, datetime, time, socket, io, h5py, argparse, shutil, io 2 | import queue 3 | import numpy as np 4 | import scipy 5 | from nltk.translate.bleu_score import sentence_bleu 6 | from nltk.translate.bleu_score import SmoothingFunction 7 | import matplotlib.pyplot as plt 8 | 9 | """ 10 | AUTHOR: 11 | Sean Xiang Gao (xiag@microsoft.com) at Microsoft Research 12 | """ 13 | 14 | SOS_token = '_SOS_' 15 | EOS_token = '_EOS_' 16 | UNK_token = '_UNK_' 17 | 18 | # just for arxiv 19 | EQN_token = '_eqn_' 20 | CITE_token = '_cite_' 21 | IX_token = '_ix_' 22 | 23 | hostname = socket.gethostname() 24 | SKIP_VIS = hostname not in ['MININT-3LHNLKS', 'xiag-0228'] 25 | FIT_VERBOSE = 1 26 | 27 | DATA_PATH = 'data/' 28 | OUT_PATH = 'out/' 29 | 30 | print('@'*20) 31 | print('hostname: %s'%hostname) 32 | print('data_path: %s'%DATA_PATH) 33 | print('out_path: %s'%OUT_PATH) 34 | print('@'*20) 35 | 36 | PHILLY = False 37 | BATCH_SIZE = 128#256 38 | 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument('--data_name', type=str, default='toy') 41 | parser.add_argument('--batch_size', type=int, default=100) 42 | parser.add_argument('--max_n_trained', type=int, default=int(10e6)) 43 | parser.add_argument('--lr', type=float, default=3e-4) 44 | parser.add_argument('--token_embed_dim', type=int, default=1000) 45 | parser.add_argument('--rnn_units', type=int, default=1000) 46 | parser.add_argument('--encoder_depth', type=int, default=2) 47 | parser.add_argument('--decoder_depth', type=int, default=2) 48 | parser.add_argument('--stddev', type=float, default=0.1) 49 | parser.add_argument('--wt_dist', type=float, default=1.) 50 | parser.add_argument('--debug','-d', action='store_true') 51 | parser.add_argument('--max_ctxt_len', type=int, default=90) 52 | parser.add_argument('--max_resp_len', type=int, default=30) 53 | parser.add_argument('--fld_suffix', type=str, default='') 54 | parser.add_argument('--conv_mix_ratio', type=float, default=0.0) 55 | parser.add_argument('--nonc_mix_ratio', type=float, default=1.0) 56 | parser.add_argument('--clf_name', type=str, default='holmes') 57 | parser.add_argument('--model_class', type=str, default='fuse') 58 | parser.add_argument('--restore', type=str, default='') 59 | parser.add_argument('--noisy_vocab', type=int, default=-1) 60 | parser.add_argument('--reld', action='store_true') 61 | parser.add_argument('--ablation', '-abl', action='store_true') 62 | parser.add_argument('--path_test', type=str) 63 | 64 | if hostname == 'MININT-3LHNLKS': 65 | parser.add_argument('--cpu_only', '-c', action='store_true') 66 | 67 | def reset_rand(RAND_SEED=9): 68 | random.seed(RAND_SEED) 69 | np.random.seed(RAND_SEED) 70 | 71 | reset_rand() 72 | 73 | def str2bool(s): 74 | return {'true':True, 'false':False}[s.lower()] 75 | 76 | def strmap(x): 77 | if 'nan' in str(x): 78 | return 'nan' 79 | if isinstance(x, str): 80 | return x 81 | if int(x) == x: 82 | return '%i'%x 83 | return '%.4f'%x 84 | 85 | 86 | def int2str(i): 87 | if i < 1000: 88 | return str(i) 89 | else: 90 | k = i/1000 91 | if int(k) == k: 92 | return '%ik'%k 93 | else: 94 | return '%.1fk'%k 95 | 96 | 97 | def makedirs(fld): 98 | if not os.path.exists(fld): 99 | os.makedirs(fld) 100 | 101 | 102 | def rand_latent(center, r, limit=True): 103 | if r == 0: 104 | return center 105 | 106 | noise = np.random.normal(size=center.shape) 107 | r_raw = np.sqrt(np.sum(np.power(noise, 2))) 108 | sampled = center + noise/r_raw*r 109 | if limit: 110 | return np.minimum(1, np.maximum(-1, sampled)) 111 | else: 112 | return sampled 113 | 114 | 115 | def calc_nltk_bleu(ref, hyp, max_ngram=4, smoothing_function=None): 116 | return sentence_bleu( 117 | [ref.split()], 118 | hyp.split(), 119 | weights=[1./max_ngram]*max_ngram, 120 | smoothing_function=smoothing_function, 121 | ) 122 | 123 | 124 | def calc_nltk_bleu_smoothed(ref, hyp, max_ngram=4): 125 | return calc_nltk_bleu(ref, hyp, max_ngram, 126 | smoothing_function=SmoothingFunction().method7) 127 | 128 | def euc_dist(a, b): 129 | # Euclidean distance 130 | if len(a.shape) == 1: 131 | return np.sqrt(np.sum(np.power(a - b, 2))) 132 | return np.sqrt(np.sum(np.power(a - b, 2), axis=1)) 133 | 134 | 135 | def now(): 136 | return datetime.datetime.now() 137 | 138 | -------------------------------------------------------------------------------- /classifier/Reddit_vs_Holmes/neural/train.log: -------------------------------------------------------------------------------- 1 | 2 | ***** Epoch 1/1, trained 0.0000M ***** 3 | 4 | start: 2019-04-19 15:40:01 5 | spent: 8 sec 6 | train: 0.6344 7 | vali: 0.7235 8 | vali_acc: 0.4910 9 | 10 | ***** Epoch 1/1, trained 0.0128M ***** 11 | 12 | start: 2019-04-19 15:40:11 13 | spent: 5 sec 14 | train: 0.4045 15 | vali: 1.1514 16 | vali_acc: 0.4910 17 | 18 | ***** Epoch 1/1, trained 0.0256M ***** 19 | 20 | start: 2019-04-19 15:40:17 21 | spent: 5 sec 22 | train: 0.3360 23 | vali: 1.1246 24 | vali_acc: 0.4910 25 | 26 | ***** Epoch 1/1, trained 0.0384M ***** 27 | 28 | start: 2019-04-19 15:40:23 29 | spent: 5 sec 30 | train: 0.3240 31 | vali: 1.0559 32 | vali_acc: 0.4910 33 | 34 | ***** Epoch 1/1, trained 0.0512M ***** 35 | 36 | start: 2019-04-19 15:40:29 37 | spent: 5 sec 38 | train: 0.2792 39 | vali: 1.0022 40 | vali_acc: 0.5020 41 | 42 | ***** Epoch 1/1, trained 0.0640M ***** 43 | 44 | start: 2019-04-19 15:40:34 45 | spent: 5 sec 46 | train: 0.2594 47 | vali: 0.8556 48 | vali_acc: 0.5690 49 | 50 | ***** Epoch 1/1, trained 0.0768M ***** 51 | 52 | start: 2019-04-19 15:40:40 53 | spent: 5 sec 54 | train: 0.2195 55 | vali: 0.6895 56 | vali_acc: 0.6870 57 | 58 | ***** Epoch 1/1, trained 0.0896M ***** 59 | 60 | start: 2019-04-19 15:40:46 61 | spent: 5 sec 62 | train: 0.1928 63 | vali: 0.5804 64 | vali_acc: 0.7410 65 | 66 | ***** Epoch 1/1, trained 0.1024M ***** 67 | 68 | start: 2019-04-19 15:40:51 69 | spent: 5 sec 70 | train: 0.1737 71 | vali: 0.5683 72 | vali_acc: 0.7690 73 | 74 | ***** Epoch 1/1, trained 0.1152M ***** 75 | 76 | start: 2019-04-19 15:40:57 77 | spent: 5 sec 78 | train: 0.1599 79 | vali: 0.4971 80 | vali_acc: 0.7890 81 | 82 | ***** Epoch 1/1, trained 0.1280M ***** 83 | 84 | start: 2019-04-19 15:41:03 85 | spent: 5 sec 86 | train: 0.1487 87 | vali: 0.5141 88 | vali_acc: 0.7890 89 | 90 | ***** Epoch 1/1, trained 0.1408M ***** 91 | 92 | start: 2019-04-19 15:41:09 93 | spent: 5 sec 94 | train: 0.1464 95 | vali: 0.5520 96 | vali_acc: 0.7670 97 | 98 | ***** Epoch 1/1, trained 0.1536M ***** 99 | 100 | start: 2019-04-19 15:41:14 101 | spent: 5 sec 102 | train: 0.1415 103 | vali: 0.5008 104 | vali_acc: 0.7920 105 | 106 | ***** Epoch 1/1, trained 0.1664M ***** 107 | 108 | start: 2019-04-19 15:41:20 109 | spent: 5 sec 110 | train: 0.1358 111 | vali: 0.4912 112 | vali_acc: 0.7860 113 | 114 | ***** Epoch 1/1, trained 0.1792M ***** 115 | 116 | start: 2019-04-19 15:41:26 117 | spent: 5 sec 118 | train: 0.1246 119 | vali: 0.5099 120 | vali_acc: 0.7830 121 | 122 | ***** Epoch 1/1, trained 0.1920M ***** 123 | 124 | start: 2019-04-19 15:41:32 125 | spent: 5 sec 126 | train: 0.1308 127 | vali: 0.4687 128 | vali_acc: 0.7970 129 | 130 | ***** Epoch 1/1, trained 0.2048M ***** 131 | 132 | start: 2019-04-19 15:41:37 133 | spent: 4 sec 134 | train: 0.1280 135 | vali: 0.4688 136 | vali_acc: 0.8000 137 | 138 | ***** Epoch 1/1, trained 0.2176M ***** 139 | 140 | start: 2019-04-19 15:41:43 141 | spent: 5 sec 142 | train: 0.1244 143 | vali: 0.4730 144 | vali_acc: 0.8020 145 | 146 | ***** Epoch 1/1, trained 0.2304M ***** 147 | 148 | start: 2019-04-19 15:41:48 149 | spent: 5 sec 150 | train: 0.1283 151 | vali: 0.4422 152 | vali_acc: 0.8120 153 | 154 | ***** Epoch 1/1, trained 0.2432M ***** 155 | 156 | start: 2019-04-19 15:41:54 157 | spent: 5 sec 158 | train: 0.1234 159 | vali: 0.4532 160 | vali_acc: 0.8020 161 | 162 | ***** Epoch 1/1, trained 0.2560M ***** 163 | 164 | start: 2019-04-19 15:42:00 165 | spent: 4 sec 166 | train: 0.1204 167 | vali: 0.4438 168 | vali_acc: 0.8120 169 | 170 | ***** Epoch 1/1, trained 0.2688M ***** 171 | 172 | start: 2019-04-19 15:42:06 173 | spent: 4 sec 174 | train: 0.1203 175 | vali: 0.4519 176 | vali_acc: 0.8060 177 | 178 | ***** Epoch 1/1, trained 0.2816M ***** 179 | 180 | start: 2019-04-19 15:42:11 181 | spent: 4 sec 182 | train: 0.1230 183 | vali: 0.4629 184 | vali_acc: 0.8080 185 | 186 | ***** Epoch 1/1, trained 0.2944M ***** 187 | 188 | start: 2019-04-19 15:42:17 189 | spent: 5 sec 190 | train: 0.1107 191 | vali: 0.4673 192 | vali_acc: 0.8040 193 | 194 | ***** Epoch 1/1, trained 0.3072M ***** 195 | 196 | start: 2019-04-19 15:42:22 197 | spent: 4 sec 198 | train: 0.1111 199 | vali: 0.4503 200 | vali_acc: 0.8090 201 | 202 | ***** Epoch 1/1, trained 0.3200M ***** 203 | 204 | start: 2019-04-19 15:42:28 205 | spent: 5 sec 206 | train: 0.1156 207 | vali: 0.4594 208 | vali_acc: 0.8080 209 | 210 | ***** Epoch 1/1, trained 0.3328M ***** 211 | 212 | start: 2019-04-19 15:42:34 213 | -------------------------------------------------------------------------------- /data/arXiv/arxiv.py: -------------------------------------------------------------------------------- 1 | """ 2 | AUTHOR: 3 | Xiang Gao (xiag@microsoft.com) at Microsoft Research 4 | """ 5 | 6 | import re, os, subprocess 7 | from nltk.tokenize import TweetTokenizer 8 | 9 | EQN_token = '_eqn_' 10 | CITE_token = '_cite_' 11 | IX_token = '_ix_' 12 | MAX_UTT_LEN = 30 # maximum length of utterance allowed. if longer, ignore 13 | 14 | 15 | def norm_sentence(txt): 16 | txt = txt.lower() 17 | 18 | # url and tag 19 | words = [] 20 | for word in txt.lower().split(): 21 | if word[0] == '#': # don't allow tag 22 | continue 23 | i = word.find('http') 24 | if i >= 0: 25 | word = word[:i] + ' ' + '__url__' 26 | words.append(word.strip()) 27 | txt = ' '.join(words) 28 | 29 | # remove illegal char 30 | txt = txt.replace(chr(92),'') # chr(92) = '\'. as twitter has 'b\/c' rather than 'b/c' 31 | txt = txt.replace("b/c","because").replace('j/k','just kidding').replace('w/o','without').replace('w/','with') 32 | txt = re.sub('__mention__','MENTION',txt) 33 | txt = re.sub('__url__','URL',txt) 34 | txt = re.sub(r"[^A-Za-z0-9():,.!?_'“” ]", " ", txt) 35 | txt = re.sub('MENTION','__mention__',txt) 36 | txt = re.sub('URL','__url__',txt) 37 | 38 | # contraction 39 | add_space = ["'s", "'m", "'re", "n't", "'ll","'ve","'d","'em"] 40 | tokenizer = TweetTokenizer(preserve_case=False) 41 | txt = ' ' + ' '.join(tokenizer.tokenize(txt)) + ' ' 42 | txt = txt.replace(" won't ", " will n't ") 43 | txt = txt.replace(" can't ", " can n't ") 44 | for a in add_space: 45 | txt = txt.replace(a+' ', ' '+a+' ') 46 | 47 | # remove un-necessary space 48 | return ' '.join(txt.split()) 49 | 50 | def arxiv_del_bib(path): 51 | lines = [] 52 | for line in open(path, encoding='utf-8', errors='ignore'): 53 | stop = False 54 | for c in ['begin{references}', 'begin{enumerate}', 'begin{thebibliography}']: 55 | if c in line: 56 | stop = True 57 | if stop: 58 | lines.append(chr(92) + 'end{document}\n\n') 59 | break 60 | else: 61 | lines.append(line.strip('\n')) 62 | with open(path+'.delbib', 'w', encoding='utf-8') as f: 63 | f.write('\n'.join(lines)) 64 | 65 | 66 | def arxiv_pandoc(fld): 67 | # preprocess arxiv latex file with pandoc 68 | # http://www.cs.cornell.edu/projects/kddcup/datasets.html 69 | # http://pandoc.org/index.html 70 | 71 | n = 0 72 | for fname in os.listdir(fld): 73 | if '.' in fname: 74 | continue 75 | 76 | path = fld + '/' + fname 77 | arxiv_del_bib(path) 78 | cmd = [ 79 | 'pandoc', 80 | '-f', 'latex', 81 | '-o', path + '_pandoc.txt', 82 | path + '.delbib' 83 | ] 84 | process = subprocess.Popen(cmd, stdout=subprocess.PIPE) 85 | process.communicate() 86 | print('='*10) 87 | print(path) 88 | 89 | n += 1 90 | #if n == 100: 91 | # break 92 | 93 | 94 | def arxiv_paragraph(path): 95 | def lines2paragraph(lines): 96 | p = ' '.join(lines) 97 | if len(p) == 0: 98 | return None 99 | if not p[0].isalpha(): 100 | return None 101 | return arxiv_clean(p) 102 | 103 | paragraphs = [] 104 | lines = [] 105 | for line in open(path, encoding='utf-8', errors='ignore'): 106 | line = line.strip('\n').strip() 107 | if len(line) == 0: 108 | paragraph = lines2paragraph(lines) 109 | if paragraph is not None: 110 | paragraphs.append(paragraph) 111 | lines = [] 112 | else: 113 | if len(re.sub(r"[^A-Za-z0-9]", "", line)) > 0: 114 | lines.append(line) 115 | 116 | paragraph = lines2paragraph(lines) 117 | if paragraph is not None: 118 | paragraphs.append(paragraph) 119 | 120 | with open(path+'.paragraph','w', encoding='utf-8') as f: 121 | f.write('\n'.join(paragraphs)) 122 | 123 | 124 | 125 | def arxiv_clean(p): 126 | # deal with equations and citations 127 | 128 | is_math = False 129 | i = 0 130 | s = '' 131 | while i < len(p): 132 | flag = False 133 | if p[i] == '$': 134 | flag = True 135 | if i + 1 < len(p) and p[i+1] == '$': 136 | i = i+1 137 | if flag: 138 | if not is_math: 139 | s += ' %s '%EQN_token 140 | is_math = not is_math 141 | elif not is_math: 142 | s += p[i] 143 | i += 1 144 | 145 | ww = [] 146 | for w in s.split(): 147 | if w == EQN_token: 148 | if len(ww) > 0 and ww[-1] == EQN_token: 149 | continue 150 | if len(ww) > 1 and ww[-1] == '.' and ww[-2] == EQN_token: 151 | continue 152 | if chr(92)+'[' in w or chr(92) + ']' in w or '[@' in w or w.startswith('@'): 153 | # citation 154 | w_ = CITE_token 155 | if w[-1]!=']': 156 | w_ += w[-1] 157 | w = w_ 158 | if len(ww) > 0 and ww[-1].startswith(CITE_token): 159 | continue 160 | else: 161 | if w[0] == '(' and len(w) > 1 and w[1].isnumeric(): 162 | w_ = IX_token 163 | if w[-1]!=')': 164 | w_ += w[-1] 165 | w = w_ 166 | w = w.replace('[**','').replace('[*','').replace('**]','').replace('*]','') 167 | ww.append(w) 168 | 169 | return ' '.join(ww) 170 | 171 | 172 | def arxiv_paragraph_all(fld): 173 | n = 0 174 | for fname in os.listdir(fld): 175 | if fname.endswith('_pandoc.txt'): 176 | path = fld + '/' + fname 177 | print(path) 178 | arxiv_paragraph(path) 179 | n += 1 180 | 181 | def arxiv_utts(path): 182 | utts = [] 183 | for p in open(path, encoding='utf-8'): 184 | p = p.strip('\n').replace(chr(92),'. ') 185 | for utt in p.split('. '): 186 | utt += '.' 187 | alpha = re.sub(r"[^a-z]", "", utt.lower()) 188 | if len(alpha) < 5: 189 | continue 190 | utt = norm_sentence(utt) 191 | if len(utt.split()) > MAX_UTT_LEN: 192 | continue 193 | utts.append(utt) 194 | with open(path+'.utt', 'w', encoding='utf-8') as f: 195 | f.write('\n'.join(utts)) 196 | return utts 197 | 198 | def arxiv_utts_all(fld): 199 | utts = [] 200 | n = 0 201 | for fname in os.listdir(fld): 202 | if fname.endswith('.paragraph'): 203 | path = fld + '/' + fname 204 | print(path) 205 | utts_ = arxiv_utts(path) 206 | for utt in utts_: 207 | if len(utt) > 10: 208 | utts.append(utt) 209 | utts = sorted(utts) 210 | with open(fld + '/../all.utt', 'w', encoding='utf-8') as f: 211 | f.write('\n'.join(utts)) 212 | 213 | def arxiv_filter(path): 214 | # make sure: 1) starts with some word, 2) longer than some min 215 | lines = [] 216 | for line in open(path, encoding='utf-8'): 217 | ww = line.strip('\n').replace('“','"').replace('”','"').split() 218 | i0 = None 219 | for i in range(len(ww)): 220 | alpha = re.sub(r"[^a-z]", "", ww[i].lower()) 221 | if len(alpha) > 1: 222 | i0 = i 223 | break 224 | if i0 is None: 225 | continue 226 | ww = ww[i0:] 227 | if len(re.sub(r"[^a-z]", "", ''.join(ww))) > 5: 228 | lines.append(' '.join(ww)) 229 | with open(path+'.filtered', 'w', encoding='utf-8') as f: 230 | f.write('\n'.join(lines)) 231 | 232 | 233 | if __name__ == '__main__': 234 | years = range(1998, 2002+1) 235 | for year in years: 236 | fld = 'hep-th-%i/%i/'%(year, year) 237 | arxiv_pandoc(fld) 238 | arxiv_paragraph_all(fld) 239 | arxiv_utts_all(fld) 240 | for year in years: 241 | print(year) 242 | arxiv_filter('hep-th-%i/all.utt'%(year)) 243 | -------------------------------------------------------------------------------- /src/main.py: -------------------------------------------------------------------------------- 1 | from shared import * 2 | from tf_lib import * 3 | from dataset import * 4 | from model import * 5 | #from dialog_gui import * 6 | from classifier import load_classifier 7 | 8 | """ 9 | AUTHOR: Xiang Gao (xiag@microsoft.com) at Microsoft Research 10 | """ 11 | 12 | 13 | def run_master(mode, args): 14 | 15 | if mode not in ['train','continue'] and args.restore != '': 16 | aa = args.restore.split('/') 17 | bb = [] 18 | for a in aa: 19 | if len(a) > 0: 20 | bb.append(a) 21 | fld = '/'.join(bb[:-1]) 22 | if mode in ['vali','vis']: 23 | vocab_only = False 24 | fld_data, _, _ = get_model_fld(args) 25 | path_bias_vocab = fld_data + '/vocab_bias.txt' 26 | else: 27 | vocab_only = True 28 | fld_data = fld 29 | path_bias_vocab = fld + '/vocab_bias.txt' 30 | else: 31 | vocab_only = False 32 | fld_data, fld_model, subfld = get_model_fld(args) 33 | fld = fld_model + '/' + subfld 34 | path_bias_vocab = fld_data + '/vocab_bias.txt' 35 | 36 | if os.path.exists(path_bias_vocab): 37 | allowed_words = [line.strip('\n').strip('\r') for line in open(path_bias_vocab, encoding='utf-8')] 38 | else: 39 | allowed_words = None 40 | 41 | 42 | model_class = args.model_class.lower() 43 | if model_class.startswith('fuse'): 44 | Master = StyleFusion 45 | elif model_class == 'mtask': 46 | Master = VanillaMTask 47 | elif model_class == 's2s': 48 | Master = Seq2Seq 49 | elif model_class == 'lm': 50 | Master = LanguageModel 51 | elif model_class == 's2s+lm': 52 | pass 53 | else: 54 | raise ValueError 55 | 56 | 57 | if model_class == 's2s+lm': 58 | master = Seq2SeqLM(args, allowed_words) 59 | else: 60 | dataset = Dataset(fld_data, 61 | max_ctxt_len=args.max_ctxt_len, max_resp_len=args.max_resp_len, 62 | vocab_only=vocab_only, noisy_vocab=args.noisy_vocab) 63 | 64 | master = Master(dataset, fld, args, new=(mode=='train'), allowed_words=allowed_words) 65 | if mode != 'train': 66 | if args.restore.endswith('.npz') or model_class == 's2s+lm': 67 | restore_path = args.restore 68 | else: 69 | restore_path = master.fld + '/models/%s.npz'%args.restore 70 | master.load_weights(restore_path) 71 | 72 | if mode in ['vis', 'load']: 73 | return master 74 | 75 | if args.clf_name.lower() == 'holmes': 76 | CLF_NAMES = ['classifier/Reddit_vs_Holmes/neural', 'classifier/Reddit_vs_Holmes/ngram'] 77 | elif args.clf_name.lower() == 'arxiv': 78 | CLF_NAMES = ['classifier/Reddit_vs_arxiv/neural', 'classifier/Reddit_vs_arxiv/ngram'] 79 | else: 80 | CLF_NAMES = [args.clf_name] 81 | print('loading classifiers '+str(CLF_NAMES)) 82 | master.clf_names = CLF_NAMES 83 | master.classifiers = [] 84 | for clf_name in CLF_NAMES: 85 | master.classifiers.append(load_classifier(clf_name)) 86 | 87 | print('\n'+fld+'\n') 88 | if mode in ['continue', 'train']: 89 | 90 | ss = ['', mode + ' @ %i'%time.time()] 91 | for k in sorted(args.__dict__.keys()): 92 | ss.append('%s = %s'%(k, args.__dict__[k])) 93 | with open(master.fld + '/args.txt', 'a') as f: 94 | f.write('\n'.join(ss)+'\n') 95 | 96 | if args.debug: 97 | batch_per_load = 1 98 | else: 99 | if PHILLY: 100 | n_sample = 1280 # philly unstable for large memory 101 | else: 102 | n_sample = 2560 103 | batch_per_load = int(n_sample/BATCH_SIZE) 104 | if mode == 'continue': 105 | master.vali() 106 | master.train(batch_per_load) 107 | 108 | elif 'summary' == mode: 109 | print(master.model.summary()) 110 | 111 | elif mode in ['cmd', 'test', 'vali']: 112 | classifiers = [] 113 | for clf_name in CLF_NAMES: 114 | classifiers.append(load_classifier(clf_name)) 115 | 116 | if 'vali' == mode: 117 | data = master.get_vali_data() 118 | s_decoded = eval_decoded(master, data, 119 | classifiers=classifiers, corr_by_tgt=True, r_rand=args.r_rand, 120 | calc_retrieval=('holmes' in args.data_name.lower()) 121 | )[0] 122 | s_surrogate = eval_surrogate(master, data)[0] 123 | print(restore_path) 124 | print() 125 | print(s_decoded) 126 | print() 127 | print(s_surrogate) 128 | return 129 | 130 | """ 131 | if model_class != 's2s+lm': 132 | with tf.variable_scope('base_rankder', reuse=tf.AUTO_REUSE): 133 | fld_base_ranker = 'restore/%s/%s/pretrained/'%(args.model_class.replace('fuse1','fuse'), args.data_name) 134 | dataset_base_ranker = Dataset(fld_base_ranker, 135 | max_ctxt_len=args.max_ctxt_len, max_resp_len=args.max_resp_len, 136 | vocab_only=True, noisy_vocab=False) 137 | base_ranker = Master(dataset_base_ranker, fld_base_ranker, args, new=False, allowed_words=master.allowed_words) 138 | path = fld_base_ranker + '/' + open(fld_base_ranker+'/base_ranker.txt').readline().strip('\n') 139 | base_ranker.load_weights(path) 140 | print('*'*10 + ' base_ranker loaded from: '+path) 141 | else: 142 | """ 143 | base_ranker = None 144 | 145 | def print_results(results): 146 | ss = ['total', 'logP', 'logP_c', 'logP_b', 'rep', 'len',] + ['clf%i'%i for i in range(len(CLF_NAMES))] 147 | print('; '.join([' '*(6-len(s))+s for s in ss])) 148 | for score, resp, terms in results: 149 | print('%6.3f; '%score + '; '.join(['%6.3f'%x for x in terms]) + '; ' + resp) 150 | 151 | if 'cmd' == mode: 152 | while True: 153 | print('\n---- please input ----') 154 | inp = input() 155 | infer_args = parse_infer_args() 156 | if inp == '': 157 | break 158 | results = infer_rank(inp, master, infer_args, base_ranker=base_ranker) 159 | print_results(results) 160 | 161 | 162 | elif 'test' == mode: 163 | infer_args = parse_infer_args() 164 | path_out = args.path_test+'.hyp' 165 | open(path_out, 'w', encoding='utf-8') 166 | for line in open(args.path_test, encoding='utf-8'): 167 | line = line.strip('\n') 168 | inp = line.split('\t')[0] 169 | results = infer_rank(inp, master, infer_args, base_ranker=base_ranker) 170 | lines = [] 171 | for _, hyp, _ in results[:min(10, len(results))]: 172 | lines.append(line + '\t' + hyp.replace(' _EOS_','').strip()) 173 | with open(path_out, 'a', encoding='utf-8') as f: 174 | f.write('\n'.join(lines) + '\n') 175 | 176 | """ 177 | path_in = DATA_PATH + '/test/' + args.test_fname 178 | if not PHILLY: 179 | fld_out = master.fld + '/eval2/' 180 | else: 181 | fld_out = OUT_PATH 182 | makedirs(fld_out) 183 | npz_name = args.restore.split('/')[-1].replace('.npz','') 184 | path_out = fld_out + '/' + args.test_fname + '_' + npz_name 185 | test_master(master, path_in, path_out, max_n_src=args.test_n_max, base_ranker=base_ranker, baseline=args.baseline, r_rand=args.r_rand) 186 | """ 187 | else: 188 | raise ValueError 189 | 190 | 191 | 192 | def get_model_fld(args): 193 | data_name = args.data_name 194 | if PHILLY: 195 | data_name = data_name.replace('+','').replace('_','') 196 | fld_data = DATA_PATH +'/' + data_name 197 | 198 | master_config = 'width%s_depth%s'%( 199 | (args.token_embed_dim, args.rnn_units), 200 | (args.encoder_depth, args.decoder_depth)) 201 | if args.max_ctxt_len != 90 or args.max_resp_len != 30: 202 | master_config += '_len' + str((args.max_ctxt_len, args.max_resp_len)) 203 | 204 | master_config = master_config.replace("'",'') 205 | fld_model = OUT_PATH 206 | if args.debug: 207 | fld_model += '/debug' 208 | fld_model += '/' + args.data_name.replace('../','') + '_' + master_config 209 | 210 | subfld = [] 211 | """ 212 | if args.randmix: 213 | s_mix = 'randmix' 214 | if args.ratio05 > 0: 215 | s_mix += '(0.5=%.2f)'%args.ratio05 216 | else: 217 | """ 218 | s_mix = 'mix' 219 | model_class = args.model_class.lower() 220 | if model_class == 's2s': 221 | subfld = ['s2s_%s(%.2f)'%(s_mix, args.conv_mix_ratio)] # no conv data 222 | else: 223 | subfld = ['%s_%s(%.2f,%.2f)'%(model_class, s_mix, args.conv_mix_ratio, args.nonc_mix_ratio)] 224 | if args.noisy_vocab > 0: 225 | subfld.append('unk%.1fk'%(args.noisy_vocab/1000)) 226 | if model_class.startswith('fuse'): 227 | subfld.append('std%.1f'%args.stddev) 228 | if args.reld: 229 | subfld.append('reld') 230 | 231 | subfld.append('lr'+str(args.lr)) 232 | if len(args.fld_suffix) > 0: 233 | subfld.append(args.fld_suffix) 234 | subfld = '_'.join(subfld) 235 | 236 | return fld_data, fld_model.replace(' ',''), subfld.replace(' ','') 237 | 238 | 239 | 240 | if __name__ == '__main__': 241 | parser.add_argument('mode') 242 | parser.add_argument('--skip', type=float, default=0.0) 243 | parser.add_argument('--test_fname', default='') 244 | parser.add_argument('--r_rand', '-r', type=float, default=-1) 245 | parser.add_argument('--test_n_max', '-n', type=int, default=2000) 246 | 247 | args = parser.parse_args() 248 | run_master(args.mode, args) 249 | 250 | 251 | 252 | 253 | -------------------------------------------------------------------------------- /src/dataset.py: -------------------------------------------------------------------------------- 1 | from shared import * 2 | 3 | """ 4 | AUTHOR: 5 | Sean Xiang Gao (xiag@microsoft.com) at Microsoft Research 6 | """ 7 | 8 | 9 | def load_vocab(path): 10 | with io.open(path, encoding='utf-8') as f: 11 | lines = f.readlines() 12 | 13 | index2token = dict() 14 | token2index = dict() 15 | for i, line in enumerate(lines): 16 | token = line.strip('\n').strip() 17 | index2token[i + 1] = token # start from 1, as 0 reserved for PAD 18 | token2index[token] = i + 1 19 | 20 | assert(SOS_token in token2index) 21 | assert(EOS_token in token2index) 22 | assert(UNK_token in token2index) 23 | return index2token, token2index 24 | 25 | 26 | class Dataset: 27 | 28 | def __init__(self, 29 | fld_data, 30 | max_ctxt_len=93, 31 | max_resp_len=30, 32 | vocab_only=False, 33 | noisy_vocab=-1, 34 | noisy_AE_src=True, 35 | noisy_bias=True, # whether add UNK noise to bias data (conv and nonc, src and tgt) 36 | ): 37 | 38 | self.max_ctxt_len = max_ctxt_len 39 | self.max_resp_len = max_resp_len 40 | self.noisy_vocab = noisy_vocab 41 | self.noisy_AE_src = noisy_AE_src 42 | self.noisy_bias = noisy_bias 43 | 44 | types = ['base_conv','bias_conv', 'base_nonc', 'bias_nonc'] 45 | 46 | self.fld_data = fld_data 47 | self.path_vocab = fld_data + '/vocab.txt' 48 | self.index2token, self.token2index = load_vocab(self.path_vocab) 49 | self.num_tokens = len(self.token2index) # not including 0-th 50 | if self.noisy_vocab > 0: 51 | self.prob_keep = dict() 52 | for ix in self.index2token: 53 | self.prob_keep[ix] = np.exp(-ix/self.noisy_vocab) 54 | 55 | if vocab_only: 56 | return 57 | 58 | self.paths = dict() 59 | self.files = dict() 60 | self.n_reset = dict() 61 | for sub in ['train', 'vali', 'test']: 62 | self.paths[sub] = dict() 63 | self.files[sub] = dict() 64 | self.n_reset[sub] = dict() 65 | for tp in types: 66 | self.n_reset[sub][tp] = -1 67 | self.paths[sub][tp] = fld_data + '/%s_%s.num'%(tp, sub) 68 | self.reset(sub, tp) 69 | 70 | for k in self.files: 71 | print(k, self.files[k].keys()) 72 | 73 | def reset(self, sub, tp=None): 74 | if tp is None: 75 | types = self.files[sub].keys() 76 | else: 77 | types = [tp] 78 | for tp in types: 79 | if os.path.exists(self.paths[sub][tp]): 80 | line = open(self.paths[sub][tp]).readline().strip('\n') 81 | if len(line) > 0: 82 | self.files[sub][tp] = open(self.paths[sub][tp]) 83 | self.n_reset[sub][tp] += 1 84 | 85 | 86 | def seq2txt(self, seq): 87 | words = [] 88 | for j in seq: 89 | if j == 0: # skip PAD 90 | continue 91 | words.append(self.index2token[int(j)]) 92 | return ' '.join(words) 93 | 94 | 95 | def txt2seq(self, text): 96 | tokens = text.strip().split() 97 | seq = [] 98 | for token in tokens: 99 | seq.append(self.token2index.get(token, self.token2index[UNK_token])) 100 | return seq 101 | 102 | 103 | def seqs2enc(self, seqs, max_len): 104 | inp = np.zeros((len(seqs), max_len)) 105 | for i, seq in enumerate(seqs): 106 | for t in range(min(max_len, len(seq))): 107 | inp[i, t] = seq[t] 108 | return inp 109 | 110 | 111 | def seqs2dec(self, seqs, max_len): 112 | 113 | # len: +2 as will 1) add EOS and 2) shift to right by 1 time step 114 | # vocab: +1 as mask_zero (token_id == 0 means PAD) 115 | 116 | ix_SOS = self.token2index[SOS_token] 117 | ix_EOS = self.token2index[EOS_token] 118 | 119 | inp = np.zeros((len(seqs), max_len + 2)) 120 | out = np.zeros((len(seqs), max_len + 2, self.num_tokens + 1)) 121 | for i, seq in enumerate(seqs): 122 | seq = seq[:min(max_len, len(seq))] 123 | for t, token_index in enumerate(seq): 124 | inp[i, t + 1] = token_index # shift 1 time step 125 | out[i, t, token_index] = 1. 126 | inp[i, 0] = ix_SOS # inp starts with EOS 127 | out[i, len(seq), ix_EOS] = 1. # out ends with EOS 128 | 129 | return inp, out 130 | 131 | 132 | def skip(self, max_n, mix_ratio, conv_only=False): 133 | sub = 'train' 134 | if isinstance(mix_ratio, int) or isinstance(mix_ratio, float): 135 | mix_ratio = (mix_ratio,) 136 | 137 | def _read(tp, n, m): 138 | for _ in self.files[sub][tp]: 139 | if m >= n: 140 | break 141 | m += 1 142 | if m%1e5 == 0: 143 | print('%s skipped %.2f M'%(tp, m/1e6)) 144 | return m 145 | 146 | m = dict() 147 | suffix = ['conv'] 148 | if not conv_only: 149 | suffix.append('nonc') 150 | 151 | for i in range(len(suffix)): 152 | suf = suffix[i] 153 | for tp, n in [ 154 | ('base_'+suf, max_n * (1. - mix_ratio[i])), 155 | ('bias_'+suf, max_n * mix_ratio[i]) 156 | ]: 157 | m[tp] = 0 158 | if n < 1 or tp not in self.files[sub]: 159 | continue 160 | while m[tp] < n: 161 | m_ = _read(tp, n, m[tp]) 162 | if m_ == m[tp]: 163 | self.reset(sub, tp) 164 | m[tp] = m_ 165 | if m_ >= n: 166 | break 167 | 168 | print('conv skipped %.2f M'%((m['base_conv'] + m['bias_conv'])/1e6)) 169 | if not conv_only: 170 | print('nonc skipped %.2f M'%((m['base_nonc'] + m['bias_nonc'])/1e6)) 171 | 172 | 173 | def add_unk_noise(self, seqs): 174 | if self.noisy_vocab < 0 or len(seqs) == 0: 175 | return seqs 176 | ix_unk = self.token2index[UNK_token] 177 | ret = [] 178 | n = 0 179 | old_n_unk = 0 180 | new_n_unk = 0 181 | for seq in seqs: 182 | noisy = [] 183 | n += len(seq) 184 | for ix in seq: 185 | old_n_unk += (ix == ix_unk) 186 | if np.random.random() > self.prob_keep[ix]: 187 | noisy.append(ix_unk) 188 | else: 189 | noisy.append(ix) 190 | new_n_unk += (noisy[-1] == ix_unk) 191 | ret.append(noisy) 192 | print('unk increased from %.2f to %.2f'%(old_n_unk/n, new_n_unk/n)) 193 | return ret 194 | 195 | 196 | def feed_data(self, sub, max_n, check_src=False, mix_ratio=(0.,0.), conv_only=False): 197 | if isinstance(mix_ratio, int) or isinstance(mix_ratio, float): 198 | mix_ratio = (mix_ratio,) 199 | print('loading data, check_src = %s, mix_ratio = %s'%(check_src, mix_ratio)) 200 | 201 | # load conversation data ------------- 202 | 203 | def _read_conv(tp, n, prev_ctxt, seqs): 204 | for line in self.files[sub][tp]: 205 | if len(seqs) >= n: 206 | break 207 | tt = line.strip('\n').split('\t') 208 | if len(tt) != 2: 209 | continue 210 | seq_ctxt, seq_resp = tt 211 | if check_src and (seq_ctxt == prev_ctxt): 212 | continue 213 | prev_ctxt = seq_ctxt 214 | seq_ctxt = [int(k) for k in seq_ctxt.split()] 215 | seq_resp = [int(k) for k in seq_resp.split()] 216 | seq_ctxt = seq_ctxt[-min(len(seq_ctxt), self.max_ctxt_len):] 217 | seq_resp = seq_resp[:min(len(seq_resp), self.max_resp_len)] 218 | 219 | seqs.append((seq_ctxt, seq_resp)) 220 | return seqs, prev_ctxt 221 | 222 | # get conv from different tp 223 | seqs = dict() 224 | for tp, n in [('base_conv', max_n * (1. - mix_ratio[0])), ('bias_conv', max_n * mix_ratio[0])]: 225 | seqs[tp] = [] 226 | if n < 1 or tp not in self.files[sub]: 227 | continue 228 | prev_ctxt = '' 229 | while True: 230 | m = len(seqs[tp]) 231 | seqs[tp], prev_ctxt = _read_conv(tp, n, prev_ctxt, seqs[tp]) 232 | if len(seqs[tp]) >= n: 233 | break 234 | if len(seqs[tp]) == m: 235 | self.reset(sub, tp) 236 | print('conv from %s: %i/%i'%(tp, len(seqs[tp]), n)) 237 | if 'bias_conv' in seqs and self.noisy_bias: 238 | seqs_ctxt = self.add_unk_noise([seq for seq, _ in seqs['bias_conv']]) 239 | seqs_resp = self.add_unk_noise([seq for _, seq in seqs['bias_conv']]) 240 | seqs['bias_conv'] = [(seqs_ctxt[i], seqs_resp[i]) for i in range(len(seqs['bias_conv']))] 241 | 242 | # then mix them 243 | ids = [] 244 | for tp in seqs: 245 | ids += [(tp, i) for i in range(len(seqs[tp]))] 246 | np.random.shuffle(ids) 247 | seqs_ctxt = [] 248 | seqs_resp = [] 249 | for tp, i in ids: 250 | seqs_ctxt.append(seqs[tp][i][0]) 251 | seqs_resp.append(seqs[tp][i][1]) 252 | 253 | inp_enc_ctxt = self.seqs2enc(seqs_ctxt, self.max_ctxt_len) 254 | if self.noisy_AE_src: 255 | inp_enc_resp = self.seqs2enc(self.add_unk_noise(seqs_resp), self.max_resp_len) 256 | else: 257 | inp_enc_resp = self.seqs2enc(seqs_resp, self.max_resp_len) 258 | inp_dec_resp, out_dec_resp = self.seqs2dec(seqs_resp, self.max_resp_len) 259 | 260 | n_sample_conv = len(ids) 261 | d_inp_enc = {'ctxt':inp_enc_ctxt, 'resp':inp_enc_resp} 262 | d_inp_dec = {'resp':inp_dec_resp} 263 | d_out_dec = {'resp':out_dec_resp} 264 | 265 | def get_ret(n, dd): 266 | n = BATCH_SIZE * int(n/BATCH_SIZE) 267 | ret = {'n_sample':n} 268 | for d_name in dd: 269 | d = dd[d_name] 270 | for k in d: 271 | if isinstance(d[k], list): 272 | d[k] = d[k][:n] 273 | else: 274 | d[k] = d[k][:n, :] 275 | ret[d_name] = d 276 | return ret 277 | 278 | if conv_only: 279 | return get_ret(n_sample_conv, { 280 | 'inp_enc':d_inp_enc, 281 | 'inp_dec':d_inp_dec, 282 | 'out_dec':d_out_dec, 283 | 'seqs':{'resp':seqs_resp}, 284 | }) 285 | 286 | # load non-conversation (nonc) data ------------- 287 | 288 | def _read_nonc(tp, n, seqs): 289 | for line in self.files[sub][tp]: 290 | if len(seqs) >= n: 291 | break 292 | seq = [int(k) for k in line.strip('\n').split()] 293 | seq = seq[:min(len(seq), self.max_resp_len)] 294 | seqs.append(seq) 295 | return seqs 296 | 297 | # get nonc from different tp 298 | seqs = dict() 299 | for tp, n in [('base_nonc', max_n * (1. - mix_ratio[1])), ('bias_nonc', max_n * mix_ratio[1])]: 300 | seqs[tp] = [] 301 | if n < 1 or tp not in self.files[sub]: 302 | continue 303 | while True: 304 | m = len(seqs[tp]) 305 | seqs[tp] = _read_nonc(tp, n, seqs[tp]) 306 | if len(seqs[tp]) >= n: 307 | break 308 | if len(seqs[tp]) == m: 309 | self.reset(sub, tp) 310 | print('nonc from %s: %i/%i'%(tp, len(seqs[tp]), n)) 311 | if 'bias_nonc' in seqs and self.noisy_bias: 312 | seqs['bias_nonc'] = self.add_unk_noise(seqs['bias_nonc']) 313 | 314 | seqs_nonc = seqs['base_nonc'] + seqs['bias_nonc'] 315 | np.random.shuffle(seqs_nonc) 316 | 317 | if self.noisy_AE_src: 318 | inp_enc_nonc = self.seqs2enc(self.add_unk_noise(seqs_nonc), self.max_resp_len) 319 | else: 320 | inp_enc_nonc = self.seqs2enc(seqs_nonc, self.max_resp_len) 321 | inp_dec_nonc, out_dec_nonc = self.seqs2dec(seqs_nonc, self.max_resp_len) 322 | 323 | d_inp_enc['nonc'] = inp_enc_nonc 324 | d_inp_dec['nonc'] = inp_dec_nonc 325 | d_out_dec['nonc'] = out_dec_nonc 326 | n_sample = min(n_sample_conv, len(seqs_nonc)) 327 | return get_ret(n_sample, { 328 | 'inp_enc':d_inp_enc, 329 | 'inp_dec':d_inp_dec, 330 | 'out_dec':d_out_dec, 331 | 'seqs':{'resp':seqs_resp, 'nonc':seqs_nonc}, 332 | }) 333 | 334 | -------------------------------------------------------------------------------- /src/vis.py: -------------------------------------------------------------------------------- 1 | from shared import * 2 | from tf_lib import * 3 | from main import run_master, get_model_fld 4 | from scipy.optimize import fmin_powell as fmin 5 | from mpl_toolkits.mplot3d import Axes3D 6 | from sklearn import manifold 7 | import scipy 8 | 9 | """ 10 | AUTHOR: 11 | Sean Xiang Gao (xiag@microsoft.com) at Microsoft Research 12 | """ 13 | 14 | 15 | 16 | 17 | 18 | def dist_mat(coord): 19 | n = coord.shape[0] 20 | dist_T2T = np.zeros((n, n)) 21 | for i in range(n): 22 | for j in range(i + 1, n): 23 | d = euc_dist(coord[i, :], coord[j, :]) 24 | dist_T2T[i, j] = d 25 | dist_T2T[j, i] = d 26 | return dist_T2T 27 | 28 | 29 | 30 | 31 | def interp(master, model_name, fld_save, type_='resp'): 32 | 33 | n = 1000 34 | print('building data...') 35 | _, d_inp_enc, d_inp_dec, d_out_dec, _ = master.dataset.feed_data('test', max_n=n, check_src=True) 36 | if type_ == 'resp': 37 | vec_u0 = master.model_encoder['S2S'].predict(d_inp_enc['ctxt']) 38 | vec_u1 = master.model_encoder['AE'].predict(d_inp_enc['resp']) 39 | elif type_ == 'stry': 40 | vec_u0 = master.model_encoder['AE'].predict(d_inp_enc['resp']) 41 | vec_u1 = master.model_encoder['AE'].predict(d_inp_enc['stry']) 42 | else: 43 | raise ValueError 44 | 45 | print('evaluating...') 46 | uu = np.linspace(0, 1, 11) 47 | NLL = [] 48 | for u in uu: 49 | latent = vec_u0 + u * np.ones(vec_u0.shape) * (vec_u1 - vec_u0) 50 | NLL_resp = master.model_decoder_tf.evaluate( 51 | [latent, d_inp_dec['resp']], 52 | d_out_dec['resp'], 53 | verbose=0) 54 | if type_ == 'resp': 55 | NLL_ = NLL_resp 56 | else: 57 | NLL_stry = master.model_decoder_tf.evaluate( 58 | [latent, d_inp_dec['stry']], 59 | d_out_dec['stry'], 60 | verbose=0) 61 | NLL_ = NLL_resp * (1. - u) + u * NLL_stry 62 | print('u = %.3f, NLL = %.3f'%(u, NLL_)) 63 | NLL.append(NLL_) 64 | 65 | fig = plt.figure(figsize=(6,3)) 66 | ax = fig.add_subplot(111) 67 | ax.plot(uu, NLL,'k.-') 68 | print(uu) 69 | print(NLL) 70 | ax.plot(0, NLL[0], 'ro') 71 | ax.plot(1, NLL[-1], 'bo') 72 | 73 | ax.text(0, NLL[0] + 0.5, ' '+r'$S$', color='r') 74 | ax.text(1, NLL[-1], ' '+r'$T$', color='b') 75 | 76 | plt.xlabel(r'$u$') 77 | plt.ylabel('NLL') 78 | plt.title(model_name+'\nNLL of interpolation: '+r'$S+u(T-S)$') 79 | plt.subplots_adjust(top=0.8) 80 | plt.subplots_adjust(bottom=0.2) 81 | plt.savefig(fld_save+'/interp_%s.png'%type_) 82 | 83 | with open(fld_save+'/interp_%s.tsv'%type_,'w') as f: 84 | f.write('\t'.join(['u'] + ['%.3f'%u for u in uu])+'\n') 85 | f.write('\t'.join(['NLL'] + ['%.3f'%l for l in NLL])+'\n') 86 | 87 | plt.show() 88 | 89 | 90 | 91 | 92 | 93 | 94 | def clusters(master, model_name, fld_save, D=2, use_bias=True, n_batch=1): 95 | 96 | n_sample = BATCH_SIZE * n_batch 97 | method = 'MDS' 98 | #method = 'tSNE' 99 | #method = 'isomap' 100 | 101 | latent_d = dict() 102 | colors = { 103 | 'base_conv': 'y', 104 | 'base_resp': 'r', 105 | 'bias_conv': 'k', 106 | 'bias_nonc': 'b', 107 | } 108 | 109 | print('building data...') 110 | d_inp_enc = master.dataset.feed_data('test', max_n=n_sample, check_src=True, mix_ratio=(0.,1.))['inp_enc'] 111 | latent_d['base_conv'] = master.model_encoder['S2S'].predict(d_inp_enc['ctxt']) 112 | if use_bias and 'AE' in master.prefix: 113 | latent_d['bias_nonc'] = master.model_encoder['AE'].predict(d_inp_enc['nonc']) 114 | #if use_bias and 'bias_conv' in master.dataset.files['test']: 115 | # d_inp_enc = master.dataset.feed_data('test', max_n=n_sample, check_src=True, mix_ratio=(1.,0.))['inp_enc'] 116 | # latent_d['bias_conv'] = master.model_encoder['S2S'].predict(d_inp_enc['ctxt']) 117 | #else: 118 | d_inp_enc = master.dataset.feed_data('test', max_n=n_sample, check_src=True, mix_ratio=(0.,0.))['inp_enc'] 119 | if 'AE' in master.prefix: 120 | #latent_d['base_nonc'] = master.model_encoder['AE'].predict(d_inp_enc['nonc']) 121 | latent_d['base_resp'] = master.model_encoder['AE'].predict(d_inp_enc['resp']) 122 | 123 | labels = list(sorted(latent_d.keys())) 124 | fname_suffix = args.restore.split('/')[-1].replace('.npz','') 125 | if use_bias: 126 | fname_suffix += '_wbias' 127 | n_labels = len(labels) 128 | latent = np.concatenate([latent_d[k] for k in labels], axis=0) 129 | print('latent.shape',latent.shape) 130 | 131 | print('plotting bit hist...') 132 | bins = np.linspace(-1,1,31) 133 | for k in latent_d: 134 | l = latent_d[k].ravel() 135 | freq, _, _ = plt.hist(l, bins=bins, color='w') 136 | plt.plot(bins[:-1], 100.*freq/sum(freq), colors[k]+'.-') 137 | plt.ylim([0,50]) 138 | plt.savefig(fld_save+'/hist_%s.png'%fname_suffix) 139 | plt.close() 140 | 141 | print('plotting dist mat...') 142 | d_norm = np.sqrt(latent.shape[1]) 143 | f, ax = plt.subplots() 144 | cax = ax.imshow(dist_mat(latent)/d_norm, cmap='bwr') 145 | #ax.set_title(model_name) 146 | f.colorbar(cax) 147 | 148 | ticks = [] 149 | ticklabels = [] 150 | n_prev = 0 151 | for i in range(n_labels): 152 | ticks.append(n_prev + n_sample/2) 153 | ticklabels.append(labels[i]+'\n') 154 | ticks.append(n_prev + n_sample) 155 | ticklabels.append('%i'%(n_sample * (i+1))) 156 | n_prev = n_prev + n_sample 157 | ax.set_xticks(ticks) 158 | ax.set_xticklabels(ticklabels) 159 | ax.xaxis.tick_top() 160 | ax.set_yticks(ticks) 161 | ax.set_yticklabels([s.strip('\n') for s in ticklabels]) 162 | 163 | plt.savefig(fld_save+'/dist_%s.png'%fname_suffix) 164 | plt.close() 165 | 166 | if method == 'tSNE': 167 | approx = manifold.TSNE(init='pca', verbose=1).fit_transform(latent) 168 | elif method == 'MDS': 169 | approx = manifold.MDS(D, verbose=1, max_iter=500, n_init=1).fit_transform(latent) 170 | elif method == 'isomap': 171 | approx = manifold.Isomap().fit_transform(latent) 172 | else: 173 | raise ValueError 174 | 175 | f, ax = plt.subplots() 176 | for k in labels: 177 | ax.plot(np.nan, np.nan, colors[k]+'.', label=k) 178 | 179 | jj = list(range(approx.shape[0])) 180 | np.random.shuffle(jj) 181 | for j in jj: 182 | i_label = int(j/n_sample) 183 | ax.plot(approx[j, 0], approx[j, 1], colors[labels[i_label]]+'.') 184 | 185 | #plt.legend(loc='best') 186 | plt.title(model_name) 187 | #ax.set_xticks([]) 188 | #ax.set_yticks([]) 189 | plt.savefig(fld_save+'/%s_%s.png'%(method, fname_suffix)) 190 | plt.show() 191 | 192 | 193 | 194 | 195 | 196 | 197 | def cos_sim(a, b): 198 | #return 1. - scipy.spatial.distance.cosine(a, b) 199 | return np.inner(a, b)/np.linalg.norm(a)/np.linalg.norm(b) 200 | 201 | 202 | def angel_hist(master, model_name, fld_save): 203 | from rand_decode import load_1toN_data 204 | data = load_1toN_data(master.dataset.generator['test']) 205 | angel = [] 206 | n_sample = 1000 207 | 208 | extra_info = [] 209 | 210 | for i in range(n_sample): 211 | if i%10 == 0: 212 | print(i) 213 | d = data[i] 214 | src_seq = np.reshape(d['src_seq'], [1,-1]) 215 | latent_src = np.ravel(master.model_encoder['dial'].predict(src_seq)) 216 | diff = [] 217 | for ref_seq in d['ref_seqs']: 218 | ref_seq = np.reshape(ref_seq, [1,-1]) 219 | latent_ref = np.ravel(master.model_encoder['auto'].predict(ref_seq)) 220 | diff.append(latent_ref - latent_src) 221 | 222 | for i in range(len(diff) - 1): 223 | for j in range(i + 1, len(diff)): 224 | if str(d['ref_seqs'][i]) == str(d['ref_seqs'][j]): 225 | continue 226 | angel.append(cos_sim(diff[i], diff[j])) 227 | extra_info.append('%i\t%i'%(i, len(d['ref_seqs']))) 228 | 229 | with open(fld_save+'/angel.txt', 'w') as f: 230 | f.write('\n'.join([str(a) for a in angel])) 231 | with open(fld_save+'/angel_extra.txt', 'w') as f: 232 | f.write('\n'.join(extra_info)) 233 | 234 | plt.hist(angel, bins=30) 235 | plt.title(model_name) 236 | plt.savefig(fld_save+'/angel.png') 237 | plt.show() 238 | 239 | 240 | 241 | 242 | def plot_history(paths, labels, k, ix=-1, ax=None): 243 | if isinstance(paths, str): 244 | paths = [paths] 245 | import matplotlib.pyplot as plt 246 | 247 | def MA(y): 248 | window = 30 249 | ret = [np.nan] * len(y) 250 | for i in range(window, len(y)): 251 | ret[i] = np.mean(y[max(0, i - window + 1): i + 1]) 252 | return ret 253 | 254 | def read_log(path, k): 255 | trained = np.nan 256 | xx = [] 257 | yy = [[] for _ in range(4)] 258 | m = None 259 | for line in open(path): 260 | if line.startswith('***** trained '): 261 | trained = float(line.split(',')[0].split()[-2]) 262 | if line.startswith(k): 263 | vv = [float(v) for v in line.replace(':','=').split('=')[-1].split(',')] 264 | if m is None: 265 | m = len(vv) 266 | print('expecting %i values'%m) 267 | else: 268 | if m!=len(vv): 269 | continue 270 | xx.append(trained) 271 | for i in range(len(vv)): 272 | yy[i].append(vv[i]) 273 | return xx, yy[:m] 274 | 275 | if ax is None: 276 | _, ax = plt.subplots() 277 | 278 | color = ['r','b','k','m'] 279 | if len(paths) > 0: 280 | for i, path in enumerate(paths): 281 | xx, yy = read_log(path, k) 282 | ss = path.split('/') 283 | label = ss[-1].replace('.txt','') 284 | ax.plot(xx, yy[ix], color=color[i], linestyle=':', alpha=0.5) 285 | ax.plot(xx, MA(yy[ix]), color=color[i], label=labels[i]) 286 | ax.set_title(k + '[%i]'%ix) 287 | 288 | else: 289 | xx, yy = read_log(paths[0], k) 290 | for i in range(len(yy)): 291 | ax.plot(xx, yy[i], color=color[i], linestyle=':') 292 | ax.plot(xx, MA(yy[i]), color=color[i], label=str(i + 1)) 293 | 294 | 295 | 296 | def plot_multiple(kk, paths, labels): 297 | 298 | n_col = 4 299 | n_row = int(len(kk)/n_col) 300 | n_row = int(np.ceil(len(kk)/n_col)) 301 | print('n_row = %i'%n_row) 302 | _, axs = plt.subplots(n_row, n_col, sharex=True) 303 | for i in range(len(kk)): 304 | k = kk[i] 305 | col = i%n_col 306 | row = int(i/n_col) 307 | ax = axs[row][col] 308 | if k.startswith('bleu') or k.startswith('corr'): 309 | ix = 2 310 | else: 311 | ix = -1 312 | plot_history(paths, labels, k, ix, ax=ax) 313 | #if i == 0: 314 | # ax.legend(loc='best') 315 | ax.grid(True) 316 | plt.show() 317 | 318 | 319 | if __name__ == '__main__': 320 | parser.add_argument('--vis_tp', default='clusters') 321 | parser.add_argument('--use_bias', type=int, default=1) 322 | parser.add_argument('--n_batch', type=int, default=5) 323 | args = parser.parse_args() 324 | print('>>>>> Not using GPU') 325 | os.environ["CUDA_VISIBLE_DEVICES"]="-1" 326 | 327 | master = run_master('vis', args) 328 | 329 | #if args.cpu_only: 330 | 331 | #fld = os.path.join(fld_model, model_name, 'vis') 332 | model_name = '' 333 | fld = master.fld + '/vis' 334 | print(fld) 335 | makedirs(fld) 336 | 337 | if args.vis_tp.startswith('interp'): 338 | if 'stry' in args.vis_tp: 339 | interp(master, model_name, fld, type_='stry') 340 | else: 341 | interp(master, model_name, fld, type_='resp') 342 | elif args.vis_tp == 'clusters': 343 | clusters(master, model_name, fld, use_bias=bool(args.use_bias), n_batch=args.n_batch) 344 | elif args.vis_tp == 'angel': 345 | angel_hist(master, model_name, fld) 346 | else: 347 | raise ValueError -------------------------------------------------------------------------------- /src/classifier.py: -------------------------------------------------------------------------------- 1 | from shared import * 2 | from tf_lib import * 3 | import json 4 | from dataset import load_vocab 5 | from sklearn import linear_model 6 | import pickle 7 | 8 | """ 9 | AUTHOR: 10 | Sean Xiang Gao (xiag@microsoft.com) at Microsoft Research 11 | """ 12 | 13 | class ClassifierNeural(): 14 | 15 | def __init__(self, fld): 16 | params = json.load(open(fld + '/args.json')) 17 | if params['tgt_only']: 18 | self.prefix = ['tgt'] 19 | else: 20 | self.prefix = ['src','tgt'] 21 | self.encoder_depth = params['encoder_depth'] 22 | self.rnn_units = params['rnn_units'] 23 | self.mlp_depth = params['mlp_depth'] 24 | self.mlp_units = params['mlp_units'] 25 | self.include_punc = params['include_punc'] 26 | self.index2token, self.token2index = load_vocab(fld + '/vocab.txt') 27 | 28 | self.fld = fld 29 | self.load() 30 | 31 | 32 | def load(self): 33 | self.build_model() 34 | self.model.load_weights(self.fld+'/model.h5') 35 | 36 | 37 | def _create_layers(self): 38 | layers = dict() 39 | 40 | layers['embedding'] = Embedding( 41 | max(self.index2token.keys()) + 1, # +1 as mask_zero 42 | self.rnn_units, mask_zero=True, 43 | name='embedding') 44 | 45 | for prefix in self.prefix: 46 | for i in range(self.encoder_depth): 47 | name = '%s_encoder_rnn_%i'%(prefix, i) 48 | layers[name] = GRU( 49 | self.rnn_units, 50 | return_state=True, 51 | return_sequences=True, 52 | name=name) 53 | 54 | for i in range(self.mlp_depth - 1): 55 | name = 'mlp_%i'%i 56 | layers[name] = Dense( 57 | self.mlp_units, 58 | activation='tanh', name=name) 59 | 60 | name = 'mlp_%i'%(self.mlp_depth - 1) 61 | layers[name] = Dense(1, activation='sigmoid', name=name) 62 | return layers 63 | 64 | 65 | def _stacked_rnn(self, rnns, inputs, initial_states=None): 66 | if initial_states is None: 67 | initial_states = [None] * len(rnns) 68 | 69 | outputs, state = rnns[0](inputs, initial_state=initial_states[0]) 70 | states = [state] 71 | for i in range(1, len(rnns)): 72 | outputs, state = rnns[i](outputs, initial_state=initial_states[i]) 73 | states.append(state) 74 | return outputs, states 75 | 76 | 77 | def _build_encoder(self, inputs, layers, prefix): 78 | _, encoder_states = self._stacked_rnn( 79 | [layers['%s_encoder_rnn_%i'%(prefix, i)] for i in range(self.encoder_depth)], 80 | layers['embedding'](inputs)) 81 | latent = encoder_states[-1] 82 | return latent 83 | 84 | 85 | def build_model(self): 86 | layers = self._create_layers() 87 | 88 | encoder_inputs = dict() 89 | latents = [] 90 | for prefix in self.prefix: 91 | encoder_inputs[prefix] = Input(shape=(None,), name=prefix+'_encoder_inputs') 92 | latents.append(self._build_encoder(encoder_inputs[prefix], layers, prefix=prefix)) 93 | 94 | if len(self.prefix) > 1: 95 | out = Concatenate()(latents) 96 | inp = [encoder_inputs['src'], encoder_inputs['tgt']] 97 | else: 98 | out = latents[0] 99 | inp = encoder_inputs[self.prefix[0]] 100 | for i in range(self.mlp_depth): 101 | out = layers['mlp_%i'%i](out) 102 | 103 | self.model = Model(inp, out) 104 | self.model.compile(optimizer=Adam(lr=0), loss='binary_crossentropy') 105 | 106 | 107 | def txt2seq(self, txt): 108 | tokens = txt.strip().split(' ') 109 | seq = [] 110 | ix_unk = self.token2index[UNK_token] 111 | for token in tokens: 112 | if self.include_punc or is_word(token): # skip punctuation if necessary 113 | seq.append(self.token2index.get(token, ix_unk)) 114 | return seq 115 | 116 | def seq2txt(self, seq): 117 | return ' '.join([self.index2token[i] for i in seq]) 118 | 119 | def txts2mat(self, txts, max_len=30): 120 | if isinstance(txts, str): 121 | txts = [txts] 122 | data = np.zeros((len(txts), max_len)) 123 | for j, txt in enumerate(txts): 124 | seq = self.txt2seq(txt.strip(EOS_token).strip()) # stripped EOS_token here 125 | for t in range(min(max_len, len(seq))): 126 | data[j, t] = seq[t] 127 | return data 128 | 129 | def predict(self, txts): 130 | mat = self.txts2mat(txts) 131 | return self.model.predict(mat).ravel() 132 | 133 | 134 | 135 | 136 | class ClassifierNgram: 137 | 138 | def __init__(self, fld, ngram, include_punc=False): 139 | self.fld = fld 140 | self.ngram2ix = dict() 141 | self.ngram = ngram 142 | self.include_punc = include_punc 143 | 144 | fname = '%igram'%ngram 145 | if include_punc: 146 | fname += '.include_punc' 147 | self.path_prefix = fld + '/' + fname 148 | for i, line in enumerate(open(self.path_prefix + '.txt', encoding='utf-8')): 149 | ngram = line.strip('\n') 150 | self.ngram2ix[ngram] = i 151 | assert(self.ngram == len(ngram.split())) 152 | self.vocab_size = i + 1 153 | print('loaded %i %igram'%(self.vocab_size, self.ngram)) 154 | #self.model = LogisticRegression(solver='sag')#, max_iter=10) 155 | self.model = linear_model.SGDClassifier(loss='log', random_state=9, max_iter=1, tol=1e-3) 156 | 157 | def txts2mat(self, txts): 158 | X = np.zeros((len(txts), self.vocab_size)) 159 | for i, txt in enumerate(txts): 160 | ww = txt2ww(txt, self.include_punc) 161 | for t in range(self.ngram, len(ww) + 1): 162 | ngram = ' '.join(ww[t - self.ngram: t]) 163 | j = self.ngram2ix.get(ngram, None) 164 | if j is not None: 165 | X[i, j] = 1. 166 | return X 167 | 168 | def load(self): 169 | self.model = pickle.load(open(self.path_prefix + '.p', 'rb')) 170 | 171 | def predict(self, txts): 172 | data = self.txts2mat(txts) 173 | prob = self.model.predict_proba(data) 174 | return prob[:,1] 175 | 176 | 177 | 178 | 179 | class ClassifierNgramEnsemble: 180 | 181 | def __init__(self, fld, include_punc=False, max_ngram=4): 182 | self.fld = fld 183 | self.children = dict() 184 | self.wt = dict() 185 | for ngram in range(1, max_ngram + 1): 186 | self.children[ngram] = ClassifierNgram(fld, ngram, include_punc) 187 | self.children[ngram].load() 188 | acc = float(open(self.children[ngram].path_prefix + '.acc').readline().strip('\n')) 189 | self.wt[ngram] = 2. * max(0, acc - 0.5) 190 | 191 | def predict(self, txts): 192 | avg_scores = np.array([0.] * len(txts)) 193 | for ngram in self.children: 194 | scores = self.children[ngram].predict(txts) 195 | avg_scores += scores * self.wt[ngram] 196 | return avg_scores / sum(self.wt.values()) 197 | 198 | 199 | def is_word(token): 200 | for c in token: 201 | if c.isalpha(): 202 | return True 203 | return False 204 | 205 | 206 | 207 | 208 | def load_classifier(fld, args=None): 209 | if fld.endswith('ngram'): 210 | return ClassifierNgramEnsemble(fld) 211 | elif fld.endswith('neural'): 212 | return ClassifierNeural(fld) 213 | else: 214 | raise ValueError 215 | 216 | 217 | def clf_interact(fld): 218 | clf = load_classifier(fld) 219 | while True: 220 | print('\n---- please input ----') 221 | txt = input() 222 | if txt == '': 223 | break 224 | score = clf.predict([txt])[0] 225 | print('%.4f'%score) 226 | 227 | 228 | def clf_eval(clf_fld, path): 229 | # path is a tsv, last col is hyp 230 | clf = load_classifier(clf_fld) 231 | sum_score = 0 232 | n = 0 233 | for line in open(path, encoding='utf-8'): 234 | txt = line.strip('\n').split('\t')[-1].lower() 235 | sum_score += clf.predict([txt])[0] 236 | n += 1 237 | if n % 100 == 0: 238 | print('eval %i lines'%n) 239 | print('finally %i samples'%n) 240 | print('avg style score: %.4f'%(sum_score/n)) 241 | 242 | 243 | def txt2ww(txt, include_punc): 244 | ww = [SOS_token] 245 | for w in txt.split(): 246 | if include_punc or is_word(w): 247 | ww.append(w) 248 | ww.append(EOS_token) 249 | return ww 250 | 251 | 252 | def score_file(path, name, col=1): 253 | clf = load_classifier(name) 254 | txts = [] 255 | for line in open(path, encoding='utf-8'): 256 | txts.append(line.strip('\n').split('\t')[col]) 257 | if len(txts) == 1500: 258 | break 259 | print('scoring...') 260 | print(np.mean(clf.predict(txts))) 261 | 262 | 263 | class Classifier1gramCount: 264 | def __init__(self, fld): 265 | self.fld = fld 266 | 267 | def fit(self, min_freq=60, max_n=1e5): 268 | scores = dict() 269 | n = 0 270 | for line in open(self.fld + '/all.txt', encoding='utf-8'): 271 | n += 1 272 | cells = line.strip('\n').split('\t') 273 | if len(cells) != 2: 274 | print(cells) 275 | exit() 276 | txt, score = cells 277 | for w in set(txt.strip().split()): 278 | if is_word(w): 279 | if w not in scores: 280 | scores[w] = [] 281 | scores[w].append(float(score)) 282 | if n == max_n: 283 | break 284 | 285 | 286 | lines = ['\t'.join(['word', 'avg', 'se', 'count'])] 287 | for w in scores: 288 | count = len(scores[w]) 289 | if count < min_freq: 290 | continue 291 | avg = np.mean(scores[w]) 292 | se = np.std(scores[w])/np.sqrt(count) 293 | lines.append('\t'.join([w, '%.4f'%avg, '%.4f'%se, '%i'%count])) 294 | 295 | with open(self.fld + '/count.tsv', 'w', encoding='utf-8') as f: 296 | f.write('\n'.join(lines)) 297 | 298 | def load(self): 299 | self.coef = dict() 300 | f = open(self.fld + '/count.tsv', encoding='utf-8') 301 | header = f.readline() 302 | for line in f: 303 | w, avg = line.strip('\n').split('\t')[:2] 304 | self.coef[w] = float(avg) 305 | 306 | def corpus_score(self, txts, kw=100): 307 | scores = [] 308 | coef_w = [] 309 | for w in self.coef: 310 | coef_w.append((self.coef[w], w)) 311 | coef_w = sorted(coef_w, reverse=True)[:kw] 312 | print('last:',coef_w[-1]) 313 | keywords = set([w for _, w in coef_w]) 314 | 315 | #total_joint = 0 316 | #total = 0 317 | 318 | for txt in txts: 319 | words = set() 320 | for w in txt.strip().split(): 321 | if is_word(w): 322 | words.add(w) 323 | joint = words & keywords 324 | scores.append(len(joint)/len(words)) 325 | #total_joint += len(joint) 326 | #total += len(words) 327 | return np.mean(scores), np.std(scores)/np.sqrt(len(scores)) 328 | #return total_joint/total 329 | 330 | 331 | def test(self, kw=100): 332 | import matplotlib.pyplot as plt 333 | 334 | txts = [] 335 | labels = [] 336 | for line in open(self.fld + '/sorted_avg.tsv', encoding='utf-8'): 337 | txt, label = line.strip('\n').split('\t') 338 | txts.append(txt) 339 | labels.append(float(label)) 340 | 341 | i0 = 0 342 | human = [] 343 | pred = [] 344 | while True: 345 | i1 = i0 + 100 346 | if i1 >= len(txts): 347 | break 348 | human.append(np.mean(labels[i0:i1])) 349 | pred.append(self.corpus_score(txts[i0:i1], kw=kw)) 350 | i0 = i1 351 | 352 | plt.plot(human, pred, '.') 353 | plt.xlabel('human') 354 | plt.xlabel('metric (ratio of keywords)') 355 | plt.title('corr = %.4f'%np.corrcoef(human, pred)[0][1]) 356 | plt.savefig(self.fld + '/test_corr_kw%i.png'%kw) 357 | 358 | 359 | if __name__ == '__main__': 360 | # e.g. `python src/classifier.py classifier/Reddit_vs_arXiv/neural' for interaction 361 | # e.g. `python src/classifier.py classifier/Reddit_vs_arXiv/neural path/to/hyp/file.tsv' for evaluating a file 362 | fld_model = sys.argv[1] # e.g. 363 | if len(sys.argv) == 2: 364 | clf_interact(fld_model) 365 | elif len(sys.argv) == 3: 366 | path_hyp = sys.argv[2] 367 | clf_eval(fld_model, path_hyp) -------------------------------------------------------------------------------- /src/decode.py: -------------------------------------------------------------------------------- 1 | from shared import * 2 | from nltk.translate.bleu_score import SmoothingFunction 3 | """ 4 | AUTHOR: 5 | Sean Xiang Gao (xiag@microsoft.com) at Microsoft Research 6 | """ 7 | 8 | class Decoder: 9 | 10 | def __init__(self, dataset, model, decoder_depth, latent_dim, allowed_words=None): 11 | self.dataset = dataset 12 | self.model = model 13 | self.decoder_depth = decoder_depth 14 | self.latent_dim = latent_dim 15 | 16 | if allowed_words is None: 17 | self.mask = np.array([1.] * (self.dataset.num_tokens + 1)) 18 | else: 19 | self.mask = np.array([0.] * (self.dataset.num_tokens + 1)) 20 | for word in allowed_words: 21 | ix = self._ix(word) 22 | if ix is not None: 23 | self.mask[ix] = 1. 24 | 25 | print('allowed words %i/%i'%(sum(self.mask), len(self.mask))) 26 | 27 | default_forbid = [UNK_token, '(', '__url__', ')', EQN_token, CITE_token, IX_token] #+ ['queer', 'holmes', 'sherlock', 'john', 'watson', 'bannister'] 28 | for word in default_forbid: 29 | ix = self._ix(word) 30 | if ix is not None: 31 | self.mask[ix] = 0. # in either case, UNK is not allowed 32 | 33 | 34 | def _ix(self, token): 35 | return self.dataset.token2index.get(token, None) 36 | 37 | 38 | def predict(self, latents, sampling=False, softmax_temperature=1, lm_wt=None): 39 | # autoregressive in parallel, greedy or softmax sampling 40 | 41 | latents = np.reshape(latents, (-1, self.latent_dim)) # (n, dim) 42 | n = latents.shape[0] 43 | n_vocab = len(self.mask) 44 | prev = np.zeros((n, 1)) + self._ix(SOS_token) 45 | states = [latents] * self.decoder_depth # list of state, each is [n, dim] 46 | mask = np.repeat(np.reshape(self.mask, (1, -1)), n, axis=0) # (n, vocab) 47 | logP = [0.] * n 48 | stop = [False] * n 49 | hyp = [] 50 | for _ in range(n): 51 | hyp.append([]) 52 | 53 | def sample_token_index_softmax(prob): 54 | if softmax_temperature != 1: 55 | prob = np.exp(np.log(prob) * softmax_temperature) 56 | return np.random.choice(n_vocab, 1, p=prob/sum(prob))[0] 57 | def sample_token_index_greedy(prob): 58 | return np.argmax(prob) 59 | if sampling: 60 | sample_token_index = sample_token_index_softmax 61 | else: 62 | sample_token_index = sample_token_index_greedy 63 | 64 | for _ in range(self.dataset.max_resp_len): 65 | out = self.model.predict([prev] + states) 66 | states = out[1:] 67 | tokens_proba = np.squeeze(out[0]) * mask # squeeze: (n, 1, vocab) => (n, vocab) 68 | 69 | prev = [0] * n 70 | for i in range(n): 71 | if stop[i]: 72 | continue 73 | prob = tokens_proba[i,:].ravel() 74 | 75 | ix = sample_token_index(prob) 76 | logP[i] += np.log(prob[ix]) 77 | hyp[i].append(ix) 78 | prev[i] = ix 79 | if ix == self._ix(EOS_token): 80 | stop[i] = True 81 | prev = np.reshape(prev, (n, 1)) 82 | 83 | return [logP[i]/len(hyp[i]) for i in range(n)], hyp 84 | 85 | 86 | def evaluate(self, latents, tgt_seqs): 87 | # teacher-forcing in parallel 88 | 89 | latents = np.reshape(latents, (-1, self.latent_dim)) # (n, dim) 90 | n = latents.shape[0] 91 | states = [latents] * self.decoder_depth # list of state, each is [n, dim] 92 | logP = [0.] * n 93 | prev = np.zeros((n, 1)) + self._ix(SOS_token) 94 | lens = [len(seq) for seq in tgt_seqs] 95 | epsilon = 1e-6 96 | 97 | for t in range(self.dataset.max_resp_len): 98 | out = self.model.predict([prev] + states) 99 | states = out[1:] 100 | tokens_proba = np.reshape(out[0], (n, -1)) # squeeze: (n, 1, vocab) => (n, vocab) 101 | prev = [0] * n 102 | for i in range(n): 103 | if t < lens[i]: 104 | ix = tgt_seqs[i][t] 105 | logP[i] += np.log(max(epsilon, tokens_proba[i, ix])) 106 | prev[i] = ix 107 | prev = np.reshape(prev, (n, 1)) 108 | 109 | return [logP[i]/lens[i] for i in range(n)] 110 | #return [logP[i]/self.dataset.max_resp_len for i in range(n)] 111 | 112 | 113 | def predict_beam(self, latents, beam_width=10, n_child=3, max_n_hyp=100): 114 | # multi-head beam search, not yet parallel 115 | 116 | prev = np.atleast_2d([self._ix(SOS_token)]) 117 | beam = [] 118 | for latent in latents: 119 | latent = np.atleast_2d(latent) 120 | states = [latent] * self.decoder_depth 121 | node = {'states':states[:], 'prev':prev, 'logP':0, 'hyp':[]} 122 | beam.append(node) 123 | print('beam search initial n = %i'%len(beam)) 124 | 125 | results = queue.PriorityQueue() 126 | t = 0 127 | while True: 128 | t += 1 129 | if t > 20:#self.dataset.max_tgt_len: 130 | break 131 | if len(beam) == 0: 132 | break 133 | 134 | pq = queue.PriorityQueue() 135 | for node in beam: 136 | out = self.model.predict([node['prev']] + node['states']) 137 | tokens_proba = out[0].ravel() 138 | states = out[1:] 139 | 140 | tokens_proba = tokens_proba * self.mask 141 | tokens_proba = tokens_proba/sum(tokens_proba) 142 | top_tokens = np.argsort(-tokens_proba) 143 | for ix in top_tokens[:n_child]: 144 | 145 | logP = node['logP'] + np.log(tokens_proba[ix]) 146 | hyp = node['hyp'][:] + [ix] 147 | if ix == self._ix(EOS_token): 148 | results.put((logP/t, hyp)) 149 | if results.qsize() > max_n_hyp: 150 | results.get() # pop the hyp of lowest logP/t 151 | continue 152 | 153 | pq.put(( 154 | logP, # no need to normalize to logP/t as every node is at the same t 155 | np.random.random(), # to avoid the case logP is the same 156 | { 157 | 'states':states, 158 | 'prev':np.atleast_2d([ix]), 159 | 'logP':logP, 160 | 'hyp':hyp, 161 | } 162 | )) 163 | if pq.qsize() > beam_width: 164 | pq.get() # pop the node of lowest logP to maintain at most beam_width nodes => but this will encourage bland response 165 | 166 | beam = [] 167 | while not pq.empty(): 168 | _, _, node = pq.get() 169 | beam.append(node) 170 | 171 | logPs = [] 172 | hyps = [] 173 | while not results.empty(): 174 | logP, hyp = results.get() 175 | logPs.append(logP) 176 | hyps.append(hyp) 177 | return logPs, hyps 178 | 179 | 180 | 181 | 182 | def rank_nbest(hyps, logP, logP_center, master, inp, infer_args=dict(), base_ranker=None): 183 | # make sure hyps are list of str, and inp is str 184 | # as base_ranker, master, and clf may not share the same vocab 185 | 186 | assert(isinstance(hyps, list)) 187 | assert(isinstance(hyps[0], str)) 188 | assert(isinstance(inp, str)) 189 | 190 | hyps_no_ie = [] 191 | for hyp in hyps[:]: 192 | hyps_no_ie.append((' '+hyp+' ').replace(' i . e . ,',' ').replace(' i . e. ',' ').strip()) 193 | hyps = hyps_no_ie[:] 194 | 195 | wt_clf = infer_args.get('wt_clf', 0) / len(master.classifiers) 196 | wt_rep = infer_args.get('wt_rep', 0) 197 | wt_len = infer_args.get('wt_len', 0) 198 | wt_center = infer_args.get('wt_center', 0) 199 | wt_base = infer_args.get('wt_base', 0) 200 | 201 | n = len(logP) 202 | clf_score = [] 203 | max_tgt_len = 30 204 | for clf in master.classifiers: 205 | clf_score.append(clf.predict(hyps).ravel()) 206 | 207 | if base_ranker is not None: 208 | hyp_seqs_base = [base_ranker.dataset.txt2seq(hyp) for hyp in hyps] 209 | inp_seq_base = base_ranker.dataset.txt2seq(inp) 210 | latent_base = base_ranker.model_encoder['S2S'].predict(np.atleast_2d(inp_seq_base)) 211 | logP_base = base_ranker.decoder.evaluate([latent_base]*n, hyp_seqs_base) 212 | else: 213 | logP_base = [0] * n 214 | 215 | pq = queue.PriorityQueue() 216 | for i in range(n): 217 | hyp = hyps[i] 218 | rep = repetition_penalty(hyp) 219 | l = min(max_tgt_len, len(hyp.split()))/max_tgt_len 220 | score = logP[i] + wt_center * logP_center[i] + wt_rep * rep + wt_len * l + wt_base * logP_base[i] 221 | 222 | clf_score_ = [] 223 | for k in range(len(master.classifiers)): 224 | s = clf_score[k][i] 225 | score += wt_clf * s 226 | clf_score_.append(s) 227 | pq.put((-score, hyp, (logP[i], logP_center[i], logP_base[i], rep, l) + tuple(clf_score_))) 228 | 229 | results = [] 230 | while not pq.empty(): 231 | neg_score, hyp, terms = pq.get() 232 | #if len(set(['queer', 'holmes', 'sherlock', 'john', 'watson', 'bannister']) & set(hyp.split())) > 0: 233 | # continue 234 | hyp = (' ' + hyp + ' ').replace(' to day ',' today ').replace(' to morrow ',' tomorrow ')#.replace('mr barker','') 235 | results.append((-neg_score, hyp, terms)) 236 | return results 237 | 238 | 239 | def repetition_penalty(hyp): 240 | # simplified from https://sunlamp.visualstudio.com/sunlamp/_git/sunlamp?path=%2Fsunlamp%2Fpython%2Fdynamic_decoder_custom.py&version=GBmaster 241 | # ratio of unique 1-gram 242 | ww = hyp.split() 243 | return np.log(min(1.0, len(set(ww)) / len(ww))) 244 | 245 | 246 | def infer(latent, master, method='greedy', beam_width=10, n_rand=20, r_rand=1.5, softmax_temperature=1, lm_wt=0.5): 247 | if method == 'greedy': 248 | return master.decoder.predict(latent, lm_wt=lm_wt) 249 | elif method == 'softmax': 250 | return master.decoder.predict([latent] * n_rand, sampling=True, lm_wt=lm_wt) 251 | elif method == 'beam': 252 | return master.decoder.predict_beam([latent], beam_width=beam_width) 253 | elif method.startswith('latent'): 254 | latents = [] 255 | if r_rand >= 0: 256 | rr = [r_rand] * n_rand 257 | else: 258 | rr = np.linspace(0, 5, n_rand) 259 | for r in rr: 260 | latents.append(rand_latent(latent, r, limit=True)) 261 | if 'beam' in method: 262 | return master.decoder.predict_beam(latents, beam_width=beam_width) 263 | else: 264 | return master.decoder.predict(latents, sampling=('softmax' in method), softmax_temperature=softmax_temperature, lm_wt=lm_wt) 265 | else: 266 | raise ValueError 267 | 268 | def infer_comb(inp, master): 269 | inp_seq = master.dataset.txt2seq(inp) 270 | latent = master.model_encoder['S2S'].predict(np.atleast_2d(inp_seq)) 271 | reset_rand() 272 | logP, hyp_seqs = infer(latent, master, method='latent', n_rand=10, r_rand=-1) 273 | logP, hyp_seqs = remove_duplicate_unfished(logP, hyp_seqs, master.dataset.token2index[EOS_token]) 274 | results = sorted(zip(logP, hyp_seqs), reverse=True) 275 | 276 | s = '-'*10 + '\n' + inp + '\n' 277 | for i, (logP, seq) in enumerate(results): 278 | hyp = master.dataset.seq2txt(seq) 279 | s += '%.3f'%logP + '\t' + hyp + '\n' 280 | if i == 4: 281 | break 282 | s += '-'*5 + '\n' 283 | return s 284 | 285 | 286 | def remove_duplicate_unfished(logP, hyp_seqs, ix_EOS): 287 | d = dict() 288 | for i in range(len(logP)): 289 | k = tuple(hyp_seqs[i]) 290 | if k[-1] != ix_EOS: 291 | continue 292 | if k not in d or logP[i] > d[k]: 293 | d[k] = logP[i] 294 | logP0, hyp0 = logP[0], hyp_seqs[0][:] 295 | logP = [] 296 | hyp_seqs = [] 297 | for k in d: 298 | logP.append(d[k]) 299 | hyp_seqs.append(list(k)) 300 | if len(logP) == 0: 301 | return [logP0], [hyp0] 302 | else: 303 | return logP, hyp_seqs 304 | 305 | 306 | def parse_infer_args(): 307 | arg = {'prefix':'S2S'} 308 | for line in open('src/infer_args.csv'): 309 | if line.startswith('#'): 310 | continue 311 | if ',' not in line: 312 | continue 313 | k, v = line.strip('\n').split(',') 314 | if k != 'method': 315 | if k in ['beam_width', 'n_rand']: 316 | v = int(v) 317 | else: 318 | v = float(v) 319 | arg[k] = v 320 | return arg 321 | 322 | 323 | def infer_rank(inp, master, infer_args, base_ranker=None, unique=True, verbose=True): 324 | if verbose: 325 | print('infer_args = '+str(infer_args)) 326 | inp_seq = master.dataset.txt2seq(inp) 327 | latent = master.model_encoder['S2S'].predict(np.atleast_2d(inp_seq)) 328 | reset_rand() 329 | 330 | if verbose: 331 | print('infering...') 332 | t0 = datetime.datetime.now() 333 | logP, hyp_seqs = infer(latent, master, method=infer_args['method'], 334 | beam_width=infer_args.get('beam_width'), n_rand=infer_args.get('n_rand'), r_rand=infer_args.get('r_rand'), 335 | softmax_temperature=infer_args.get('softmax_temperature'), lm_wt=infer_args.get('lm_wt')) 336 | t1 = datetime.datetime.now() 337 | if verbose: 338 | print('*'*10 + ' infer spent: '+str(t1-t0)) 339 | 340 | n_raw = len(logP) 341 | logP, hyp_seqs = remove_duplicate_unfished(logP, hyp_seqs, master.dataset.token2index[EOS_token]) 342 | if verbose: 343 | print('kept %i/%i after remove deuplication/unfisihed'%(len(logP), n_raw)) 344 | 345 | hyps = [master.dataset.seq2txt(seq) for seq in hyp_seqs] 346 | if len(hyps) == 0: 347 | return [] 348 | 349 | n_results = len(logP) 350 | if infer_args['method'] == 'latent' and infer_args['r_rand'] > 0: 351 | if verbose: 352 | print('calculating tf_logP...') 353 | logP_center = master.decoder.evaluate([latent]*n_results, hyp_seqs) 354 | else: 355 | logP_center = logP 356 | 357 | t2 = datetime.datetime.now() 358 | if verbose: 359 | print('*'*10 + ' logP_center spent: '+str(t2-t1)) 360 | 361 | wts_classifier = [] 362 | for clf_name in master.clf_names: 363 | wts_classifier.append(infer_args.get(clf_name, 0)) 364 | 365 | if verbose: 366 | print('ranking...') 367 | results = rank_nbest(hyps, logP, logP_center, master, inp, infer_args, base_ranker) 368 | t3 = datetime.datetime.now() 369 | if verbose: 370 | print('*'*10 + ' ranking spent: '+str(t3-t2)) 371 | return results 372 | 373 | -------------------------------------------------------------------------------- /classifier/Reddit_vs_arxiv/neural/train.log: -------------------------------------------------------------------------------- 1 | 2 | ***** Epoch 1/1, trained 0.0000M ***** 3 | 4 | start: 2019-04-19 16:41:52 5 | spent: 8 sec 6 | train: 0.6876 7 | vali: 0.6743 8 | vali_acc: 0.6150 9 | 10 | ***** Epoch 1/1, trained 0.0128M ***** 11 | 12 | start: 2019-04-19 16:42:02 13 | spent: 5 sec 14 | train: 0.5827 15 | vali: 0.3854 16 | vali_acc: 0.9180 17 | 18 | ***** Epoch 1/1, trained 0.0256M ***** 19 | 20 | start: 2019-04-19 16:42:08 21 | spent: 5 sec 22 | train: 0.2182 23 | vali: 0.1507 24 | vali_acc: 0.9540 25 | 26 | ***** Epoch 1/1, trained 0.0384M ***** 27 | 28 | start: 2019-04-19 16:42:13 29 | spent: 5 sec 30 | train: 0.1291 31 | vali: 0.1179 32 | vali_acc: 0.9650 33 | 34 | ***** Epoch 1/1, trained 0.0512M ***** 35 | 36 | start: 2019-04-19 16:42:19 37 | spent: 4 sec 38 | train: 0.1128 39 | vali: 0.0960 40 | vali_acc: 0.9710 41 | 42 | ***** Epoch 1/1, trained 0.0640M ***** 43 | 44 | start: 2019-04-19 16:42:25 45 | spent: 4 sec 46 | train: 0.0952 47 | vali: 0.0815 48 | vali_acc: 0.9750 49 | 50 | ***** Epoch 1/1, trained 0.0768M ***** 51 | 52 | start: 2019-04-19 16:42:30 53 | spent: 4 sec 54 | train: 0.0815 55 | vali: 0.0703 56 | vali_acc: 0.9820 57 | 58 | ***** Epoch 1/1, trained 0.0896M ***** 59 | 60 | start: 2019-04-19 16:42:36 61 | spent: 4 sec 62 | train: 0.0746 63 | vali: 0.0650 64 | vali_acc: 0.9800 65 | 66 | ***** Epoch 1/1, trained 0.1024M ***** 67 | 68 | start: 2019-04-19 16:42:41 69 | spent: 4 sec 70 | train: 0.0703 71 | vali: 0.0595 72 | vali_acc: 0.9840 73 | 74 | ***** Epoch 1/1, trained 0.1152M ***** 75 | 76 | start: 2019-04-19 16:42:47 77 | spent: 4 sec 78 | train: 0.0688 79 | vali: 0.0556 80 | vali_acc: 0.9850 81 | 82 | ***** Epoch 1/1, trained 0.1280M ***** 83 | 84 | start: 2019-04-19 16:42:52 85 | spent: 4 sec 86 | train: 0.0599 87 | vali: 0.0532 88 | vali_acc: 0.9860 89 | 90 | ***** Epoch 1/1, trained 0.1408M ***** 91 | 92 | start: 2019-04-19 16:42:58 93 | spent: 4 sec 94 | train: 0.0594 95 | vali: 0.0513 96 | vali_acc: 0.9850 97 | 98 | ***** Epoch 1/1, trained 0.1536M ***** 99 | 100 | start: 2019-04-19 16:43:03 101 | spent: 5 sec 102 | train: 0.0556 103 | vali: 0.0484 104 | vali_acc: 0.9860 105 | 106 | ***** Epoch 1/1, trained 0.1664M ***** 107 | 108 | start: 2019-04-19 16:43:09 109 | spent: 5 sec 110 | train: 0.0537 111 | vali: 0.0474 112 | vali_acc: 0.9870 113 | 114 | ***** Epoch 1/1, trained 0.1792M ***** 115 | 116 | start: 2019-04-19 16:43:15 117 | spent: 5 sec 118 | train: 0.0516 119 | vali: 0.0482 120 | vali_acc: 0.9880 121 | 122 | ***** Epoch 1/1, trained 0.1920M ***** 123 | 124 | start: 2019-04-19 16:43:22 125 | spent: 5 sec 126 | train: 0.0511 127 | vali: 0.0436 128 | vali_acc: 0.9890 129 | 130 | ***** Epoch 1/1, trained 0.2048M ***** 131 | 132 | start: 2019-04-19 16:43:28 133 | spent: 5 sec 134 | train: 0.0490 135 | vali: 0.0422 136 | vali_acc: 0.9890 137 | 138 | ***** Epoch 1/1, trained 0.2176M ***** 139 | 140 | start: 2019-04-19 16:43:34 141 | spent: 11 sec 142 | train: 0.0446 143 | vali: 0.0411 144 | vali_acc: 0.9910 145 | 146 | ***** Epoch 1/1, trained 0.2304M ***** 147 | 148 | start: 2019-04-19 16:43:46 149 | spent: 5 sec 150 | train: 0.0453 151 | vali: 0.0405 152 | vali_acc: 0.9920 153 | 154 | ***** Epoch 1/1, trained 0.2432M ***** 155 | 156 | start: 2019-04-19 16:43:52 157 | spent: 5 sec 158 | train: 0.0467 159 | vali: 0.0397 160 | vali_acc: 0.9920 161 | 162 | ***** Epoch 1/1, trained 0.2560M ***** 163 | 164 | start: 2019-04-19 16:43:58 165 | spent: 5 sec 166 | train: 0.0429 167 | vali: 0.0382 168 | vali_acc: 0.9920 169 | 170 | ***** Epoch 1/1, trained 0.2688M ***** 171 | 172 | start: 2019-04-19 16:44:04 173 | spent: 5 sec 174 | train: 0.0478 175 | vali: 0.0377 176 | vali_acc: 0.9920 177 | 178 | ***** Epoch 1/1, trained 0.2816M ***** 179 | 180 | start: 2019-04-19 16:44:10 181 | spent: 5 sec 182 | train: 0.0463 183 | vali: 0.0358 184 | vali_acc: 0.9920 185 | 186 | ***** Epoch 1/1, trained 0.2944M ***** 187 | 188 | start: 2019-04-19 16:44:16 189 | spent: 5 sec 190 | train: 0.0422 191 | vali: 0.0367 192 | vali_acc: 0.9900 193 | 194 | ***** Epoch 1/1, trained 0.3072M ***** 195 | 196 | start: 2019-04-19 16:44:22 197 | spent: 5 sec 198 | train: 0.0360 199 | vali: 0.0368 200 | vali_acc: 0.9910 201 | 202 | ***** Epoch 1/1, trained 0.3200M ***** 203 | 204 | start: 2019-04-19 16:44:28 205 | spent: 5 sec 206 | train: 0.0410 207 | vali: 0.0351 208 | vali_acc: 0.9930 209 | 210 | ***** Epoch 1/1, trained 0.3328M ***** 211 | 212 | start: 2019-04-19 16:44:34 213 | spent: 5 sec 214 | train: 0.0407 215 | vali: 0.0368 216 | vali_acc: 0.9910 217 | 218 | ***** Epoch 1/1, trained 0.3456M ***** 219 | 220 | start: 2019-04-19 16:44:39 221 | spent: 5 sec 222 | train: 0.0368 223 | vali: 0.0341 224 | vali_acc: 0.9930 225 | 226 | ***** Epoch 1/1, trained 0.3584M ***** 227 | 228 | start: 2019-04-19 16:44:45 229 | spent: 5 sec 230 | train: 0.0395 231 | vali: 0.0339 232 | vali_acc: 0.9920 233 | 234 | ***** Epoch 1/1, trained 0.3712M ***** 235 | 236 | start: 2019-04-19 16:44:51 237 | spent: 4 sec 238 | train: 0.0376 239 | vali: 0.0327 240 | vali_acc: 0.9940 241 | 242 | ***** Epoch 1/1, trained 0.3840M ***** 243 | 244 | start: 2019-04-19 16:44:57 245 | spent: 4 sec 246 | train: 0.0405 247 | vali: 0.0326 248 | vali_acc: 0.9920 249 | 250 | ***** Epoch 1/1, trained 0.3968M ***** 251 | 252 | start: 2019-04-19 16:45:02 253 | spent: 5 sec 254 | train: 0.0389 255 | vali: 0.0336 256 | vali_acc: 0.9910 257 | 258 | ***** Epoch 1/1, trained 0.4096M ***** 259 | 260 | start: 2019-04-19 16:45:08 261 | spent: 4 sec 262 | train: 0.0366 263 | vali: 0.0322 264 | vali_acc: 0.9910 265 | 266 | ***** Epoch 1/1, trained 0.4224M ***** 267 | 268 | start: 2019-04-19 16:45:13 269 | spent: 4 sec 270 | train: 0.0396 271 | vali: 0.0337 272 | vali_acc: 0.9920 273 | 274 | ***** Epoch 1/1, trained 0.4352M ***** 275 | 276 | start: 2019-04-19 16:45:19 277 | spent: 5 sec 278 | train: 0.0397 279 | vali: 0.0304 280 | vali_acc: 0.9920 281 | 282 | ***** Epoch 1/1, trained 0.4480M ***** 283 | 284 | start: 2019-04-19 16:45:25 285 | spent: 4 sec 286 | train: 0.0369 287 | vali: 0.0297 288 | vali_acc: 0.9930 289 | 290 | ***** Epoch 1/1, trained 0.4608M ***** 291 | 292 | start: 2019-04-19 16:45:30 293 | spent: 5 sec 294 | train: 0.0355 295 | vali: 0.0301 296 | vali_acc: 0.9940 297 | 298 | ***** Epoch 1/1, trained 0.4736M ***** 299 | 300 | start: 2019-04-19 16:45:36 301 | spent: 5 sec 302 | train: 0.0360 303 | vali: 0.0284 304 | vali_acc: 0.9930 305 | 306 | ***** Epoch 1/1, trained 0.4864M ***** 307 | 308 | start: 2019-04-19 16:45:42 309 | spent: 5 sec 310 | train: 0.0344 311 | vali: 0.0288 312 | vali_acc: 0.9910 313 | 314 | ***** Epoch 1/1, trained 0.4992M ***** 315 | 316 | start: 2019-04-19 16:45:48 317 | spent: 5 sec 318 | train: 0.0336 319 | vali: 0.0277 320 | vali_acc: 0.9930 321 | 322 | ***** Epoch 1/1, trained 0.5120M ***** 323 | 324 | start: 2019-04-19 16:45:53 325 | spent: 5 sec 326 | train: 0.0342 327 | vali: 0.0266 328 | vali_acc: 0.9940 329 | 330 | ***** Epoch 1/1, trained 0.5248M ***** 331 | 332 | start: 2019-04-19 16:45:59 333 | spent: 5 sec 334 | train: 0.0326 335 | vali: 0.0267 336 | vali_acc: 0.9950 337 | 338 | ***** Epoch 1/1, trained 0.5376M ***** 339 | 340 | start: 2019-04-19 16:46:05 341 | spent: 5 sec 342 | train: 0.0299 343 | vali: 0.0263 344 | vali_acc: 0.9950 345 | 346 | ***** Epoch 1/1, trained 0.5504M ***** 347 | 348 | start: 2019-04-19 16:46:11 349 | spent: 4 sec 350 | train: 0.0323 351 | vali: 0.0265 352 | vali_acc: 0.9930 353 | 354 | ***** Epoch 1/1, trained 0.5632M ***** 355 | 356 | start: 2019-04-19 16:46:17 357 | spent: 5 sec 358 | train: 0.0326 359 | vali: 0.0255 360 | vali_acc: 0.9940 361 | 362 | ***** Epoch 1/1, trained 0.5760M ***** 363 | 364 | start: 2019-04-19 16:46:23 365 | spent: 5 sec 366 | train: 0.0352 367 | vali: 0.0248 368 | vali_acc: 0.9950 369 | 370 | ***** Epoch 1/1, trained 0.5888M ***** 371 | 372 | start: 2019-04-19 16:46:29 373 | spent: 4 sec 374 | train: 0.0331 375 | vali: 0.0275 376 | vali_acc: 0.9930 377 | 378 | ***** Epoch 1/1, trained 0.6016M ***** 379 | 380 | start: 2019-04-19 16:46:35 381 | spent: 4 sec 382 | train: 0.0326 383 | vali: 0.0266 384 | vali_acc: 0.9930 385 | 386 | ***** Epoch 1/1, trained 0.6144M ***** 387 | 388 | start: 2019-04-19 16:46:40 389 | spent: 5 sec 390 | train: 0.0289 391 | vali: 0.0251 392 | vali_acc: 0.9940 393 | 394 | ***** Epoch 1/1, trained 0.6272M ***** 395 | 396 | start: 2019-04-19 16:46:46 397 | spent: 4 sec 398 | train: 0.0308 399 | vali: 0.0236 400 | vali_acc: 0.9950 401 | 402 | ***** Epoch 1/1, trained 0.6400M ***** 403 | 404 | start: 2019-04-19 16:46:52 405 | spent: 4 sec 406 | train: 0.0354 407 | vali: 0.0246 408 | vali_acc: 0.9940 409 | 410 | ***** Epoch 1/1, trained 0.6528M ***** 411 | 412 | start: 2019-04-19 16:46:57 413 | spent: 5 sec 414 | train: 0.0339 415 | vali: 0.0248 416 | vali_acc: 0.9930 417 | 418 | ***** Epoch 1/1, trained 0.6656M ***** 419 | 420 | start: 2019-04-19 16:47:03 421 | spent: 5 sec 422 | train: 0.0288 423 | vali: 0.0255 424 | vali_acc: 0.9930 425 | 426 | ***** Epoch 1/1, trained 0.6784M ***** 427 | 428 | start: 2019-04-19 16:47:09 429 | spent: 4 sec 430 | train: 0.0295 431 | vali: 0.0243 432 | vali_acc: 0.9940 433 | 434 | ***** Epoch 1/1, trained 0.6912M ***** 435 | 436 | start: 2019-04-19 16:47:15 437 | spent: 4 sec 438 | train: 0.0292 439 | vali: 0.0243 440 | vali_acc: 0.9940 441 | 442 | ***** Epoch 1/1, trained 0.7040M ***** 443 | 444 | start: 2019-04-19 16:47:20 445 | spent: 5 sec 446 | train: 0.0272 447 | vali: 0.0243 448 | vali_acc: 0.9940 449 | 450 | ***** Epoch 1/1, trained 0.7168M ***** 451 | 452 | start: 2019-04-19 16:47:26 453 | spent: 5 sec 454 | train: 0.0368 455 | vali: 0.0236 456 | vali_acc: 0.9950 457 | 458 | ***** Epoch 1/1, trained 0.7296M ***** 459 | 460 | start: 2019-04-19 16:47:31 461 | spent: 4 sec 462 | train: 0.0292 463 | vali: 0.0245 464 | vali_acc: 0.9950 465 | 466 | ***** Epoch 1/1, trained 0.7424M ***** 467 | 468 | start: 2019-04-19 16:47:37 469 | spent: 5 sec 470 | train: 0.0284 471 | vali: 0.0257 472 | vali_acc: 0.9930 473 | 474 | ***** Epoch 1/1, trained 0.7552M ***** 475 | 476 | start: 2019-04-19 16:47:43 477 | spent: 5 sec 478 | train: 0.0298 479 | vali: 0.0235 480 | vali_acc: 0.9940 481 | 482 | ***** Epoch 1/1, trained 0.7680M ***** 483 | 484 | start: 2019-04-19 16:47:48 485 | spent: 5 sec 486 | train: 0.0284 487 | vali: 0.0250 488 | vali_acc: 0.9940 489 | 490 | ***** Epoch 1/1, trained 0.7808M ***** 491 | 492 | start: 2019-04-19 16:47:54 493 | spent: 5 sec 494 | train: 0.0277 495 | vali: 0.0238 496 | vali_acc: 0.9940 497 | 498 | ***** Epoch 1/1, trained 0.7936M ***** 499 | 500 | start: 2019-04-19 16:48:00 501 | spent: 5 sec 502 | train: 0.0269 503 | vali: 0.0247 504 | vali_acc: 0.9930 505 | 506 | ***** Epoch 1/1, trained 0.8064M ***** 507 | 508 | start: 2019-04-19 16:48:06 509 | spent: 5 sec 510 | train: 0.0300 511 | vali: 0.0235 512 | vali_acc: 0.9950 513 | 514 | ***** Epoch 1/1, trained 0.8192M ***** 515 | 516 | start: 2019-04-19 16:48:12 517 | spent: 5 sec 518 | train: 0.0279 519 | vali: 0.0249 520 | vali_acc: 0.9930 521 | 522 | ***** Epoch 1/1, trained 0.8320M ***** 523 | 524 | start: 2019-04-19 16:48:17 525 | spent: 4 sec 526 | train: 0.0297 527 | vali: 0.0264 528 | vali_acc: 0.9940 529 | 530 | ***** Epoch 1/1, trained 0.8448M ***** 531 | 532 | start: 2019-04-19 16:48:23 533 | spent: 4 sec 534 | train: 0.0336 535 | vali: 0.0248 536 | vali_acc: 0.9940 537 | 538 | ***** Epoch 1/1, trained 0.8576M ***** 539 | 540 | start: 2019-04-19 16:48:29 541 | spent: 5 sec 542 | train: 0.0271 543 | vali: 0.0245 544 | vali_acc: 0.9930 545 | 546 | ***** Epoch 1/1, trained 0.8704M ***** 547 | 548 | start: 2019-04-19 16:48:34 549 | spent: 5 sec 550 | train: 0.0288 551 | vali: 0.0234 552 | vali_acc: 0.9940 553 | 554 | ***** Epoch 1/1, trained 0.8832M ***** 555 | 556 | start: 2019-04-19 16:48:40 557 | spent: 5 sec 558 | train: 0.0283 559 | vali: 0.0249 560 | vali_acc: 0.9930 561 | 562 | ***** Epoch 1/1, trained 0.8960M ***** 563 | 564 | start: 2019-04-19 16:48:46 565 | spent: 5 sec 566 | train: 0.0276 567 | vali: 0.0231 568 | vali_acc: 0.9940 569 | 570 | ***** Epoch 1/1, trained 0.9088M ***** 571 | 572 | start: 2019-04-19 16:48:51 573 | spent: 4 sec 574 | train: 0.0289 575 | vali: 0.0224 576 | vali_acc: 0.9940 577 | 578 | ***** Epoch 1/1, trained 0.9216M ***** 579 | 580 | start: 2019-04-19 16:48:57 581 | spent: 4 sec 582 | train: 0.0309 583 | vali: 0.0218 584 | vali_acc: 0.9940 585 | 586 | ***** Epoch 1/1, trained 0.9344M ***** 587 | 588 | start: 2019-04-19 16:49:03 589 | spent: 4 sec 590 | train: 0.0246 591 | vali: 0.0218 592 | vali_acc: 0.9940 593 | 594 | ***** Epoch 1/1, trained 0.9472M ***** 595 | 596 | start: 2019-04-19 16:49:08 597 | spent: 4 sec 598 | train: 0.0262 599 | vali: 0.0217 600 | vali_acc: 0.9930 601 | 602 | ***** Epoch 1/1, trained 0.9600M ***** 603 | 604 | start: 2019-04-19 16:49:14 605 | spent: 4 sec 606 | train: 0.0274 607 | vali: 0.0222 608 | vali_acc: 0.9930 609 | 610 | ***** Epoch 1/1, trained 0.9728M ***** 611 | 612 | start: 2019-04-19 16:49:19 613 | spent: 5 sec 614 | train: 0.0291 615 | vali: 0.0225 616 | vali_acc: 0.9930 617 | 618 | ***** Epoch 1/1, trained 0.9856M ***** 619 | 620 | start: 2019-04-19 16:49:25 621 | spent: 5 sec 622 | train: 0.0290 623 | vali: 0.0225 624 | vali_acc: 0.9930 625 | 626 | ***** Epoch 1/1, trained 0.9984M ***** 627 | 628 | start: 2019-04-19 16:49:31 629 | spent: 5 sec 630 | train: 0.0272 631 | vali: 0.0223 632 | vali_acc: 0.9930 633 | 634 | ***** Epoch 1/1, trained 1.0112M ***** 635 | 636 | start: 2019-04-19 16:49:37 637 | spent: 4 sec 638 | train: 0.0297 639 | vali: 0.0236 640 | vali_acc: 0.9930 641 | 642 | ***** Epoch 1/1, trained 1.0240M ***** 643 | 644 | start: 2019-04-19 16:49:42 645 | spent: 4 sec 646 | train: 0.0255 647 | vali: 0.0234 648 | vali_acc: 0.9930 649 | 650 | ***** Epoch 1/1, trained 1.0368M ***** 651 | 652 | start: 2019-04-19 16:49:48 653 | spent: 4 sec 654 | train: 0.0244 655 | vali: 0.0216 656 | vali_acc: 0.9930 657 | 658 | ***** Epoch 1/1, trained 1.0496M ***** 659 | 660 | start: 2019-04-19 16:49:53 661 | spent: 4 sec 662 | train: 0.0266 663 | vali: 0.0215 664 | vali_acc: 0.9930 665 | 666 | ***** Epoch 1/1, trained 1.0624M ***** 667 | 668 | start: 2019-04-19 16:49:59 669 | spent: 4 sec 670 | train: 0.0302 671 | vali: 0.0216 672 | vali_acc: 0.9930 673 | 674 | ***** Epoch 1/1, trained 1.0752M ***** 675 | 676 | start: 2019-04-19 16:50:05 677 | spent: 4 sec 678 | train: 0.0257 679 | vali: 0.0217 680 | vali_acc: 0.9930 681 | 682 | ***** Epoch 1/1, trained 1.0880M ***** 683 | 684 | start: 2019-04-19 16:50:10 685 | spent: 4 sec 686 | train: 0.0310 687 | vali: 0.0219 688 | vali_acc: 0.9940 689 | 690 | ***** Epoch 1/1, trained 1.1008M ***** 691 | 692 | start: 2019-04-19 16:50:16 693 | spent: 4 sec 694 | train: 0.0302 695 | vali: 0.0230 696 | vali_acc: 0.9930 697 | 698 | ***** Epoch 1/1, trained 1.1136M ***** 699 | 700 | start: 2019-04-19 16:50:21 701 | spent: 4 sec 702 | train: 0.0277 703 | vali: 0.0209 704 | vali_acc: 0.9960 705 | 706 | ***** Epoch 1/1, trained 1.1264M ***** 707 | 708 | start: 2019-04-19 16:50:27 709 | spent: 4 sec 710 | train: 0.0247 711 | vali: 0.0203 712 | vali_acc: 0.9950 713 | 714 | ***** Epoch 1/1, trained 1.1392M ***** 715 | 716 | start: 2019-04-19 16:50:33 717 | spent: 4 sec 718 | train: 0.0270 719 | vali: 0.0200 720 | vali_acc: 0.9960 721 | 722 | ***** Epoch 1/1, trained 1.1520M ***** 723 | 724 | start: 2019-04-19 16:50:38 725 | spent: 5 sec 726 | train: 0.0279 727 | vali: 0.0205 728 | vali_acc: 0.9950 729 | 730 | ***** Epoch 1/1, trained 1.1648M ***** 731 | 732 | start: 2019-04-19 16:50:44 733 | spent: 4 sec 734 | train: 0.0253 735 | vali: 0.0213 736 | vali_acc: 0.9940 737 | 738 | ***** Epoch 1/1, trained 1.1776M ***** 739 | 740 | start: 2019-04-19 16:50:49 741 | spent: 4 sec 742 | train: 0.0306 743 | vali: 0.0196 744 | vali_acc: 0.9990 745 | 746 | ***** Epoch 1/1, trained 1.1904M ***** 747 | 748 | start: 2019-04-19 16:50:55 749 | spent: 5 sec 750 | train: 0.0288 751 | vali: 0.0198 752 | vali_acc: 0.9960 753 | 754 | ***** Epoch 1/1, trained 1.2032M ***** 755 | 756 | start: 2019-04-19 16:51:01 757 | spent: 5 sec 758 | train: 0.0268 759 | vali: 0.0203 760 | vali_acc: 0.9960 761 | 762 | ***** Epoch 1/1, trained 1.2160M ***** 763 | 764 | start: 2019-04-19 16:51:06 765 | spent: 4 sec 766 | train: 0.0247 767 | vali: 0.0193 768 | vali_acc: 0.9990 769 | 770 | ***** Epoch 1/1, trained 1.2288M ***** 771 | 772 | start: 2019-04-19 16:51:12 773 | spent: 5 sec 774 | train: 0.0279 775 | vali: 0.0196 776 | vali_acc: 0.9990 777 | 778 | ***** Epoch 1/1, trained 1.2416M ***** 779 | 780 | start: 2019-04-19 16:51:18 781 | spent: 5 sec 782 | train: 0.0295 783 | vali: 0.0199 784 | vali_acc: 0.9950 785 | 786 | ***** Epoch 1/1, trained 1.2544M ***** 787 | 788 | start: 2019-04-19 16:51:24 789 | spent: 5 sec 790 | train: 0.0266 791 | vali: 0.0198 792 | vali_acc: 0.9970 793 | 794 | ***** Epoch 1/1, trained 1.2672M ***** 795 | 796 | start: 2019-04-19 16:51:30 797 | spent: 5 sec 798 | train: 0.0227 799 | vali: 0.0191 800 | vali_acc: 0.9980 801 | 802 | ***** Epoch 1/1, trained 1.2800M ***** 803 | 804 | start: 2019-04-19 16:51:36 805 | spent: 6 sec 806 | train: 0.0265 807 | vali: 0.0190 808 | vali_acc: 0.9960 809 | 810 | ***** Epoch 1/1, trained 1.2928M ***** 811 | 812 | start: 2019-04-19 16:51:44 813 | -------------------------------------------------------------------------------- /data/toy/base_conv_vali.num: -------------------------------------------------------------------------------- 1 | 10 13 4 9 18 5 18 25 29 6 2 | 10 13 4 9 18 5 18 25 27 6 3 | 10 14 4 3 5 27 6 4 | 10 14 4 3 5 26 33 27 6 5 | 13 4 9 7 12 5 19 6 6 | 13 4 9 7 12 5 24 8 8 6 7 | 10 13 4 9 18 5 18 25 29 6 8 | 10 13 4 9 18 5 18 25 27 6 9 | 10 14 4 3 5 27 6 10 | 10 14 4 3 5 26 33 27 6 11 | 13 4 9 7 12 5 19 6 12 | 13 4 9 7 12 5 24 8 8 6 13 | 10 13 4 9 18 5 18 25 29 6 14 | 10 13 4 9 18 5 18 25 27 6 15 | 10 14 4 3 5 27 6 16 | 10 14 4 3 5 26 33 27 6 17 | 13 4 9 7 12 5 19 6 18 | 13 4 9 7 12 5 24 8 8 6 19 | 10 13 4 9 18 5 18 25 29 6 20 | 10 13 4 9 18 5 18 25 27 6 21 | 10 14 4 3 5 27 6 22 | 10 14 4 3 5 26 33 27 6 23 | 13 4 9 7 12 5 19 6 24 | 13 4 9 7 12 5 24 8 8 6 25 | 10 13 4 9 18 5 18 25 29 6 26 | 10 13 4 9 18 5 18 25 27 6 27 | 10 14 4 3 5 27 6 28 | 10 14 4 3 5 26 33 27 6 29 | 13 4 9 7 12 5 19 6 30 | 13 4 9 7 12 5 24 8 8 6 31 | 10 13 4 9 18 5 18 25 29 6 32 | 10 13 4 9 18 5 18 25 27 6 33 | 10 14 4 3 5 27 6 34 | 10 14 4 3 5 26 33 27 6 35 | 13 4 9 7 12 5 19 6 36 | 13 4 9 7 12 5 24 8 8 6 37 | 10 13 4 9 18 5 18 25 29 6 38 | 10 13 4 9 18 5 18 25 27 6 39 | 10 14 4 3 5 27 6 40 | 10 14 4 3 5 26 33 27 6 41 | 13 4 9 7 12 5 19 6 42 | 13 4 9 7 12 5 24 8 8 6 43 | 10 13 4 9 18 5 18 25 29 6 44 | 10 13 4 9 18 5 18 25 27 6 45 | 10 14 4 3 5 27 6 46 | 10 14 4 3 5 26 33 27 6 47 | 13 4 9 7 12 5 19 6 48 | 13 4 9 7 12 5 24 8 8 6 49 | 10 13 4 9 18 5 18 25 29 6 50 | 10 13 4 9 18 5 18 25 27 6 51 | 10 14 4 3 5 27 6 52 | 10 14 4 3 5 26 33 27 6 53 | 13 4 9 7 12 5 19 6 54 | 13 4 9 7 12 5 24 8 8 6 55 | 10 13 4 9 18 5 18 25 29 6 56 | 10 13 4 9 18 5 18 25 27 6 57 | 10 14 4 3 5 27 6 58 | 10 14 4 3 5 26 33 27 6 59 | 13 4 9 7 12 5 19 6 60 | 13 4 9 7 12 5 24 8 8 6 61 | 10 13 4 9 18 5 18 25 29 6 62 | 10 13 4 9 18 5 18 25 27 6 63 | 10 14 4 3 5 27 6 64 | 10 14 4 3 5 26 33 27 6 65 | 13 4 9 7 12 5 19 6 66 | 13 4 9 7 12 5 24 8 8 6 67 | 10 13 4 9 18 5 18 25 29 6 68 | 10 13 4 9 18 5 18 25 27 6 69 | 10 14 4 3 5 27 6 70 | 10 14 4 3 5 26 33 27 6 71 | 13 4 9 7 12 5 19 6 72 | 13 4 9 7 12 5 24 8 8 6 73 | 10 13 4 9 18 5 18 25 29 6 74 | 10 13 4 9 18 5 18 25 27 6 75 | 10 14 4 3 5 27 6 76 | 10 14 4 3 5 26 33 27 6 77 | 13 4 9 7 12 5 19 6 78 | 13 4 9 7 12 5 24 8 8 6 79 | 10 13 4 9 18 5 18 25 29 6 80 | 10 13 4 9 18 5 18 25 27 6 81 | 10 14 4 3 5 27 6 82 | 10 14 4 3 5 26 33 27 6 83 | 13 4 9 7 12 5 19 6 84 | 13 4 9 7 12 5 24 8 8 6 85 | 10 13 4 9 18 5 18 25 29 6 86 | 10 13 4 9 18 5 18 25 27 6 87 | 10 14 4 3 5 27 6 88 | 10 14 4 3 5 26 33 27 6 89 | 13 4 9 7 12 5 19 6 90 | 13 4 9 7 12 5 24 8 8 6 91 | 10 13 4 9 18 5 18 25 29 6 92 | 10 13 4 9 18 5 18 25 27 6 93 | 10 14 4 3 5 27 6 94 | 10 14 4 3 5 26 33 27 6 95 | 13 4 9 7 12 5 19 6 96 | 13 4 9 7 12 5 24 8 8 6 97 | 10 13 4 9 18 5 18 25 29 6 98 | 10 13 4 9 18 5 18 25 27 6 99 | 10 14 4 3 5 27 6 100 | 10 14 4 3 5 26 33 27 6 101 | 13 4 9 7 12 5 19 6 102 | 13 4 9 7 12 5 24 8 8 6 103 | 10 13 4 9 18 5 18 25 29 6 104 | 10 13 4 9 18 5 18 25 27 6 105 | 10 14 4 3 5 27 6 106 | 10 14 4 3 5 26 33 27 6 107 | 13 4 9 7 12 5 19 6 108 | 13 4 9 7 12 5 24 8 8 6 109 | 10 13 4 9 18 5 18 25 29 6 110 | 10 13 4 9 18 5 18 25 27 6 111 | 10 14 4 3 5 27 6 112 | 10 14 4 3 5 26 33 27 6 113 | 13 4 9 7 12 5 19 6 114 | 13 4 9 7 12 5 24 8 8 6 115 | 10 13 4 9 18 5 18 25 29 6 116 | 10 13 4 9 18 5 18 25 27 6 117 | 10 14 4 3 5 27 6 118 | 10 14 4 3 5 26 33 27 6 119 | 13 4 9 7 12 5 19 6 120 | 13 4 9 7 12 5 24 8 8 6 121 | 10 13 4 9 18 5 18 25 29 6 122 | 10 13 4 9 18 5 18 25 27 6 123 | 10 14 4 3 5 27 6 124 | 10 14 4 3 5 26 33 27 6 125 | 13 4 9 7 12 5 19 6 126 | 13 4 9 7 12 5 24 8 8 6 127 | 10 13 4 9 18 5 18 25 29 6 128 | 10 13 4 9 18 5 18 25 27 6 129 | 10 14 4 3 5 27 6 130 | 10 14 4 3 5 26 33 27 6 131 | 13 4 9 7 12 5 19 6 132 | 13 4 9 7 12 5 24 8 8 6 133 | 10 13 4 9 18 5 18 25 29 6 134 | 10 13 4 9 18 5 18 25 27 6 135 | 10 14 4 3 5 27 6 136 | 10 14 4 3 5 26 33 27 6 137 | 13 4 9 7 12 5 19 6 138 | 13 4 9 7 12 5 24 8 8 6 139 | 10 13 4 9 18 5 18 25 29 6 140 | 10 13 4 9 18 5 18 25 27 6 141 | 10 14 4 3 5 27 6 142 | 10 14 4 3 5 26 33 27 6 143 | 13 4 9 7 12 5 19 6 144 | 13 4 9 7 12 5 24 8 8 6 145 | 10 13 4 9 18 5 18 25 29 6 146 | 10 13 4 9 18 5 18 25 27 6 147 | 10 14 4 3 5 27 6 148 | 10 14 4 3 5 26 33 27 6 149 | 13 4 9 7 12 5 19 6 150 | 13 4 9 7 12 5 24 8 8 6 151 | 10 13 4 9 18 5 18 25 29 6 152 | 10 13 4 9 18 5 18 25 27 6 153 | 10 14 4 3 5 27 6 154 | 10 14 4 3 5 26 33 27 6 155 | 13 4 9 7 12 5 19 6 156 | 13 4 9 7 12 5 24 8 8 6 157 | 10 13 4 9 18 5 18 25 29 6 158 | 10 13 4 9 18 5 18 25 27 6 159 | 10 14 4 3 5 27 6 160 | 10 14 4 3 5 26 33 27 6 161 | 13 4 9 7 12 5 19 6 162 | 13 4 9 7 12 5 24 8 8 6 163 | 10 13 4 9 18 5 18 25 29 6 164 | 10 13 4 9 18 5 18 25 27 6 165 | 10 14 4 3 5 27 6 166 | 10 14 4 3 5 26 33 27 6 167 | 13 4 9 7 12 5 19 6 168 | 13 4 9 7 12 5 24 8 8 6 169 | 10 13 4 9 18 5 18 25 29 6 170 | 10 13 4 9 18 5 18 25 27 6 171 | 10 14 4 3 5 27 6 172 | 10 14 4 3 5 26 33 27 6 173 | 13 4 9 7 12 5 19 6 174 | 13 4 9 7 12 5 24 8 8 6 175 | 10 13 4 9 18 5 18 25 29 6 176 | 10 13 4 9 18 5 18 25 27 6 177 | 10 14 4 3 5 27 6 178 | 10 14 4 3 5 26 33 27 6 179 | 13 4 9 7 12 5 19 6 180 | 13 4 9 7 12 5 24 8 8 6 181 | 10 13 4 9 18 5 18 25 29 6 182 | 10 13 4 9 18 5 18 25 27 6 183 | 10 14 4 3 5 27 6 184 | 10 14 4 3 5 26 33 27 6 185 | 13 4 9 7 12 5 19 6 186 | 13 4 9 7 12 5 24 8 8 6 187 | 10 13 4 9 18 5 18 25 29 6 188 | 10 13 4 9 18 5 18 25 27 6 189 | 10 14 4 3 5 27 6 190 | 10 14 4 3 5 26 33 27 6 191 | 13 4 9 7 12 5 19 6 192 | 13 4 9 7 12 5 24 8 8 6 193 | 10 13 4 9 18 5 18 25 29 6 194 | 10 13 4 9 18 5 18 25 27 6 195 | 10 14 4 3 5 27 6 196 | 10 14 4 3 5 26 33 27 6 197 | 13 4 9 7 12 5 19 6 198 | 13 4 9 7 12 5 24 8 8 6 199 | 10 13 4 9 18 5 18 25 29 6 200 | 10 13 4 9 18 5 18 25 27 6 201 | 10 14 4 3 5 27 6 202 | 10 14 4 3 5 26 33 27 6 203 | 13 4 9 7 12 5 19 6 204 | 13 4 9 7 12 5 24 8 8 6 205 | 10 13 4 9 18 5 18 25 29 6 206 | 10 13 4 9 18 5 18 25 27 6 207 | 10 14 4 3 5 27 6 208 | 10 14 4 3 5 26 33 27 6 209 | 13 4 9 7 12 5 19 6 210 | 13 4 9 7 12 5 24 8 8 6 211 | 10 13 4 9 18 5 18 25 29 6 212 | 10 13 4 9 18 5 18 25 27 6 213 | 10 14 4 3 5 27 6 214 | 10 14 4 3 5 26 33 27 6 215 | 13 4 9 7 12 5 19 6 216 | 13 4 9 7 12 5 24 8 8 6 217 | 10 13 4 9 18 5 18 25 29 6 218 | 10 13 4 9 18 5 18 25 27 6 219 | 10 14 4 3 5 27 6 220 | 10 14 4 3 5 26 33 27 6 221 | 13 4 9 7 12 5 19 6 222 | 13 4 9 7 12 5 24 8 8 6 223 | 10 13 4 9 18 5 18 25 29 6 224 | 10 13 4 9 18 5 18 25 27 6 225 | 10 14 4 3 5 27 6 226 | 10 14 4 3 5 26 33 27 6 227 | 13 4 9 7 12 5 19 6 228 | 13 4 9 7 12 5 24 8 8 6 229 | 10 13 4 9 18 5 18 25 29 6 230 | 10 13 4 9 18 5 18 25 27 6 231 | 10 14 4 3 5 27 6 232 | 10 14 4 3 5 26 33 27 6 233 | 13 4 9 7 12 5 19 6 234 | 13 4 9 7 12 5 24 8 8 6 235 | 10 13 4 9 18 5 18 25 29 6 236 | 10 13 4 9 18 5 18 25 27 6 237 | 10 14 4 3 5 27 6 238 | 10 14 4 3 5 26 33 27 6 239 | 13 4 9 7 12 5 19 6 240 | 13 4 9 7 12 5 24 8 8 6 241 | 10 13 4 9 18 5 18 25 29 6 242 | 10 13 4 9 18 5 18 25 27 6 243 | 10 14 4 3 5 27 6 244 | 10 14 4 3 5 26 33 27 6 245 | 13 4 9 7 12 5 19 6 246 | 13 4 9 7 12 5 24 8 8 6 247 | 10 13 4 9 18 5 18 25 29 6 248 | 10 13 4 9 18 5 18 25 27 6 249 | 10 14 4 3 5 27 6 250 | 10 14 4 3 5 26 33 27 6 251 | 13 4 9 7 12 5 19 6 252 | 13 4 9 7 12 5 24 8 8 6 253 | 10 13 4 9 18 5 18 25 29 6 254 | 10 13 4 9 18 5 18 25 27 6 255 | 10 14 4 3 5 27 6 256 | 10 14 4 3 5 26 33 27 6 257 | 13 4 9 7 12 5 19 6 258 | 13 4 9 7 12 5 24 8 8 6 259 | 10 13 4 9 18 5 18 25 29 6 260 | 10 13 4 9 18 5 18 25 27 6 261 | 10 14 4 3 5 27 6 262 | 10 14 4 3 5 26 33 27 6 263 | 13 4 9 7 12 5 19 6 264 | 13 4 9 7 12 5 24 8 8 6 265 | 10 13 4 9 18 5 18 25 29 6 266 | 10 13 4 9 18 5 18 25 27 6 267 | 10 14 4 3 5 27 6 268 | 10 14 4 3 5 26 33 27 6 269 | 13 4 9 7 12 5 19 6 270 | 13 4 9 7 12 5 24 8 8 6 271 | 10 13 4 9 18 5 18 25 29 6 272 | 10 13 4 9 18 5 18 25 27 6 273 | 10 14 4 3 5 27 6 274 | 10 14 4 3 5 26 33 27 6 275 | 13 4 9 7 12 5 19 6 276 | 13 4 9 7 12 5 24 8 8 6 277 | 10 13 4 9 18 5 18 25 29 6 278 | 10 13 4 9 18 5 18 25 27 6 279 | 10 14 4 3 5 27 6 280 | 10 14 4 3 5 26 33 27 6 281 | 13 4 9 7 12 5 19 6 282 | 13 4 9 7 12 5 24 8 8 6 283 | 10 13 4 9 18 5 18 25 29 6 284 | 10 13 4 9 18 5 18 25 27 6 285 | 10 14 4 3 5 27 6 286 | 10 14 4 3 5 26 33 27 6 287 | 13 4 9 7 12 5 19 6 288 | 13 4 9 7 12 5 24 8 8 6 289 | 10 13 4 9 18 5 18 25 29 6 290 | 10 13 4 9 18 5 18 25 27 6 291 | 10 14 4 3 5 27 6 292 | 10 14 4 3 5 26 33 27 6 293 | 13 4 9 7 12 5 19 6 294 | 13 4 9 7 12 5 24 8 8 6 295 | 10 13 4 9 18 5 18 25 29 6 296 | 10 13 4 9 18 5 18 25 27 6 297 | 10 14 4 3 5 27 6 298 | 10 14 4 3 5 26 33 27 6 299 | 13 4 9 7 12 5 19 6 300 | 13 4 9 7 12 5 24 8 8 6 301 | 10 13 4 9 18 5 18 25 29 6 302 | 10 13 4 9 18 5 18 25 27 6 303 | 10 14 4 3 5 27 6 304 | 10 14 4 3 5 26 33 27 6 305 | 13 4 9 7 12 5 19 6 306 | 13 4 9 7 12 5 24 8 8 6 307 | 10 13 4 9 18 5 18 25 29 6 308 | 10 13 4 9 18 5 18 25 27 6 309 | 10 14 4 3 5 27 6 310 | 10 14 4 3 5 26 33 27 6 311 | 13 4 9 7 12 5 19 6 312 | 13 4 9 7 12 5 24 8 8 6 313 | 10 13 4 9 18 5 18 25 29 6 314 | 10 13 4 9 18 5 18 25 27 6 315 | 10 14 4 3 5 27 6 316 | 10 14 4 3 5 26 33 27 6 317 | 13 4 9 7 12 5 19 6 318 | 13 4 9 7 12 5 24 8 8 6 319 | 10 13 4 9 18 5 18 25 29 6 320 | 10 13 4 9 18 5 18 25 27 6 321 | 10 14 4 3 5 27 6 322 | 10 14 4 3 5 26 33 27 6 323 | 13 4 9 7 12 5 19 6 324 | 13 4 9 7 12 5 24 8 8 6 325 | 10 13 4 9 18 5 18 25 29 6 326 | 10 13 4 9 18 5 18 25 27 6 327 | 10 14 4 3 5 27 6 328 | 10 14 4 3 5 26 33 27 6 329 | 13 4 9 7 12 5 19 6 330 | 13 4 9 7 12 5 24 8 8 6 331 | 10 13 4 9 18 5 18 25 29 6 332 | 10 13 4 9 18 5 18 25 27 6 333 | 10 14 4 3 5 27 6 334 | 10 14 4 3 5 26 33 27 6 335 | 13 4 9 7 12 5 19 6 336 | 13 4 9 7 12 5 24 8 8 6 337 | 10 13 4 9 18 5 18 25 29 6 338 | 10 13 4 9 18 5 18 25 27 6 339 | 10 14 4 3 5 27 6 340 | 10 14 4 3 5 26 33 27 6 341 | 13 4 9 7 12 5 19 6 342 | 13 4 9 7 12 5 24 8 8 6 343 | 10 13 4 9 18 5 18 25 29 6 344 | 10 13 4 9 18 5 18 25 27 6 345 | 10 14 4 3 5 27 6 346 | 10 14 4 3 5 26 33 27 6 347 | 13 4 9 7 12 5 19 6 348 | 13 4 9 7 12 5 24 8 8 6 349 | 10 13 4 9 18 5 18 25 29 6 350 | 10 13 4 9 18 5 18 25 27 6 351 | 10 14 4 3 5 27 6 352 | 10 14 4 3 5 26 33 27 6 353 | 13 4 9 7 12 5 19 6 354 | 13 4 9 7 12 5 24 8 8 6 355 | 10 13 4 9 18 5 18 25 29 6 356 | 10 13 4 9 18 5 18 25 27 6 357 | 10 14 4 3 5 27 6 358 | 10 14 4 3 5 26 33 27 6 359 | 13 4 9 7 12 5 19 6 360 | 13 4 9 7 12 5 24 8 8 6 361 | 10 13 4 9 18 5 18 25 29 6 362 | 10 13 4 9 18 5 18 25 27 6 363 | 10 14 4 3 5 27 6 364 | 10 14 4 3 5 26 33 27 6 365 | 13 4 9 7 12 5 19 6 366 | 13 4 9 7 12 5 24 8 8 6 367 | 10 13 4 9 18 5 18 25 29 6 368 | 10 13 4 9 18 5 18 25 27 6 369 | 10 14 4 3 5 27 6 370 | 10 14 4 3 5 26 33 27 6 371 | 13 4 9 7 12 5 19 6 372 | 13 4 9 7 12 5 24 8 8 6 373 | 10 13 4 9 18 5 18 25 29 6 374 | 10 13 4 9 18 5 18 25 27 6 375 | 10 14 4 3 5 27 6 376 | 10 14 4 3 5 26 33 27 6 377 | 13 4 9 7 12 5 19 6 378 | 13 4 9 7 12 5 24 8 8 6 379 | 10 13 4 9 18 5 18 25 29 6 380 | 10 13 4 9 18 5 18 25 27 6 381 | 10 14 4 3 5 27 6 382 | 10 14 4 3 5 26 33 27 6 383 | 13 4 9 7 12 5 19 6 384 | 13 4 9 7 12 5 24 8 8 6 385 | 10 13 4 9 18 5 18 25 29 6 386 | 10 13 4 9 18 5 18 25 27 6 387 | 10 14 4 3 5 27 6 388 | 10 14 4 3 5 26 33 27 6 389 | 13 4 9 7 12 5 19 6 390 | 13 4 9 7 12 5 24 8 8 6 391 | 10 13 4 9 18 5 18 25 29 6 392 | 10 13 4 9 18 5 18 25 27 6 393 | 10 14 4 3 5 27 6 394 | 10 14 4 3 5 26 33 27 6 395 | 13 4 9 7 12 5 19 6 396 | 13 4 9 7 12 5 24 8 8 6 397 | 10 13 4 9 18 5 18 25 29 6 398 | 10 13 4 9 18 5 18 25 27 6 399 | 10 14 4 3 5 27 6 400 | 10 14 4 3 5 26 33 27 6 401 | 13 4 9 7 12 5 19 6 402 | 13 4 9 7 12 5 24 8 8 6 403 | 10 13 4 9 18 5 18 25 29 6 404 | 10 13 4 9 18 5 18 25 27 6 405 | 10 14 4 3 5 27 6 406 | 10 14 4 3 5 26 33 27 6 407 | 13 4 9 7 12 5 19 6 408 | 13 4 9 7 12 5 24 8 8 6 409 | 10 13 4 9 18 5 18 25 29 6 410 | 10 13 4 9 18 5 18 25 27 6 411 | 10 14 4 3 5 27 6 412 | 10 14 4 3 5 26 33 27 6 413 | 13 4 9 7 12 5 19 6 414 | 13 4 9 7 12 5 24 8 8 6 415 | 10 13 4 9 18 5 18 25 29 6 416 | 10 13 4 9 18 5 18 25 27 6 417 | 10 14 4 3 5 27 6 418 | 10 14 4 3 5 26 33 27 6 419 | 13 4 9 7 12 5 19 6 420 | 13 4 9 7 12 5 24 8 8 6 421 | 10 13 4 9 18 5 18 25 29 6 422 | 10 13 4 9 18 5 18 25 27 6 423 | 10 14 4 3 5 27 6 424 | 10 14 4 3 5 26 33 27 6 425 | 13 4 9 7 12 5 19 6 426 | 13 4 9 7 12 5 24 8 8 6 427 | 10 13 4 9 18 5 18 25 29 6 428 | 10 13 4 9 18 5 18 25 27 6 429 | 10 14 4 3 5 27 6 430 | 10 14 4 3 5 26 33 27 6 431 | 13 4 9 7 12 5 19 6 432 | 13 4 9 7 12 5 24 8 8 6 433 | 10 13 4 9 18 5 18 25 29 6 434 | 10 13 4 9 18 5 18 25 27 6 435 | 10 14 4 3 5 27 6 436 | 10 14 4 3 5 26 33 27 6 437 | 13 4 9 7 12 5 19 6 438 | 13 4 9 7 12 5 24 8 8 6 439 | 10 13 4 9 18 5 18 25 29 6 440 | 10 13 4 9 18 5 18 25 27 6 441 | 10 14 4 3 5 27 6 442 | 10 14 4 3 5 26 33 27 6 443 | 13 4 9 7 12 5 19 6 444 | 13 4 9 7 12 5 24 8 8 6 445 | 10 13 4 9 18 5 18 25 29 6 446 | 10 13 4 9 18 5 18 25 27 6 447 | 10 14 4 3 5 27 6 448 | 10 14 4 3 5 26 33 27 6 449 | 13 4 9 7 12 5 19 6 450 | 13 4 9 7 12 5 24 8 8 6 451 | 10 13 4 9 18 5 18 25 29 6 452 | 10 13 4 9 18 5 18 25 27 6 453 | 10 14 4 3 5 27 6 454 | 10 14 4 3 5 26 33 27 6 455 | 13 4 9 7 12 5 19 6 456 | 13 4 9 7 12 5 24 8 8 6 457 | 10 13 4 9 18 5 18 25 29 6 458 | 10 13 4 9 18 5 18 25 27 6 459 | 10 14 4 3 5 27 6 460 | 10 14 4 3 5 26 33 27 6 461 | 13 4 9 7 12 5 19 6 462 | 13 4 9 7 12 5 24 8 8 6 463 | 10 13 4 9 18 5 18 25 29 6 464 | 10 13 4 9 18 5 18 25 27 6 465 | 10 14 4 3 5 27 6 466 | 10 14 4 3 5 26 33 27 6 467 | 13 4 9 7 12 5 19 6 468 | 13 4 9 7 12 5 24 8 8 6 469 | 10 13 4 9 18 5 18 25 29 6 470 | 10 13 4 9 18 5 18 25 27 6 471 | 10 14 4 3 5 27 6 472 | 10 14 4 3 5 26 33 27 6 473 | 13 4 9 7 12 5 19 6 474 | 13 4 9 7 12 5 24 8 8 6 475 | 10 13 4 9 18 5 18 25 29 6 476 | 10 13 4 9 18 5 18 25 27 6 477 | 10 14 4 3 5 27 6 478 | 10 14 4 3 5 26 33 27 6 479 | 13 4 9 7 12 5 19 6 480 | 13 4 9 7 12 5 24 8 8 6 481 | 10 13 4 9 18 5 18 25 29 6 482 | 10 13 4 9 18 5 18 25 27 6 483 | 10 14 4 3 5 27 6 484 | 10 14 4 3 5 26 33 27 6 485 | 13 4 9 7 12 5 19 6 486 | 13 4 9 7 12 5 24 8 8 6 487 | 10 13 4 9 18 5 18 25 29 6 488 | 10 13 4 9 18 5 18 25 27 6 489 | 10 14 4 3 5 27 6 490 | 10 14 4 3 5 26 33 27 6 491 | 13 4 9 7 12 5 19 6 492 | 13 4 9 7 12 5 24 8 8 6 493 | 10 13 4 9 18 5 18 25 29 6 494 | 10 13 4 9 18 5 18 25 27 6 495 | 10 14 4 3 5 27 6 496 | 10 14 4 3 5 26 33 27 6 497 | 13 4 9 7 12 5 19 6 498 | 13 4 9 7 12 5 24 8 8 6 499 | 10 13 4 9 18 5 18 25 29 6 500 | 10 13 4 9 18 5 18 25 27 6 501 | 10 14 4 3 5 27 6 502 | 10 14 4 3 5 26 33 27 6 503 | 13 4 9 7 12 5 19 6 504 | 13 4 9 7 12 5 24 8 8 6 505 | 10 13 4 9 18 5 18 25 29 6 506 | 10 13 4 9 18 5 18 25 27 6 507 | 10 14 4 3 5 27 6 508 | 10 14 4 3 5 26 33 27 6 509 | 13 4 9 7 12 5 19 6 510 | 13 4 9 7 12 5 24 8 8 6 511 | 10 13 4 9 18 5 18 25 29 6 512 | 10 13 4 9 18 5 18 25 27 6 513 | 10 14 4 3 5 27 6 514 | 10 14 4 3 5 26 33 27 6 515 | 13 4 9 7 12 5 19 6 516 | 13 4 9 7 12 5 24 8 8 6 517 | 10 13 4 9 18 5 18 25 29 6 518 | 10 13 4 9 18 5 18 25 27 6 519 | 10 14 4 3 5 27 6 520 | 10 14 4 3 5 26 33 27 6 521 | 13 4 9 7 12 5 19 6 522 | 13 4 9 7 12 5 24 8 8 6 523 | 10 13 4 9 18 5 18 25 29 6 524 | 10 13 4 9 18 5 18 25 27 6 525 | 10 14 4 3 5 27 6 526 | 10 14 4 3 5 26 33 27 6 527 | 13 4 9 7 12 5 19 6 528 | 13 4 9 7 12 5 24 8 8 6 529 | 10 13 4 9 18 5 18 25 29 6 530 | 10 13 4 9 18 5 18 25 27 6 531 | 10 14 4 3 5 27 6 532 | 10 14 4 3 5 26 33 27 6 533 | 13 4 9 7 12 5 19 6 534 | 13 4 9 7 12 5 24 8 8 6 535 | 10 13 4 9 18 5 18 25 29 6 536 | 10 13 4 9 18 5 18 25 27 6 537 | 10 14 4 3 5 27 6 538 | 10 14 4 3 5 26 33 27 6 539 | 13 4 9 7 12 5 19 6 540 | 13 4 9 7 12 5 24 8 8 6 541 | 10 13 4 9 18 5 18 25 29 6 542 | 10 13 4 9 18 5 18 25 27 6 543 | 10 14 4 3 5 27 6 544 | 10 14 4 3 5 26 33 27 6 545 | 13 4 9 7 12 5 19 6 546 | 13 4 9 7 12 5 24 8 8 6 547 | 10 13 4 9 18 5 18 25 29 6 548 | 10 13 4 9 18 5 18 25 27 6 549 | 10 14 4 3 5 27 6 550 | 10 14 4 3 5 26 33 27 6 551 | 13 4 9 7 12 5 19 6 552 | 13 4 9 7 12 5 24 8 8 6 553 | 10 13 4 9 18 5 18 25 29 6 554 | 10 13 4 9 18 5 18 25 27 6 555 | 10 14 4 3 5 27 6 556 | 10 14 4 3 5 26 33 27 6 557 | 13 4 9 7 12 5 19 6 558 | 13 4 9 7 12 5 24 8 8 6 559 | 10 13 4 9 18 5 18 25 29 6 560 | 10 13 4 9 18 5 18 25 27 6 561 | 10 14 4 3 5 27 6 562 | 10 14 4 3 5 26 33 27 6 563 | 13 4 9 7 12 5 19 6 564 | 13 4 9 7 12 5 24 8 8 6 565 | 10 13 4 9 18 5 18 25 29 6 566 | 10 13 4 9 18 5 18 25 27 6 567 | 10 14 4 3 5 27 6 568 | 10 14 4 3 5 26 33 27 6 569 | 13 4 9 7 12 5 19 6 570 | 13 4 9 7 12 5 24 8 8 6 571 | 10 13 4 9 18 5 18 25 29 6 572 | 10 13 4 9 18 5 18 25 27 6 573 | 10 14 4 3 5 27 6 574 | 10 14 4 3 5 26 33 27 6 575 | 13 4 9 7 12 5 19 6 576 | 13 4 9 7 12 5 24 8 8 6 577 | 10 13 4 9 18 5 18 25 29 6 578 | 10 13 4 9 18 5 18 25 27 6 579 | 10 14 4 3 5 27 6 580 | 10 14 4 3 5 26 33 27 6 581 | 13 4 9 7 12 5 19 6 582 | 13 4 9 7 12 5 24 8 8 6 583 | 10 13 4 9 18 5 18 25 29 6 584 | 10 13 4 9 18 5 18 25 27 6 585 | 10 14 4 3 5 27 6 586 | 10 14 4 3 5 26 33 27 6 587 | 13 4 9 7 12 5 19 6 588 | 13 4 9 7 12 5 24 8 8 6 589 | 10 13 4 9 18 5 18 25 29 6 590 | 10 13 4 9 18 5 18 25 27 6 591 | 10 14 4 3 5 27 6 592 | 10 14 4 3 5 26 33 27 6 593 | 13 4 9 7 12 5 19 6 594 | 13 4 9 7 12 5 24 8 8 6 595 | 10 13 4 9 18 5 18 25 29 6 596 | 10 13 4 9 18 5 18 25 27 6 597 | 10 14 4 3 5 27 6 598 | 10 14 4 3 5 26 33 27 6 599 | 13 4 9 7 12 5 19 6 600 | 13 4 9 7 12 5 24 8 8 6 601 | 10 13 4 9 18 5 18 25 29 6 602 | 10 13 4 9 18 5 18 25 27 6 603 | 10 14 4 3 5 27 6 604 | 10 14 4 3 5 26 33 27 6 605 | 13 4 9 7 12 5 19 6 606 | 13 4 9 7 12 5 24 8 8 6 607 | 10 13 4 9 18 5 18 25 29 6 608 | 10 13 4 9 18 5 18 25 27 6 609 | 10 14 4 3 5 27 6 610 | 10 14 4 3 5 26 33 27 6 611 | 13 4 9 7 12 5 19 6 612 | 13 4 9 7 12 5 24 8 8 6 613 | 10 13 4 9 18 5 18 25 29 6 614 | 10 13 4 9 18 5 18 25 27 6 615 | 10 14 4 3 5 27 6 616 | 10 14 4 3 5 26 33 27 6 617 | 13 4 9 7 12 5 19 6 618 | 13 4 9 7 12 5 24 8 8 6 619 | 10 13 4 9 18 5 18 25 29 6 620 | 10 13 4 9 18 5 18 25 27 6 621 | 10 14 4 3 5 27 6 622 | 10 14 4 3 5 26 33 27 6 623 | 13 4 9 7 12 5 19 6 624 | 13 4 9 7 12 5 24 8 8 6 625 | 10 13 4 9 18 5 18 25 29 6 626 | 10 13 4 9 18 5 18 25 27 6 627 | 10 14 4 3 5 27 6 628 | 10 14 4 3 5 26 33 27 6 629 | 13 4 9 7 12 5 19 6 630 | 13 4 9 7 12 5 24 8 8 6 631 | 10 13 4 9 18 5 18 25 29 6 632 | 10 13 4 9 18 5 18 25 27 6 633 | 10 14 4 3 5 27 6 634 | 10 14 4 3 5 26 33 27 6 635 | 13 4 9 7 12 5 19 6 636 | 13 4 9 7 12 5 24 8 8 6 637 | 10 13 4 9 18 5 18 25 29 6 638 | 10 13 4 9 18 5 18 25 27 6 639 | 10 14 4 3 5 27 6 640 | 10 14 4 3 5 26 33 27 6 641 | 13 4 9 7 12 5 19 6 642 | 13 4 9 7 12 5 24 8 8 6 643 | 10 13 4 9 18 5 18 25 29 6 644 | 10 13 4 9 18 5 18 25 27 6 645 | 10 14 4 3 5 27 6 646 | 10 14 4 3 5 26 33 27 6 647 | 13 4 9 7 12 5 19 6 648 | 13 4 9 7 12 5 24 8 8 6 649 | 10 13 4 9 18 5 18 25 29 6 650 | 10 13 4 9 18 5 18 25 27 6 651 | 10 14 4 3 5 27 6 652 | 10 14 4 3 5 26 33 27 6 653 | 13 4 9 7 12 5 19 6 654 | 13 4 9 7 12 5 24 8 8 6 655 | 10 13 4 9 18 5 18 25 29 6 656 | 10 13 4 9 18 5 18 25 27 6 657 | 10 14 4 3 5 27 6 658 | 10 14 4 3 5 26 33 27 6 659 | 13 4 9 7 12 5 19 6 660 | 13 4 9 7 12 5 24 8 8 6 661 | 10 13 4 9 18 5 18 25 29 6 662 | 10 13 4 9 18 5 18 25 27 6 663 | 10 14 4 3 5 27 6 664 | 10 14 4 3 5 26 33 27 6 665 | 13 4 9 7 12 5 19 6 666 | 13 4 9 7 12 5 24 8 8 6 667 | 10 13 4 9 18 5 18 25 29 6 668 | 10 13 4 9 18 5 18 25 27 6 669 | 10 14 4 3 5 27 6 670 | 10 14 4 3 5 26 33 27 6 671 | 13 4 9 7 12 5 19 6 672 | 13 4 9 7 12 5 24 8 8 6 673 | 10 13 4 9 18 5 18 25 29 6 674 | 10 13 4 9 18 5 18 25 27 6 675 | 10 14 4 3 5 27 6 676 | 10 14 4 3 5 26 33 27 6 677 | 13 4 9 7 12 5 19 6 678 | 13 4 9 7 12 5 24 8 8 6 679 | 10 13 4 9 18 5 18 25 29 6 680 | 10 13 4 9 18 5 18 25 27 6 681 | 10 14 4 3 5 27 6 682 | 10 14 4 3 5 26 33 27 6 683 | 13 4 9 7 12 5 19 6 684 | 13 4 9 7 12 5 24 8 8 6 685 | 10 13 4 9 18 5 18 25 29 6 686 | 10 13 4 9 18 5 18 25 27 6 687 | 10 14 4 3 5 27 6 688 | 10 14 4 3 5 26 33 27 6 689 | 13 4 9 7 12 5 19 6 690 | 13 4 9 7 12 5 24 8 8 6 691 | 10 13 4 9 18 5 18 25 29 6 692 | 10 13 4 9 18 5 18 25 27 6 693 | 10 14 4 3 5 27 6 694 | 10 14 4 3 5 26 33 27 6 695 | 13 4 9 7 12 5 19 6 696 | 13 4 9 7 12 5 24 8 8 6 697 | 10 13 4 9 18 5 18 25 29 6 698 | 10 13 4 9 18 5 18 25 27 6 699 | 10 14 4 3 5 27 6 700 | 10 14 4 3 5 26 33 27 6 701 | 13 4 9 7 12 5 19 6 702 | 13 4 9 7 12 5 24 8 8 6 703 | 10 13 4 9 18 5 18 25 29 6 704 | 10 13 4 9 18 5 18 25 27 6 705 | 10 14 4 3 5 27 6 706 | 10 14 4 3 5 26 33 27 6 707 | 13 4 9 7 12 5 19 6 708 | 13 4 9 7 12 5 24 8 8 6 709 | 10 13 4 9 18 5 18 25 29 6 710 | 10 13 4 9 18 5 18 25 27 6 711 | 10 14 4 3 5 27 6 712 | 10 14 4 3 5 26 33 27 6 713 | 13 4 9 7 12 5 19 6 714 | 13 4 9 7 12 5 24 8 8 6 715 | 10 13 4 9 18 5 18 25 29 6 716 | 10 13 4 9 18 5 18 25 27 6 717 | 10 14 4 3 5 27 6 718 | 10 14 4 3 5 26 33 27 6 719 | 13 4 9 7 12 5 19 6 720 | 13 4 9 7 12 5 24 8 8 6 -------------------------------------------------------------------------------- /data/toy/base_conv_test.num: -------------------------------------------------------------------------------- 1 | 11 15 10 13 4 9 18 5 29 6 2 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 3 | 11 15 10 25 18 3 5 27 6 4 | 11 15 10 25 18 3 5 8 8 6 5 | 13 4 9 7 3 5 17 8 23 6 6 | 13 4 9 7 3 5 18 25 29 6 7 | 11 15 10 13 4 9 18 5 29 6 8 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 9 | 11 15 10 25 18 3 5 27 6 10 | 11 15 10 25 18 3 5 8 8 6 11 | 13 4 9 7 3 5 17 8 23 6 12 | 13 4 9 7 3 5 18 25 29 6 13 | 11 15 10 13 4 9 18 5 29 6 14 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 15 | 11 15 10 25 18 3 5 27 6 16 | 11 15 10 25 18 3 5 8 8 6 17 | 13 4 9 7 3 5 17 8 23 6 18 | 13 4 9 7 3 5 18 25 29 6 19 | 11 15 10 13 4 9 18 5 29 6 20 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 21 | 11 15 10 25 18 3 5 27 6 22 | 11 15 10 25 18 3 5 8 8 6 23 | 13 4 9 7 3 5 17 8 23 6 24 | 13 4 9 7 3 5 18 25 29 6 25 | 11 15 10 13 4 9 18 5 29 6 26 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 27 | 11 15 10 25 18 3 5 27 6 28 | 11 15 10 25 18 3 5 8 8 6 29 | 13 4 9 7 3 5 17 8 23 6 30 | 13 4 9 7 3 5 18 25 29 6 31 | 11 15 10 13 4 9 18 5 29 6 32 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 33 | 11 15 10 25 18 3 5 27 6 34 | 11 15 10 25 18 3 5 8 8 6 35 | 13 4 9 7 3 5 17 8 23 6 36 | 13 4 9 7 3 5 18 25 29 6 37 | 11 15 10 13 4 9 18 5 29 6 38 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 39 | 11 15 10 25 18 3 5 27 6 40 | 11 15 10 25 18 3 5 8 8 6 41 | 13 4 9 7 3 5 17 8 23 6 42 | 13 4 9 7 3 5 18 25 29 6 43 | 11 15 10 13 4 9 18 5 29 6 44 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 45 | 11 15 10 25 18 3 5 27 6 46 | 11 15 10 25 18 3 5 8 8 6 47 | 13 4 9 7 3 5 17 8 23 6 48 | 13 4 9 7 3 5 18 25 29 6 49 | 11 15 10 13 4 9 18 5 29 6 50 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 51 | 11 15 10 25 18 3 5 27 6 52 | 11 15 10 25 18 3 5 8 8 6 53 | 13 4 9 7 3 5 17 8 23 6 54 | 13 4 9 7 3 5 18 25 29 6 55 | 11 15 10 13 4 9 18 5 29 6 56 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 57 | 11 15 10 25 18 3 5 27 6 58 | 11 15 10 25 18 3 5 8 8 6 59 | 13 4 9 7 3 5 17 8 23 6 60 | 13 4 9 7 3 5 18 25 29 6 61 | 11 15 10 13 4 9 18 5 29 6 62 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 63 | 11 15 10 25 18 3 5 27 6 64 | 11 15 10 25 18 3 5 8 8 6 65 | 13 4 9 7 3 5 17 8 23 6 66 | 13 4 9 7 3 5 18 25 29 6 67 | 11 15 10 13 4 9 18 5 29 6 68 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 69 | 11 15 10 25 18 3 5 27 6 70 | 11 15 10 25 18 3 5 8 8 6 71 | 13 4 9 7 3 5 17 8 23 6 72 | 13 4 9 7 3 5 18 25 29 6 73 | 11 15 10 13 4 9 18 5 29 6 74 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 75 | 11 15 10 25 18 3 5 27 6 76 | 11 15 10 25 18 3 5 8 8 6 77 | 13 4 9 7 3 5 17 8 23 6 78 | 13 4 9 7 3 5 18 25 29 6 79 | 11 15 10 13 4 9 18 5 29 6 80 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 81 | 11 15 10 25 18 3 5 27 6 82 | 11 15 10 25 18 3 5 8 8 6 83 | 13 4 9 7 3 5 17 8 23 6 84 | 13 4 9 7 3 5 18 25 29 6 85 | 11 15 10 13 4 9 18 5 29 6 86 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 87 | 11 15 10 25 18 3 5 27 6 88 | 11 15 10 25 18 3 5 8 8 6 89 | 13 4 9 7 3 5 17 8 23 6 90 | 13 4 9 7 3 5 18 25 29 6 91 | 11 15 10 13 4 9 18 5 29 6 92 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 93 | 11 15 10 25 18 3 5 27 6 94 | 11 15 10 25 18 3 5 8 8 6 95 | 13 4 9 7 3 5 17 8 23 6 96 | 13 4 9 7 3 5 18 25 29 6 97 | 11 15 10 13 4 9 18 5 29 6 98 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 99 | 11 15 10 25 18 3 5 27 6 100 | 11 15 10 25 18 3 5 8 8 6 101 | 13 4 9 7 3 5 17 8 23 6 102 | 13 4 9 7 3 5 18 25 29 6 103 | 11 15 10 13 4 9 18 5 29 6 104 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 105 | 11 15 10 25 18 3 5 27 6 106 | 11 15 10 25 18 3 5 8 8 6 107 | 13 4 9 7 3 5 17 8 23 6 108 | 13 4 9 7 3 5 18 25 29 6 109 | 11 15 10 13 4 9 18 5 29 6 110 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 111 | 11 15 10 25 18 3 5 27 6 112 | 11 15 10 25 18 3 5 8 8 6 113 | 13 4 9 7 3 5 17 8 23 6 114 | 13 4 9 7 3 5 18 25 29 6 115 | 11 15 10 13 4 9 18 5 29 6 116 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 117 | 11 15 10 25 18 3 5 27 6 118 | 11 15 10 25 18 3 5 8 8 6 119 | 13 4 9 7 3 5 17 8 23 6 120 | 13 4 9 7 3 5 18 25 29 6 121 | 11 15 10 13 4 9 18 5 29 6 122 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 123 | 11 15 10 25 18 3 5 27 6 124 | 11 15 10 25 18 3 5 8 8 6 125 | 13 4 9 7 3 5 17 8 23 6 126 | 13 4 9 7 3 5 18 25 29 6 127 | 11 15 10 13 4 9 18 5 29 6 128 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 129 | 11 15 10 25 18 3 5 27 6 130 | 11 15 10 25 18 3 5 8 8 6 131 | 13 4 9 7 3 5 17 8 23 6 132 | 13 4 9 7 3 5 18 25 29 6 133 | 11 15 10 13 4 9 18 5 29 6 134 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 135 | 11 15 10 25 18 3 5 27 6 136 | 11 15 10 25 18 3 5 8 8 6 137 | 13 4 9 7 3 5 17 8 23 6 138 | 13 4 9 7 3 5 18 25 29 6 139 | 11 15 10 13 4 9 18 5 29 6 140 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 141 | 11 15 10 25 18 3 5 27 6 142 | 11 15 10 25 18 3 5 8 8 6 143 | 13 4 9 7 3 5 17 8 23 6 144 | 13 4 9 7 3 5 18 25 29 6 145 | 11 15 10 13 4 9 18 5 29 6 146 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 147 | 11 15 10 25 18 3 5 27 6 148 | 11 15 10 25 18 3 5 8 8 6 149 | 13 4 9 7 3 5 17 8 23 6 150 | 13 4 9 7 3 5 18 25 29 6 151 | 11 15 10 13 4 9 18 5 29 6 152 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 153 | 11 15 10 25 18 3 5 27 6 154 | 11 15 10 25 18 3 5 8 8 6 155 | 13 4 9 7 3 5 17 8 23 6 156 | 13 4 9 7 3 5 18 25 29 6 157 | 11 15 10 13 4 9 18 5 29 6 158 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 159 | 11 15 10 25 18 3 5 27 6 160 | 11 15 10 25 18 3 5 8 8 6 161 | 13 4 9 7 3 5 17 8 23 6 162 | 13 4 9 7 3 5 18 25 29 6 163 | 11 15 10 13 4 9 18 5 29 6 164 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 165 | 11 15 10 25 18 3 5 27 6 166 | 11 15 10 25 18 3 5 8 8 6 167 | 13 4 9 7 3 5 17 8 23 6 168 | 13 4 9 7 3 5 18 25 29 6 169 | 11 15 10 13 4 9 18 5 29 6 170 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 171 | 11 15 10 25 18 3 5 27 6 172 | 11 15 10 25 18 3 5 8 8 6 173 | 13 4 9 7 3 5 17 8 23 6 174 | 13 4 9 7 3 5 18 25 29 6 175 | 11 15 10 13 4 9 18 5 29 6 176 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 177 | 11 15 10 25 18 3 5 27 6 178 | 11 15 10 25 18 3 5 8 8 6 179 | 13 4 9 7 3 5 17 8 23 6 180 | 13 4 9 7 3 5 18 25 29 6 181 | 11 15 10 13 4 9 18 5 29 6 182 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 183 | 11 15 10 25 18 3 5 27 6 184 | 11 15 10 25 18 3 5 8 8 6 185 | 13 4 9 7 3 5 17 8 23 6 186 | 13 4 9 7 3 5 18 25 29 6 187 | 11 15 10 13 4 9 18 5 29 6 188 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 189 | 11 15 10 25 18 3 5 27 6 190 | 11 15 10 25 18 3 5 8 8 6 191 | 13 4 9 7 3 5 17 8 23 6 192 | 13 4 9 7 3 5 18 25 29 6 193 | 11 15 10 13 4 9 18 5 29 6 194 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 195 | 11 15 10 25 18 3 5 27 6 196 | 11 15 10 25 18 3 5 8 8 6 197 | 13 4 9 7 3 5 17 8 23 6 198 | 13 4 9 7 3 5 18 25 29 6 199 | 11 15 10 13 4 9 18 5 29 6 200 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 201 | 11 15 10 25 18 3 5 27 6 202 | 11 15 10 25 18 3 5 8 8 6 203 | 13 4 9 7 3 5 17 8 23 6 204 | 13 4 9 7 3 5 18 25 29 6 205 | 11 15 10 13 4 9 18 5 29 6 206 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 207 | 11 15 10 25 18 3 5 27 6 208 | 11 15 10 25 18 3 5 8 8 6 209 | 13 4 9 7 3 5 17 8 23 6 210 | 13 4 9 7 3 5 18 25 29 6 211 | 11 15 10 13 4 9 18 5 29 6 212 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 213 | 11 15 10 25 18 3 5 27 6 214 | 11 15 10 25 18 3 5 8 8 6 215 | 13 4 9 7 3 5 17 8 23 6 216 | 13 4 9 7 3 5 18 25 29 6 217 | 11 15 10 13 4 9 18 5 29 6 218 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 219 | 11 15 10 25 18 3 5 27 6 220 | 11 15 10 25 18 3 5 8 8 6 221 | 13 4 9 7 3 5 17 8 23 6 222 | 13 4 9 7 3 5 18 25 29 6 223 | 11 15 10 13 4 9 18 5 29 6 224 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 225 | 11 15 10 25 18 3 5 27 6 226 | 11 15 10 25 18 3 5 8 8 6 227 | 13 4 9 7 3 5 17 8 23 6 228 | 13 4 9 7 3 5 18 25 29 6 229 | 11 15 10 13 4 9 18 5 29 6 230 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 231 | 11 15 10 25 18 3 5 27 6 232 | 11 15 10 25 18 3 5 8 8 6 233 | 13 4 9 7 3 5 17 8 23 6 234 | 13 4 9 7 3 5 18 25 29 6 235 | 11 15 10 13 4 9 18 5 29 6 236 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 237 | 11 15 10 25 18 3 5 27 6 238 | 11 15 10 25 18 3 5 8 8 6 239 | 13 4 9 7 3 5 17 8 23 6 240 | 13 4 9 7 3 5 18 25 29 6 241 | 11 15 10 13 4 9 18 5 29 6 242 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 243 | 11 15 10 25 18 3 5 27 6 244 | 11 15 10 25 18 3 5 8 8 6 245 | 13 4 9 7 3 5 17 8 23 6 246 | 13 4 9 7 3 5 18 25 29 6 247 | 11 15 10 13 4 9 18 5 29 6 248 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 249 | 11 15 10 25 18 3 5 27 6 250 | 11 15 10 25 18 3 5 8 8 6 251 | 13 4 9 7 3 5 17 8 23 6 252 | 13 4 9 7 3 5 18 25 29 6 253 | 11 15 10 13 4 9 18 5 29 6 254 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 255 | 11 15 10 25 18 3 5 27 6 256 | 11 15 10 25 18 3 5 8 8 6 257 | 13 4 9 7 3 5 17 8 23 6 258 | 13 4 9 7 3 5 18 25 29 6 259 | 11 15 10 13 4 9 18 5 29 6 260 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 261 | 11 15 10 25 18 3 5 27 6 262 | 11 15 10 25 18 3 5 8 8 6 263 | 13 4 9 7 3 5 17 8 23 6 264 | 13 4 9 7 3 5 18 25 29 6 265 | 11 15 10 13 4 9 18 5 29 6 266 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 267 | 11 15 10 25 18 3 5 27 6 268 | 11 15 10 25 18 3 5 8 8 6 269 | 13 4 9 7 3 5 17 8 23 6 270 | 13 4 9 7 3 5 18 25 29 6 271 | 11 15 10 13 4 9 18 5 29 6 272 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 273 | 11 15 10 25 18 3 5 27 6 274 | 11 15 10 25 18 3 5 8 8 6 275 | 13 4 9 7 3 5 17 8 23 6 276 | 13 4 9 7 3 5 18 25 29 6 277 | 11 15 10 13 4 9 18 5 29 6 278 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 279 | 11 15 10 25 18 3 5 27 6 280 | 11 15 10 25 18 3 5 8 8 6 281 | 13 4 9 7 3 5 17 8 23 6 282 | 13 4 9 7 3 5 18 25 29 6 283 | 11 15 10 13 4 9 18 5 29 6 284 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 285 | 11 15 10 25 18 3 5 27 6 286 | 11 15 10 25 18 3 5 8 8 6 287 | 13 4 9 7 3 5 17 8 23 6 288 | 13 4 9 7 3 5 18 25 29 6 289 | 11 15 10 13 4 9 18 5 29 6 290 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 291 | 11 15 10 25 18 3 5 27 6 292 | 11 15 10 25 18 3 5 8 8 6 293 | 13 4 9 7 3 5 17 8 23 6 294 | 13 4 9 7 3 5 18 25 29 6 295 | 11 15 10 13 4 9 18 5 29 6 296 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 297 | 11 15 10 25 18 3 5 27 6 298 | 11 15 10 25 18 3 5 8 8 6 299 | 13 4 9 7 3 5 17 8 23 6 300 | 13 4 9 7 3 5 18 25 29 6 301 | 11 15 10 13 4 9 18 5 29 6 302 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 303 | 11 15 10 25 18 3 5 27 6 304 | 11 15 10 25 18 3 5 8 8 6 305 | 13 4 9 7 3 5 17 8 23 6 306 | 13 4 9 7 3 5 18 25 29 6 307 | 11 15 10 13 4 9 18 5 29 6 308 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 309 | 11 15 10 25 18 3 5 27 6 310 | 11 15 10 25 18 3 5 8 8 6 311 | 13 4 9 7 3 5 17 8 23 6 312 | 13 4 9 7 3 5 18 25 29 6 313 | 11 15 10 13 4 9 18 5 29 6 314 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 315 | 11 15 10 25 18 3 5 27 6 316 | 11 15 10 25 18 3 5 8 8 6 317 | 13 4 9 7 3 5 17 8 23 6 318 | 13 4 9 7 3 5 18 25 29 6 319 | 11 15 10 13 4 9 18 5 29 6 320 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 321 | 11 15 10 25 18 3 5 27 6 322 | 11 15 10 25 18 3 5 8 8 6 323 | 13 4 9 7 3 5 17 8 23 6 324 | 13 4 9 7 3 5 18 25 29 6 325 | 11 15 10 13 4 9 18 5 29 6 326 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 327 | 11 15 10 25 18 3 5 27 6 328 | 11 15 10 25 18 3 5 8 8 6 329 | 13 4 9 7 3 5 17 8 23 6 330 | 13 4 9 7 3 5 18 25 29 6 331 | 11 15 10 13 4 9 18 5 29 6 332 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 333 | 11 15 10 25 18 3 5 27 6 334 | 11 15 10 25 18 3 5 8 8 6 335 | 13 4 9 7 3 5 17 8 23 6 336 | 13 4 9 7 3 5 18 25 29 6 337 | 11 15 10 13 4 9 18 5 29 6 338 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 339 | 11 15 10 25 18 3 5 27 6 340 | 11 15 10 25 18 3 5 8 8 6 341 | 13 4 9 7 3 5 17 8 23 6 342 | 13 4 9 7 3 5 18 25 29 6 343 | 11 15 10 13 4 9 18 5 29 6 344 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 345 | 11 15 10 25 18 3 5 27 6 346 | 11 15 10 25 18 3 5 8 8 6 347 | 13 4 9 7 3 5 17 8 23 6 348 | 13 4 9 7 3 5 18 25 29 6 349 | 11 15 10 13 4 9 18 5 29 6 350 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 351 | 11 15 10 25 18 3 5 27 6 352 | 11 15 10 25 18 3 5 8 8 6 353 | 13 4 9 7 3 5 17 8 23 6 354 | 13 4 9 7 3 5 18 25 29 6 355 | 11 15 10 13 4 9 18 5 29 6 356 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 357 | 11 15 10 25 18 3 5 27 6 358 | 11 15 10 25 18 3 5 8 8 6 359 | 13 4 9 7 3 5 17 8 23 6 360 | 13 4 9 7 3 5 18 25 29 6 361 | 11 15 10 13 4 9 18 5 29 6 362 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 363 | 11 15 10 25 18 3 5 27 6 364 | 11 15 10 25 18 3 5 8 8 6 365 | 13 4 9 7 3 5 17 8 23 6 366 | 13 4 9 7 3 5 18 25 29 6 367 | 11 15 10 13 4 9 18 5 29 6 368 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 369 | 11 15 10 25 18 3 5 27 6 370 | 11 15 10 25 18 3 5 8 8 6 371 | 13 4 9 7 3 5 17 8 23 6 372 | 13 4 9 7 3 5 18 25 29 6 373 | 11 15 10 13 4 9 18 5 29 6 374 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 375 | 11 15 10 25 18 3 5 27 6 376 | 11 15 10 25 18 3 5 8 8 6 377 | 13 4 9 7 3 5 17 8 23 6 378 | 13 4 9 7 3 5 18 25 29 6 379 | 11 15 10 13 4 9 18 5 29 6 380 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 381 | 11 15 10 25 18 3 5 27 6 382 | 11 15 10 25 18 3 5 8 8 6 383 | 13 4 9 7 3 5 17 8 23 6 384 | 13 4 9 7 3 5 18 25 29 6 385 | 11 15 10 13 4 9 18 5 29 6 386 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 387 | 11 15 10 25 18 3 5 27 6 388 | 11 15 10 25 18 3 5 8 8 6 389 | 13 4 9 7 3 5 17 8 23 6 390 | 13 4 9 7 3 5 18 25 29 6 391 | 11 15 10 13 4 9 18 5 29 6 392 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 393 | 11 15 10 25 18 3 5 27 6 394 | 11 15 10 25 18 3 5 8 8 6 395 | 13 4 9 7 3 5 17 8 23 6 396 | 13 4 9 7 3 5 18 25 29 6 397 | 11 15 10 13 4 9 18 5 29 6 398 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 399 | 11 15 10 25 18 3 5 27 6 400 | 11 15 10 25 18 3 5 8 8 6 401 | 13 4 9 7 3 5 17 8 23 6 402 | 13 4 9 7 3 5 18 25 29 6 403 | 11 15 10 13 4 9 18 5 29 6 404 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 405 | 11 15 10 25 18 3 5 27 6 406 | 11 15 10 25 18 3 5 8 8 6 407 | 13 4 9 7 3 5 17 8 23 6 408 | 13 4 9 7 3 5 18 25 29 6 409 | 11 15 10 13 4 9 18 5 29 6 410 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 411 | 11 15 10 25 18 3 5 27 6 412 | 11 15 10 25 18 3 5 8 8 6 413 | 13 4 9 7 3 5 17 8 23 6 414 | 13 4 9 7 3 5 18 25 29 6 415 | 11 15 10 13 4 9 18 5 29 6 416 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 417 | 11 15 10 25 18 3 5 27 6 418 | 11 15 10 25 18 3 5 8 8 6 419 | 13 4 9 7 3 5 17 8 23 6 420 | 13 4 9 7 3 5 18 25 29 6 421 | 11 15 10 13 4 9 18 5 29 6 422 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 423 | 11 15 10 25 18 3 5 27 6 424 | 11 15 10 25 18 3 5 8 8 6 425 | 13 4 9 7 3 5 17 8 23 6 426 | 13 4 9 7 3 5 18 25 29 6 427 | 11 15 10 13 4 9 18 5 29 6 428 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 429 | 11 15 10 25 18 3 5 27 6 430 | 11 15 10 25 18 3 5 8 8 6 431 | 13 4 9 7 3 5 17 8 23 6 432 | 13 4 9 7 3 5 18 25 29 6 433 | 11 15 10 13 4 9 18 5 29 6 434 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 435 | 11 15 10 25 18 3 5 27 6 436 | 11 15 10 25 18 3 5 8 8 6 437 | 13 4 9 7 3 5 17 8 23 6 438 | 13 4 9 7 3 5 18 25 29 6 439 | 11 15 10 13 4 9 18 5 29 6 440 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 441 | 11 15 10 25 18 3 5 27 6 442 | 11 15 10 25 18 3 5 8 8 6 443 | 13 4 9 7 3 5 17 8 23 6 444 | 13 4 9 7 3 5 18 25 29 6 445 | 11 15 10 13 4 9 18 5 29 6 446 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 447 | 11 15 10 25 18 3 5 27 6 448 | 11 15 10 25 18 3 5 8 8 6 449 | 13 4 9 7 3 5 17 8 23 6 450 | 13 4 9 7 3 5 18 25 29 6 451 | 11 15 10 13 4 9 18 5 29 6 452 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 453 | 11 15 10 25 18 3 5 27 6 454 | 11 15 10 25 18 3 5 8 8 6 455 | 13 4 9 7 3 5 17 8 23 6 456 | 13 4 9 7 3 5 18 25 29 6 457 | 11 15 10 13 4 9 18 5 29 6 458 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 459 | 11 15 10 25 18 3 5 27 6 460 | 11 15 10 25 18 3 5 8 8 6 461 | 13 4 9 7 3 5 17 8 23 6 462 | 13 4 9 7 3 5 18 25 29 6 463 | 11 15 10 13 4 9 18 5 29 6 464 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 465 | 11 15 10 25 18 3 5 27 6 466 | 11 15 10 25 18 3 5 8 8 6 467 | 13 4 9 7 3 5 17 8 23 6 468 | 13 4 9 7 3 5 18 25 29 6 469 | 11 15 10 13 4 9 18 5 29 6 470 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 471 | 11 15 10 25 18 3 5 27 6 472 | 11 15 10 25 18 3 5 8 8 6 473 | 13 4 9 7 3 5 17 8 23 6 474 | 13 4 9 7 3 5 18 25 29 6 475 | 11 15 10 13 4 9 18 5 29 6 476 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 477 | 11 15 10 25 18 3 5 27 6 478 | 11 15 10 25 18 3 5 8 8 6 479 | 13 4 9 7 3 5 17 8 23 6 480 | 13 4 9 7 3 5 18 25 29 6 481 | 11 15 10 13 4 9 18 5 29 6 482 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 483 | 11 15 10 25 18 3 5 27 6 484 | 11 15 10 25 18 3 5 8 8 6 485 | 13 4 9 7 3 5 17 8 23 6 486 | 13 4 9 7 3 5 18 25 29 6 487 | 11 15 10 13 4 9 18 5 29 6 488 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 489 | 11 15 10 25 18 3 5 27 6 490 | 11 15 10 25 18 3 5 8 8 6 491 | 13 4 9 7 3 5 17 8 23 6 492 | 13 4 9 7 3 5 18 25 29 6 493 | 11 15 10 13 4 9 18 5 29 6 494 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 495 | 11 15 10 25 18 3 5 27 6 496 | 11 15 10 25 18 3 5 8 8 6 497 | 13 4 9 7 3 5 17 8 23 6 498 | 13 4 9 7 3 5 18 25 29 6 499 | 11 15 10 13 4 9 18 5 29 6 500 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 501 | 11 15 10 25 18 3 5 27 6 502 | 11 15 10 25 18 3 5 8 8 6 503 | 13 4 9 7 3 5 17 8 23 6 504 | 13 4 9 7 3 5 18 25 29 6 505 | 11 15 10 13 4 9 18 5 29 6 506 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 507 | 11 15 10 25 18 3 5 27 6 508 | 11 15 10 25 18 3 5 8 8 6 509 | 13 4 9 7 3 5 17 8 23 6 510 | 13 4 9 7 3 5 18 25 29 6 511 | 11 15 10 13 4 9 18 5 29 6 512 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 513 | 11 15 10 25 18 3 5 27 6 514 | 11 15 10 25 18 3 5 8 8 6 515 | 13 4 9 7 3 5 17 8 23 6 516 | 13 4 9 7 3 5 18 25 29 6 517 | 11 15 10 13 4 9 18 5 29 6 518 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 519 | 11 15 10 25 18 3 5 27 6 520 | 11 15 10 25 18 3 5 8 8 6 521 | 13 4 9 7 3 5 17 8 23 6 522 | 13 4 9 7 3 5 18 25 29 6 523 | 11 15 10 13 4 9 18 5 29 6 524 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 525 | 11 15 10 25 18 3 5 27 6 526 | 11 15 10 25 18 3 5 8 8 6 527 | 13 4 9 7 3 5 17 8 23 6 528 | 13 4 9 7 3 5 18 25 29 6 529 | 11 15 10 13 4 9 18 5 29 6 530 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 531 | 11 15 10 25 18 3 5 27 6 532 | 11 15 10 25 18 3 5 8 8 6 533 | 13 4 9 7 3 5 17 8 23 6 534 | 13 4 9 7 3 5 18 25 29 6 535 | 11 15 10 13 4 9 18 5 29 6 536 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 537 | 11 15 10 25 18 3 5 27 6 538 | 11 15 10 25 18 3 5 8 8 6 539 | 13 4 9 7 3 5 17 8 23 6 540 | 13 4 9 7 3 5 18 25 29 6 541 | 11 15 10 13 4 9 18 5 29 6 542 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 543 | 11 15 10 25 18 3 5 27 6 544 | 11 15 10 25 18 3 5 8 8 6 545 | 13 4 9 7 3 5 17 8 23 6 546 | 13 4 9 7 3 5 18 25 29 6 547 | 11 15 10 13 4 9 18 5 29 6 548 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 549 | 11 15 10 25 18 3 5 27 6 550 | 11 15 10 25 18 3 5 8 8 6 551 | 13 4 9 7 3 5 17 8 23 6 552 | 13 4 9 7 3 5 18 25 29 6 553 | 11 15 10 13 4 9 18 5 29 6 554 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 555 | 11 15 10 25 18 3 5 27 6 556 | 11 15 10 25 18 3 5 8 8 6 557 | 13 4 9 7 3 5 17 8 23 6 558 | 13 4 9 7 3 5 18 25 29 6 559 | 11 15 10 13 4 9 18 5 29 6 560 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 561 | 11 15 10 25 18 3 5 27 6 562 | 11 15 10 25 18 3 5 8 8 6 563 | 13 4 9 7 3 5 17 8 23 6 564 | 13 4 9 7 3 5 18 25 29 6 565 | 11 15 10 13 4 9 18 5 29 6 566 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 567 | 11 15 10 25 18 3 5 27 6 568 | 11 15 10 25 18 3 5 8 8 6 569 | 13 4 9 7 3 5 17 8 23 6 570 | 13 4 9 7 3 5 18 25 29 6 571 | 11 15 10 13 4 9 18 5 29 6 572 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 573 | 11 15 10 25 18 3 5 27 6 574 | 11 15 10 25 18 3 5 8 8 6 575 | 13 4 9 7 3 5 17 8 23 6 576 | 13 4 9 7 3 5 18 25 29 6 577 | 11 15 10 13 4 9 18 5 29 6 578 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 579 | 11 15 10 25 18 3 5 27 6 580 | 11 15 10 25 18 3 5 8 8 6 581 | 13 4 9 7 3 5 17 8 23 6 582 | 13 4 9 7 3 5 18 25 29 6 583 | 11 15 10 13 4 9 18 5 29 6 584 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 585 | 11 15 10 25 18 3 5 27 6 586 | 11 15 10 25 18 3 5 8 8 6 587 | 13 4 9 7 3 5 17 8 23 6 588 | 13 4 9 7 3 5 18 25 29 6 589 | 11 15 10 13 4 9 18 5 29 6 590 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 591 | 11 15 10 25 18 3 5 27 6 592 | 11 15 10 25 18 3 5 8 8 6 593 | 13 4 9 7 3 5 17 8 23 6 594 | 13 4 9 7 3 5 18 25 29 6 595 | 11 15 10 13 4 9 18 5 29 6 596 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 597 | 11 15 10 25 18 3 5 27 6 598 | 11 15 10 25 18 3 5 8 8 6 599 | 13 4 9 7 3 5 17 8 23 6 600 | 13 4 9 7 3 5 18 25 29 6 601 | 11 15 10 13 4 9 18 5 29 6 602 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 603 | 11 15 10 25 18 3 5 27 6 604 | 11 15 10 25 18 3 5 8 8 6 605 | 13 4 9 7 3 5 17 8 23 6 606 | 13 4 9 7 3 5 18 25 29 6 607 | 11 15 10 13 4 9 18 5 29 6 608 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 609 | 11 15 10 25 18 3 5 27 6 610 | 11 15 10 25 18 3 5 8 8 6 611 | 13 4 9 7 3 5 17 8 23 6 612 | 13 4 9 7 3 5 18 25 29 6 613 | 11 15 10 13 4 9 18 5 29 6 614 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 615 | 11 15 10 25 18 3 5 27 6 616 | 11 15 10 25 18 3 5 8 8 6 617 | 13 4 9 7 3 5 17 8 23 6 618 | 13 4 9 7 3 5 18 25 29 6 619 | 11 15 10 13 4 9 18 5 29 6 620 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 621 | 11 15 10 25 18 3 5 27 6 622 | 11 15 10 25 18 3 5 8 8 6 623 | 13 4 9 7 3 5 17 8 23 6 624 | 13 4 9 7 3 5 18 25 29 6 625 | 11 15 10 13 4 9 18 5 29 6 626 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 627 | 11 15 10 25 18 3 5 27 6 628 | 11 15 10 25 18 3 5 8 8 6 629 | 13 4 9 7 3 5 17 8 23 6 630 | 13 4 9 7 3 5 18 25 29 6 631 | 11 15 10 13 4 9 18 5 29 6 632 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 633 | 11 15 10 25 18 3 5 27 6 634 | 11 15 10 25 18 3 5 8 8 6 635 | 13 4 9 7 3 5 17 8 23 6 636 | 13 4 9 7 3 5 18 25 29 6 637 | 11 15 10 13 4 9 18 5 29 6 638 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 639 | 11 15 10 25 18 3 5 27 6 640 | 11 15 10 25 18 3 5 8 8 6 641 | 13 4 9 7 3 5 17 8 23 6 642 | 13 4 9 7 3 5 18 25 29 6 643 | 11 15 10 13 4 9 18 5 29 6 644 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 645 | 11 15 10 25 18 3 5 27 6 646 | 11 15 10 25 18 3 5 8 8 6 647 | 13 4 9 7 3 5 17 8 23 6 648 | 13 4 9 7 3 5 18 25 29 6 649 | 11 15 10 13 4 9 18 5 29 6 650 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 651 | 11 15 10 25 18 3 5 27 6 652 | 11 15 10 25 18 3 5 8 8 6 653 | 13 4 9 7 3 5 17 8 23 6 654 | 13 4 9 7 3 5 18 25 29 6 655 | 11 15 10 13 4 9 18 5 29 6 656 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 657 | 11 15 10 25 18 3 5 27 6 658 | 11 15 10 25 18 3 5 8 8 6 659 | 13 4 9 7 3 5 17 8 23 6 660 | 13 4 9 7 3 5 18 25 29 6 661 | 11 15 10 13 4 9 18 5 29 6 662 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 663 | 11 15 10 25 18 3 5 27 6 664 | 11 15 10 25 18 3 5 8 8 6 665 | 13 4 9 7 3 5 17 8 23 6 666 | 13 4 9 7 3 5 18 25 29 6 667 | 11 15 10 13 4 9 18 5 29 6 668 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 669 | 11 15 10 25 18 3 5 27 6 670 | 11 15 10 25 18 3 5 8 8 6 671 | 13 4 9 7 3 5 17 8 23 6 672 | 13 4 9 7 3 5 18 25 29 6 673 | 11 15 10 13 4 9 18 5 29 6 674 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 675 | 11 15 10 25 18 3 5 27 6 676 | 11 15 10 25 18 3 5 8 8 6 677 | 13 4 9 7 3 5 17 8 23 6 678 | 13 4 9 7 3 5 18 25 29 6 679 | 11 15 10 13 4 9 18 5 29 6 680 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 681 | 11 15 10 25 18 3 5 27 6 682 | 11 15 10 25 18 3 5 8 8 6 683 | 13 4 9 7 3 5 17 8 23 6 684 | 13 4 9 7 3 5 18 25 29 6 685 | 11 15 10 13 4 9 18 5 29 6 686 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 687 | 11 15 10 25 18 3 5 27 6 688 | 11 15 10 25 18 3 5 8 8 6 689 | 13 4 9 7 3 5 17 8 23 6 690 | 13 4 9 7 3 5 18 25 29 6 691 | 11 15 10 13 4 9 18 5 29 6 692 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 693 | 11 15 10 25 18 3 5 27 6 694 | 11 15 10 25 18 3 5 8 8 6 695 | 13 4 9 7 3 5 17 8 23 6 696 | 13 4 9 7 3 5 18 25 29 6 697 | 11 15 10 13 4 9 18 5 29 6 698 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 699 | 11 15 10 25 18 3 5 27 6 700 | 11 15 10 25 18 3 5 8 8 6 701 | 13 4 9 7 3 5 17 8 23 6 702 | 13 4 9 7 3 5 18 25 29 6 703 | 11 15 10 13 4 9 18 5 29 6 704 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 705 | 11 15 10 25 18 3 5 27 6 706 | 11 15 10 25 18 3 5 8 8 6 707 | 13 4 9 7 3 5 17 8 23 6 708 | 13 4 9 7 3 5 18 25 29 6 709 | 11 15 10 13 4 9 18 5 29 6 710 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 711 | 11 15 10 25 18 3 5 27 6 712 | 11 15 10 25 18 3 5 8 8 6 713 | 13 4 9 7 3 5 17 8 23 6 714 | 13 4 9 7 3 5 18 25 29 6 715 | 11 15 10 13 4 9 18 5 29 6 716 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 717 | 11 15 10 25 18 3 5 27 6 718 | 11 15 10 25 18 3 5 8 8 6 719 | 13 4 9 7 3 5 17 8 23 6 720 | 13 4 9 7 3 5 18 25 29 6 721 | 11 15 10 13 4 9 18 5 29 6 722 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 723 | 11 15 10 25 18 3 5 27 6 724 | 11 15 10 25 18 3 5 8 8 6 725 | 13 4 9 7 3 5 17 8 23 6 726 | 13 4 9 7 3 5 18 25 29 6 727 | 11 15 10 13 4 9 18 5 29 6 728 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 729 | 11 15 10 25 18 3 5 27 6 730 | 11 15 10 25 18 3 5 8 8 6 731 | 13 4 9 7 3 5 17 8 23 6 732 | 13 4 9 7 3 5 18 25 29 6 733 | 11 15 10 13 4 9 18 5 29 6 734 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 735 | 11 15 10 25 18 3 5 27 6 736 | 11 15 10 25 18 3 5 8 8 6 737 | 13 4 9 7 3 5 17 8 23 6 738 | 13 4 9 7 3 5 18 25 29 6 739 | 11 15 10 13 4 9 18 5 29 6 740 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 741 | 11 15 10 25 18 3 5 27 6 742 | 11 15 10 25 18 3 5 8 8 6 743 | 13 4 9 7 3 5 17 8 23 6 744 | 13 4 9 7 3 5 18 25 29 6 745 | 11 15 10 13 4 9 18 5 29 6 746 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 747 | 11 15 10 25 18 3 5 27 6 748 | 11 15 10 25 18 3 5 8 8 6 749 | 13 4 9 7 3 5 17 8 23 6 750 | 13 4 9 7 3 5 18 25 29 6 751 | 11 15 10 13 4 9 18 5 29 6 752 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 753 | 11 15 10 25 18 3 5 27 6 754 | 11 15 10 25 18 3 5 8 8 6 755 | 13 4 9 7 3 5 17 8 23 6 756 | 13 4 9 7 3 5 18 25 29 6 757 | 11 15 10 13 4 9 18 5 29 6 758 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 759 | 11 15 10 25 18 3 5 27 6 760 | 11 15 10 25 18 3 5 8 8 6 761 | 13 4 9 7 3 5 17 8 23 6 762 | 13 4 9 7 3 5 18 25 29 6 763 | 11 15 10 13 4 9 18 5 29 6 764 | 11 15 10 13 4 9 18 5 18 25 21 30 22 6 765 | 11 15 10 25 18 3 5 27 6 766 | 11 15 10 25 18 3 5 8 8 6 767 | 13 4 9 7 3 5 17 8 23 6 768 | 13 4 9 7 3 5 18 25 29 6 -------------------------------------------------------------------------------- /data/toy/base_conv_vali.txt: -------------------------------------------------------------------------------- 1 | how do you like it ? it is boring . 2 | how do you like it ? it is good . 3 | how are you doing ? good . 4 | how are you doing ? i 'm good . 5 | do you like this game ? yes . 6 | do you like this game ? just so so . 7 | how do you like it ? it is boring . 8 | how do you like it ? it is good . 9 | how are you doing ? good . 10 | how are you doing ? i 'm good . 11 | do you like this game ? yes . 12 | do you like this game ? just so so . 13 | how do you like it ? it is boring . 14 | how do you like it ? it is good . 15 | how are you doing ? good . 16 | how are you doing ? i 'm good . 17 | do you like this game ? yes . 18 | do you like this game ? just so so . 19 | how do you like it ? it is boring . 20 | how do you like it ? it is good . 21 | how are you doing ? good . 22 | how are you doing ? i 'm good . 23 | do you like this game ? yes . 24 | do you like this game ? just so so . 25 | how do you like it ? it is boring . 26 | how do you like it ? it is good . 27 | how are you doing ? good . 28 | how are you doing ? i 'm good . 29 | do you like this game ? yes . 30 | do you like this game ? just so so . 31 | how do you like it ? it is boring . 32 | how do you like it ? it is good . 33 | how are you doing ? good . 34 | how are you doing ? i 'm good . 35 | do you like this game ? yes . 36 | do you like this game ? just so so . 37 | how do you like it ? it is boring . 38 | how do you like it ? it is good . 39 | how are you doing ? good . 40 | how are you doing ? i 'm good . 41 | do you like this game ? yes . 42 | do you like this game ? just so so . 43 | how do you like it ? it is boring . 44 | how do you like it ? it is good . 45 | how are you doing ? good . 46 | how are you doing ? i 'm good . 47 | do you like this game ? yes . 48 | do you like this game ? just so so . 49 | how do you like it ? it is boring . 50 | how do you like it ? it is good . 51 | how are you doing ? good . 52 | how are you doing ? i 'm good . 53 | do you like this game ? yes . 54 | do you like this game ? just so so . 55 | how do you like it ? it is boring . 56 | how do you like it ? it is good . 57 | how are you doing ? good . 58 | how are you doing ? i 'm good . 59 | do you like this game ? yes . 60 | do you like this game ? just so so . 61 | how do you like it ? it is boring . 62 | how do you like it ? it is good . 63 | how are you doing ? good . 64 | how are you doing ? i 'm good . 65 | do you like this game ? yes . 66 | do you like this game ? just so so . 67 | how do you like it ? it is boring . 68 | how do you like it ? it is good . 69 | how are you doing ? good . 70 | how are you doing ? i 'm good . 71 | do you like this game ? yes . 72 | do you like this game ? just so so . 73 | how do you like it ? it is boring . 74 | how do you like it ? it is good . 75 | how are you doing ? good . 76 | how are you doing ? i 'm good . 77 | do you like this game ? yes . 78 | do you like this game ? just so so . 79 | how do you like it ? it is boring . 80 | how do you like it ? it is good . 81 | how are you doing ? good . 82 | how are you doing ? i 'm good . 83 | do you like this game ? yes . 84 | do you like this game ? just so so . 85 | how do you like it ? it is boring . 86 | how do you like it ? it is good . 87 | how are you doing ? good . 88 | how are you doing ? i 'm good . 89 | do you like this game ? yes . 90 | do you like this game ? just so so . 91 | how do you like it ? it is boring . 92 | how do you like it ? it is good . 93 | how are you doing ? good . 94 | how are you doing ? i 'm good . 95 | do you like this game ? yes . 96 | do you like this game ? just so so . 97 | how do you like it ? it is boring . 98 | how do you like it ? it is good . 99 | how are you doing ? good . 100 | how are you doing ? i 'm good . 101 | do you like this game ? yes . 102 | do you like this game ? just so so . 103 | how do you like it ? it is boring . 104 | how do you like it ? it is good . 105 | how are you doing ? good . 106 | how are you doing ? i 'm good . 107 | do you like this game ? yes . 108 | do you like this game ? just so so . 109 | how do you like it ? it is boring . 110 | how do you like it ? it is good . 111 | how are you doing ? good . 112 | how are you doing ? i 'm good . 113 | do you like this game ? yes . 114 | do you like this game ? just so so . 115 | how do you like it ? it is boring . 116 | how do you like it ? it is good . 117 | how are you doing ? good . 118 | how are you doing ? i 'm good . 119 | do you like this game ? yes . 120 | do you like this game ? just so so . 121 | how do you like it ? it is boring . 122 | how do you like it ? it is good . 123 | how are you doing ? good . 124 | how are you doing ? i 'm good . 125 | do you like this game ? yes . 126 | do you like this game ? just so so . 127 | how do you like it ? it is boring . 128 | how do you like it ? it is good . 129 | how are you doing ? good . 130 | how are you doing ? i 'm good . 131 | do you like this game ? yes . 132 | do you like this game ? just so so . 133 | how do you like it ? it is boring . 134 | how do you like it ? it is good . 135 | how are you doing ? good . 136 | how are you doing ? i 'm good . 137 | do you like this game ? yes . 138 | do you like this game ? just so so . 139 | how do you like it ? it is boring . 140 | how do you like it ? it is good . 141 | how are you doing ? good . 142 | how are you doing ? i 'm good . 143 | do you like this game ? yes . 144 | do you like this game ? just so so . 145 | how do you like it ? it is boring . 146 | how do you like it ? it is good . 147 | how are you doing ? good . 148 | how are you doing ? i 'm good . 149 | do you like this game ? yes . 150 | do you like this game ? just so so . 151 | how do you like it ? it is boring . 152 | how do you like it ? it is good . 153 | how are you doing ? good . 154 | how are you doing ? i 'm good . 155 | do you like this game ? yes . 156 | do you like this game ? just so so . 157 | how do you like it ? it is boring . 158 | how do you like it ? it is good . 159 | how are you doing ? good . 160 | how are you doing ? i 'm good . 161 | do you like this game ? yes . 162 | do you like this game ? just so so . 163 | how do you like it ? it is boring . 164 | how do you like it ? it is good . 165 | how are you doing ? good . 166 | how are you doing ? i 'm good . 167 | do you like this game ? yes . 168 | do you like this game ? just so so . 169 | how do you like it ? it is boring . 170 | how do you like it ? it is good . 171 | how are you doing ? good . 172 | how are you doing ? i 'm good . 173 | do you like this game ? yes . 174 | do you like this game ? just so so . 175 | how do you like it ? it is boring . 176 | how do you like it ? it is good . 177 | how are you doing ? good . 178 | how are you doing ? i 'm good . 179 | do you like this game ? yes . 180 | do you like this game ? just so so . 181 | how do you like it ? it is boring . 182 | how do you like it ? it is good . 183 | how are you doing ? good . 184 | how are you doing ? i 'm good . 185 | do you like this game ? yes . 186 | do you like this game ? just so so . 187 | how do you like it ? it is boring . 188 | how do you like it ? it is good . 189 | how are you doing ? good . 190 | how are you doing ? i 'm good . 191 | do you like this game ? yes . 192 | do you like this game ? just so so . 193 | how do you like it ? it is boring . 194 | how do you like it ? it is good . 195 | how are you doing ? good . 196 | how are you doing ? i 'm good . 197 | do you like this game ? yes . 198 | do you like this game ? just so so . 199 | how do you like it ? it is boring . 200 | how do you like it ? it is good . 201 | how are you doing ? good . 202 | how are you doing ? i 'm good . 203 | do you like this game ? yes . 204 | do you like this game ? just so so . 205 | how do you like it ? it is boring . 206 | how do you like it ? it is good . 207 | how are you doing ? good . 208 | how are you doing ? i 'm good . 209 | do you like this game ? yes . 210 | do you like this game ? just so so . 211 | how do you like it ? it is boring . 212 | how do you like it ? it is good . 213 | how are you doing ? good . 214 | how are you doing ? i 'm good . 215 | do you like this game ? yes . 216 | do you like this game ? just so so . 217 | how do you like it ? it is boring . 218 | how do you like it ? it is good . 219 | how are you doing ? good . 220 | how are you doing ? i 'm good . 221 | do you like this game ? yes . 222 | do you like this game ? just so so . 223 | how do you like it ? it is boring . 224 | how do you like it ? it is good . 225 | how are you doing ? good . 226 | how are you doing ? i 'm good . 227 | do you like this game ? yes . 228 | do you like this game ? just so so . 229 | how do you like it ? it is boring . 230 | how do you like it ? it is good . 231 | how are you doing ? good . 232 | how are you doing ? i 'm good . 233 | do you like this game ? yes . 234 | do you like this game ? just so so . 235 | how do you like it ? it is boring . 236 | how do you like it ? it is good . 237 | how are you doing ? good . 238 | how are you doing ? i 'm good . 239 | do you like this game ? yes . 240 | do you like this game ? just so so . 241 | how do you like it ? it is boring . 242 | how do you like it ? it is good . 243 | how are you doing ? good . 244 | how are you doing ? i 'm good . 245 | do you like this game ? yes . 246 | do you like this game ? just so so . 247 | how do you like it ? it is boring . 248 | how do you like it ? it is good . 249 | how are you doing ? good . 250 | how are you doing ? i 'm good . 251 | do you like this game ? yes . 252 | do you like this game ? just so so . 253 | how do you like it ? it is boring . 254 | how do you like it ? it is good . 255 | how are you doing ? good . 256 | how are you doing ? i 'm good . 257 | do you like this game ? yes . 258 | do you like this game ? just so so . 259 | how do you like it ? it is boring . 260 | how do you like it ? it is good . 261 | how are you doing ? good . 262 | how are you doing ? i 'm good . 263 | do you like this game ? yes . 264 | do you like this game ? just so so . 265 | how do you like it ? it is boring . 266 | how do you like it ? it is good . 267 | how are you doing ? good . 268 | how are you doing ? i 'm good . 269 | do you like this game ? yes . 270 | do you like this game ? just so so . 271 | how do you like it ? it is boring . 272 | how do you like it ? it is good . 273 | how are you doing ? good . 274 | how are you doing ? i 'm good . 275 | do you like this game ? yes . 276 | do you like this game ? just so so . 277 | how do you like it ? it is boring . 278 | how do you like it ? it is good . 279 | how are you doing ? good . 280 | how are you doing ? i 'm good . 281 | do you like this game ? yes . 282 | do you like this game ? just so so . 283 | how do you like it ? it is boring . 284 | how do you like it ? it is good . 285 | how are you doing ? good . 286 | how are you doing ? i 'm good . 287 | do you like this game ? yes . 288 | do you like this game ? just so so . 289 | how do you like it ? it is boring . 290 | how do you like it ? it is good . 291 | how are you doing ? good . 292 | how are you doing ? i 'm good . 293 | do you like this game ? yes . 294 | do you like this game ? just so so . 295 | how do you like it ? it is boring . 296 | how do you like it ? it is good . 297 | how are you doing ? good . 298 | how are you doing ? i 'm good . 299 | do you like this game ? yes . 300 | do you like this game ? just so so . 301 | how do you like it ? it is boring . 302 | how do you like it ? it is good . 303 | how are you doing ? good . 304 | how are you doing ? i 'm good . 305 | do you like this game ? yes . 306 | do you like this game ? just so so . 307 | how do you like it ? it is boring . 308 | how do you like it ? it is good . 309 | how are you doing ? good . 310 | how are you doing ? i 'm good . 311 | do you like this game ? yes . 312 | do you like this game ? just so so . 313 | how do you like it ? it is boring . 314 | how do you like it ? it is good . 315 | how are you doing ? good . 316 | how are you doing ? i 'm good . 317 | do you like this game ? yes . 318 | do you like this game ? just so so . 319 | how do you like it ? it is boring . 320 | how do you like it ? it is good . 321 | how are you doing ? good . 322 | how are you doing ? i 'm good . 323 | do you like this game ? yes . 324 | do you like this game ? just so so . 325 | how do you like it ? it is boring . 326 | how do you like it ? it is good . 327 | how are you doing ? good . 328 | how are you doing ? i 'm good . 329 | do you like this game ? yes . 330 | do you like this game ? just so so . 331 | how do you like it ? it is boring . 332 | how do you like it ? it is good . 333 | how are you doing ? good . 334 | how are you doing ? i 'm good . 335 | do you like this game ? yes . 336 | do you like this game ? just so so . 337 | how do you like it ? it is boring . 338 | how do you like it ? it is good . 339 | how are you doing ? good . 340 | how are you doing ? i 'm good . 341 | do you like this game ? yes . 342 | do you like this game ? just so so . 343 | how do you like it ? it is boring . 344 | how do you like it ? it is good . 345 | how are you doing ? good . 346 | how are you doing ? i 'm good . 347 | do you like this game ? yes . 348 | do you like this game ? just so so . 349 | how do you like it ? it is boring . 350 | how do you like it ? it is good . 351 | how are you doing ? good . 352 | how are you doing ? i 'm good . 353 | do you like this game ? yes . 354 | do you like this game ? just so so . 355 | how do you like it ? it is boring . 356 | how do you like it ? it is good . 357 | how are you doing ? good . 358 | how are you doing ? i 'm good . 359 | do you like this game ? yes . 360 | do you like this game ? just so so . 361 | how do you like it ? it is boring . 362 | how do you like it ? it is good . 363 | how are you doing ? good . 364 | how are you doing ? i 'm good . 365 | do you like this game ? yes . 366 | do you like this game ? just so so . 367 | how do you like it ? it is boring . 368 | how do you like it ? it is good . 369 | how are you doing ? good . 370 | how are you doing ? i 'm good . 371 | do you like this game ? yes . 372 | do you like this game ? just so so . 373 | how do you like it ? it is boring . 374 | how do you like it ? it is good . 375 | how are you doing ? good . 376 | how are you doing ? i 'm good . 377 | do you like this game ? yes . 378 | do you like this game ? just so so . 379 | how do you like it ? it is boring . 380 | how do you like it ? it is good . 381 | how are you doing ? good . 382 | how are you doing ? i 'm good . 383 | do you like this game ? yes . 384 | do you like this game ? just so so . 385 | how do you like it ? it is boring . 386 | how do you like it ? it is good . 387 | how are you doing ? good . 388 | how are you doing ? i 'm good . 389 | do you like this game ? yes . 390 | do you like this game ? just so so . 391 | how do you like it ? it is boring . 392 | how do you like it ? it is good . 393 | how are you doing ? good . 394 | how are you doing ? i 'm good . 395 | do you like this game ? yes . 396 | do you like this game ? just so so . 397 | how do you like it ? it is boring . 398 | how do you like it ? it is good . 399 | how are you doing ? good . 400 | how are you doing ? i 'm good . 401 | do you like this game ? yes . 402 | do you like this game ? just so so . 403 | how do you like it ? it is boring . 404 | how do you like it ? it is good . 405 | how are you doing ? good . 406 | how are you doing ? i 'm good . 407 | do you like this game ? yes . 408 | do you like this game ? just so so . 409 | how do you like it ? it is boring . 410 | how do you like it ? it is good . 411 | how are you doing ? good . 412 | how are you doing ? i 'm good . 413 | do you like this game ? yes . 414 | do you like this game ? just so so . 415 | how do you like it ? it is boring . 416 | how do you like it ? it is good . 417 | how are you doing ? good . 418 | how are you doing ? i 'm good . 419 | do you like this game ? yes . 420 | do you like this game ? just so so . 421 | how do you like it ? it is boring . 422 | how do you like it ? it is good . 423 | how are you doing ? good . 424 | how are you doing ? i 'm good . 425 | do you like this game ? yes . 426 | do you like this game ? just so so . 427 | how do you like it ? it is boring . 428 | how do you like it ? it is good . 429 | how are you doing ? good . 430 | how are you doing ? i 'm good . 431 | do you like this game ? yes . 432 | do you like this game ? just so so . 433 | how do you like it ? it is boring . 434 | how do you like it ? it is good . 435 | how are you doing ? good . 436 | how are you doing ? i 'm good . 437 | do you like this game ? yes . 438 | do you like this game ? just so so . 439 | how do you like it ? it is boring . 440 | how do you like it ? it is good . 441 | how are you doing ? good . 442 | how are you doing ? i 'm good . 443 | do you like this game ? yes . 444 | do you like this game ? just so so . 445 | how do you like it ? it is boring . 446 | how do you like it ? it is good . 447 | how are you doing ? good . 448 | how are you doing ? i 'm good . 449 | do you like this game ? yes . 450 | do you like this game ? just so so . 451 | how do you like it ? it is boring . 452 | how do you like it ? it is good . 453 | how are you doing ? good . 454 | how are you doing ? i 'm good . 455 | do you like this game ? yes . 456 | do you like this game ? just so so . 457 | how do you like it ? it is boring . 458 | how do you like it ? it is good . 459 | how are you doing ? good . 460 | how are you doing ? i 'm good . 461 | do you like this game ? yes . 462 | do you like this game ? just so so . 463 | how do you like it ? it is boring . 464 | how do you like it ? it is good . 465 | how are you doing ? good . 466 | how are you doing ? i 'm good . 467 | do you like this game ? yes . 468 | do you like this game ? just so so . 469 | how do you like it ? it is boring . 470 | how do you like it ? it is good . 471 | how are you doing ? good . 472 | how are you doing ? i 'm good . 473 | do you like this game ? yes . 474 | do you like this game ? just so so . 475 | how do you like it ? it is boring . 476 | how do you like it ? it is good . 477 | how are you doing ? good . 478 | how are you doing ? i 'm good . 479 | do you like this game ? yes . 480 | do you like this game ? just so so . 481 | how do you like it ? it is boring . 482 | how do you like it ? it is good . 483 | how are you doing ? good . 484 | how are you doing ? i 'm good . 485 | do you like this game ? yes . 486 | do you like this game ? just so so . 487 | how do you like it ? it is boring . 488 | how do you like it ? it is good . 489 | how are you doing ? good . 490 | how are you doing ? i 'm good . 491 | do you like this game ? yes . 492 | do you like this game ? just so so . 493 | how do you like it ? it is boring . 494 | how do you like it ? it is good . 495 | how are you doing ? good . 496 | how are you doing ? i 'm good . 497 | do you like this game ? yes . 498 | do you like this game ? just so so . 499 | how do you like it ? it is boring . 500 | how do you like it ? it is good . 501 | how are you doing ? good . 502 | how are you doing ? i 'm good . 503 | do you like this game ? yes . 504 | do you like this game ? just so so . 505 | how do you like it ? it is boring . 506 | how do you like it ? it is good . 507 | how are you doing ? good . 508 | how are you doing ? i 'm good . 509 | do you like this game ? yes . 510 | do you like this game ? just so so . 511 | how do you like it ? it is boring . 512 | how do you like it ? it is good . 513 | how are you doing ? good . 514 | how are you doing ? i 'm good . 515 | do you like this game ? yes . 516 | do you like this game ? just so so . 517 | how do you like it ? it is boring . 518 | how do you like it ? it is good . 519 | how are you doing ? good . 520 | how are you doing ? i 'm good . 521 | do you like this game ? yes . 522 | do you like this game ? just so so . 523 | how do you like it ? it is boring . 524 | how do you like it ? it is good . 525 | how are you doing ? good . 526 | how are you doing ? i 'm good . 527 | do you like this game ? yes . 528 | do you like this game ? just so so . 529 | how do you like it ? it is boring . 530 | how do you like it ? it is good . 531 | how are you doing ? good . 532 | how are you doing ? i 'm good . 533 | do you like this game ? yes . 534 | do you like this game ? just so so . 535 | how do you like it ? it is boring . 536 | how do you like it ? it is good . 537 | how are you doing ? good . 538 | how are you doing ? i 'm good . 539 | do you like this game ? yes . 540 | do you like this game ? just so so . 541 | how do you like it ? it is boring . 542 | how do you like it ? it is good . 543 | how are you doing ? good . 544 | how are you doing ? i 'm good . 545 | do you like this game ? yes . 546 | do you like this game ? just so so . 547 | how do you like it ? it is boring . 548 | how do you like it ? it is good . 549 | how are you doing ? good . 550 | how are you doing ? i 'm good . 551 | do you like this game ? yes . 552 | do you like this game ? just so so . 553 | how do you like it ? it is boring . 554 | how do you like it ? it is good . 555 | how are you doing ? good . 556 | how are you doing ? i 'm good . 557 | do you like this game ? yes . 558 | do you like this game ? just so so . 559 | how do you like it ? it is boring . 560 | how do you like it ? it is good . 561 | how are you doing ? good . 562 | how are you doing ? i 'm good . 563 | do you like this game ? yes . 564 | do you like this game ? just so so . 565 | how do you like it ? it is boring . 566 | how do you like it ? it is good . 567 | how are you doing ? good . 568 | how are you doing ? i 'm good . 569 | do you like this game ? yes . 570 | do you like this game ? just so so . 571 | how do you like it ? it is boring . 572 | how do you like it ? it is good . 573 | how are you doing ? good . 574 | how are you doing ? i 'm good . 575 | do you like this game ? yes . 576 | do you like this game ? just so so . 577 | how do you like it ? it is boring . 578 | how do you like it ? it is good . 579 | how are you doing ? good . 580 | how are you doing ? i 'm good . 581 | do you like this game ? yes . 582 | do you like this game ? just so so . 583 | how do you like it ? it is boring . 584 | how do you like it ? it is good . 585 | how are you doing ? good . 586 | how are you doing ? i 'm good . 587 | do you like this game ? yes . 588 | do you like this game ? just so so . 589 | how do you like it ? it is boring . 590 | how do you like it ? it is good . 591 | how are you doing ? good . 592 | how are you doing ? i 'm good . 593 | do you like this game ? yes . 594 | do you like this game ? just so so . 595 | how do you like it ? it is boring . 596 | how do you like it ? it is good . 597 | how are you doing ? good . 598 | how are you doing ? i 'm good . 599 | do you like this game ? yes . 600 | do you like this game ? just so so . 601 | how do you like it ? it is boring . 602 | how do you like it ? it is good . 603 | how are you doing ? good . 604 | how are you doing ? i 'm good . 605 | do you like this game ? yes . 606 | do you like this game ? just so so . 607 | how do you like it ? it is boring . 608 | how do you like it ? it is good . 609 | how are you doing ? good . 610 | how are you doing ? i 'm good . 611 | do you like this game ? yes . 612 | do you like this game ? just so so . 613 | how do you like it ? it is boring . 614 | how do you like it ? it is good . 615 | how are you doing ? good . 616 | how are you doing ? i 'm good . 617 | do you like this game ? yes . 618 | do you like this game ? just so so . 619 | how do you like it ? it is boring . 620 | how do you like it ? it is good . 621 | how are you doing ? good . 622 | how are you doing ? i 'm good . 623 | do you like this game ? yes . 624 | do you like this game ? just so so . 625 | how do you like it ? it is boring . 626 | how do you like it ? it is good . 627 | how are you doing ? good . 628 | how are you doing ? i 'm good . 629 | do you like this game ? yes . 630 | do you like this game ? just so so . 631 | how do you like it ? it is boring . 632 | how do you like it ? it is good . 633 | how are you doing ? good . 634 | how are you doing ? i 'm good . 635 | do you like this game ? yes . 636 | do you like this game ? just so so . 637 | how do you like it ? it is boring . 638 | how do you like it ? it is good . 639 | how are you doing ? good . 640 | how are you doing ? i 'm good . 641 | do you like this game ? yes . 642 | do you like this game ? just so so . 643 | how do you like it ? it is boring . 644 | how do you like it ? it is good . 645 | how are you doing ? good . 646 | how are you doing ? i 'm good . 647 | do you like this game ? yes . 648 | do you like this game ? just so so . 649 | how do you like it ? it is boring . 650 | how do you like it ? it is good . 651 | how are you doing ? good . 652 | how are you doing ? i 'm good . 653 | do you like this game ? yes . 654 | do you like this game ? just so so . 655 | how do you like it ? it is boring . 656 | how do you like it ? it is good . 657 | how are you doing ? good . 658 | how are you doing ? i 'm good . 659 | do you like this game ? yes . 660 | do you like this game ? just so so . 661 | how do you like it ? it is boring . 662 | how do you like it ? it is good . 663 | how are you doing ? good . 664 | how are you doing ? i 'm good . 665 | do you like this game ? yes . 666 | do you like this game ? just so so . 667 | how do you like it ? it is boring . 668 | how do you like it ? it is good . 669 | how are you doing ? good . 670 | how are you doing ? i 'm good . 671 | do you like this game ? yes . 672 | do you like this game ? just so so . 673 | how do you like it ? it is boring . 674 | how do you like it ? it is good . 675 | how are you doing ? good . 676 | how are you doing ? i 'm good . 677 | do you like this game ? yes . 678 | do you like this game ? just so so . 679 | how do you like it ? it is boring . 680 | how do you like it ? it is good . 681 | how are you doing ? good . 682 | how are you doing ? i 'm good . 683 | do you like this game ? yes . 684 | do you like this game ? just so so . 685 | how do you like it ? it is boring . 686 | how do you like it ? it is good . 687 | how are you doing ? good . 688 | how are you doing ? i 'm good . 689 | do you like this game ? yes . 690 | do you like this game ? just so so . 691 | how do you like it ? it is boring . 692 | how do you like it ? it is good . 693 | how are you doing ? good . 694 | how are you doing ? i 'm good . 695 | do you like this game ? yes . 696 | do you like this game ? just so so . 697 | how do you like it ? it is boring . 698 | how do you like it ? it is good . 699 | how are you doing ? good . 700 | how are you doing ? i 'm good . 701 | do you like this game ? yes . 702 | do you like this game ? just so so . 703 | how do you like it ? it is boring . 704 | how do you like it ? it is good . 705 | how are you doing ? good . 706 | how are you doing ? i 'm good . 707 | do you like this game ? yes . 708 | do you like this game ? just so so . 709 | how do you like it ? it is boring . 710 | how do you like it ? it is good . 711 | how are you doing ? good . 712 | how are you doing ? i 'm good . 713 | do you like this game ? yes . 714 | do you like this game ? just so so . 715 | how do you like it ? it is boring . 716 | how do you like it ? it is good . 717 | how are you doing ? good . 718 | how are you doing ? i 'm good . 719 | do you like this game ? yes . 720 | do you like this game ? just so so . 721 | -------------------------------------------------------------------------------- /src/model.py: -------------------------------------------------------------------------------- 1 | from shared import * 2 | from tf_lib import * 3 | from dataset import * 4 | from decode import * 5 | from evaluate import * 6 | 7 | 8 | """ 9 | AUTHOR: Xiang Gao (xiag@microsoft.com) at Microsoft Research 10 | """ 11 | 12 | 13 | class ModelBase: 14 | 15 | def __init__(self): 16 | self.fld = None # str 17 | self.n_trained = None # int 18 | self.max_n_trained = None # int 19 | self.dataset = None # Dataset obj 20 | self.extra = None # list of str 21 | self.vali_data = None # dict of list 22 | self.layers = None 23 | 24 | 25 | def init_log(self, new, args): 26 | 27 | # deal with existing fld 28 | if new and os.path.exists(self.fld): 29 | if PHILLY: 30 | suffix = 0 31 | while True: 32 | fld = self.fld + '_%i'%suffix 33 | if not os.path.exists(fld): 34 | self.fld = fld 35 | break 36 | else: 37 | if not PHILLY and not self.debug: 38 | print('%s\nalready exists, do you want to delete the folder? (y/n)'%self.fld) 39 | ans = input() 40 | if not ans.lower() == 'y': 41 | exit() 42 | 43 | print('deleting fld: '+self.fld) 44 | shutil.rmtree(self.fld) 45 | time.sleep(0.1) 46 | print('fld deleted') 47 | 48 | self.log_train = self.fld + '/train.txt' 49 | 50 | if new or PHILLY or hostname != 'MININT-3LHNLKS': 51 | makedirs(os.path.join(self.fld, 'models')) 52 | open(self.log_train, 'w') 53 | if not os.path.exists(self.fld + '/vocab.txt'): 54 | shutil.copyfile(self.dataset.path_vocab, self.fld + '/vocab.txt') 55 | 56 | ss = [] 57 | for k in sorted(args.__dict__.keys()): 58 | ss.append('%s = %s'%(k, args.__dict__[k])) 59 | with open(self.fld + '/args.txt', 'w') as f: 60 | f.write('\n'.join(ss)) 61 | 62 | if PHILLY: 63 | with open(self.log_train, 'a') as f: 64 | f.write('hostname: %s\n'%hostname) 65 | f.write('data_path: %s\n'%DATA_PATH) 66 | f.write('out_path: %s\n'%OUT_PATH) 67 | 68 | 69 | def train(self, batch_per_load=100): 70 | self.vali() 71 | while self.n_trained < self.max_n_trained: 72 | s = '\n***** trained %.3f M'%(self.n_trained/1e6) 73 | for tp in self.dataset.n_reset['train']: 74 | s += ', %s = %i'%(tp, self.dataset.n_reset['train'][tp]) 75 | s += ' *****' 76 | write_log(self.log_train, s) 77 | self.train_a_load(batch_per_load) 78 | if self.debug: 79 | exit() 80 | 81 | 82 | def load_weights(self, path): 83 | self.prev_wt_fuse = None 84 | print('loading weights from %s'%path) 85 | npz = np.load(path, encoding='latin1', allow_pickle=True) 86 | print(npz.files) 87 | weights = npz['layers'].item() 88 | for k in weights: 89 | s = ' '*(20-len(k)) + k + ': %i params: '%len(weights[k]) 90 | for wt in weights[k]: 91 | s += str(wt.shape) + ', ' 92 | print(s) 93 | for attr in self.extra: 94 | if attr in npz: 95 | if attr not in ['name']: 96 | setattr(self, attr, npz[attr]) 97 | else: 98 | print('WARNING! attr %s not in npz'%attr) 99 | self.build_model(weights) 100 | self.build_model_test() 101 | 102 | 103 | def extract_weights(self): 104 | weights = dict() 105 | if self.layers is None: 106 | return weights 107 | for k in self.layers: 108 | weights[k] = self.layers[k].get_weights() 109 | return weights 110 | 111 | 112 | def save_weights(self): 113 | path = self.fld + '/models/%.1fM.npz'%(self.n_trained/1e6) 114 | weights = self.extract_weights() 115 | to_save = {'layers':weights} 116 | for attr in self.extra: 117 | to_save[attr] = getattr(self, attr) 118 | n_try = 0 119 | while n_try < 3: 120 | try: 121 | np.savez(path, **to_save) 122 | print('saved to: '+path) 123 | break 124 | except: 125 | n_try += 1 126 | print('cannot save, try %i'%n_try) 127 | return path 128 | 129 | def build_model_test(self): 130 | pass 131 | def build_model(self, weights=dict()): 132 | pass 133 | def train_a_load(self, batch_per_load): 134 | pass 135 | def set_extra(self, npz): 136 | pass 137 | 138 | 139 | 140 | 141 | class Seq2SeqBase(ModelBase): 142 | 143 | def __init__(self, dataset, fld, args, new=False, allowed_words=None): 144 | 145 | self.dataset = dataset 146 | self.fld = fld 147 | self.allowed_words = allowed_words 148 | self.layers = None 149 | self.history = LossHistory() 150 | self.vali_data = None 151 | self.classifiers = [] 152 | 153 | self.n_batch = 0 154 | self.prev_n_batch = 0 155 | self.dn_batch_vali = 100 156 | 157 | self.bias_conv = False # hasattr(self.dataset, 'files') and ('bias_conv' in self.dataset.files['train']) 158 | self.debug = args.debug 159 | self.token_embed_dim = args.token_embed_dim 160 | self.rnn_units = args.rnn_units 161 | self.encoder_depth = args.encoder_depth 162 | self.decoder_depth = args.decoder_depth 163 | self.lr = args.lr 164 | self.max_n_trained = args.max_n_trained 165 | self.randmix = False 166 | self.mix_ratio = (args.conv_mix_ratio, args.nonc_mix_ratio) 167 | 168 | if not self.bias_conv: 169 | assert(args.conv_mix_ratio == 0.) 170 | 171 | self.extra = ['name'] 172 | self.init_extra(args) 173 | 174 | if hasattr(args, 'skip'): 175 | skip = int(1e6*args.skip) 176 | else: 177 | skip = 0 178 | self.dataset.skip(skip, self.mix_ratio, conv_only=(self.name=='s2s')) 179 | self.n_trained = skip 180 | self.init_log(new, args) 181 | self.build_model() 182 | 183 | 184 | def get_mix_ratio(self): 185 | if self.randmix: 186 | ret = [] 187 | for ratio in self.mix_ratio: 188 | p = [1. - ratio, ratio] 189 | ret.append(np.random.choice([0.,1.], 1, p=p)[0]) 190 | return tuple(ret) 191 | else: 192 | return self.mix_ratio 193 | 194 | 195 | def fit(self, inputs, outputs): 196 | n_try = 0 197 | if self.debug: 198 | self.model.fit( 199 | inputs, 200 | outputs, 201 | batch_size=BATCH_SIZE, 202 | callbacks=[self.history], 203 | verbose=FIT_VERBOSE) 204 | return 205 | 206 | while n_try < 3: 207 | try: 208 | self.model.fit( 209 | inputs, 210 | outputs, 211 | batch_size=BATCH_SIZE, 212 | callbacks=[self.history], 213 | verbose=FIT_VERBOSE) 214 | return 215 | except Exception as e: 216 | print('got error, sleeping') 217 | print('E'*20) 218 | print(e) 219 | print('E'*20) 220 | time.sleep(1) 221 | n_try += 1 222 | 223 | 224 | def _stacked_rnn(self, rnns, inputs, initial_states=None): 225 | if initial_states is None: 226 | initial_states = [None] * len(rnns) 227 | outputs, state = rnns[0](inputs, initial_state=initial_states[0]) 228 | states = [state] 229 | for i in range(1, len(rnns)): 230 | outputs, state = rnns[i](outputs, initial_state=initial_states[i]) 231 | states.append(state) 232 | return outputs, states 233 | 234 | 235 | def _build_encoder(self, inputs, prefix): 236 | _, encoder_states = self._stacked_rnn( 237 | [self.layers['%s_encoder_rnn_%i'%(prefix, i)] for i in range(self.encoder_depth)], 238 | self.layers['embedding'](inputs)) 239 | latent = encoder_states[-1] 240 | return latent 241 | 242 | 243 | def _build_decoder(self, input_seqs, input_states): 244 | """ 245 | for auto-regressive, states are returned and used as input for the generation of the next token 246 | for teacher-forcing, token already given, so only need init states 247 | """ 248 | 249 | decoder_outputs, decoder_states = self._stacked_rnn( 250 | [self.layers['decoder_rnn_%i'%i] for i in range(self.decoder_depth)], 251 | self.layers['embedding'](input_seqs), 252 | input_states) 253 | decoder_outputs = self.layers['decoder_softmax'](decoder_outputs) 254 | return decoder_outputs, decoder_states 255 | 256 | 257 | def _create_layers(self, weights=dict()): 258 | layers = dict() 259 | 260 | name = 'embedding' 261 | params = _params(name, weights, {'mask_zero':True}) 262 | layers[name] = Embedding( 263 | self.dataset.num_tokens + 1, # +1 as mask_zero 264 | self.token_embed_dim, 265 | **params) 266 | 267 | for i in range(self.decoder_depth): 268 | name = 'decoder_rnn_%i'%i 269 | params = _params(name, weights, {'return_state':True, 'return_sequences':True}) 270 | layers[name] = GRU( 271 | self.rnn_units, 272 | **params) 273 | 274 | for prefix in self.prefix: 275 | for i in range(self.encoder_depth): 276 | name = '%s_encoder_rnn_%i'%(prefix, i) 277 | params = _params(name, weights, {'return_state':True, 'return_sequences':True}) 278 | layers[name] = GRU( 279 | self.rnn_units, 280 | **params) 281 | 282 | name = 'decoder_softmax' 283 | params = _params(name, weights, {'activation':'softmax'}) 284 | layers[name] = Dense( 285 | self.dataset.num_tokens + 1, # +1 as mask_zero 286 | **params) 287 | 288 | return layers 289 | 290 | 291 | def build_model_test(self): 292 | 293 | #self.refresh_session() 294 | decoder_inputs = Input(shape=(None,), name='decoder_inputs') 295 | 296 | # encoder 297 | self.model_encoder = dict() 298 | self.model_tf = dict() 299 | self.tf_history = dict() 300 | for prefix in self.prefix: 301 | encoder_inputs = Input(shape=(None,), name=prefix+'_encoder_inputs') 302 | latent = self._build_encoder(encoder_inputs, prefix=prefix) 303 | self.model_encoder[prefix] = Model(encoder_inputs, latent) 304 | self.model_encoder[prefix]._make_predict_function() 305 | 306 | decoder_outputs, _ = self._build_decoder(decoder_inputs, [latent]*self.decoder_depth) 307 | self.model_tf[prefix] = Model([encoder_inputs, decoder_inputs], decoder_outputs) 308 | for layer in self.model_tf[prefix].layers: 309 | layer.trainable = False 310 | self.model_tf[prefix].compile(Adam(lr=0.), loss=_dec_loss) # lr = 0 to use '.fit', which has callbacks, as '.evaluate' 311 | self.tf_history[prefix] = LossHistory() 312 | 313 | # decoder: autoregressive 314 | decoder_inital_states = [] 315 | for i in range(self.decoder_depth): 316 | decoder_inital_states.append(Input(shape=(self.rnn_units,), name="decoder_inital_state_%i"%i)) 317 | decoder_outputs, decoder_states = self._build_decoder(decoder_inputs, decoder_inital_states) 318 | model_decoder = Model( 319 | [decoder_inputs] + decoder_inital_states, 320 | [decoder_outputs] + decoder_states) 321 | model_decoder._make_predict_function() 322 | self.decoder = Decoder(self.dataset, model_decoder, 323 | self.decoder_depth, self.rnn_units, allowed_words=self.allowed_words) 324 | 325 | 326 | 327 | def get_vali_data(self): 328 | if self.vali_data is not None: 329 | #print('returning self.vali_data', self.vali_data) 330 | return self.vali_data 331 | print('getting vali data...') 332 | 333 | def _feed_vali(k): 334 | self.dataset.reset('vali') 335 | d = self.dataset.feed_data('vali', max_n=vali_size, check_src=True, mix_ratio=k, conv_only=(self.name=='s2s')) 336 | self.dataset.reset('vali') 337 | return d 338 | 339 | if self.debug: 340 | vali_size = BATCH_SIZE 341 | else: 342 | vali_size = 1000 343 | self.vali_data = _feed_vali((0, 1)) 344 | """ 345 | self.vali_data['base'] = _feed_vali((0, 0)) 346 | self.vali_data['mix'] = _feed_vali(self.mix_ratio) 347 | if self.bias_conv: 348 | self.vali_data['bias'] = _feed_vali((1, 1)) 349 | else: 350 | self.vali_data['bias'] = _feed_vali((0, 1)) 351 | """ 352 | return self.vali_data 353 | 354 | def vali(self): 355 | 356 | self.build_model_test() 357 | ss = [] 358 | for inp in ['who is he ?', 'do you like this game ?', 'good morning .']: 359 | ss.append(infer_comb(inp, self)) 360 | write_log(self.log_train, '\n'.join(ss)) 361 | 362 | """ 363 | data = self.get_vali_data() 364 | if self.name.startswith('fuse'): 365 | r_rand = 0.1 * np.sqrt(self.rnn_units) 366 | else: 367 | r_rand = 0. 368 | #s_decoded = ''#eval_decoded(self, data, self.classifiers, r_rand=r_rand)[0] 369 | #s_surrogate = eval_surrogate(self, data)[0] 370 | #write_log(self.log_train, '\n' + s_decoded + '\n\n' + s_surrogate + '\n') 371 | """ 372 | self.prev_n_batch = self.n_batch 373 | 374 | # save -------------------- 375 | self.save_weights() 376 | 377 | 378 | def init_extra(self, args): 379 | pass 380 | 381 | 382 | def train_a_load(self, batch_per_load): 383 | 384 | mix_ratio = self.get_mix_ratio() 385 | data = self.dataset.feed_data('train', BATCH_SIZE * batch_per_load, mix_ratio=mix_ratio, conv_only=(self.name == 's2s')) 386 | n_sample, inputs, outputs = self._inp_out_data(data) 387 | 388 | t0 = datetime.datetime.now() 389 | t0_str = str(t0).split('.')[0] 390 | write_log(self.log_train, 'start: %s'%t0_str + ', mix_ratio = '+str(mix_ratio)) 391 | print('fitting...') 392 | 393 | self.fit(inputs, outputs) 394 | self.n_trained += n_sample 395 | self.n_batch += batch_per_load 396 | 397 | dt = (datetime.datetime.now() - t0).seconds 398 | loss = np.mean(self.history.losses) 399 | 400 | write_log(self.log_train, 'n_batch: %i, prev %i'%(self.n_batch, self.prev_n_batch)) 401 | ss = ['spent: %i sec'%dt, 'train: %.4f'%loss] 402 | write_log(self.log_train, '\n'.join(ss)) 403 | 404 | if not self.debug and (self.n_batch - self.prev_n_batch < self.dn_batch_vali): 405 | return 406 | 407 | # vali -------------------- 408 | self.vali() 409 | 410 | def print_loss(self, loss_weights): 411 | s = 'loss: '+'-'*20 + '\n' 412 | for i in range(len(self.loss)): 413 | loss_name = str(self.loss[i]) 414 | if loss_name.startswith(' %i'%(n_old, n_new)) 795 | 796 | new2old = dict() 797 | ix_unk_old = token2index_old[UNK_token] 798 | for ix in index2token_new: 799 | token = index2token_new[ix] 800 | new2old[ix] = token2index_old.get(token, ix_unk_old) 801 | 802 | print('loading from: '+str(path_npz_old)) 803 | npz = np.load(path_npz_old, encoding='latin1') 804 | weights = npz['layers'].item() 805 | embedding_old = weights['embedding'][0] 806 | softmax_wt_old = weights['decoder_softmax'][0] 807 | softmax_bias_old = weights['decoder_softmax'][1] 808 | 809 | n_old_loaded, dim = embedding_old.shape 810 | assert(n_old_loaded == n_old) 811 | embedding_new = np.zeros((n_new, dim)) 812 | softmax_wt_new = np.zeros((dim, n_new)) 813 | softmax_bias_new = np.zeros((n_new,)) 814 | 815 | print(' embedding: ' + str(embedding_old.shape) + ' => ' + str(embedding_new.shape)) 816 | print(' softmax_wt: ' + str(softmax_wt_old.shape) + ' => ' + str(softmax_wt_new.shape)) 817 | print('softmax_bias: ' + str(softmax_bias_old.shape) + ' => ' + str(softmax_bias_new.shape)) 818 | 819 | # PAD 820 | embedding_new[0,:] = embedding_old[0, :] 821 | softmax_wt_new[:, 0] = softmax_wt_old[:, 0] 822 | softmax_bias_new[0] = softmax_bias_old[0] 823 | 824 | for ix in index2token_new: 825 | embedding_new[ix, :] = embedding_old[new2old[ix], :] 826 | softmax_wt_new[:, ix] = softmax_wt_old[:, new2old[ix]] 827 | softmax_bias_new[ix] = softmax_bias_old[new2old[ix]] 828 | 829 | weights['embedding'] = [embedding_new] 830 | weights['decoder_softmax'] = [softmax_wt_new, softmax_bias_new] 831 | 832 | print('saving to: '+str(path_npz_new)) 833 | to_save = {'layers':weights} 834 | for k in npz.files: 835 | if k != 'layers' and 'mix' not in k: 836 | to_save[k] = npz[k] 837 | np.savez(path_npz_new, **to_save) 838 | --------------------------------------------------------------------------------