├── .gitignore ├── LICENSE ├── README.md ├── assets ├── 2016-03-20-final-hist.png ├── 2016-03-20-final-scalar.png ├── model.png ├── training-2016-03-20.png └── training-2016-03-21.png ├── batch_loader.py ├── data ├── ptb │ ├── test.txt │ ├── train.txt │ └── valid.txt └── toy │ ├── test.txt │ ├── train.txt │ └── valid.txt ├── main.py ├── models ├── __init__.py ├── base.py ├── nasm.py └── nvdm.py ├── reader.py ├── toy_generator.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | # misc 2 | logs 3 | 4 | # data 5 | samples 6 | checkpoint 7 | *.npy 8 | *.pkl 9 | *.tgz 10 | *.zip 11 | *.tar.gz 12 | 13 | 14 | # Created by https://www.gitignore.io/api/python,vim 15 | 16 | ### IPythonNotebook ### 17 | ## Temporary data 18 | .ipynb_checkpoints/ 19 | 20 | ### Python ### 21 | # Byte-compiled / optimized / DLL files 22 | __pycache__/ 23 | *.py[cod] 24 | *$py.class 25 | 26 | # C extensions 27 | *.so 28 | 29 | # Distribution / packaging 30 | .Python 31 | env/ 32 | build/ 33 | develop-eggs/ 34 | dist/ 35 | downloads/ 36 | eggs/ 37 | .eggs/ 38 | lib/ 39 | lib64/ 40 | parts/ 41 | sdist/ 42 | var/ 43 | *.egg-info/ 44 | .installed.cfg 45 | *.egg 46 | 47 | # PyInstaller 48 | # Usually these files are written by a python script from a template 49 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 50 | *.manifest 51 | *.spec 52 | 53 | # Installer logs 54 | pip-log.txt 55 | pip-delete-this-directory.txt 56 | 57 | # Unit test / coverage reports 58 | htmlcov/ 59 | .tox/ 60 | .coverage 61 | .coverage.* 62 | .cache 63 | nosetests.xml 64 | coverage.xml 65 | *,cover 66 | .hypothesis/ 67 | 68 | # Translations 69 | *.mo 70 | *.pot 71 | 72 | # Django stuff: 73 | *.log 74 | 75 | # Sphinx documentation 76 | docs/_build/ 77 | 78 | # PyBuilder 79 | target/ 80 | 81 | 82 | ### Vim ### 83 | [._]*.s[a-w][a-z] 84 | [._]s[a-w][a-z] 85 | *.un~ 86 | Session.vim 87 | .netrwhist 88 | *~ 89 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Taehoon Kim 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Neural Variational Document Model 2 | ================================= 3 | 4 | Tensorflow implementation of [Neural Variational Inference for Text Processing](http://arxiv.org/abs/1511.06038). 5 | 6 | ![model_demo](./assets/model.png) 7 | 8 | This implementation contains: 9 | 10 | 1. Neural Variational Document Model 11 | - Variational inference framework for generative model of text 12 | - Combines a stochastic document representation with a bag-of-words generative model 13 | 2. Neural Answer Selection Model (in progress) 14 | - Variational inference framework for conditional generative model of text 15 | - Combines a LSTM embeddings with an attention mechanism to extract the semantics between question and answer 16 | 17 | 18 | Prerequisites 19 | ------------- 20 | 21 | - Python 2.7 or Python 3.3+ 22 | - [NLTK](http://www.nltk.org/) 23 | - [TensorFlow](https://www.tensorflow.org/) 24 | 25 | 26 | Usage 27 | ----- 28 | 29 | To train a model with Penn Tree Bank dataset: 30 | 31 | $ python main.py --dataset ptb 32 | 33 | To test an existing model: 34 | 35 | $ python main.py --dataset ptb --forward_only True 36 | 37 | 38 | Results 39 | ------- 40 | 41 | Training details of NVDM. The best result can be achieved by onehost updates, not alternative updates. 42 | 43 | ![scalar](./assets/2016-03-20-final-scalar.png) 44 | 45 | ![histogram](./assets/2016-03-20-final-hist.png) 46 | 47 | Author 48 | ------ 49 | 50 | Taehoon Kim / [@carpedm20](http://carpedm20.github.io/) 51 | -------------------------------------------------------------------------------- /assets/2016-03-20-final-hist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carpedm20/variational-text-tensorflow/7484a5236865fc07602cc60d4e940430e349caae/assets/2016-03-20-final-hist.png -------------------------------------------------------------------------------- /assets/2016-03-20-final-scalar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carpedm20/variational-text-tensorflow/7484a5236865fc07602cc60d4e940430e349caae/assets/2016-03-20-final-scalar.png -------------------------------------------------------------------------------- /assets/model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carpedm20/variational-text-tensorflow/7484a5236865fc07602cc60d4e940430e349caae/assets/model.png -------------------------------------------------------------------------------- /assets/training-2016-03-20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carpedm20/variational-text-tensorflow/7484a5236865fc07602cc60d4e940430e349caae/assets/training-2016-03-20.png -------------------------------------------------------------------------------- /assets/training-2016-03-21.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carpedm20/variational-text-tensorflow/7484a5236865fc07602cc60d4e940430e349caae/assets/training-2016-03-21.png -------------------------------------------------------------------------------- /batch_loader.py: -------------------------------------------------------------------------------- 1 | import re 2 | import os 3 | import math 4 | import pickle 5 | import numpy as np 6 | 7 | def save(fname, obj): 8 | with open(fname, 'w') as f: 9 | pickle.dump(obj, f) 10 | 11 | def load(fname): 12 | with open(fname, 'r') as f: 13 | return pickle.load(f) 14 | 15 | class BatchLoader(object): 16 | def __init__(self, data_dir, dataset_name, batch_size, seq_length, max_word_length): 17 | train_fname = os.path.join(data_dir, dataset_name, 'train.txt') 18 | valid_fname = os.path.join(data_dir, dataset_name, 'valid.txt') 19 | test_fname = os.path.join(data_dir, dataset_name, 'test.txt') 20 | input_fnames = [train_fname, valid_fname, test_fname] 21 | 22 | vocab_fname = os.path.join(data_dir, dataset_name, 'vocab.pkl') 23 | tensor_fname = os.path.join(data_dir, dataset_name, 'data.pkl') 24 | char_fname = os.path.join(data_dir, dataset_name, 'data_char.pkl') 25 | 26 | if not os.path.exists(vocab_fname) or not os.path.exists(tensor_fname) or not os.path.exists(char_fname): 27 | print("Creating vocab...") 28 | self.text_to_tensor(input_fnames, vocab_fname, tensor_fname, char_fname, max_word_length) 29 | 30 | print("Loading vocab...") 31 | all_data = load(tensor_fname) 32 | all_data_char = load(char_fname) 33 | self.idx2word, self.word2idx, self.idx2char, self.char2idx = load(vocab_fname) 34 | vocab_size = len(self.idx2word) 35 | 36 | print("Word vocab size: %d, Char vocab size: %d" % (len(self.idx2word), len(self.idx2char))) 37 | self.max_word_length = all_data_char[0].shape[1] 38 | self.sizes = [] 39 | self.all_batches = [] 40 | 41 | print("Reshaping tensors...") 42 | for split, data in enumerate(all_data): # split = 0:train, 1:valid, 2:test 43 | length = data.shape[0] 44 | #if length % (batch_size * seq_length) != 0 and split < 2: 45 | data = data[: batch_size * seq_length * math.floor(length / (batch_size * seq_length))] 46 | ydata = np.zeros_like(data) 47 | ydata[:-1] = data[1:].copy() 48 | ydata[-1] = data[0].copy() 49 | data_char = np.zeros([data.shape[0], self.max_word_length]) 50 | 51 | for idx in xrange(data.shape[0]): 52 | data_char[idx] = all_data_char[split][idx] 53 | 54 | if split < 2: 55 | x_batches = list(data.reshape([-1, batch_size, seq_length])) 56 | y_batches = list(ydata.reshape([-1, batch_size, seq_length])) 57 | x_char_batches = list(data_char.reshape([-1, batch_size, seq_length, self.max_word_length])) 58 | self.sizes.append(len(x_batches)) 59 | else: 60 | x_batches = list(data.reshape([-1, batch_size, seq_length])) 61 | y_batches = list(ydata.reshape([-1, batch_size, seq_length])) 62 | x_char_batches = list(data_char.reshape([-1, batch_size, seq_length, self.max_word_length])) 63 | self.sizes.append(len(x_batches)) 64 | # x_batches = np.tile(data, (batch_size, 1)) 65 | # y_batches = np.tile(ydata, (batch_size, 1)) 66 | # x_char_batches = np.tile(data_char, (batch_size, 1)).reshape(batch_size, -1, data_char.shape[1]) 67 | # self.sizes.append(1) 68 | self.all_batches.append([x_batches, y_batches, x_char_batches]) 69 | 70 | self.batch_idx = [0, 0, 0] 71 | print("data load done. Number of batches in train: %d, val: %d, test: %d" \ 72 | % (self.sizes[0], self.sizes[1], self.sizes[2])) 73 | 74 | def next_batch(self, split_idx): 75 | # cycle around to beginning 76 | if self.batch_idx[split_idx] >= self.sizes[split_idx]: 77 | self.batch_idx[split_idx] = 0 78 | idx = self.batch_idx[split_idx] 79 | self.batch_idx[split_idx] = self.batch_idx[split_idx] + 1 80 | return self.all_batches[split_idx][0][idx], \ 81 | self.all_batches[split_idx][1][idx], \ 82 | self.all_batches[split_idx][2][idx] 83 | 84 | def reset_batch_pointer(self, split_idx, batch_idx=None): 85 | if batch_idx == None: 86 | batch_idx = 0 87 | self.batch_idx[split_idx] = batch_idx 88 | 89 | def text_to_tensor(self, input_files, vocab_fname, tensor_fname, char_fname, max_word_length): 90 | max_word_length_tmp = 0 91 | counts = [] 92 | 93 | for input_file in input_files: 94 | count = 0 95 | 96 | with open(input_file) as f: 97 | for line in f: 98 | line = line.replace('', '|') 99 | line = line.replace('}', '') 100 | line = line.replace('{', '') 101 | for word in line.split(): 102 | max_word_length_tmp = max(max_word_length_tmp, len(word) + 2) 103 | count += 1 104 | 105 | count += 1 # for \n 106 | counts.append(count) 107 | 108 | print("After first pass of data, max word length is: %d" % max_word_length_tmp) 109 | print("Token count: train %d, val %d, test %d" % (counts[0], counts[1], counts[2])) 110 | 111 | max_word_length = min(max_word_length_tmp, max_word_length) 112 | 113 | char2idx = {' ':0, '{': 1, '}': 2} 114 | word2idx = {'': 0} 115 | idx2char = [' ', '{', '}'] 116 | idx2word = [''] 117 | 118 | output_tensors = [] 119 | output_chars = [] 120 | 121 | for idx, input_file in enumerate(input_files): 122 | count = 0 123 | 124 | with open(input_file) as f: 125 | output_tensor = np.ndarray(counts[idx]) 126 | output_char = np.ones([counts[idx], max_word_length]) 127 | 128 | word_num = 0 129 | for line in f: 130 | line = line.replace('', '|') 131 | line = line.replace('}', '') 132 | line = line.replace('{', '') 133 | 134 | for word in line.split() + ['+']: 135 | chars = [char2idx['{']] 136 | if word[0] == '|' and len(word) > 1: 137 | word = word[2:] 138 | output_tensor[word_num] = word2idx['|'] 139 | else: 140 | if not word2idx.has_key(word): 141 | idx2word.append(word) 142 | word2idx[word] = len(idx2word) - 1 143 | output_tensor[word_num] = word2idx[word] 144 | 145 | for char in word: 146 | if not char2idx.has_key(char): 147 | idx2char.append(char) 148 | char2idx[char] = len(idx2char) - 1 149 | chars.append(char2idx[char]) 150 | chars.append(char2idx['}']) 151 | 152 | if len(chars) == max_word_length: 153 | chars[-1] = char2idx['}'] 154 | 155 | for idx in xrange(min(len(chars), max_word_length)): 156 | output_char[word_num][idx] = chars[idx] 157 | word_num += 1 158 | 159 | output_tensors.append(output_tensor) 160 | output_chars.append(output_char) 161 | 162 | save(vocab_fname, [idx2word, word2idx, idx2char, char2idx]) 163 | save(tensor_fname, output_tensors) 164 | save(char_fname, output_chars) 165 | -------------------------------------------------------------------------------- /data/toy/test.txt: -------------------------------------------------------------------------------- 1 | 0 y y y z 2 | f g h e k f i j k 3 | d b 8 e c 9 a 4 | o o l k n l m l 5 | 8 7 9 7 8 6 | z y w x 1 0 1 v 7 | k i j h j l k 8 | 9 8 b 8 8 7 8 8 a 9 | p v q p s u r t 10 | a c a b 11 | j l j k j 12 | a c b c 13 | i e g g i f h 14 | d e f 15 | 8 9 a b 7 16 | j j i k k k k 17 | p r q s o 18 | b 7 a c 8 9 6 8 6 b 19 | q r o s p 20 | g d c f a b e 21 | m j n m i n k l o n 22 | q n p p m l o k 23 | i i h g i 24 | d a c b e 25 | c d b 26 | w r u t s t r q t v r q 27 | l m n l k l i m j o 28 | e i j h g f d 29 | s t v r u 30 | 7 3 3 3 5 1 6 4 2 31 | j j k n l i m k h 32 | s u u v w v t u 33 | 6 4 4 5 34 | 2 0 1 4 3 2 35 | c f g c e b f d h 36 | p n o n 37 | r s u q t 38 | i j j h 39 | r q p t r t s 40 | 1 2 5 6 4 2 0 3 41 | c d e g f 42 | c c b c a 9 d 43 | r q l p o m n 44 | o m m n o 45 | q o q n l k p m 46 | a a b c a a 47 | a a 7 9 b 9 8 48 | m n n n o 49 | o m n q p p 50 | 1 z y 2 y 0 51 | 2 3 3 1 3 52 | j k i h g i 53 | j j h g f e i j k h 54 | o m q p m s n n r 55 | x 0 3 1 y y 2 3 z 56 | 0 y z 0 x 1 57 | 8 a 7 6 9 58 | p n o n 59 | e e f g e f 60 | k l m j n p k o 61 | u w t t t s w w w v s 62 | 2 0 z 3 y 4 1 0 63 | y 4 1 3 0 2 z 64 | u p s q u v p v t r 65 | d b 9 b c b a c 66 | m k n l l o p j 67 | l i j k m h h g 68 | g e f 69 | k l k k m 70 | e g h i f i g h 71 | 0 1 2 3 z 1 72 | w x v w 73 | i l h j k k 74 | r p o u t s p q t 75 | a c b c c 76 | e e f d 77 | 5 9 7 b 8 6 a 78 | 2 0 2 1 y z 79 | k n m k k o o l 80 | r o p s m q n 81 | 8 9 5 7 9 6 5 9 82 | 2 4 8 7 3 6 5 83 | f f g h 84 | 3 3 2 4 4 85 | u s t s t 86 | 2 5 3 0 5 4 3 z 1 87 | c c a c b 88 | y z y v x 1 0 y w 89 | z z 0 1 90 | r t q s p 91 | r q m n l o p 92 | t r s s q t p 93 | c b 9 c a 8 94 | e d f b a c 9 d e 95 | 5 5 5 3 4 1 2 5 96 | z y 1 4 4 3 4 2 2 3 0 97 | 4 5 1 2 2 3 98 | e e d c c 99 | 9 8 5 6 6 8 7 100 | 0 2 1 z y 2 4 3 4 101 | x u t v t w s y 102 | s t u s u u 103 | w y 0 x 2 1 z 104 | 3 0 0 3 4 1 2 105 | v w w w u 106 | 3 4 2 0 3 1 107 | s v q r u p t 108 | l m p o n 109 | z 1 0 3 5 4 2 110 | 4 2 3 5 1 2 111 | f c e d f b 112 | q p r q 113 | m m n n l 114 | 4 1 y z 0 2 3 115 | o m p q k n l o 116 | 3 4 4 6 5 3 2 117 | f l g j i h k l 118 | f f e g d c 119 | 2 0 y 0 3 1 z y 1 4 120 | t q q r p q s r 121 | 3 3 2 2 1 122 | j j g e f k h i 123 | q q q q r r p 124 | a b 8 9 7 125 | p q r q t p s 126 | k l m j i l 127 | g i g i h 128 | v w s t u v 129 | x v w v 130 | d c h e b f g 131 | 9 7 6 a 8 132 | o p k m n l q n p 133 | f e f f e g 134 | k h i l m j g 135 | 6 6 5 8 8 4 9 3 9 7 136 | v x u u w u t 137 | c 8 d b e a c 9 138 | y y t w v u t y t x z y 139 | l p p o p p m o n 140 | q m n r p r s o 141 | c d e f e b 142 | m j i j i k k l 143 | m k n l l o 144 | 2 x y 1 y 0 z w 0 145 | 4 3 5 4 146 | s q t p t r q q o u 147 | c e d 148 | 8 8 8 c b 9 a 149 | 7 5 9 4 a 8 9 5 8 6 4 150 | x 2 z w 1 0 y 151 | w w v v u v 152 | a b 9 f b c c e d c 153 | q q o p 154 | n m n o 155 | x w y 156 | d e c f b f h g 157 | 3 0 4 y z 1 z 2 158 | k j m i h l j n 159 | 7 5 7 7 6 160 | k l m 161 | p o n 162 | 3 6 5 2 4 4 163 | r t q s q r p 164 | h e g b f c f d c f 165 | n s o q q p m r 166 | x v y x u w y 0 u z z u 167 | h e i g e f j j d 168 | 1 z 0 169 | 3 4 2 4 170 | e h f d i j h g 171 | 5 6 3 2 8 7 8 4 172 | k l i h m j g i 173 | r t s q s p 174 | m k k k l 175 | 9 7 7 a 7 b 8 176 | r p q o n q 177 | e e d c 178 | x w w t v v u 179 | l k m i m n h j 180 | r s s q 181 | c d a e d c b g f 182 | y v y z x w x 183 | c c b d 184 | t u u v 185 | q p o p 186 | 4 5 3 3 187 | n k k o l m 188 | x 0 z 1 w w y 2 189 | 2 2 4 4 3 190 | 9 5 8 8 6 7 191 | 9 8 8 9 a 192 | 9 b 9 a 193 | v u t 194 | e e g f e 195 | f f f e g f f 196 | 8 a 7 9 6 197 | c e c f d b e b b 198 | u v w v s t t 199 | b c d c 200 | 0 y y z 0 0 201 | o r q p s 202 | f d h e g j i 203 | d b f e d c d g a 204 | w v y w x w z 205 | h g i d e c f 206 | v z x v w y 207 | c e d d g h f b 208 | f g h h f 209 | a b c 210 | y u x v v w u z 0 211 | l k m n n j 212 | n h j h l m k i 213 | 7 9 6 7 a 9 8 8 214 | o o p o o q o 215 | o n l m r p n q 216 | s r s q 217 | q s t s r q u q 218 | t v u 219 | 8 9 7 7 220 | 5 5 7 7 7 6 221 | m p n l o q k 222 | y 2 3 1 0 z y 4 223 | e h i g f 224 | 5 6 6 7 6 7 225 | d c g e h i f d 226 | 6 6 6 1 6 3 2 5 4 7 227 | u v w t x 228 | 7 8 5 9 6 8 229 | 7 a c 9 d c 8 d b a 230 | r q p q q 231 | 0 3 z 4 y 2 1 232 | d a c b e g e f 233 | b f 9 a e c d 234 | x z y t u v w w w 235 | 0 3 3 4 1 2 236 | h e f g i 237 | 5 4 3 6 6 7 2 8 238 | 1 2 3 y 4 z 0 239 | v v x v t v u w 240 | 6 4 3 4 8 7 2 5 241 | 0 z 3 2 1 4 3 1 y 3 242 | s t r t t 243 | h h i j h 244 | v w 0 z x y 1 245 | j n l h m i k 246 | k j k l i h j 247 | x z x y w 1 0 2 z 248 | m o n o o 249 | a 9 b b 250 | u w v 251 | u r t q s u 252 | o l l n p m 253 | f g h g g g 254 | 4 3 4 6 6 5 7 255 | c c b e d e f f 256 | g f h i g e c i d h 257 | 1 z 2 0 3 z 1 258 | 6 8 7 9 7 6 6 5 259 | k l j h k i 260 | i j n o k l k m 261 | q r p n o s t 262 | 3 4 4 0 4 1 2 263 | 8 7 6 7 264 | f d f h g g e 265 | f j f i g k k h l 266 | d f e f b c d d e 267 | a e d c b a 268 | l n j o m k i l 269 | p r q q t s 270 | 3 4 6 3 5 2 3 271 | o n p q r o 272 | n p o p p p 273 | f h j i f e g d 274 | a a 9 8 a 275 | x y z y 0 0 1 0 1 276 | a 4 6 4 8 5 4 9 7 5 277 | u t x u u w w s y v 278 | b f g e c b h d 279 | j j i l l m k 280 | 2 3 4 4 281 | a 9 c c 7 8 d b b 282 | y z 0 283 | m l n l p o l l 284 | w v x u t x 285 | r n p s q q n t n o 286 | q u v w t s r v 287 | y x t y v u z y w t 288 | s u v r s v t 289 | y z 1 z y 0 2 290 | m n n l 291 | u p s o t r q 292 | e g f 293 | u t v 294 | 4 4 4 2 3 2 295 | 8 6 4 7 5 296 | x 2 y 1 x 0 y 3 z 0 297 | 9 7 8 7 7 298 | h f f g f e f d 299 | 1 3 2 5 4 300 | 2 0 3 4 5 1 z 4 301 | m m g j h i l g g k 302 | m n m o n 303 | t p u s o r q o u 304 | x y x y x y w 305 | u t v t w x 306 | q q r q s q q r 307 | 1 x y 0 z 2 3 x 308 | c a e b 9 8 d 309 | z y 2 z w 1 0 x 310 | y u x v t z w 311 | c e h g b e f d c 312 | 0 y z 313 | i m n l h m j k 314 | 3 2 1 315 | e f d a b 9 9 c 316 | 1 6 5 4 3 2 1 0 317 | y x w t v u v s v w 318 | b a 7 6 b c 8 9 c b 319 | m n o 320 | u z w t v w y x w 321 | c a e f b f d 9 c 322 | e e b a d a e c 323 | 1 3 5 3 0 z 2 4 324 | m k o k n l j k p l 325 | a 6 8 9 8 7 326 | l n p m o r q r 327 | b c d 328 | f i h g j j k l g 329 | t q q r s p 330 | 9 b 8 a c c 331 | b d e c f 332 | t q p r n o s 333 | u s t w v 334 | 5 8 7 5 5 9 6 335 | 5 2 3 4 2 1 336 | 9 a b d c f e c 337 | d d e c e 338 | 9 6 7 b c 8 9 a 339 | x w w y 340 | u s t r s s q 341 | y y z w x w w v 342 | 0 y 0 x 3 1 3 2 z 343 | w w w v v x 344 | b e c e d a 345 | o o q p 346 | 5 7 7 4 5 4 8 6 347 | n n q p r r o 348 | q r r r s r s 349 | e h i f g 350 | p m o n p q m 351 | 7 a 7 8 6 a 9 352 | r v u t r s 353 | e f h k g j i 354 | i j i i l k h l 355 | e d e h g f 356 | 0 z 2 w 1 x y 357 | 7 8 b 9 a d b c 358 | g e c f g d e 359 | l m l l k 360 | l m o n k l k n 361 | f h e i j d g 362 | k g h j f e f i k 363 | x w z 0 1 x y z v w 364 | 0 4 6 5 1 5 2 3 365 | 5 6 a 8 9 7 4 366 | v w y x t z u z 367 | s p p q o r s 368 | 6 5 5 5 2 6 7 4 3 8 369 | 2 1 0 2 1 2 0 370 | 5 1 6 3 4 4 2 3 7 3 371 | y 1 1 0 z x 0 0 y 372 | 5 3 2 4 5 6 373 | d b a c c c 9 374 | f d c a g e b 375 | l n k j m 376 | a a 9 8 377 | 1 0 z 378 | 2 3 1 2 379 | 3 8 6 5 8 4 7 9 4 3 8 380 | a 9 b d c 381 | 0 x w z y 382 | 0 3 1 2 2 z 2 383 | 0 0 1 z 0 z 384 | m j l n h i k 385 | u u t w v y z y x y 386 | s q s r p t 387 | o o r s q p 388 | q m q o l l k p n 389 | f c e d c b 390 | a 8 c a b 9 391 | e h c d e g f c b 392 | 8 7 9 393 | 0 y v u x z w 0 394 | o m p l n 395 | m k k o m l n 396 | 9 6 7 7 8 5 397 | 0 2 2 1 2 398 | z 1 0 399 | s r s t 400 | z w z v x y x 401 | b a c 8 9 402 | n o m m 403 | i f h g h e f f 404 | s s u v r r s w x t 405 | k j l l 406 | 6 4 8 6 2 4 5 3 6 7 407 | p u s t q v r 408 | 5 5 3 4 3 409 | v w v v y z z x 410 | h j l j h k i 411 | m q p r s o n m 412 | 5 4 4 4 3 413 | f g k l i h j 414 | r q s o p s s 415 | s w v u u u s t 416 | y y z 0 x w z y 417 | 8 9 c b 8 b a 7 a b 6 418 | k j k i k 419 | u v p s r q s t 420 | 2 5 7 4 8 6 3 421 | d c b 9 c c c a 422 | 9 9 7 8 b a 423 | 1 2 0 424 | r n p q o t r s p 425 | m l i k j 426 | j j k l 427 | e a a e 9 d c 8 b 428 | a b a e 9 c f d 429 | 7 2 5 3 6 4 8 3 4 8 430 | g j j i h k 431 | i g j f h 432 | b b c d 433 | y w x x x y 434 | l l p m n l o 435 | 7 3 5 4 6 436 | 4 6 4 5 437 | j h i 438 | x v v x w 439 | 9 a 8 7 8 9 b 440 | l o k j i n m 441 | m l m k 442 | h g g i h g 443 | j k i 444 | l m n 445 | y w u 0 y x z v 446 | 1 1 1 z 0 447 | r s q r r p r t 448 | g f h 449 | c d b 450 | r s t v u 451 | j h g k l m i h 452 | 5 7 8 6 9 453 | v t u v u 454 | s p s t r q 455 | j i k i 456 | t s r 457 | 8 c 7 b 9 8 d a 458 | 1 0 2 3 4 1 2 3 2 459 | 3 6 7 6 5 4 460 | v w x 461 | g h f f 462 | 4 7 5 3 7 6 463 | s v u r t v 464 | 1 5 2 7 3 5 4 1 4 6 465 | a 8 a 9 6 7 466 | 1 0 y 3 x y 2 z y 1 0 467 | n m n l n 468 | 2 0 4 1 3 469 | x u w v t 470 | 1 0 0 2 1 471 | f e b 9 f d c 9 a f 472 | p o p t s r q u q 473 | 5 3 2 6 4 3 5 6 474 | o q u q s t p r 475 | u x y v w 476 | 7 b a 5 9 5 b 7 6 8 477 | 1 1 z 3 y y y 0 4 1 z 2 478 | j i k h l l l 479 | j j h g k i j 480 | 4 1 5 2 3 481 | e a 9 8 b d 9 c 482 | q p m p k l o n 483 | 6 5 4 4 6 6 7 3 484 | k j i h l g g g m 485 | l j n o k m i 486 | k k j l j 487 | l h k i j i k 488 | w v v x 489 | q q s q r 490 | i h i h g g 491 | 5 6 a 8 9 7 b 492 | 4 1 3 2 0 2 493 | z z 0 3 3 2 1 494 | 3 3 2 1 495 | k j l l k j 496 | 7 3 7 4 6 5 4 7 497 | x y 1 z 0 498 | d 8 a 9 8 e c b a a a 499 | t u q r s o p 500 | o l p n m p k q 501 | 1 y z 0 2 2 2 x 3 502 | c a b d 9 503 | 8 7 9 9 9 7 504 | 2 2 4 2 3 3 4 505 | w v z 1 y 1 0 x v 506 | k l k j 507 | c 7 a 9 a 8 d 7 b 508 | o q n r p 509 | p q s v v u r t 510 | s s t u r q 511 | n m l l m m 512 | 0 y y 2 1 2 z 2 513 | 3 1 4 2 0 4 0 514 | h f g 515 | l j k h n i m l 516 | k k j h g i g i h h 517 | q s r p t s 518 | l m n l 519 | s u t 520 | 5 5 7 6 7 521 | b a e c d e 522 | 4 6 1 5 0 0 3 2 523 | k k j k l 524 | m m k k m l 525 | o n n o o p 526 | c b c a c c c 527 | k l j g i k h m 528 | s u s s u t 529 | 9 8 d b a 7 8 c 530 | 0 1 1 2 1 531 | v t x w t u r w s 532 | m n m n l n 533 | i i m m l j k l 534 | o r q s q p 535 | f e c d g 536 | f e d f e 537 | r s p t r q s p 538 | 5 2 8 3 4 6 7 539 | i h j 540 | h i g 541 | z z y 0 542 | 3 8 8 9 7 5 6 4 543 | b d g b f a b c e e 544 | 4 5 3 1 3 2 545 | 3 6 5 4 3 2 546 | 3 5 4 547 | 7 9 a 6 6 8 548 | 9 b a c b 7 6 8 549 | v u t v 550 | y 3 0 z 4 0 2 1 3 551 | o p p r q p s o p 552 | j i j k 553 | i k j 554 | 4 5 3 z 0 1 z 2 555 | 3 0 1 2 4 556 | u t s q t r 557 | f h d d f b c e g 558 | w 1 0 y z x v 559 | h j j l k m i j g 560 | d f i e g h j 561 | l l j j k 562 | p q o n o r 563 | 2 z 0 x 1 w z y 564 | 4 3 5 565 | m k n o k l p n n l j 566 | s w v y v u u x t 567 | i j f d g e h 568 | h i h i g 569 | x w y 570 | 5 7 9 6 6 8 571 | 3 7 6 4 5 6 6 5 572 | q s t q p r 573 | q w u s v s s r q t 574 | 0 1 5 6 3 6 2 4 575 | c d e e 576 | o m o n 577 | c 8 b 9 7 c a 6 9 578 | 6 6 7 6 8 8 579 | x w z t u v y w w 580 | n k l m o 581 | 8 7 9 b 6 c a c 582 | c d b a f c e g 583 | 7 6 6 8 9 8 a 584 | 3 1 z 1 2 0 585 | 1 0 0 3 4 2 586 | b e c h d f h g 587 | w y w v x u u 588 | h h f g h h l k j i 589 | 6 7 7 6 8 7 590 | 4 7 5 6 7 8 9 3 591 | h m k i l i n j i 592 | 4 3 5 4 4 593 | q p r 594 | q l p m n m o k 595 | 2 3 3 4 z 0 1 5 596 | t s v r u 597 | 3 2 z 0 1 3 598 | 3 0 2 5 4 1 z 599 | 1 3 1 2 1 1 600 | r o p s p q 601 | m m k i j g h h l 602 | k j j j j l j 603 | t u v t 604 | t t u v 605 | a d 9 c 8 d e d b d 606 | p p r q r q r 607 | k l h g f i k j i h 608 | f g i h j 609 | s r q 610 | e f f h i g 611 | g h f d c i c e 612 | l m n k o n m m 613 | a f e 9 b c d e 614 | 4 4 6 5 615 | 3 4 1 1 4 0 0 2 3 616 | z 0 3 4 2 1 y 4 3 4 617 | f f f f g h 618 | 7 2 3 4 6 2 2 8 5 619 | q o p o 620 | 9 a 8 a 621 | s t r s s 622 | o m n 623 | d f g i h e c 624 | m o n n o 625 | 8 9 7 9 8 6 a 626 | r q r q p 627 | 2 0 x z y 1 y w 628 | p o q 629 | j i h 630 | 4 z 0 3 3 1 5 2 z z 631 | 5 3 4 6 7 9 8 632 | q r q p 633 | e e d f 634 | 0 1 2 635 | 2 3 4 1 0 4 2 636 | v s t x u r w u t 637 | 2 4 3 0 y 0 1 z 638 | 1 y v v v v w x v 0 z 639 | 8 6 7 640 | u w w v w v 641 | f h j g i l k 642 | v u w r v s v t x 643 | i j k l h 644 | w u v w y x y 645 | 5 6 4 4 646 | 6 5 7 7 7 647 | t r r s u u v t t 648 | c 9 9 d 9 b a 649 | e i j k h f e g k k 650 | x v v w 651 | m n o o 652 | v s v u w r t u x 653 | r r s o p q r 654 | n o m 655 | b f e c f d g g a 656 | g k g h i j 657 | p q o s p r r 658 | z 0 z z 1 659 | y 0 0 1 3 2 4 z 660 | m k g h l k i m j 661 | t r u s v v s 662 | 6 5 2 4 3 7 8 663 | r w t w w s r u x v 664 | 8 7 6 8 5 4 665 | p o n 666 | 7 8 5 8 6 8 4 667 | o n p 668 | z y z 1 z 0 1 x 669 | 8 7 9 670 | 1 y v 0 z w x 671 | q u p t p s p r o p 672 | i h l k j l 673 | 5 6 7 8 5 4 674 | o j m m l n k p m j 675 | t u s o p r q p 676 | g k j k i h 677 | 5 b 6 9 8 b a 6 7 678 | q p o 679 | x w x y w 680 | w w y x 681 | p o q n m p 682 | z w y z x v 683 | b h h g f d d e c 684 | u v w 685 | l m k h g j i j 686 | j k m n l h i l 687 | k j i l k l m 688 | 3 5 0 4 5 z 2 1 689 | z 0 3 1 2 690 | b d a d c f g b e 691 | g k m j l i l h k 692 | 9 7 8 a 8 9 6 693 | z y x y y y 694 | 1 3 1 2 695 | 0 z 3 1 2 2 y 4 696 | a 9 b c d d 697 | m n l n l 698 | n m o m q p 699 | z y 0 x y 0 w w y 700 | 8 7 6 7 701 | r p r q q r 702 | g d i e h f j 703 | g f h d b h g c e 704 | 1 0 0 z 705 | t s t r u v 706 | l m l m n 707 | s r s q 708 | 2 2 3 4 3 3 2 709 | 0 y z w w x x 1 0 v 710 | p l m o o n n 711 | g h i g f e j d i 712 | 8 9 8 6 8 8 5 7 6 713 | i i h j j 714 | h e f i d g c e 715 | h g f g h h f 716 | 6 4 5 4 717 | e c d 718 | z 0 z 1 z 719 | p s t r t t q 720 | o p n m l 721 | g h g i 722 | y v z w x 723 | r s o q p s 724 | 2 1 z 3 2 0 725 | h g d f j e i 726 | v v t v v u 727 | 8 9 7 9 7 728 | 4 5 2 2 3 4 1 729 | o n m 730 | o p n 731 | j m l h n i h k 732 | 0 3 1 3 2 2 6 0 4 5 733 | 1 2 w x z y w 0 734 | y 2 x y w 0 z 1 z 735 | d e h h g f g 736 | i j e g f d f h d 737 | 3 2 2 z 1 0 738 | c b 9 b d 7 8 b a 739 | 9 b c 8 d d 8 a 7 9 740 | y 0 x 1 3 2 z 741 | x v v w w 742 | v z z w u t y x 743 | 8 7 9 7 a b 744 | k l o j m j p l l n 745 | 3 5 4 8 2 4 6 7 746 | x t z u w v y y 747 | l o p n p l p l m 748 | 4 4 3 6 7 4 5 749 | 2 2 3 4 5 1 4 750 | y 0 z 0 z 751 | b b c d c d b 752 | a 8 a 9 753 | 0 3 1 0 y 4 2 z 0 754 | 8 d c b 9 7 a 755 | z y 0 0 1 2 z 2 756 | z 1 0 1 x 3 1 2 y 2 757 | e c d d e b a 758 | l k j j 759 | m l n 760 | c b d f e c 761 | g i h h 762 | k i j o i n l m 763 | b c b a a 764 | k i m n o l j i 765 | k l j n m l 766 | k l k m 767 | i k h j l h 768 | w x w y 769 | x z y 2 2 0 1 3 z 770 | 1 2 z 3 y x 0 771 | 9 b b a 772 | h n k j l j i m 773 | y v t w v v u z x 774 | s y v y t u x y t w 775 | i g m h l j k 776 | 3 5 5 1 5 2 4 5 777 | q u r r s t 778 | s u t t t 779 | s q r q q 780 | n k m l j o j i 781 | x y z 782 | z z 1 y 2 2 0 z 783 | h j j i 784 | f g g g g h 785 | z 2 0 4 3 1 y 786 | f h g i i f j e k 787 | 0 y z 788 | 8 b e a d c a 9 9 a c 789 | j k g m i l m h l 790 | g h e i f e e 791 | s u v w t u 792 | b c a a 793 | 8 8 6 9 a 7 6 6 794 | l k m j i k 795 | 7 7 8 8 9 796 | j j i h j j j i 797 | f g c e b d a 798 | w y x 0 x z x w 799 | j k l j j 800 | a d 7 8 c 7 7 b 9 9 801 | 7 7 5 6 5 802 | i h f e j d g h 803 | x z t x y w w v u t 804 | l j j k k 805 | n o m m 806 | 6 7 5 807 | 2 1 3 2 3 808 | 2 5 2 3 4 6 3 809 | 9 a a b 8 e c d 810 | x z y 811 | 9 6 6 7 a 8 6 6 812 | n q o l m k p 813 | b d e e c 9 a f 814 | w x y u s t v s s 815 | n o p p 816 | j k m i k l 817 | r t x t w s u v 818 | 0 x z 1 z y z 819 | 2 y 0 2 3 z x y 1 820 | 4 6 7 3 5 8 2 6 821 | h f e f d g 822 | v x y w z 1 0 823 | a 7 8 6 9 a 8 b 5 8 824 | 6 7 9 4 a 5 7 7 8 825 | l k o p m n q 826 | m k l k k 827 | g h e h c i d e f 828 | d e f f d 829 | 2 2 1 3 1 830 | v w w s v t t u 831 | 3 2 1 3 832 | 6 7 4 4 3 4 5 4 4 833 | x z x z y 834 | 2 0 3 1 z 835 | r m l n o q p 836 | z y x 837 | 8 7 6 3 9 4 5 838 | m o l p k q n 839 | 2 x z w 1 y z 2 0 1 840 | p o q q 841 | z 1 2 0 3 0 842 | h e f f g d d h 843 | f d e e b c e 844 | v w u u 845 | d a 8 7 9 c b b 846 | y 0 2 z 1 1 2 847 | 1 2 0 y z z x w 848 | o q n r p p p 849 | 7 6 6 7 5 850 | 6 5 7 4 3 7 851 | d e e f e e e 852 | w x 0 x y y z 853 | 3 4 1 5 2 7 1 6 1 854 | x z y y x y z 855 | n k o i m l j 856 | x y 0 1 z 857 | m p s r o n q 858 | y z y y y x z 859 | h h g f 860 | a d 8 8 c 9 b e 8 a d 861 | x v v x x y v u w 862 | x x y w 863 | 4 5 6 6 864 | 8 b 9 9 b a 7 865 | t s u u u 866 | o q o m p n 867 | q s r 868 | d e f c e b 869 | 2 4 0 z y 0 3 1 870 | 1 1 3 2 871 | t u q r w s v t 872 | p n q o p m q 873 | e b c a d d a 874 | n o m n 875 | o n m r p q l 876 | e d c a b 877 | f g h h h 878 | j i h h 879 | y w x 880 | 5 7 9 8 6 7 881 | 6 7 9 5 8 8 882 | i h c g g e f d i 883 | 1 0 z z z 884 | m m j l k m i 885 | g h f g e f i 886 | z x y w w v 887 | 2 2 6 2 4 2 5 3 888 | w x v x w 889 | e c b d b f b c 890 | t n s r q p o 891 | l l n l n m 892 | v s u v w t u u 893 | k j j k l 894 | p p p n n n p o 895 | d b c 9 8 a e a 896 | f h l g k j i 897 | w u v u 898 | i o n j l k m 899 | k m l n o 900 | b 9 8 a c c 901 | x y w 902 | i m n n o l j k 903 | x x w v 904 | q m n o l p r 905 | 7 8 6 a 9 9 9 906 | d d d d f e 907 | 6 8 7 7 908 | d f d g h e 909 | g h h g f 910 | x u 0 z w u y 0 v 911 | r q p 912 | f h g h e d 913 | f e d d 914 | l l m k n o 915 | g f f f e f f 916 | 2 3 1 4 4 0 917 | d g b e c f h e 918 | a e e g f d e c b 919 | j k l m i k k m l 920 | 6 7 6 5 921 | 9 5 a 6 8 7 4 5 922 | 6 5 4 7 8 923 | a 8 c 6 7 7 9 6 b 924 | x y w 925 | 5 2 3 4 1 5 926 | 1 0 z z 927 | 4 8 3 9 5 4 5 7 6 928 | 7 4 2 6 3 5 7 1 929 | r p q t s 930 | a a 9 9 b 931 | c e e d e 932 | 3 2 1 5 5 2 6 0 4 933 | f e c g e d 934 | j g i h f 935 | 3 6 7 1 2 5 4 5 936 | k g j i l h m g 937 | x x x w y 938 | k k l i j m 939 | d g h f c i e h h f i 940 | 2 y z 1 0 z 941 | m i k m j l 942 | f g h h 943 | b 8 b b c 9 e a e d 9 944 | 4 2 8 7 6 5 3 2 945 | o o q k p l m k n 946 | w y w x w 947 | z x 2 1 3 0 y z 948 | y 0 z 0 y 949 | i h f j e g g i d 950 | i j k k 951 | l i k j h l 952 | j l l k l l 953 | f d g e g h 954 | c b e f d 955 | 5 3 4 1 4 2 956 | f g h g 957 | u v s w t q v w r 958 | m o k n p j n l l p n 959 | n p n n o 960 | c e b 9 a d a 8 c 961 | a e b 9 f a c d 962 | s w y t s s v u y x 963 | 4 7 8 6 3 9 5 8 964 | r s s t 965 | k i i k j k j i 966 | x z y 1 0 z 967 | 7 8 5 4 2 3 6 5 968 | e f d e 969 | p q o l q l o m n r n 970 | p q p q o 971 | c d e e e c 972 | n o s r s t q p 973 | 2 3 z 0 1 2 z 2 974 | z y w 0 x w 975 | 9 9 7 7 8 976 | u w x v u u y 977 | o q p p n m 978 | j e e k f i h j g 979 | r t s 980 | k m i k k j l k 981 | n q r l o p m r 982 | t s u 983 | 2 2 3 1 3 2 984 | t s s r 985 | d h g e f d d 986 | 1 z 0 1 z 987 | 3 3 5 6 4 7 988 | o n m 989 | m l q p m m n o n r 990 | 4 3 2 0 y y 3 1 z 0 991 | s t u u r x v w 992 | l k m m m 993 | z v y x w 994 | l k m k o o n 995 | t u x w u v v 996 | 0 1 z z 997 | f h g h 998 | t v v u 999 | q r q q p 1000 | d e f d e 1001 | u z v y t w x w 1002 | h g f g 1003 | v u t t x w y s 1004 | q q s q s r 1005 | n l k m h j i i 1006 | g h i m i m l k m j 1007 | s s t r p q q q t 1008 | 6 7 5 5 5 6 1009 | r q q q p 1010 | r r p s t r q s 1011 | 2 2 4 z 1 3 1 4 y 0 1012 | g h e j f d i 1013 | k n l j m 1014 | n n l m p l o 1015 | 0 u x w v z y 1016 | 8 5 8 a 6 9 b 7 1017 | w z u t v y w x t 1018 | 9 a 9 8 1019 | i k l j k m 1020 | 5 7 6 5 1021 | 5 3 4 2 6 4 5 1022 | a 7 8 b 9 1023 | e a e f b f 9 b c d 1024 | 2 3 2 1 3 0 4 1025 | z z 0 1 1 1026 | 2 1 x 0 3 2 y z 1027 | t s u 1028 | g d g f h g e f 1029 | c b d f d a d e f 9 1030 | h g f h 1031 | 2 1 2 0 3 z 1032 | r r s q 1033 | l q m k k o n p 1034 | v s v r t u 1035 | 3 0 4 3 1 2 3 1036 | 0 2 1 2 1 1037 | y y 0 y w x z 0 y 1038 | q t r p u s v 1039 | 6 4 4 5 6 1040 | 0 4 3 2 z 5 1 z 1041 | i j k 1042 | g h h f f f 1043 | 2 5 z 0 z 1 4 3 1044 | 1 z v y 0 w x 1045 | c a 8 b c 8 9 1046 | q n m l p l o r 1047 | r t v u v s t v s 1048 | 6 4 7 6 3 5 5 1049 | u t t v u t 1050 | b c a 1051 | 0 6 5 3 6 2 4 1 3 1052 | 5 3 4 3 4 3 1053 | s r u v t 1054 | q m m n o p m 1055 | z x w y w 0 1056 | a d c 9 9 f e e b e e d 1057 | 7 a 9 b 8 1058 | r t s 1059 | d d b c 1060 | q l n m r o p n 1061 | 3 1 0 4 2 3 1 0 1 1 1062 | k m n j h i j l j 1063 | r p r s q o q 1064 | r p p o q r s 1065 | g f g i h e 1066 | w y x 0 z w 1067 | m n o p q 1068 | v w u v 1069 | m p m o q m n 1070 | 3 2 1 1071 | w x t t w u v 1072 | z z 1 0 0 z 1073 | 5 6 6 6 4 5 1074 | k p o l p m j n n p 1075 | 1 3 0 y z 2 x 3 1076 | 7 7 6 5 7 1077 | 5 6 9 5 5 8 7 1078 | y 0 y y y z 1079 | p n n o 1080 | 9 8 7 1081 | e g f g 1082 | n n l m 1083 | 4 3 4 6 5 6 2 1084 | f c g e d g g g 1085 | o p q r p r n 1086 | r t s 1087 | t v u u r s 1088 | y z 0 y z z 1089 | m j l n m i n k m h 1090 | y 1 0 z x 1091 | u u w y x v 1092 | j h i 1093 | y x y y z 1094 | 2 5 1 5 3 3 2 4 1095 | 4 2 2 3 2 1096 | i j h i i 1097 | k o m n j i l 1098 | k i l h f j g 1099 | k i j k j 1100 | h e f f i e d j g 1101 | i h g g 1102 | z 1 0 0 1103 | c e d 1104 | o q p m n 1105 | j f h k g i e 1106 | s t s u 1107 | m r m m p q n s o m 1108 | f g h i j j 1109 | f e e d d e 1110 | t y u x v w t z 1111 | p m q n o 1112 | 5 6 7 4 1 3 2 3 1113 | y w x v u w 1114 | w y x x y 1115 | c g i f e c d i c h 1116 | a g d e b a f c 1117 | 4 3 2 4 2 2 1118 | v x v w 1119 | 7 2 3 4 5 8 6 1120 | j n o i k m l 1121 | j m k o i n l 1122 | b 5 7 5 a 6 8 9 5 9 1123 | a 7 5 4 6 9 8 4 1124 | g j k k k h i 1125 | h h g h f h h 1126 | 7 5 7 3 6 4 1127 | 0 z x w v 0 y u u 1128 | 2 1 0 x w z y 1129 | e f g f f e 1130 | e g c b d c f d h g 1131 | y x w x 1132 | a a a c b 8 9 d 7 1133 | 7 6 5 7 6 6 1134 | b c d d 1135 | z 1 0 2 z 1 1 y 1136 | l l n m m l m 1137 | u v w u 1138 | 7 8 b d a 7 c 9 7 7 7 c 1139 | f h i g e 1140 | 2 3 3 4 1141 | 6 7 5 7 1142 | 2 6 3 3 3 5 4 6 1143 | i j k i j 1144 | b a 9 b 1145 | j k l k 1146 | h g g g f 1147 | z y 2 0 z 0 x 1 w 1148 | 2 3 6 4 5 1149 | i j h j g f 1150 | 2 1 1 4 3 7 6 7 7 5 3 1151 | 8 7 9 1152 | 7 9 7 b 8 a 1153 | 6 8 7 8 1154 | y w x z v t u 1155 | g f e f d c 1156 | y z z 0 z 1157 | 6 6 3 7 4 5 6 1158 | t v p u r v q s 1159 | q s o q r p t o n 1160 | 3 3 2 1 1161 | q p o n m 1162 | 1 2 1 1 2 3 1163 | 7 8 6 8 4 5 1164 | 9 b e a c e a d 8 1165 | 0 z z 1 5 3 2 4 1166 | 1 z x 0 y x w 2 1167 | f c h i g e d g 1168 | r o p n o q 1169 | v s u t w 1170 | j f h g i k e 1171 | v w x v v v z y y 1172 | 8 7 4 4 6 5 1173 | 5 9 8 7 4 6 3 9 9 7 4 1174 | i h j j j k l 1175 | z x 1 y 0 1176 | y 1 1 1 x 0 z z 1177 | 1 0 0 2 1178 | l j j j k 1179 | o s t t p u r q t 1180 | r s t q t p r 1181 | t s u 1182 | 0 2 3 1 0 z 4 y 2 z 3 1183 | y x x v u w 1184 | d c b f b e 1185 | l l m j i i k 1186 | 0 z y z 1187 | m k j l n n l 1188 | p q r t t s v r u 1189 | v w x z y z 1190 | w z x 0 y x 1191 | p m l p k k q n o l 1192 | n q p r r n o 1193 | v u s u t w 1194 | f k g e h f j i g 1195 | 9 a b a 7 8 9 1196 | e d f e d 1197 | 0 z 3 2 2 1 1198 | 6 7 8 8 4 4 9 5 a 1199 | k j i g h 1200 | 4 y 2 z 3 1 3 0 1201 | s u r t r v 1202 | i j g k l h m 1203 | 2 3 4 1204 | z w t u v x y z u y 1205 | n j m j k l j 1206 | z 0 y y 1207 | e f g 1208 | z 1 y 1 1 0 2 1209 | n q m o o p n 1210 | t w t x v u 1211 | 7 4 6 5 3 5 5 1212 | h j i f g 1213 | 7 4 6 8 9 5 a 1214 | u w v 1215 | w x y 1216 | p q r s q s q t p q 1217 | p n o 1218 | g e g f 1219 | r q r o p r n s m 1220 | 8 6 7 3 4 4 5 9 1221 | v v u u r s t 1222 | k g f e i j h j i 1223 | k l n m j n n 1224 | k l h j i 1225 | n o n t p n r q p s 1226 | 7 6 5 3 4 3 3 1227 | c e f b d c 1228 | q r t v w u s 1229 | 3 z z 4 0 3 y 3 1 2 3 1230 | g f g h 1231 | h i l j h k 1232 | e g h f h i 1233 | l k n i j m o n o 1234 | r s x w u t v 1235 | m l n n n 1236 | v w u 1237 | d d e e e f 1238 | f g h j e i e d 1239 | 7 9 9 a 6 8 1240 | w u s t v t 1241 | b d c c 1242 | p s r o o q 1243 | f g d h e 1244 | z 0 1 3 x y 1 2 1245 | w x v u t 1246 | 1 0 z w v x y 0 1247 | i g h j k 1248 | z x w w 0 y 1249 | g h g i k j 1250 | p m q q n p n o o n 1251 | h f g f 1252 | d c f g d g e 1253 | v v w u v v v 1254 | d e c f f g 1255 | p q o p p 1256 | v v y z w x x 1257 | o t s u q r p 1258 | 3 z 1 0 2 3 1259 | 4 3 6 5 8 7 5 4 4 9 1260 | d 9 b c 8 b a e 1261 | g e g f f 1262 | n n l m 1263 | n o p p r r q 1264 | c f b a 9 d e 1265 | a 9 a 8 8 9 1266 | y x w w 1267 | z v y z x w y y 1268 | 2 4 3 2 1269 | t s r t 1270 | c a b 8 9 c c 1271 | y z x 1 w v y 0 1272 | 7 8 6 5 3 9 4 1273 | s t u s s u s 1274 | u s u t s u 1275 | b a 8 6 9 6 c b 7 1276 | x w w w y 1277 | 4 5 6 6 4 4 5 1278 | 2 5 7 3 4 6 8 1279 | 6 9 5 7 8 1280 | 6 8 9 8 5 a 7 b 8 8 8 1281 | b a d d c 9 1282 | 0 1 y z 2 1283 | h j l i i k h h 1284 | 3 5 4 1285 | r p q n l m o 1286 | w u y u x v 1287 | 9 7 6 6 a 8 1288 | 5 6 5 7 5 1289 | 0 w y 0 0 1 x z v 1290 | e d f d d 1291 | 4 4 1 3 0 2 4 1292 | j k l k k 1293 | 8 6 7 5 2 3 4 1294 | z y 0 y 1295 | 9 6 7 4 5 8 a 1296 | e d e f g c 1297 | y y x y w w 1298 | q p q u p s v v t r 1299 | 8 b a 9 7 1300 | q n o p r p 1301 | a b c 1302 | 7 4 5 8 6 8 8 1303 | y 0 z 1 y y 2 1304 | n k j m k l k 1305 | d b a 9 9 c 1306 | t t x t u v w 1307 | o q n r p 1308 | t y z v x w u 1309 | o n p m q 1310 | r p q p o s r 1311 | q r s 1312 | 2 0 1 1313 | 7 9 5 b 9 a 6 8 1314 | f e e e d 1315 | x u v w y v 1316 | i j j k 1317 | p q r q q 1318 | 9 8 7 9 1319 | n k l j j m l 1320 | v t u t 1321 | 2 3 1 2 0 4 1322 | v q p t r u r q s 1323 | 9 c 6 b a 8 a 8 6 6 7 1324 | k m i j l 1325 | c c c d c e b a 1326 | l o m p n j k o 1327 | 7 3 6 5 4 8 9 1328 | r q r s 1329 | r s v w u w q t v u 1330 | f l h i k g j k 1331 | v v t u 1332 | b f g d c e a 1333 | i g h i 1334 | 1 1 0 2 2 1335 | l m n n l m 1336 | p p p q o p 1337 | 0 2 1 2 4 3 4 1338 | 6 8 8 5 9 4 5 7 a 1339 | o q n r o p p 1340 | s r p m q o n 1341 | d e e d d c 1342 | a d 9 d 8 a 7 b 8 9 c 1343 | w y x 1344 | m q q q m n p s r o 1345 | d f d e f e e 1346 | g h d e i h j f h 1347 | i g g h 1348 | 9 9 7 8 1349 | m q n q o l r p l m 1350 | k h j g i m l 1351 | p t u q s v r 1352 | q q r q p p 1353 | x t w u s x r v v v 1354 | 1 2 0 1355 | o q r n p r p 1356 | i e f d c c h i g 1357 | d f e d f d d 1358 | e d f c c g d 1359 | g g e g f 1360 | y v w t u t s x v y 1361 | n o m l k 1362 | m g k j l i g h 1363 | j k l j 1364 | f j h i k e g 1365 | p q q o o 1366 | r v u s r t u 1367 | 2 3 4 3 2 1368 | 9 a b 1369 | k k l j 1370 | 4 7 5 1 3 2 6 1371 | h h i j 1372 | b c f e d 1373 | l k m j i 1374 | m n o 1375 | p s q r t 1376 | i m k l j 1377 | v u s y t w x s 1378 | a c b c a 1379 | 2 3 4 4 2 1380 | 2 4 3 z 5 3 4 0 1 5 1381 | n p o 1382 | l i o k m l m j j n 1383 | g c f d e 1384 | v v x y u w 1385 | v w x x x 1386 | 7 a 9 9 6 8 9 5 4 1387 | 1 0 4 2 1 y z 3 1388 | r r u q r t s 1389 | h g g f h 1390 | y 1 0 x 0 z 1391 | t u v r s 1392 | k p m j o l n 1393 | 4 3 2 2 1394 | 7 7 8 9 1395 | a 8 7 9 6 a a 1396 | 7 4 3 5 6 4 1397 | 0 u w z x v y 1398 | 7 5 7 8 b 6 9 a a 1399 | g l j m j i k h 1400 | d b c d 1401 | q q q s t r u q 1402 | 5 7 9 6 7 8 5 5 1403 | i i j h j 1404 | 9 8 b 8 a 9 7 1405 | o l r q r p p p m n 1406 | d c e 1407 | u r t r t t o q s p 1408 | b d d c 9 a 1409 | 1 x 0 w y y z 2 1410 | b 9 a 9 1411 | u u v w s t 1412 | z y v x w 1413 | e b c c d 9 d a f 1414 | 4 3 1 2 5 1415 | 3 4 1 0 2 1416 | 8 8 7 8 9 1417 | v w u v 1418 | o r t p n s q s 1419 | g h e f d 1420 | l k j 1421 | 6 6 5 7 1422 | 7 c 8 b a 8 9 d 1423 | b e c g e c d f a 1424 | l p o m q n r o 1425 | 2 z 1 0 3 1426 | q t q p n o s r q 1427 | a e b d c c c 1428 | g k h j i i k 1429 | t s u v r 1430 | u u s t 1431 | v w r u v t t q s w 1432 | k m l n j l 1433 | x x x y x z 1434 | b d 7 8 a c c 9 1435 | h f g j i 1436 | d 9 c a f e a b 1437 | z 0 y 1438 | 2 4 2 1 3 5 3 1439 | 2 4 6 7 2 8 3 5 1440 | p q r q s p t 1441 | z 3 2 5 0 1 4 1442 | u w s v q u t v r 1443 | z 3 1 3 2 y x 0 1444 | 1 2 3 0 z 3 1445 | a b c b 1446 | p q p r 1447 | 5 3 4 5 1448 | z z 0 z 1 1449 | 9 6 5 6 b 8 7 a 1450 | k j l 1451 | 8 7 6 b c a 8 9 8 1452 | f g e 1453 | i l k k m i i m j 1454 | 3 4 2 1455 | e f g g d c 1456 | e f h i g h 1457 | h g h f 1458 | p v s v q u r v t 1459 | x s y v u t w 1460 | t s u t t 1461 | r q s r p t n r o o 1462 | w x u t v 1463 | n l m 1464 | n p l q r m o p 1465 | c b a 1466 | 9 6 5 8 7 1467 | g g h l f i j i k 1468 | l o l k l n p m q n 1469 | w q q t v u r w w s 1470 | o m n k j l j i j 1471 | m p o n o q l k 1472 | s p m r q m n o 1473 | i e j e d f h f j g i 1474 | 8 8 b 9 7 a 7 1475 | j h h j h i 1476 | z 1 0 1477 | b f c f e d 1478 | u x v w u y 0 z 1479 | d i g c i e e f h 1480 | y z x w 0 x x 1481 | z z z z y x 1482 | s r p o q 1483 | p n o n l k m q l n 1484 | 1 2 2 1 4 3 2 5 1485 | 0 z 0 1 1486 | 6 3 3 4 5 2 2 1487 | i k m l n o j i 1488 | i k j j 1489 | f g j j i j i h 1490 | 4 6 2 3 5 4 1491 | u s w w t v s s s 1492 | 7 5 6 5 1493 | k k j j l k 1494 | f e e g g 1495 | 7 7 7 8 9 a 6 7 1496 | c f a b e e d g 1497 | i g f j h j 1498 | f g e h d 1499 | 9 8 9 a b 7 1500 | 5 2 0 4 z 0 3 1 1501 | 0 y x w z x z 1502 | 0 3 2 1 5 z 4 1503 | p r p o q p n s t 1504 | f c g d f g e c 1505 | x w x y 1506 | 8 6 7 4 5 4 1507 | a b 9 d 8 7 c 1508 | p q t o r p u p s 1509 | o m l l q q p p p r n 1510 | x w v w 1511 | y x w 2 0 x z 1 1512 | a 7 8 d c b 9 c d 1513 | b f c e f h d g b g b 1514 | v w x v z y 1515 | u w x t y v s 1516 | g b d h f e d c 1517 | u s t t r q q 1518 | t v r u v w s x r 1519 | c b a f f e d 9 9 e 1520 | 5 3 1 z 2 4 5 0 5 1521 | r r q s 1522 | n i j k l m o 1523 | 6 7 5 8 4 4 4 1524 | 4 5 5 6 7 1 2 1 3 1525 | j l k j k 1526 | 7 a 9 7 8 6 1527 | k q o n k p m l o n n 1528 | j j j k l l 1529 | r n q m l p o n p 1530 | f f h f f g g 1531 | z y 3 0 4 1 2 3 1532 | n q m o p 1533 | b 9 a d b a c 1534 | h e i g f 1535 | z 1 0 1 y 2 0 1536 | 8 9 a 7 6 1537 | a 6 6 7 9 8 c b 1538 | p r s o q t u 1539 | 4 3 7 5 3 6 2 8 1540 | f c d e b a a f g f 1541 | 5 4 5 2 6 1 0 4 3 1542 | u r o t r s p s q q 1543 | 2 2 3 4 4 1544 | 5 z z z 4 0 4 3 2 1 1545 | w u s t v 1546 | y w 0 y x z 1547 | j k n l l m 1548 | l m n p o o 1549 | 9 a 8 a a a 8 1550 | a 9 b b a 1551 | q s w q t r q v u 1552 | r t q r p s 1553 | 0 4 6 2 5 1 3 1554 | r s x u v t w 1555 | x x w v 1556 | 7 7 6 9 8 a 9 5 6 a 4 1557 | b f c e d 1558 | z y x y z w y 0 1559 | s u t w v t 1560 | 1 0 z 4 3 2 5 1561 | g i h i 1562 | 3 5 4 4 1563 | s u w t v v 1564 | n l k m o 1565 | g i j m k h l 1566 | n i k l h j m k 1567 | d e d c f b c 1568 | 8 9 8 a 1569 | 1 6 1 3 2 5 4 0 1570 | q o r t p s n 1571 | c g e a a f d b c 1572 | 5 2 6 3 7 8 4 4 1573 | h f j i g 1574 | l n m m 1575 | 8 6 4 2 3 5 7 1576 | g d h f e d d 1577 | 3 2 2 1 1 1578 | 4 5 3 5 1579 | o j n k p l m p 1580 | v z 0 y 1 w x 1581 | l i l l g k h k j m 1582 | o p r q t u q s r 1583 | j h h i 1584 | f h i j g 1585 | x v v y w u 1586 | 5 1 7 5 6 4 2 3 1587 | 1 3 5 1 2 4 1588 | 4 9 7 5 8 6 4 4 a 1589 | r u u t v s u s 1590 | e g f 1591 | 8 9 6 7 a 6 1592 | 8 8 6 7 8 1593 | f c h i i e d h g 1594 | 3 3 2 6 0 0 5 4 1 1595 | h j f h g j f i 1596 | c b a a c 1597 | p r r s r q o 1598 | o p o o p p q 1599 | y 3 4 y 0 1 z y y 2 1600 | h f g i j g g 1601 | v r q u t s p v 1602 | b h c d e g f g 1603 | 8 6 7 4 5 9 3 8 1604 | 7 8 7 9 8 7 1605 | 5 3 5 2 1 4 1606 | t t s u s s t 1607 | v r u s t 1608 | o t r p n q s 1609 | d e c d 1610 | e d f c h g i 1611 | s s t s t q p r 1612 | 1 0 z 1613 | m n o n q p p 1614 | b c a g e c f d 1615 | h d j e g h f i j 1616 | y 1 z 0 y x 1617 | e h g i f 1618 | e d c d 1619 | o n m l k 1620 | 0 2 3 z 1 y 1 4 1621 | 3 4 2 3 5 2 1 1622 | 0 2 w w 1 x z y 1623 | g l k h h j i m i 1624 | o q p 1625 | q o n o m n m n p 1626 | 9 9 9 8 7 8 1627 | h i g g 1628 | n q r m s p o r 1629 | 5 6 7 7 1630 | q p r t s 1631 | i i e j g d j f h 1632 | 4 5 6 7 2 3 1 1633 | z w 0 x y 0 0 1 w 2 z 1634 | h f g g 1635 | c c e a b 9 d f c f a 1636 | 9 c a f d b e 1637 | a b a b 8 c 9 c b 6 7 8 1638 | n n l m n p o m m 1639 | v s u r t v 1640 | o r q m l q n m p 1641 | c b e b d c f e 1642 | 0 0 z 0 1 1 1643 | 5 6 4 1644 | t s u t 1645 | x z y x 0 1 z 1646 | z x w y v 1647 | 9 6 4 5 7 3 8 1648 | b 8 9 b b c a c 1649 | b a a c d 9 1650 | s t s u 1651 | d f c g a f e b e 1652 | t w w v w u y u x z 1653 | v t z w y x u t 1654 | m o o l p m n o 1655 | y 0 0 z 1656 | i k i j h l 1657 | k j j l n m l 1658 | g d f f g h e f 1659 | g a b g c f d e 1660 | n r o p s m q q m 1661 | z y x v w 1662 | e d f b c b d 1663 | k l l j 1664 | 4 5 6 6 2 2 2 8 8 7 3 1665 | g c h f e i g d 1666 | s q p r s o 1667 | a c b b 1668 | c e d h f h e g b 1669 | 2 3 6 4 5 1670 | t u t v 1671 | 4 7 a 5 9 6 8 1672 | t r s q q u 1673 | y v u x w 1674 | i h j k g 1675 | g f g f e 1676 | 7 9 8 9 9 1677 | p q r p p 1678 | w t v u s 1679 | w x z u x u v y w t 1680 | h j f e i e g d h 1681 | x w v y x z 1682 | t v x r s u w v w 1683 | t u u q s r 1684 | u w x z v z x 0 y 1685 | 8 a 9 9 7 6 a 1686 | 7 8 9 8 1687 | n q m l o k q p 1688 | 7 b 8 a 9 1689 | n m o 1690 | e c d b a 1691 | r s s t u v v s 1692 | t w x u v 1693 | m o m m n o 1694 | 3 5 4 1695 | o k n l m m k 1696 | v w v t t u x u 1697 | 9 9 a a 8 9 1698 | i k j 1699 | k n j l m 1700 | h h h g h g i g 1701 | d f e d e d f 1702 | p m n n l o o o o 1703 | l j k m n 1704 | 1 1 2 4 1 5 3 4 1705 | l m l j l n k 1706 | 9 8 a 9 1707 | h i j j i 1708 | 5 6 7 8 8 9 5 1709 | 6 8 8 7 1710 | x v w y 1 z 1 v 0 1711 | h g f h h h 1712 | u s y x w t v 1713 | l j k 1714 | c d d b b b a e 1715 | f j j h i g 1716 | 1 z 0 z 1717 | 9 8 a 9 1718 | p o p q r n 1719 | w z 1 x y 0 2 1720 | g h f 1721 | a b c 9 8 8 b 8 1722 | k j i 1723 | g d i f j h e 1724 | j h i 1725 | 7 7 9 9 7 7 8 8 7 1726 | 3 1 0 z 2 1727 | o n o m 1728 | a 9 b 9 1729 | h j k e f i g f 1730 | y x 2 x z 3 0 1 1731 | w y z 0 x w 1732 | 2 z 0 3 1 1733 | y w x z w x 0 1734 | b d d a c 9 c 1735 | 2 4 8 5 3 7 6 6 1736 | g h j h f i k l l 1737 | u u s t 1738 | q s t v u w w r 1739 | r u w q v s s t 1740 | 9 7 6 8 a 7 6 1741 | z 0 z 3 2 0 1 z 1742 | y 4 2 2 3 4 0 2 z 1 1743 | 9 9 8 7 b c 8 d d 7 a 1744 | f a 9 e d a c b 1745 | z w v y w x z 1746 | y w x 1747 | e g d c b f f h 1748 | r r s t u v 1749 | c a b a d e 1750 | g c g e f d d d 1751 | 0 0 0 1 1 2 1752 | l k m n j 1753 | k j m m n o j p l 1754 | o q p m o p q n 1755 | 0 z y 1 z 2 1756 | 5 3 4 2 0 6 1 1 1757 | t t s t u t 1758 | q p o u q r r t s 1759 | 6 3 8 2 8 5 7 4 7 1760 | k i k j 1761 | o n k k m n k l k 1762 | d 9 9 b a c 1763 | o m n 1764 | 8 4 6 6 7 5 1765 | 6 8 8 9 5 7 8 1766 | j f h g e j e j d e i 1767 | u u v w q t s r 1768 | 5 3 6 4 7 4 8 4 9 4 1769 | f g i g j h 1770 | 8 5 7 6 7 9 6 5 9 1771 | v z w v v x y v 1772 | u q t r o q p s 1773 | 9 b a a 1774 | b d b a e 9 c b f 1775 | 2 w w x 2 1 0 y z w 1776 | 4 2 4 3 4 3 1777 | p r r o r r s q 1778 | n o p 1779 | q o m r m s n p 1780 | f e d 1781 | d b e h c g f 1782 | 5 7 6 a 9 6 6 8 b 6 1783 | x z 0 z 1 y z 1784 | 2 z 1 y 0 z 1785 | 0 2 1 z y 1786 | 3 3 3 2 4 1 5 1 1787 | 1 0 6 2 5 5 3 4 1788 | i h k l m j g 1789 | b d c 1790 | s v t u v w t 1791 | d h g i f e c 1792 | v u w y x u 1793 | 6 6 7 5 7 7 1794 | 3 2 7 4 5 6 4 1 1795 | f e c b c d c 1796 | e f f d 1797 | a 7 7 7 8 b 9 1798 | r r r t r s r 1799 | p u s q r t q q v 1800 | j p k m j n m o l 1801 | b a a 6 5 9 8 7 1802 | g k l i i j h f l 1803 | 6 a 7 9 b c 8 1804 | y t z w u x t z v 1805 | 0 y w z x w 1806 | l n o l m p n p 1807 | p s p p r s o s q p 1808 | q p p p p r 1809 | n k j o l m j k p 1810 | k i k f j g h i l l f 1811 | n m o m m p m l 1812 | 4 6 7 5 7 5 8 1813 | g i h 1814 | d h f g e d b c 1815 | 9 a b e d e 8 c 1816 | 9 a b b a b 1817 | s q r v w t u 1818 | i o m k j n l 1819 | y v w y x y z 1820 | f h g f f f 1821 | l m l l k l 1822 | 8 7 7 7 9 9 7 1823 | i j d d h g f e 1824 | f i g h j h 1825 | v y w w x 1 w 0 z 1826 | 6 4 3 5 7 3 7 1827 | m i m l j k 1828 | a 9 9 a b 1829 | c e d f b e 1830 | q r p 1831 | u v u r t w x s u 1832 | b c d 1833 | a 5 8 5 7 9 4 6 1834 | d f e e c b 1835 | 6 5 5 4 1836 | y 0 x z 3 1 1 2 1837 | 7 3 4 7 6 7 5 1838 | 6 a 9 8 a 7 1839 | q v p r u t p s 1840 | 1 0 1 1 2 1841 | 4 4 4 3 2 1842 | c a c 9 b d c 1843 | j h g k l m i 1844 | v u u y w x 1845 | k l m n l o k 1846 | n r q q o s p t q 1847 | d c e a b e f c g 1848 | l j k k j 1849 | 8 a 6 9 6 9 7 9 1850 | 5 3 2 4 1 4 1851 | 1 0 0 1 z 1852 | a 8 b 7 9 9 7 1853 | j k i k g m j h g l 1854 | 4 0 1 3 3 1 4 2 1855 | x 0 z y w 2 y 1 1856 | m k h j g j i l l 1857 | 7 c a 9 b d 8 a 9 1858 | o p l m n l 1859 | l n j k n m m 1860 | 4 4 3 5 1861 | l i m j k 1862 | u t s r o s q p s 1863 | o o n m q p 1864 | k j i m n l h 1865 | r o s p q n o m 1866 | t r s t 1867 | 5 5 4 3 1868 | 0 w y z x y 1869 | n o m o q o k l p 1870 | q p r 1871 | k l h n j i m 1872 | 4 1 2 3 5 1 1873 | o r s q p s q r 1874 | 7 6 8 8 1875 | i f g e h 1876 | q o p q q o o 1877 | j k i k 1878 | y w u u x v 1879 | 6 7 2 3 4 5 8 8 1880 | o p p p n 1881 | p m q p p o n 1882 | r u v s t 1883 | r s w s t u s s v x 1884 | m n m m l 1885 | o p k o m o l j n 1886 | n o o m o 1887 | w x y 1888 | x x z x y 1889 | g g f h 1890 | s r q p o 1891 | x z 0 w y 1892 | e b d c h f g 1893 | i c e d f g h 1894 | a d 7 8 b c 9 a 1895 | p p o p q 1896 | 6 7 5 a 9 4 8 1897 | k g e f f i j h 1898 | 4 2 1 3 2 5 1899 | 3 4 3 7 5 6 6 6 1900 | o m n n 1901 | 2 z 3 1 0 1902 | 2 3 3 1 1903 | 3 2 4 3 2 1904 | c e d e 1905 | 7 5 6 7 1906 | n m o m o o n m 1907 | g c e b f d a 1908 | j m l k n l n n l 1909 | d d f e e 1910 | u w x x v t w 1911 | l j k j 1912 | k i n m h j j n l 1913 | 1 1 0 2 1914 | 6 a 8 c 7 8 b 9 1915 | 0 y z v 0 u 0 u x w u 1916 | 0 1 4 3 2 2 4 1917 | d c b a e 1918 | l h k j i 1919 | o p q p q t p n s r 1920 | i h g g g 1921 | 9 8 6 7 a 8 1922 | h g i j k j 1923 | m o l k k l m n 1924 | t p r t q t o q u s 1925 | c b f f d e 1926 | w v t u t x z y t 1927 | f d d f e 1928 | f h g 1929 | y t u x u z w v 1930 | f c e d 9 a b 1931 | b 9 b a b 1932 | a b a a a c a 1933 | x z 3 y 2 1 0 1 1934 | 2 3 0 0 1 0 4 4 1935 | f h c g d e i 1936 | g i k e j f h h 1937 | o n n m 1938 | 1 1 z 4 3 3 0 0 5 2 0 1939 | d e f 1940 | 0 z y y 1941 | 2 0 0 1 1942 | 5 a 6 7 8 8 9 b 1943 | s v v u w t 1944 | 5 4 3 5 1945 | 8 a 9 6 7 8 9 1946 | n p o p 1947 | t u s r q s v w q 1948 | e a c b c d c 1949 | b d c d d d b 1950 | h g j i h k 1951 | 0 1 z 2 3 1952 | n j m l k m m 1953 | w v u z y x y t 1954 | u x z z 0 y w v 1955 | d g c e f e d 1956 | 8 7 7 b c d 9 9 c a c d 1957 | 8 5 6 9 7 7 1958 | t w s x y y v u 1959 | y z z x 1960 | x z y 1961 | e d f b c c c 1962 | i j j f g g h 1963 | p t n o p t o n r r q n s 1964 | u s s t u 1965 | y v x u w 1966 | x z y 3 2 x 1 0 1967 | 7 9 5 8 6 1968 | 2 3 1 4 0 1969 | r s v w v x u t 1970 | 9 c c b e d a 8 1971 | r t u s r v u 1972 | m m m n l 1973 | b c b b d 1974 | i g h g j f h g 1975 | 3 6 7 1 2 4 5 1976 | 4 z 0 0 y 3 3 2 1 1977 | w x w y y 0 y z 1978 | 0 y z y z 1979 | m n l r q p o 1980 | p n o n o 1981 | w y x x 1982 | w y w x 1983 | 3 y z 1 4 0 2 1984 | m j m n j l o p k 1985 | 5 4 3 3 5 2 1 1986 | l m k 1987 | c g c e e d f 1988 | h j i i 1989 | g e c f g f b a d 1990 | n p s o s r m q 1991 | i g h g 1992 | r s t p q u v p 1993 | 3 y y 1 x 0 z 2 1994 | 1 1 3 2 1 1995 | f b d a e c b c g 1996 | x w v 1997 | c d b 1998 | s u r t x w v v 1999 | 3 z 0 2 z 1 2 2000 | r s u q q s p r o r t 2001 | -------------------------------------------------------------------------------- /data/toy/valid.txt: -------------------------------------------------------------------------------- 1 | v v x w 2 | 2 6 5 4 7 1 7 3 3 | 4 5 3 1 z z 4 2 0 z 4 | 7 7 7 5 6 5 | r r q p 6 | 9 7 8 7 7 | o n m o o m 8 | p n n p o 9 | v u u s v t v w 10 | z 0 y z z y y 11 | 7 9 7 6 a c 8 b c 12 | 1 z 0 1 3 2 13 | t u r u s v u 14 | o p q q q 15 | f e d d e d 16 | h f e b g d b b c 17 | i h f j j k g e 18 | s t q p r t 19 | o m l k o m n 20 | 5 7 6 8 b a 9 21 | 4 3 3 y 0 z 1 2 22 | j k l l 23 | u x w t r x v t s 24 | r q t u r t q s 25 | 9 8 7 8 26 | 0 3 2 z 0 2 1 3 2 z 27 | j g f i h 28 | 4 3 1 4 7 6 5 2 29 | s w v t x x y u 30 | t u r p s o q 31 | 1 0 2 32 | o o p n p 33 | 2 8 3 5 7 6 4 3 34 | 0 w x z y y 35 | q r p q s o m n o 36 | t q p o n r r s t 37 | l i m j m k 38 | 8 6 8 a 7 9 39 | j e f h g i k j 40 | f e d h e f c g g e b f 41 | 1 2 4 0 2 3 42 | 7 6 5 4 5 5 4 8 9 3 8 6 5 43 | g j h g f i 44 | z 1 y x w 2 z 0 45 | i l f h j g h k k 46 | 9 a 7 8 b 47 | 8 b e 9 a d c 48 | 4 2 3 3 2 49 | k k k i j 50 | z 0 1 51 | t o n r p q s 52 | v t w u w x 53 | d e c c 54 | v w u w 55 | g h d f h h e 56 | 0 1 4 4 3 2 57 | c a a b 58 | 6 8 4 a 8 9 9 a 5 7 59 | f g g g f h 60 | q m r q n o s p 61 | 2 3 z 0 1 62 | u v t x w w s y 63 | 8 6 6 9 5 5 7 64 | 7 5 3 4 8 6 9 8 7 3 65 | h j j e f k g i 66 | g g f e 67 | 3 0 z 1 3 2 68 | g g i h 69 | c e d 70 | 5 4 5 2 7 1 3 6 71 | 6 7 6 6 4 5 3 72 | 0 2 4 4 3 1 73 | g i f f i e h 74 | n o o m n 75 | h j i 76 | l l o k n m l n 77 | x v w 78 | 0 1 0 z 79 | d d g g f c e 80 | 1 5 3 0 4 2 0 z 81 | 4 0 3 6 2 1 5 82 | x w s y t v u 83 | j k j l k n m 84 | r n n p m r o q s 85 | g k k i h j 86 | 1 4 2 2 2 5 3 87 | c b a b b a 88 | 6 a a 9 7 8 5 4 89 | 3 2 4 7 6 8 4 5 8 5 8 90 | d e c g e f 91 | z 2 0 1 y x w 92 | m o n l p p 93 | 6 6 8 7 8 6 94 | f d e f 95 | t u z x v y u w 96 | t v v w u x y x s y 97 | 5 3 6 0 4 1 2 5 98 | 2 y 1 y 1 4 z 0 z 3 1 99 | d f e f 100 | r r r t s 101 | c b a c 102 | j f i h g 103 | 5 7 7 6 104 | o l j k n i m o 105 | m p o m n l 106 | x 0 y z 1 y 107 | 7 3 5 4 3 1 7 2 3 6 108 | j n i o l n k m 109 | a b b 9 b 110 | z 0 y 0 111 | w x v 112 | e f i h g 113 | c a a a b c 114 | 0 z w x y z 115 | h g i h 116 | p s p p q r t 117 | w v x v u x y w 0 y v z v 118 | m l k m 119 | d b e a c d 120 | q r p 121 | b 8 6 9 5 a 7 a a 122 | m m i j k l 123 | 5 5 2 4 3 2 6 124 | 5 4 5 6 125 | z y z 0 0 126 | 1 5 2 4 6 3 7 1 127 | n r q o q p m l r 128 | w v t s u 129 | k i j 130 | m h i g h j h k l 131 | n q k o l p m q 132 | q o p p o p p 133 | c f b e f d d 134 | u y x u z w x z 0 v 0 135 | 6 8 7 6 8 136 | c d d e b d c a 137 | c 7 a 8 b 9 d 138 | w s u v t 139 | y x 1 w 0 z v 140 | v t u t 141 | k k h k l i j 142 | a 7 b 7 9 8 143 | t v u u v u 144 | t s r q q o p u t 145 | 3 2 3 1 146 | c d e e 147 | x y z z 0 y w 148 | l m k o m n 149 | e f e e g 150 | g g h g g i g 151 | t o n s q p r s 152 | u w t v w s s 153 | o r q o s u p t 154 | 2 3 4 4 155 | b d 9 a c b d 156 | 9 7 8 7 157 | 3 0 0 0 1 2 1 z 158 | f e f d 159 | e f d f e 160 | g h e d f g b c 161 | f g f h 162 | a b c d 7 8 b 9 163 | f e g h g e d 164 | v r s p u t t q 165 | b a 9 8 7 166 | d f d e 167 | 3 0 2 z 1 168 | u t r p u u o q s 169 | 3 1 1 2 3 170 | x w x x u v t s r 171 | d e c 172 | 2 0 1 1 173 | 9 b c e a d 9 f 174 | d c d e d 175 | n l j i m k o m 176 | p t q v p s u s r 177 | j h f d g e i 178 | u w v y s x t 179 | o n m 180 | f f e f g 181 | h d c e i f g 182 | q p o q p 183 | 6 5 a a 8 9 6 6 5 4 7 7 184 | g k j h i e i f g k 185 | t u r s q s 186 | w z v v y x w y 187 | u v v t v u 188 | 1 2 0 1 189 | c b e d f 190 | o s q r s t n p 191 | r s q 192 | s u q u w w v r u t 193 | e g f 194 | 1 1 2 2 3 195 | 6 5 8 4 7 196 | y y x w y 197 | d c d e c 198 | p p q r 199 | w z x z z 0 y y 200 | r x v u r s s w x t 201 | e f d e f 202 | 3 1 x z 2 y x 1 0 203 | r t q w v s r w u 204 | a a b 9 205 | i f f f d j e h g 206 | f g c b e d h 207 | l j n i m k o 208 | w w u v 209 | h b f d f c g h e 210 | s t r s s 211 | k l j i h 212 | l j l k l 213 | 3 1 3 2 3 214 | d c a 9 9 c d b 215 | k g h i j 216 | z 2 1 0 4 3 y 217 | u t u u u u v 218 | s v v t r u x w w u u 219 | v x s w u u y x w t x 220 | u x t y z v z z u w 221 | w x v t s r u 222 | 4 3 3 2 3 223 | 2 0 1 2 1 2 224 | 6 7 8 225 | 8 7 6 9 a 7 a 226 | 7 8 4 8 7 6 4 5 8 227 | w x x w x y 228 | 3 3 2 4 229 | l p o r m p n q 230 | 8 6 4 4 7 6 7 7 5 231 | 9 8 5 6 6 3 4 7 232 | s v t x s y u w w 233 | 7 4 7 8 6 5 7 234 | y z 0 x y 1 235 | a 8 8 9 236 | p s r n n q p t o 237 | 7 6 8 5 9 238 | s p v p r r t u t q 239 | 3 2 4 3 240 | q p r v q t u q s 241 | e b b f d c b f 242 | u t s s u 243 | a c b 244 | r r s q p o m n 245 | o m m n 246 | d a d e e d a b c 247 | s r t t s 248 | 0 x y z w 249 | b a c 250 | n o s t n r r p q 251 | l g m h i j k i 252 | r q n p m o n s 253 | f b e 9 c a d 254 | y 1 x 0 z z y z 255 | o m p l o p n p 256 | r s s p q o 257 | x 0 z y z x w 258 | 2 2 2 1 3 2 259 | 8 6 7 9 a a 5 6 4 260 | m o l k n l p n j 261 | k k k j l 262 | o q p p o 263 | f g g e f 264 | 4 1 2 0 1 3 4 z 5 265 | y w u y v v x 266 | 0 0 z 3 1 2 267 | k m j l n 268 | 0 4 3 1 0 0 2 269 | w u u x t v 270 | d b c 271 | t t s p r q t 272 | b b a a a 9 273 | c 8 d a 9 b e 274 | 6 5 4 275 | p q u t r q s t o 276 | o q p q q 277 | u x v y w 278 | y x y s t w t v t u 279 | b e c d f 280 | 8 a 9 8 7 6 281 | s t s u u 282 | z y v 0 x w u 283 | h i j 284 | j g i k j h 285 | o r u p s q t 286 | m m l k 287 | h g i 288 | l j j k 289 | o r q p l m n 290 | d c e 291 | n r p q o p n 292 | q q r p 293 | x v z y 1 1 x 0 w 294 | s r q q 295 | g h j k g i 296 | z y x 1 0 297 | o p r s q s 298 | f h i d e e g c 299 | q t u p r s v 300 | 7 8 6 7 9 5 5 4 3 301 | 3 5 2 1 1 4 3 2 4 302 | y z x 303 | 0 2 4 3 1 2 5 5 4 6 304 | g e f 305 | 9 8 b a c 9 8 306 | i k k k k j 307 | 9 8 8 a 9 a 308 | 1 z 3 4 4 2 0 5 309 | t t s u u 310 | y 2 0 1 z 311 | z 1 y x 3 3 3 0 z 2 312 | 0 0 y z 313 | 7 8 7 6 314 | b 7 b a 6 8 5 b 9 315 | l k m i j h k g 316 | 7 6 7 7 5 317 | r s p q o 318 | 5 6 7 4 1 3 1 4 2 319 | q t s q r p 320 | r q q r q s 321 | 7 6 9 4 a 8 5 322 | 5 4 3 3 1 2 3 323 | f d e d f 324 | 8 a 7 b b 9 325 | j k i k 326 | j f i g h 327 | 1 2 0 328 | e d h g h j f e i 329 | x z y 330 | 3 5 z 0 1 2 2 z 4 331 | y z z x z 332 | w s w t x r u v 333 | w y s x t v u t 334 | 8 9 9 7 335 | j m p k n o l 336 | 6 5 7 7 337 | r t q s p 338 | 2 y 3 4 0 1 z 339 | b 9 f a e d c d a d 340 | d c h e g b d d f b 341 | h f g i g j 342 | k i k l h j 343 | r u u w q v s t 344 | c a b c b 345 | b 8 a a 7 9 7 346 | s v x t u v y w y 347 | h f j i g 348 | 3 4 9 6 6 9 8 5 7 7 349 | 3 1 2 350 | c b e d d e f c 351 | 8 9 a 352 | 2 1 4 1 0 3 353 | u w v 354 | z 0 1 y 2 2 2 y 355 | x w x y x 356 | 1 1 3 2 357 | p q p t s p q r p 358 | 6 7 5 7 4 8 8 359 | n o m p k l q 360 | f f g f h 361 | n o p p 362 | k m n l o 363 | 2 4 4 3 2 364 | 1 4 6 3 5 2 6 4 0 365 | w y x z v 366 | u w v r x t s 367 | d d g e h f d h 368 | v x w 369 | u t u s r v 370 | i h l j h h k j m n 371 | 1 z 0 z 2 0 3 372 | 3 9 4 5 7 8 6 373 | t q s v u w r r 374 | 5 4 6 5 375 | f d d e 376 | t t v u 377 | p p p m p m l p o n 378 | o m p l n m l 379 | c b a 380 | 2 3 0 4 1 6 5 381 | g f f e f 382 | 5 8 4 6 7 5 8 383 | e f e f d e 384 | s v r q t t u p 385 | l m r q p n n o 386 | s r t 387 | a 9 a b 388 | n k m i j h m l m 389 | w v w u v 390 | 3 0 1 1 4 2 391 | 6 8 6 4 5 4 7 392 | l k m l k i i j h g 393 | w y x y 394 | s q q r q t u s t 395 | c d c g a b c e a b f 396 | x v w w 397 | 2 2 3 4 2 398 | 7 8 9 399 | i h j i 400 | f d e f e 401 | a d d c a b 9 402 | t v w w x x u 403 | m n l l l m 404 | p n r o q o o 405 | s t w u v s v 406 | 3 5 2 4 1 2 407 | v x v v w x v v 408 | 1 3 4 0 5 2 z 409 | s r t u q s r s r 410 | b b a c b 411 | w x v z y u t 412 | x t w w w v w s u r r 413 | y 1 0 z 2 4 z 3 414 | w 1 0 y 1 w y 2 x z 415 | i n m k j l n h i 416 | m n o 417 | a b b c 418 | x y z 419 | z x y x 420 | e f d d e 421 | c b d e a d f g 422 | x y w 0 z x 423 | l m o o n k l 424 | j f g g i h 425 | a a a e b a c d 426 | 8 9 7 8 a 8 b 427 | 3 1 0 2 3 4 428 | d b b c d 429 | 3 4 2 4 5 1 1 3 430 | l p q n m q q o q r o 431 | 9 8 7 432 | g h i i g 433 | k g f f j l h i 434 | u t s 435 | 4 5 3 5 436 | o n s q n r p t 437 | p o q m p n m 438 | o o n q o p n m p p 439 | h j h i 440 | n m k j o l i 441 | u w t u w v s 442 | j m i j i k l 443 | 4 4 3 3 4 2 444 | s r q t q u 445 | w v y w v x u w y 446 | u u v w 447 | n l p o o m m 448 | j l k 449 | i l j l k m l 450 | 3 2 1 2 1 451 | u y v w z 0 x 452 | 1 4 3 2 2 0 4 2 0 453 | g i j h g f k k h l 454 | 1 z 2 3 y 0 4 0 455 | v u s w r x t r 456 | v t p q s r u s 457 | 9 c b d 8 e a 458 | z 4 2 3 0 y 1 459 | y x 1 z 3 0 2 x 460 | p u r s q v t 461 | y z 1 v x y 0 w v 462 | 7 7 6 5 5 463 | m o n 464 | z z 0 2 3 1 2 465 | 9 8 7 b a c d a 466 | j h j h i 467 | l l k m n k o n k 468 | i n k o j m n l 469 | 1 z 1 0 z 470 | x 2 3 y 0 1 z 471 | h g h i m l j k 472 | j e f k i h e g f 473 | 6 9 8 a 4 7 5 a 4 474 | 1 6 0 3 2 5 4 475 | n m l l 476 | 9 d a c a b a 477 | t p p q r s 478 | 1 7 6 4 2 6 3 1 1 5 479 | x y w z w 0 480 | 1 0 3 2 z 481 | o q q p o 482 | d b e c a 483 | v y z w y z x x 484 | b d c e f c c 485 | 0 1 y 2 1 z 3 x y 486 | z 1 0 0 487 | w z y x x w 0 488 | 2 z 0 z 1 3 489 | g j f h i 490 | 2 y 1 1 z 0 491 | l l k j m j i 492 | f e d 493 | d d b b c 494 | m i m h m n j l k 495 | a b a c b c 496 | f e g h i 497 | 4 5 3 2 4 6 498 | i g h f j f e d 499 | d c d e e g f d 500 | v v u w u 501 | a 9 c 9 9 8 b 502 | s u v x w v t r w 503 | 0 z 3 2 1 2 3 0 504 | r q r q s 505 | q t r r p v s u u 506 | k g j i j h 507 | y y 0 y y x 1 0 z 508 | 5 3 4 509 | b c 9 8 a b 8 8 510 | q u r s t t 511 | r o s q p q 512 | b f e e d e c 513 | g j e e d f i i h 514 | 5 4 3 3 5 515 | c b d 9 e 8 a d c 516 | q o p 517 | 4 6 5 518 | 9 c a 8 8 b 8 519 | f f g e f 520 | 2 z y x 3 1 0 0 521 | g h f f i j f 522 | 7 8 8 9 6 a 9 523 | 4 6 7 5 3 524 | e f d b b d b d c 525 | e f d f d 526 | d b b c 527 | 4 6 5 5 528 | v w v u 529 | 1 0 z w x 0 2 y 530 | q r s u t 531 | v z w x 1 1 y 1 0 532 | e f d b f c 533 | u s t q u r 534 | f c c g e d 535 | z 1 2 0 y 536 | y 1 1 x w 0 x x z y 2 537 | 7 6 7 b 5 7 7 9 8 8 a 538 | z 1 0 4 2 2 y 3 539 | 7 8 5 5 8 9 6 8 540 | n m m m o m o 541 | 8 b c 9 7 6 a 542 | 4 1 2 1 0 3 543 | x y z v u u 0 w v 544 | k n l h i j m 545 | j l l j k 546 | 1 7 6 4 3 5 2 547 | z y x y 548 | 5 6 4 549 | 6 2 5 3 4 5 550 | v w u v w 551 | p t s r q q p q 552 | v w u w 553 | i h e f g 554 | a 6 9 7 b 8 5 a b 555 | z 4 1 0 3 2 y 556 | b 9 e c a 9 d f e 557 | t u w s v 558 | 0 5 2 z 3 1 4 559 | f g h 560 | m l k i k j j 561 | u r t v w x s 562 | 8 6 5 7 4 563 | z x y y x y 564 | g g h f 565 | y z x 566 | i m k j l 567 | r p q 568 | h g g f f h 569 | 8 4 7 8 6 5 570 | y u w y y v w y x 571 | w x x y 572 | b d c e a e 573 | 7 9 8 8 9 574 | g l k i f k j h 575 | c e c d 576 | c b b d 577 | n m k k o l k k 578 | 8 7 6 8 579 | 8 7 6 580 | 0 1 z y x 581 | 1 0 2 582 | 2 0 3 4 1 2 583 | 0 z 0 0 1 z 0 584 | e g i d i i h f j 585 | 3 7 5 2 8 3 5 6 4 586 | h i l j k g m 587 | a c f e e b g d b 588 | b d c f c d e 589 | f h h f g 590 | 7 6 6 7 7 5 7 7 591 | x w v 592 | i f h f j g 593 | w x x x v v 594 | 8 4 5 9 9 3 6 7 595 | y z z 0 y z 596 | g f e a c d b d 597 | 4 3 4 5 4 598 | k l l m l 599 | 3 0 4 1 2 0 600 | 3 8 2 7 7 8 6 5 4 601 | p n k l m q o l 602 | m n l o l p p 603 | o l p o l p m n 604 | g e d h h f 605 | x y v y w x 0 u z v 606 | s s r q t p 607 | f i g h j 608 | q r n r q o p 609 | 9 a 8 9 8 610 | 0 z 0 0 y 611 | 9 8 7 7 a 8 6 7 612 | g d e f f c 613 | 9 8 7 614 | 2 4 3 2 615 | c b 8 a c b 9 616 | c d b a 9 617 | 9 9 8 7 618 | d c e f b b 619 | w w w u v u w v 620 | c d b b c c 621 | c b f d e 622 | b 8 9 7 9 6 c 6 a 623 | a c b 624 | i n m l h m j k 625 | b g b e c f d e a 626 | b b 8 a a 8 8 5 8 6 9 7 627 | 2 3 2 4 628 | 6 4 8 5 9 8 7 a 629 | 4 4 5 3 630 | c c e d f c b 631 | h k j e f i e g 632 | u t w t v x 633 | 5 8 7 3 4 2 8 5 6 5 634 | u v t v t v 635 | f d g a b d e c 636 | 4 3 2 6 3 5 637 | n o q m p r s 638 | w s v w x x t r u 639 | l m k k h g j i k 640 | 7 8 a 7 9 b c d a 641 | 3 3 1 3 2 2 642 | i j k m l l 643 | 3 2 0 1 5 4 z 644 | j l g f k g h i 645 | z 2 1 z 3 0 z 0 z 1 646 | j l j i k j i l m 647 | 6 8 a 9 7 4 5 9 9 7 648 | 9 7 c 9 6 a 8 b a 7 649 | z 0 1 0 650 | 5 4 6 8 2 6 3 7 651 | k f i g e j e h e 652 | a 8 7 a 6 9 653 | e d c e c d 654 | 8 a b c c 9 a 655 | v u s t r 656 | 8 6 7 657 | k i l g h m j h 658 | w 0 y z v x 1 659 | p j k l o n m l p 660 | j g h e f k i 661 | m m o n 662 | g f c h b g e d f 663 | w y y z x v z 664 | z x 0 1 y 3 1 2 665 | s p o r p q 666 | a d c 8 c b a 9 e e 667 | 6 8 6 7 668 | q p r p p 669 | t v u u 670 | j h e f g d i 671 | 9 b a c 7 6 b b 8 672 | 7 a 7 8 b 9 9 b 673 | p o t n q r s q r 674 | l m m j k n 675 | 7 3 5 4 6 8 3 9 676 | 0 1 1 1 1 y z z 1 x z 1 677 | 5 5 6 7 678 | i e f g h h 679 | 4 3 4 2 3 680 | e d f e f 681 | v w z y y x y 682 | f h g g 683 | h f g 684 | f g h 685 | f g d c i h e 686 | o s n p q r m o 687 | d e c e a b 688 | 1 1 z 0 689 | m o n r s p q 690 | f d b e f c 691 | 8 a 7 9 a 5 6 b 692 | 2 6 4 7 1 5 3 693 | b c g d f g c f e h h 694 | h i j i 695 | h i l j g k m i 696 | z 1 0 3 4 y 2 697 | y v u y u w x 698 | 4 z 0 5 2 5 1 3 1 699 | g f j h i f 700 | j p l o n k m l 701 | c d b d c d e a 702 | p r r q 703 | i j h 704 | 8 6 7 7 705 | b 8 7 8 8 a 8 9 706 | z 1 0 z z 707 | n o o m 708 | f f g f e 709 | t v u w s 710 | 1 y z 1 z z 0 x x 711 | 2 2 4 7 2 3 6 6 1 5 712 | g h h f 713 | b a f d c e g 714 | g e f e g 715 | y 0 1 1 z x 1 0 716 | p q s o r o 717 | m l m m k k m m 718 | i h l k l j i 719 | l p p l m n o 720 | p o m j k l k j n l 721 | d f g h e 722 | z z 2 1 0 3 z 723 | 2 4 4 3 724 | 0 1 y z x z 725 | d b c 726 | u u v u u t 727 | 1 x z 2 w y 0 1 w 728 | c f e d b c 9 a 729 | s s t q t u r t 730 | v z 0 w u v v y x 731 | 0 5 4 3 1 0 3 0 6 2 1 3 732 | 5 a 9 b 7 7 8 8 6 9 733 | 8 a a 9 a 8 734 | n s p q t s r s o 735 | h g j i f g f i 736 | w v u w 737 | t v x w u 738 | 7 5 6 6 5 739 | q m p s p r o n s p 740 | g k j i h 741 | e f f d 742 | 7 9 8 8 7 743 | n q o r p m s 744 | 4 5 0 1 6 2 6 3 745 | j i g h f 746 | w y 0 x z 747 | m i l k j 748 | g h k k i j 749 | c f h e g b c d g c 750 | l m k l l n n m j o p 751 | e b d 8 a b c 9 9 b 752 | u y v z x w 0 753 | s t r u v u 754 | e d e b a c 755 | b 5 9 7 6 a 8 756 | a 9 c 9 6 8 b 6 7 b 757 | 6 9 8 7 7 5 7 758 | q n p q m o o 759 | o r n o q p t s 760 | 3 5 9 6 8 5 7 8 4 7 761 | d g f c e e g b a a 762 | 0 y 2 z 1 763 | h i g g 764 | 8 9 8 b a c 765 | e d c e d d 766 | d c a 9 c c b d 767 | j l i m k m 768 | t v u v v w s 769 | 1 0 2 770 | d e e d f 771 | 2 2 0 2 1 772 | 4 5 4 z 2 z 1 0 3 4 1 773 | 8 e b 9 b c d a 774 | 6 8 7 6 775 | l m n n 776 | q l n l r r m p o 777 | 0 z 2 1 3 1 1 778 | i k e g j h f k 779 | v u t v w s w 780 | d i e h c f g 781 | 6 6 2 4 1 7 3 5 7 782 | k n m p l o q 783 | s v w y x w t u 784 | 0 0 1 z 2 y 1 785 | s v t r u r u s v 786 | v u t v 787 | j k l k 788 | u t v 789 | w x t v u 790 | k o o m o n l 791 | p q n k q p o k m l 792 | g f f d e c 793 | t r s 794 | 3 5 5 6 4 6 6 2 795 | x w v v w 796 | 4 6 5 2 6 2 8 3 7 797 | l n m p o p q p k 798 | y 0 z 2 y 1 2 799 | 3 2 5 8 7 4 2 6 800 | h h e i j f g i k 801 | p n l q k o n m n 802 | j m k l n j n n 803 | 9 a 8 6 7 6 804 | o p n 805 | 5 7 4 6 3 3 3 806 | g g f e g 807 | v u s q r t p 808 | n l m m 809 | m j k l i 810 | j k l 811 | p o n 812 | x y w w x x x 813 | b c e b b b b a b b d 814 | 6 7 8 815 | d c c e 816 | c 9 b a a d 817 | a a 8 9 818 | 2 0 1 z z 1 3 3 819 | 9 5 9 6 4 3 9 5 5 7 8 820 | 4 5 3 3 821 | m m l p n o j k n 822 | t s o p t r r q s u u 823 | c e c f i h i i g i d 824 | 8 a 9 8 8 8 825 | s u x w v v t y s w 826 | p o p o n 827 | o m m m n m 828 | e e d c a e b 829 | d 8 9 8 a c 7 b c 830 | a a a 5 8 9 7 a 6 b 7 831 | r u v t s 832 | o m n m m n m 833 | p p q p r 834 | 9 7 8 7 6 5 b a 9 835 | n q o r m m m p l 836 | i h f c g d e 837 | 6 4 7 5 3 838 | 2 4 5 6 3 2 839 | r u s t q 840 | o m n 841 | p m n n q n o 842 | c a d f b e g 843 | 6 7 5 3 4 4 844 | e c c d 845 | 4 6 7 8 8 5 846 | a b c 9 c d 847 | h i g i 848 | x r u w s t w v 849 | t p p v p q v u r s 850 | 6 4 5 851 | 0 2 2 1 852 | n p q s p m o r 853 | z y x z x z w 0 x 854 | n m n n n l 855 | f g e f e 856 | v t u r s 857 | f e c g d d 858 | 4 1 3 2 6 5 4 0 859 | g e h d f 860 | l m k m l 861 | 6 8 8 7 6 4 5 862 | p n o m q 863 | a 5 9 8 6 b a 7 864 | 2 4 3 865 | 7 b 9 b 8 a b b 866 | a 9 8 7 9 c 6 b 867 | r p u o t q s 868 | 1 0 3 3 z 3 2 z 869 | 7 b 8 c 6 7 a 9 9 870 | w 1 z x z 0 v y 871 | 7 6 8 b 7 a 9 5 b 872 | 9 4 7 5 8 4 3 6 873 | 6 8 5 7 5 9 6 6 874 | q r p 875 | c a b a a 876 | 0 z x z z 0 y w z 877 | 7 8 7 6 878 | h m k l j i n 879 | f e i g h 880 | c d e e c a 8 9 e b 881 | b 8 7 b b a 9 882 | q s r r s 883 | f f d e 884 | u z v y x t w 885 | 1 z 1 x 1 0 1 1 y y 886 | p q s r o q 887 | f d e e 888 | u v v z w x y t 889 | z y x 890 | c d e c 891 | 3 4 2 2 4 2 892 | b c d b 9 a b 893 | 3 2 y 1 0 1 0 x 3 z 894 | 2 3 2 1 895 | j g h k i g 896 | 4 7 6 5 7 3 897 | q p r q 898 | m p q m o n n m 899 | p o r r q s 900 | l k m 901 | 2 0 4 2 1 1 3 1 902 | w x w y w 903 | 4 0 1 z 3 2 y 0 904 | q p q p r q 905 | y y x y z 906 | z x y x 907 | z y x 908 | p n m m q o 909 | f a b e g c e d 910 | o q p s r 911 | z 2 0 3 1 912 | 6 7 6 3 5 4 913 | w y x w w w v z 914 | m r q p o n q l 915 | 7 8 9 9 916 | 9 d c a b 917 | g h e d f 918 | 7 8 9 9 7 9 7 919 | m l n m l n 920 | t v u 921 | 7 6 5 922 | a a 9 8 923 | 2 1 2 2 0 924 | z 0 1 z 0 925 | i j g i f l h j k 926 | e c d f c c g 927 | 9 d d 8 b 7 a c 9 928 | 4 0 3 1 2 929 | 5 a 7 7 6 8 9 4 930 | q o p n n r n 931 | l m m n l m 932 | q r r p q p 933 | w x w v 934 | 9 b 8 a 7 935 | b a b e c d 936 | b c d a 9 9 b b 937 | j k l m i 938 | 6 4 5 939 | r s r q 940 | q p q r 941 | m h j l i j n k 942 | c d e b b e a 943 | o k m k k p q n k o l 944 | h j g i f 945 | k j m h g i l 946 | v y x 0 w 0 1 z z 947 | h g i 948 | d b c a a e c 949 | r t p q s v u 950 | i j i h 951 | 4 1 3 5 2 2 952 | 7 6 5 6 953 | 9 8 7 9 954 | p l j m n m k o 955 | a 7 8 9 6 7 956 | z z 1 x 3 2 0 y 957 | n p p o n p 958 | e h g g g d f 959 | w z y v x z w 960 | e f e e g 961 | v w x 962 | w x y 963 | j j l j m j j k i 964 | a 9 4 5 6 7 8 a 9 9 965 | 9 a 8 9 9 966 | p n o p p 967 | n o m 968 | y z y w 2 x 1 y 0 969 | g f h j i k e 970 | 3 5 4 2 6 6 971 | n o m 972 | k l m k n p k j k o 973 | i j j l k h i 974 | 7 a a 9 a a 6 8 6 975 | q s r t s u 976 | o p r p q s 977 | j l k h h n m j i 978 | 7 9 8 979 | t t t r s 980 | d d d b c 981 | i k j h g f e 982 | c e b d f 983 | t v u r x s w r 984 | q o p u s t r 985 | f g h 986 | p u q v s t r 987 | 9 9 6 b 7 8 7 a c 988 | h j n k m l i 989 | w t s u v u s u 990 | d j h f i e g 991 | 8 5 8 6 4 5 7 5 992 | d d b d a c e g f 993 | z z 2 0 0 3 y 1 x 994 | 7 7 7 6 8 995 | w v t z x u y 996 | r t p u t v q s r 997 | q n r m p s m o 998 | 1 1 0 2 999 | y 0 0 z 1 x 0 x y 1000 | 4 3 7 5 6 8 9 1001 | a 7 9 9 6 6 8 1002 | o l n m p 1003 | a 9 8 a 9 1004 | k m l o l n k m 1005 | s u v t w v x v w r 1006 | g k h f k l j i 1007 | x 0 y w y z 1008 | q o p p q m n o 1009 | 0 0 z z y 0 1010 | p m o p n q s r 1011 | w v u 1012 | d e c 1013 | m k m m l 1014 | m j j j l k i 1015 | 2 0 1 1 2 2 1016 | q n n o o p o m 1017 | 6 5 7 4 7 8 5 1018 | i k j l j m 1019 | x v w y u v w x 1020 | n r p s t n q o 1021 | p l o n m m 1022 | c a d 9 9 d b 1023 | k j i 1024 | p n o 1025 | r t s 1026 | 1 3 6 4 5 2 7 4 1027 | s r q p o 1028 | s r q t u s w v 1029 | q t r q o n s p q 1030 | w u v w 1031 | 9 9 9 b a 9 1032 | c 7 d 9 8 a 8 b c 1033 | 4 3 2 1 0 1034 | k l m j n 1035 | r p q m l o n 1036 | 0 z y y z 1037 | h i f i g c f e d 1038 | 7 4 5 5 8 4 6 8 1039 | w x w y 1040 | r u t t q s 1041 | 6 4 5 6 5 1042 | e i d j h f d d h g 1043 | v x s s t w u r 1044 | z 2 1 1 z y 0 z 1045 | b a a 9 1046 | u v w w v 1047 | t v u t x w 1048 | i k h e f g f j 1049 | n o q p t r q s 1050 | 4 5 2 6 4 6 3 3 1051 | l m n j k 1052 | r s r q s 1053 | p p o q p 1054 | u x v z w 0 y v 1055 | c e d 1056 | g i j k h f e 1057 | s v w t t u 1058 | e g d g c f d 1059 | q m o p n 1060 | 6 8 7 8 1061 | 6 6 4 8 9 3 7 5 1062 | 2 1 y z 0 1063 | n m k l m o 1064 | s u t 1065 | 3 5 3 3 3 3 6 4 2 4 1066 | u v t w v x 1067 | d d c e f g 1068 | a 7 6 a 8 6 a 4 5 9 1069 | 1 1 1 2 1 0 1070 | d 9 a b e c 8 8 1071 | f f e f d f 1072 | z 1 z z 0 z 1073 | s w s t v u 1074 | a 7 9 6 b 8 c 1075 | c d c d e 1076 | r q p t s r r u v u u 1077 | l m k m m 1078 | z 1 0 1 1079 | 6 7 7 5 5 1080 | p o p m n l o m 1081 | g e g d f d d c 1082 | a 9 b b 1083 | q m r l o n r p n 1084 | h i g 1085 | 7 6 3 5 4 1086 | h h c e f d g i 1087 | 6 6 5 5 5 8 9 7 1088 | a 5 9 6 5 7 6 4 8 1089 | z 5 2 3 1 4 z 0 3 1090 | u v s v w v t 1091 | 4 4 7 3 4 5 6 1092 | 0 1 2 1093 | q o r p s 1094 | j l j k 1095 | o n p m l k q 1096 | q o p s n m r 1097 | 8 7 7 5 3 2 4 3 6 1098 | 2 4 3 2 3 3 1099 | d b 7 c d 8 a d a 7 a a 9 1100 | m o n m m 1101 | m n q p o 1102 | u t t v v 1103 | 6 5 5 4 7 3 1104 | j k m n l l 1105 | 5 7 6 7 1106 | j h m n l m k i 1107 | d c c b b d 1108 | g h g f 1109 | t v t t u 1110 | j k l j j l j 1111 | d a c c c b d 9 b 1112 | m k j n m l 1113 | 4 5 3 2 1 1114 | l l n l n l m 1115 | p r t q s 1116 | 3 3 2 4 0 1 1117 | u r s t v 1118 | 8 7 7 9 1119 | a b 5 8 6 9 5 7 1120 | s o p q r r 1121 | r o s r s p u q t 1122 | e d e f f 1123 | m n i l k h j 1124 | 5 5 4 5 6 5 4 1125 | y v w x u 1126 | 1 0 2 1 0 1 0 1127 | h g j f i k l 1128 | n j m k n k m m l 1129 | j h l i k k 1130 | b a b 9 8 7 5 6 1131 | u u x t y w s v 1132 | a e 9 f d 9 b b a c 1133 | c c e c d c 1134 | h g k f i k j l 1135 | p q s r p o 1136 | s t u u x w y v s 1137 | w v s u t v 1138 | 1 0 1 y x z 1 0 x 1139 | 8 6 7 4 4 5 9 3 1140 | 2 0 1 0 1141 | b e b a c b d c c 1142 | v x x x w 1143 | 9 7 8 7 1144 | s r p q t 1145 | d e c 1146 | t u u z v y u y x w v 1147 | v v u r s t x r w v 1148 | u t t v 1149 | c f d b e c b e 1150 | 0 1 5 5 5 4 2 z 3 1151 | o q p r n 1152 | 8 9 5 7 6 4 3 1153 | d c f d e e b 1154 | 2 4 0 z y 3 3 1 1155 | c a b b b 1156 | 6 4 5 7 3 5 5 1157 | 9 a d b c 1158 | 6 5 4 8 4 4 7 1159 | u u q r s t r 1160 | 9 3 7 5 7 8 4 9 6 1161 | o o q p p 1162 | 4 5 3 3 3 1163 | d e e f d 1164 | 0 y z 1165 | l l l k j 1166 | 2 1 3 z 0 1 1167 | m n m l 1168 | d e f f f 1169 | 4 5 7 a 9 6 8 1170 | f j j e g k f h i 1171 | r q p t s 1172 | s t r u v 1173 | u t y x x s v w 1174 | g h f h j i 1175 | n o o p n 1176 | q t s q r p 1177 | 8 c a 9 b 1178 | r p q o n t o s 1179 | v w x y v z v 1180 | x w x u v w t 1181 | z y x u v u t w z x 1182 | k g m h i j l g k 1183 | p s o r q q q 1184 | 7 8 9 a b 1185 | a c 9 b d b c 1186 | p n r o p q 1187 | l n m m 1188 | z y x y y y 1189 | l k m 1190 | p q r r s s r t 1191 | t s t s r 1192 | w w 1 2 y x z 0 1193 | m k k l k n o 1194 | x 1 z 3 0 y 1 z 2 1195 | r t s u v 1196 | 5 5 6 7 6 1197 | g g e f d c g 1198 | l n o m o k k 1199 | y z 0 w 1 x 2 1200 | z 4 4 1 3 0 y 2 4 1201 | e h f g d e 1202 | 0 1 y x z z 0 1203 | k k l j l 1204 | p s o p p q q r 1205 | 3 3 4 2 4 1206 | g e f f 1207 | l k i j g h f 1208 | l f j h k j i g h 1209 | 7 8 7 6 5 9 6 1210 | 9 c d a a b 1211 | b 7 a 8 9 7 1212 | 6 7 a 9 8 6 9 1213 | 6 8 9 8 7 a 1214 | v x v w y z 0 1 1215 | c c 7 9 6 6 a 8 b a a 1216 | q q p q r q 1217 | x y 0 2 2 z w 1 z 1218 | 6 a 9 8 7 9 b c 1219 | 8 d e 8 a b a 8 c b 9 1220 | q o o p 1221 | y y z w 0 1 v x 1222 | d e e e c 1223 | l n k j m o i 1224 | z 1 1 0 1225 | z 1 y 2 y 0 z 1226 | z 1 0 3 y 2 x 1227 | l m l k 1228 | a c 9 9 b 9 c d 1229 | y x w v z z 1 0 1230 | 2 0 z z 1 z y 2 1231 | n o o p 1232 | 9 a b 9 1233 | 4 5 3 6 4 7 4 1234 | u w v 1235 | 5 2 3 4 5 6 6 1236 | m l n 1237 | s s w u v t 1238 | u u v t 1239 | o q p p o 1240 | r u w q t s v 1241 | 6 6 3 2 5 6 4 1242 | r q s 1243 | 3 2 3 5 6 4 1244 | s r q r s p o 1245 | p p o o p n 1246 | 1 3 2 3 1247 | q s r u t 1248 | n k i i n o m j l 1249 | d b c 1250 | 8 5 7 6 9 1251 | 0 z 2 1 0 y 0 w x 1252 | d f h e g 1253 | q n m s o o q r q p 1254 | i i h g h 1255 | 4 2 2 6 5 3 7 8 1256 | 4 2 2 3 1257 | s q r q 1258 | n o p n n 1259 | n j n o o l k m p 1260 | s t u q r 1261 | f g h i e 1262 | d c b d 1263 | p q t s r q v u 1264 | i j h j 1265 | n p o o p 1266 | p o m q n 1267 | 7 4 4 5 5 8 6 1268 | s u r v t 1269 | o p p n o l m j k 1270 | f d g h e e f b c 1271 | q p p r p 1272 | 9 c a 8 b 8 1273 | w s t v q r u 1274 | l l l j l k 1275 | r u t t q s 1276 | 9 9 8 9 a 9 1277 | 1 3 3 3 2 1278 | b a 7 8 d 9 c 1279 | 3 4 2 6 5 2 1280 | w u y v y x w 1281 | k g j e f h i 1282 | e e d b c f 1283 | l o n n m k q n p q 1284 | t r q r s v u v w 1285 | q p r 1286 | 0 0 y 0 z 1287 | 1 1 z 0 1288 | 7 6 5 1289 | r m o n q s p 1290 | e d f 1291 | i m l i j k 1292 | d c c e 1293 | z z 0 z 1 1294 | y z z y 2 1 0 2 1 2 1295 | r q o n p n s m 1296 | e d e d c 1297 | e h d i c h g f 1298 | 5 2 6 2 3 4 1299 | 7 8 9 b a 1300 | t v w u r q s 1301 | n l m m o p n n 1302 | 1 2 5 4 3 4 1303 | 5 5 1 2 4 4 3 1304 | 5 4 6 4 1305 | 4 3 0 5 1 3 z 5 2 5 1306 | o p n o o 1307 | v t u s u r r 1308 | g h j f k k k l i 1309 | q r t o q n s p 1310 | 0 0 x 0 z w y 1311 | a c b 9 d d 1312 | j m k n l 1313 | 3 5 4 2 6 3 1314 | t u s s 1315 | x v x w x 1316 | z y 1 x x 0 0 1317 | k g i j h 1318 | 5 7 5 3 4 6 1319 | f d e e 1320 | x 1 0 w 2 y z 1321 | q r s 1322 | r p q q 1323 | m o p r n s q 1324 | s s u t q r u 1325 | h i d f g g c c e 1326 | 5 5 6 4 5 1327 | a b c c 1328 | g e h b d c f h 1329 | x w u t s v y 1330 | t t q u s p o r t u 1331 | o o n p r q 1332 | w u t s v 1333 | o m n m 1334 | q o n p m 1335 | k m l 1336 | u v v t 1337 | 5 2 1 4 1 3 1338 | 0 3 4 1 2 2 3 1339 | u u s q w v r t 1340 | x u v s w y w t 1341 | s u t u 1342 | f f b d c e 1343 | 0 y x z w 1344 | j g d i h e f 1345 | j k i h l 1346 | 0 6 1 0 4 3 2 3 5 1347 | z z 2 0 3 1 5 4 1348 | o l q k m n p 1349 | 0 6 1 5 4 3 2 1350 | y x v y t w u s 1351 | 3 1 4 2 4 4 4 4 4 0 1352 | s t s t r r 1353 | t w y s x u v v 1354 | c a e a b d 1355 | p v t s q r u q 1356 | a e b 9 d b 8 8 c 8 1357 | 3 0 2 6 4 1 5 6 1358 | s v t t u w 1359 | e d d f 1360 | f d e f d 1361 | 3 4 1 5 2 1362 | c d g c h f e i 1363 | t u v t r s 1364 | 6 8 6 8 8 7 7 1365 | j h h i 1366 | h g h g h h i 1367 | 7 8 6 7 1368 | e g h f i h 1369 | f c c f f d g g e i h 1370 | h e f i i g j k h h 1371 | x v w u y 1372 | 5 4 4 3 1373 | b b a c a 1374 | c e h g d f i e f e 1375 | y z 0 1376 | f i g f h e f 1377 | b a f e 9 b d c c 1378 | q q n n o p m 1379 | s t u s s 1380 | 1 0 z 1 1381 | t s r s 1382 | k l m k 1383 | f d e f f 1384 | 1 3 0 z 3 3 2 1385 | 4 4 2 2 3 4 1386 | x z 0 y w 2 z 1 1387 | h f d i h j e g h 1388 | x y x w w 1389 | p q o 1390 | 1 2 z 3 0 4 y 4 1391 | a 9 8 a 8 1392 | 7 7 7 8 6 1393 | m m n l m 1394 | l k m o n 1395 | y x x w 1396 | u x w u t v 1397 | 4 3 5 2 3 1 1398 | o p l m p k q q n 1399 | 5 3 6 6 4 7 1400 | a 9 6 7 a 8 a 1401 | l j k j 1402 | k l h j i 1403 | 6 6 5 6 4 1404 | 2 3 6 1 4 5 0 2 6 1405 | k j i 1406 | v x x x w 1407 | u w v w x v v y 1408 | 0 2 1 y z 1409 | i h i g g i 1410 | r s u r q p v t s s 1411 | 6 7 7 8 9 9 9 9 5 1412 | 8 9 a 8 9 1413 | r t t s r s r 1414 | j h j e i g f k 1415 | b d c 1416 | y 0 z 1 x 1 1417 | 8 6 7 1418 | 2 z y 0 2 0 1 z 1419 | d d c c b 1420 | 9 8 7 6 6 6 5 6 1421 | j k h l m m g g i 1422 | v s u v r s t s 1423 | l n n m 1424 | n l j m k m m 1425 | 8 c 8 9 9 a 7 d b 1426 | 6 6 4 5 3 6 7 1427 | a c b 1428 | r s r q 1429 | w v y u 0 w z x y 1430 | n o p 1431 | 7 8 4 5 6 1432 | f k h g l l j i g 1433 | c d d e d 1434 | a a b a 9 1435 | j l l k 1436 | i k m l i j j 1437 | 5 7 6 6 1438 | q u u s o t p q q r 1439 | h i g f f h j j k g l 1440 | c g b c d e h b f 1441 | 3 2 4 2 1442 | 7 8 8 6 6 6 6 1443 | x 2 3 2 1 z 0 y 1444 | 8 b 7 b 6 7 c a 9 8 1445 | z z v y x v w 1446 | c e b b a c d 1447 | u v v w t v x y z 1448 | 4 0 1 2 0 3 1449 | 7 a a b 9 8 1450 | a c c a b c 1451 | 1 3 3 5 2 0 1 6 4 6 1452 | 8 a 9 1453 | w v y z t u x 1454 | g g f e 1455 | f i d j g h e g 1456 | i g g g h h g 1457 | q o r n p 1458 | 6 9 9 7 7 8 5 1459 | 5 6 5 4 6 5 1460 | u q s p v q t r s 1461 | 1 0 2 y 1 z 1 1462 | u p q t r s o o 1463 | l k m l 1464 | x z 0 y x v 1 w x 1465 | 5 3 3 4 4 1466 | i l k h m j n 1467 | w z y w u u u t x v 1468 | x z y 1469 | 5 8 9 b 8 a 6 7 1470 | 4 5 2 6 3 1471 | v w 0 y 1 z x x 1472 | u u q t t r t u s s 1473 | z y x x 1474 | g f e d c f d d 1475 | j k n m l l p o 1476 | 1 0 1 2 2 1 3 z 1477 | k m l l 1478 | q r s o u t p 1479 | 3 4 2 4 3 1480 | 7 8 9 9 1481 | x v w x 1482 | p r q r p 1483 | n n q p o r 1484 | w u t t t x v 1485 | q s r p t 1486 | 8 7 6 7 7 1487 | h d c d e g h f i 1488 | s v u s r t 1489 | a 9 8 b c 1490 | f g g e g g 1491 | n m p l o q r 1492 | u t t v 1493 | 0 z y z 1494 | 0 0 0 y z 0 1495 | m l o n l o k 1496 | 8 a b c 9 c a 1497 | 8 7 9 9 1498 | j l m i k j m j 1499 | r n q p r o 1500 | g h h f f 1501 | 1 2 0 3 3 4 1502 | 1 0 1 z 1503 | y 3 2 z 0 1 y 4 1504 | 6 a 7 8 9 a 1505 | s r p t q 1506 | y 0 1 w x z x w v w 1507 | y 1 z 0 1 x 1508 | 9 8 a a 8 1509 | t u q r p v p s 1510 | z 1 0 1 1 1511 | q o t r s u u o p u 1512 | a b b 9 a b 1513 | 2 0 w w x y 1 z 1514 | o n p o 1515 | d 9 c b f c a e d 1516 | v w u 1517 | 4 3 4 4 2 1518 | i l l k j k m 1519 | s u t 1520 | e b f d g c a 1521 | j g e f i k f i h 1522 | k h i j j l 1523 | i e g f h 1524 | n o p o o q m 1525 | p r s q m n q o m 1526 | 6 4 0 3 1 0 1 5 2 1527 | f g e e 1528 | e d d c e e 1529 | 1 4 1 5 5 2 3 1530 | a c b a c 1531 | 7 6 7 5 1532 | v u y u u w x u 1533 | m k l k k k k 1534 | h j g k i f e 1535 | w u x w y v 1536 | o q r r p n n p 1537 | g h f 1538 | f e d 1539 | 8 c 9 d a e b 9 1540 | n k m l j h i l 1541 | l n m l 1542 | 7 4 6 a 5 9 7 6 8 1543 | k l k j 1544 | k i l j j m l 1545 | t t u v s s r 1546 | q p t s t q r 1547 | 8 6 7 6 8 1548 | i l k m j 1549 | b 9 a 9 1550 | 2 0 1 1551 | l m j o k l p n l 1552 | 7 3 8 2 6 5 4 1553 | s q q q s r 1554 | d b d a b c e 1555 | e j f i h g d j 1556 | w r t u q u v r s 1557 | e d c e e 1558 | b 9 a 1559 | a 9 8 a 1560 | c h g b f e h d 1561 | 7 6 3 1 2 4 5 1562 | 8 a 9 8 8 1563 | 6 7 8 8 8 8 7 1564 | p o q s r o 1565 | l l m k j n 1566 | a 7 6 9 8 6 1567 | q r p 1568 | d e f 1569 | t s t u s t u 1570 | s u v t r x w 1571 | 2 4 3 1 1 5 2 0 z 1572 | a b 8 6 9 5 8 9 7 7 1573 | h g i d g e c f 1574 | d d c b a 9 c e 8 1575 | s t u s t 1576 | 0 z y 1577 | x y y w 1578 | a c b c c 1579 | l h k k i j j 1580 | u u t v u 1581 | 5 1 4 0 3 4 2 z z 1582 | j n j o l p k m j 1583 | p q o q 1584 | j k k l 1585 | x w t v u v 1586 | z x z w y 0 1587 | w v x y y y z y 1588 | 6 5 6 5 4 5 1589 | s s q r r p t 1590 | x y y z 0 3 2 1 1591 | z u v y x t w t 1592 | g e e f 1593 | f g e g 1594 | f e g e 1595 | p o s s n q o r t s 1596 | u t u u w z y v x 1597 | n m j i k o j l j 1598 | j j j l k j j 1599 | w x y y w x 1600 | 6 6 7 a 5 9 4 a 8 1601 | 9 b a b a b 1602 | h i f g k j l 1603 | o r q s p 1604 | e g f 1605 | f h f h g g 1606 | g i h h 1607 | c c d 9 b a 1608 | s q r 1609 | z x w y 0 y y 1610 | 1 z z 0 0 z 1611 | 1 2 3 3 1612 | a 9 a 8 1613 | b a 9 a 8 c 1614 | f g f f h f 1615 | x w v z y 1616 | o p p q o 1617 | e g g f g d h 1618 | 1 z 1 3 2 0 1619 | l k m i j m 1620 | e f g i h 1621 | w u u v 1622 | r s t s t 1623 | o o m n n 1624 | e g f e h d 1625 | x x 1 0 y 2 x 3 1 z 1626 | j j k k l 1627 | g e f i k j h 1628 | n k l p l m o q 1629 | 6 4 6 5 1630 | 0 z y 0 1631 | r w t q v t s u 1632 | s q t p r 1633 | 6 8 a 8 9 7 8 b c 1634 | g e d a d c b f e 1635 | p o p k l m q n 1636 | 2 1 4 5 3 5 4 1637 | c d f c c c b e e h d g 1638 | x 2 0 z 3 0 1 y 1639 | f g h g 1640 | 4 3 2 1 5 4 4 1641 | b a c d e 1642 | t u x y w s v 1643 | p m m n l o 1644 | k i g j h f e 1645 | v y t z y x w z u 1646 | r t s p q o p t u p 1647 | g e f k f j i k h 1648 | n q o m o p m 1649 | z 0 0 1 1 1650 | m q l o k p p n n 1651 | x z v w t y t u 1652 | q q s o m r n p 1653 | i k l j m i m 1654 | d f f e b c f 1655 | o n o m 1656 | 6 3 7 4 5 1657 | h h h i j 1658 | d a b c 9 8 e b 1659 | m h k k n i l j 1660 | 8 c c 7 9 d b c b a 9 1661 | o n p m j l k 1662 | s r s r q 1663 | d g e j h f i 1664 | e g e f f 1665 | r s u u q t o p r 1666 | v s s u t w w 1667 | w s t y x t u v 1668 | 4 7 6 3 5 1669 | x v w w 1670 | 3 1 2 y y z 0 x 1671 | i f g h j 1672 | h g f 1673 | 4 0 1 1 3 2 1674 | r s t s t t 1675 | a 9 9 8 1676 | e g i h f h 1677 | 7 6 8 1678 | 2 y 1 x 0 3 z 1679 | v u t w x y z v 1680 | s o r q s p o o 1681 | 8 a 9 8 a a 1682 | 2 1 1 5 6 7 4 4 3 1 1683 | 5 6 7 5 1684 | 2 0 x 1 y z x 3 y 1685 | r m s p q o n s 1686 | i d g j f h j e 1687 | r l p o q o m n m 1688 | 0 1 z z 1689 | 8 9 a b 8 7 8 1690 | x w y w t x u s v 1691 | m p q n p o 1692 | k k i j j j 1693 | r u v s x w t 1694 | 8 9 6 8 7 5 8 9 8 8 1695 | l k m k 1696 | u q s s t r 1697 | p o n 1698 | f f e e d 1699 | h f g 1700 | m k i l j 1701 | z x y u 0 v w 1702 | 0 2 2 w z w x y 1 1703 | g g e f i h 1704 | c d b h f g e 1705 | w x 1 v w z 0 y 1706 | b 7 a 8 9 1707 | b d e a f 9 c f 1708 | q u r p u u t s o 1709 | c i e h i g f d c 1710 | a b a 9 9 1711 | 1 0 6 3 5 4 2 1712 | c e b d h b g c f 1713 | h g i g i 1714 | o n p p 1715 | 9 a 8 8 1716 | 1 w x z z 0 v y 1717 | s q p r o 1718 | z 1 2 1 y z 0 1719 | w v x u y s t y t 1720 | r r q s 1721 | d c b g e f a 1722 | k i j k 1723 | 9 b b d b 9 d c a 1724 | q s p o q r 1725 | 0 2 1 0 1726 | t w x u w v x u 1727 | s u r u t t q 1728 | o q q r p s 1729 | y y 1 z 0 3 4 2 1730 | 0 y 0 x 0 z 1 0 1731 | 2 y y 1 3 0 x z 1732 | q p o o 1733 | f b g c c d h e f h 1734 | x 0 1 2 z y 3 2 1735 | f d g b h b c h e 1736 | n m l 1737 | 5 4 3 2 1 1738 | t r r s t 1739 | s v s r u t 1740 | x x 0 y z u v w 1741 | a c 8 c b 9 1742 | h m h g k l j i 1743 | 8 7 6 6 1744 | r q p q 1745 | g c f d c e c 1746 | d e g h e h d f 1747 | 1 2 y 4 1 3 z 0 1748 | 6 8 7 7 6 1749 | n m j n n l k 1750 | n o r p s t q p r 1751 | w w 0 z x 0 y v 1 1752 | 6 7 5 b 8 9 a 1753 | i j j k l k i h k j 1754 | d a e b a c b 1755 | 6 5 7 1756 | a e b d f d c d g 1757 | r q t u q s 1758 | a 8 b 9 c 1759 | d d d f d c b e d d 1760 | k e f k f h g i j 1761 | r q p t r n n n n o s t 1762 | e f g h f f i f 1763 | p p n m o q 1764 | y z 2 x x 0 w x 1 1765 | d d c d b d 1766 | j k l h m i g 1767 | 0 y z v x w 1 y x 1768 | b c d d 1769 | 0 2 3 1 4 4 3 1770 | j p n m o k l 1771 | l m k 1772 | a a b c 1773 | 2 6 4 3 5 5 3 1774 | 5 5 5 7 7 7 6 1775 | x z 0 y 1 1776 | c e b d f b a g 1777 | a f d c e b g 1778 | 9 b b 9 a 1779 | c c b a c 1780 | w w t u t u s v 1781 | v w x 1782 | p q m q o n o 1783 | q q p q o 1784 | x 1 0 y z 0 0 1785 | 6 2 1 5 6 0 3 6 5 4 2 4 1786 | i c d f h e g 1787 | w s u t u s v v 1788 | j l m h m m g j k i 1789 | f e d c g 1790 | 2 1 0 z 2 3 0 1791 | q o p 1792 | b d c 9 a d 1793 | 0 w x x z z y v 1 z 1794 | x x v w y z x 1795 | e i k j g f h e 1796 | 2 4 2 5 6 2 3 1797 | j k m i k n l n h 1798 | 7 a 9 8 b 8 1799 | d c e c b a 9 b 8 1800 | w y v z x 1801 | r u w v s t x 1802 | z z 1 x x 0 y 1803 | m m k i i l j 1804 | 9 8 a 6 7 6 7 1805 | a g d f b e c 1806 | j h i f g j 1807 | w x 0 z y 1808 | 6 7 9 8 a 8 1809 | 0 y z 2 1 2 2 1810 | 1 1 3 y z 2 0 4 3 1811 | c b a 1812 | q t u v r s r p 1813 | n m p q p o 1814 | 9 6 4 7 3 5 8 7 1815 | 5 6 2 4 3 6 1816 | h g e d e j f i 1817 | k l k m m 1818 | 0 4 2 2 y 4 y 0 1 0 1 2 3 z 1819 | j m l k k j k i 1820 | 3 4 5 5 1821 | c b a c 8 6 7 9 1822 | k j h g e f j i 1823 | k i m l n o k j l 1824 | v v v x w x u y 1825 | l l m k m 1826 | i k j m o l n 1827 | f g c e e h i d 1828 | o k l k k n m 1829 | c d c d d d b 1830 | 6 6 7 5 1831 | p o q p r n s r m 1832 | b c c c c d 1833 | 9 a a 7 6 7 8 1834 | r w q t w s q u v s 1835 | l i k m j j j 1836 | f g e e f 1837 | 9 7 9 a 8 6 7 1838 | q r u s t 1839 | 2 3 4 3 1840 | 9 8 e d a c 9 b 1841 | k i j k 1842 | 8 7 6 1843 | 8 c b 6 7 9 b b 6 a 7 1844 | a a e d a b d c 1845 | w y w x y y y 1846 | w 0 y y 0 x z 1847 | z 0 1 x 0 y 1848 | 1 3 5 4 5 2 6 7 2 1849 | l o l m p n q k 1850 | k o n l p j m 1851 | v v r s u t v 1852 | g e f c h b c d 1853 | c c b 8 a 9 6 7 1854 | b a b 9 1855 | k g h k i f l j 1856 | x u y t z w v 1857 | l h i l k n n j m 1858 | 2 3 1 2 3 1859 | u t t t v 1860 | n n m o 1861 | q r n p o 1862 | 0 1 0 z 1863 | y y x z 1864 | l h j l l h k k i 1865 | z 0 z 1 1866 | g g g d f f e d c 1867 | c a b b a 1868 | u w u v v 1869 | 4 1 3 2 2 5 1870 | 8 8 9 7 1871 | g f e a c f g b d 1872 | t u s r v v v 1873 | e d c d 1874 | f d e e g f c g 1875 | 3 5 4 1876 | 3 3 5 4 1877 | j l l j l k 1878 | g k h j j k l f i 1879 | p n o p 1880 | v x z y w 1881 | n m o o o n 1882 | 9 7 8 a 8 a b 1883 | z 0 1 4 2 3 5 1884 | v x w y z y y w 1885 | i f d e f h g c 1886 | t w w v x x u 1887 | 3 5 2 1 2 4 2 5 5 1888 | 1 2 z 2 y 0 1889 | b b c c d 1890 | g g l h f j i i k 1891 | b d c d d d d 1892 | v x w y z w w 1893 | 8 6 6 7 1894 | 8 a 9 a a 1895 | 2 3 6 5 4 1896 | y 3 1 0 4 2 z 1897 | i i k i j 1898 | p p o n s p s q m r 1899 | j f i e g d f e h 1900 | n n o n p 1901 | d e b c b b c f 1902 | h g i i i g f e 1903 | y 0 y z 1904 | 8 6 7 6 7 1905 | p m l r q n o r r 1906 | 6 5 4 4 1907 | y z x y 1908 | i g j h k g 1909 | l n o i m k j l 1910 | b d c d c d d 1911 | 0 y z 1912 | x 0 w z y x w 1913 | s o o u p q t q r 1914 | d g h e f h 1915 | 1 4 2 4 5 5 3 1 4 1916 | x y 0 1 z 1917 | e e e d e f 1918 | k l m l 1919 | p r q t s 1920 | h k k k i j g k 1921 | w x y v u y x 1922 | z 4 0 1 2 3 2 1 5 1923 | o n m k q p k l 1924 | o p n 1925 | p u s r v q t 1926 | z y v x w 1927 | h f k k g k e i j 1928 | i j h i j 1929 | w u v y x 1930 | 1 z 0 4 y 3 2 1931 | t t r s t 1932 | l h h i j j k l j 1933 | z z 1 0 1934 | m n o o 1935 | u u v u w 1936 | 4 1 0 6 5 2 2 3 2 1937 | a 7 9 6 7 8 1938 | 2 3 8 2 4 7 4 5 6 5 4 4 1939 | o p p o n 1940 | q p r s u o q t u o 1941 | 6 5 7 6 1942 | z y x y y y x 1943 | c b d e g f b h c f 1944 | p n n n o 1945 | 0 3 1 0 z 2 1946 | 9 9 5 7 8 6 1947 | p q r v t s u 1948 | v x u t w 1949 | u y z y v y t x z w 1950 | v u u s t r 1951 | a a c b d 9 c 1952 | e b f c e d b 1953 | 2 0 1 0 1954 | j i h i 1955 | j n l j l p o l k m 1956 | m l o k m n 1957 | v 0 u w u x y z 1958 | 4 4 6 7 5 8 6 5 6 1959 | 2 3 4 4 1960 | g l i j h l f k 1961 | 4 5 6 5 1962 | g h g i g 1963 | c d d c c b d 1964 | b 9 a 9 8 7 1965 | n l m o m k m q p 1966 | f e e e e g 1967 | x y y v y y w z w 1968 | a 9 6 c 8 7 a b 1969 | 1 y 2 z y 0 y z y 1970 | b a a a a 9 1971 | 3 2 3 1 3 1 1972 | s t u s 1973 | b 7 8 a 6 a b 5 9 1974 | c d d a a b e a 1975 | n t o t r p o s q 1976 | i g h i 1977 | o m m n 1978 | v x x x w 1979 | h f c g b e d 1980 | c c d e d 1981 | y v 0 u z x w 1982 | v u w w 1983 | 7 b d b c a 7 9 8 7 c 1984 | 8 6 a 9 b 7 c 1985 | t s t v x u x r w 1986 | w w y w x v v u 1987 | r q p s o r s 1988 | 7 9 8 6 9 a 8 8 1989 | i l i g k i h i j g m 1990 | n o k o l m m 1991 | l m k k k 1992 | u v w t t s 1993 | t s t r t 1994 | 6 5 7 9 8 6 1995 | g h i f e 1996 | 5 6 9 8 7 1997 | w v u w w 1998 | y y x v w u 1999 | r t s s 2000 | o l k m n p k l j n n 2001 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | from utils import pp 6 | from models import NVDM, NASM 7 | from reader import TextReader 8 | 9 | flags = tf.app.flags 10 | flags.DEFINE_float("learning_rate", 0.001, "Learning rate of adam optimizer [0.001]") 11 | flags.DEFINE_float("decay_rate", 0.96, "Decay rate of learning rate [0.96]") 12 | flags.DEFINE_float("decay_step", 10000, "# of decay step for learning rate decaying [10000]") 13 | flags.DEFINE_integer("max_iter", 450000, "Maximum of iteration [450000]") 14 | flags.DEFINE_integer("h_dim", 50, "The dimension of latent variable [50]") 15 | flags.DEFINE_integer("embed_dim", 500, "The dimension of word embeddings [500]") 16 | flags.DEFINE_string("dataset", "ptb", "The name of dataset [ptb]") 17 | flags.DEFINE_string("model", "nvdm", "The name of model [nvdm, nasm]") 18 | flags.DEFINE_string("checkpoint_dir", "checkpoint", "Directory name to save the checkpoints [checkpoints]") 19 | flags.DEFINE_boolean("forward_only", False, "False for training, True for testing [False]") 20 | FLAGS = flags.FLAGS 21 | 22 | MODELS = { 23 | 'nvdm': NVDM, 24 | 'nasm': NASM, 25 | } 26 | 27 | def main(_): 28 | pp.pprint(flags.FLAGS.__flags) 29 | 30 | data_path = "./data/%s" % FLAGS.dataset 31 | reader = TextReader(data_path) 32 | 33 | with tf.Session() as sess: 34 | m = MODELS[FLAGS.model] 35 | model = m(sess, reader, dataset=FLAGS.dataset, 36 | embed_dim=FLAGS.embed_dim, h_dim=FLAGS.h_dim, 37 | learning_rate=FLAGS.learning_rate, max_iter=FLAGS.max_iter, 38 | checkpoint_dir=FLAGS.checkpoint_dir) 39 | 40 | if FLAGS.forward_only: 41 | model.load(FLAGS.checkpoint_dir) 42 | else: 43 | model.train(FLAGS) 44 | 45 | while True: 46 | text = raw_input(" [*] Enter text to test: ") 47 | model.sample(5, text) 48 | 49 | if __name__ == '__main__': 50 | tf.app.run() 51 | -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | from nvdm import NVDM 2 | from nasm import NASM 3 | -------------------------------------------------------------------------------- /models/base.py: -------------------------------------------------------------------------------- 1 | import os 2 | from glob import glob 3 | import tensorflow as tf 4 | 5 | class Model(object): 6 | """Abstract object representing an Reader model.""" 7 | def __init__(self): 8 | pass 9 | 10 | def get_model_dir(self): 11 | model_dir = self.dataset 12 | for attr in self._attrs: 13 | if hasattr(self, attr): 14 | model_dir += "/%s=%s" % (attr, getattr(self, attr)) 15 | return model_dir 16 | 17 | def save(self, checkpoint_dir, global_step=None): 18 | self.saver = tf.train.Saver() 19 | 20 | print(" [*] Saving checkpoints...") 21 | model_name = type(self).__name__ 22 | model_dir = self.get_model_dir() 23 | 24 | checkpoint_dir = os.path.join(checkpoint_dir, model_dir) 25 | if not os.path.exists(checkpoint_dir): 26 | os.makedirs(checkpoint_dir) 27 | self.saver.save(self.sess, 28 | os.path.join(checkpoint_dir, model_name), global_step=global_step) 29 | 30 | def initialize(self, log_dir="./logs"): 31 | self.merged_sum = tf.merge_all_summaries() 32 | self.writer = tf.train.SummaryWriter(log_dir, self.sess.graph_def) 33 | 34 | tf.initialize_all_variables().run() 35 | self.load(self.checkpoint_dir) 36 | 37 | start_iter = self.step.eval() 38 | 39 | def load(self, checkpoint_dir): 40 | self.saver = tf.train.Saver() 41 | 42 | print(" [*] Loading checkpoints...") 43 | model_dir = self.get_model_dir() 44 | checkpoint_dir = os.path.join(checkpoint_dir, model_dir) 45 | 46 | ckpt = tf.train.get_checkpoint_state(checkpoint_dir) 47 | if ckpt and ckpt.model_checkpoint_path: 48 | ckpt_name = os.path.basename(ckpt.model_checkpoint_path) 49 | self.saver.restore(self.sess, os.path.join(checkpoint_dir, ckpt_name)) 50 | print(" [*] Load SUCCESS") 51 | return True 52 | else: 53 | print(" [!] Load failed...") 54 | return False 55 | -------------------------------------------------------------------------------- /models/nasm.py: -------------------------------------------------------------------------------- 1 | import time 2 | import tensorflow as tf 3 | 4 | from base import Model 5 | 6 | class NASM(Model): 7 | """Neural Answer Selection Model""" 8 | 9 | def __init__(self, sess, reader, dataset="ptb", 10 | batch_size=20, num_steps=3, embed_dim=500, 11 | h_dim=50, learning_rate=0.01, epoch=50, 12 | checkpoint_dir="checkpoint"): 13 | """Initialize Neural Varational Document Model. 14 | 15 | params: 16 | sess: TensorFlow Session object. 17 | reader: TextReader object for training and test. 18 | dataset: The name of dataset to use. 19 | h_dim: The dimension of document representations (h). [50, 200] 20 | """ 21 | self.sess = sess 22 | self.reader = reader 23 | 24 | self.h_dim = h_dim 25 | self.embed_dim = embed_dim 26 | 27 | self.epoch = epoch 28 | self.batch_size = batch_size 29 | self.learning_rate = learning_rate 30 | self.checkpoint_dir = checkpoint_dir 31 | 32 | self.dataset="ptb" 33 | self._attrs=["batch_size", "num_steps", "embed_dim", "h_dim", "learning_rate"] 34 | 35 | raise Exception(" [!] Working in progress") 36 | self.build_model() 37 | 38 | def build_model(self): 39 | self.q = tf.placeholder(tf.float32, [self.reader.vocab_size], name="question") 40 | self.a = tf.placeholder(tf.float32, [self.reader.vocab_size], name="answer") 41 | 42 | self.build_encoder() 43 | self.build_decoder() 44 | 45 | # Kullback Leibler divergence 46 | self.e_loss = -0.5 * tf.reduce_sum(1 + self.log_sigma_sq - tf.square(self.mu) - tf.exp(self.log_sigma_sq)) 47 | 48 | # Log likelihood 49 | self.g_loss = tf.reduce_sum(tf.log(self.p_x_i)) 50 | 51 | self.loss = tf.reduce_mean(self.e_loss + self.g_loss) 52 | self.optim = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(-self.loss) 53 | 54 | _ = tf.scalar_summary("encoder loss", self.e_loss) 55 | _ = tf.scalar_summary("decoder loss", self.g_loss) 56 | _ = tf.scalar_summary("loss", self.loss) 57 | 58 | def build_encoder(self): 59 | """Inference Network. q(h|X)""" 60 | with tf.variable_scope("encoder"): 61 | q_cell = tf.nn.rnn_cell.LSTMCell(self.embed_dim, self.vocab_size) 62 | a_cell = tf.nn.rnn_cell.LSTMCell(self.embed_dim, self.vocab_size) 63 | 64 | l1 = tf.nn.relu(tf.nn.rnn_cell.linear(tf.expand_dims(self.x, 0), self.embed_dim, bias=True, scope="l1")) 65 | l2 = tf.nn.relu(tf.nn.rnn_cell.linear(l1, self.embed_dim, bias=True, scope="l2")) 66 | 67 | self.mu = tf.nn.rnn_cell.linear(l2, self.h_dim, bias=True, scope="mu") 68 | self.log_sigma_sq = tf.nn.rnn_cell.linear(l2, self.h_dim, bias=True, scope="log_sigma_sq") 69 | 70 | eps = tf.random_normal((1, self.h_dim), 0, 1, dtype=tf.float32) 71 | sigma = tf.sqrt(tf.exp(self.log_sigma_sq)) 72 | 73 | _ = tf.histogram_summary("mu", self.mu) 74 | _ = tf.histogram_summary("sigma", sigma) 75 | 76 | self.h = self.mu + sigma * eps 77 | 78 | def build_decoder(self): 79 | """Inference Network. p(X|h)""" 80 | with tf.variable_scope("decoder"): 81 | R = tf.get_variable("R", [self.reader.vocab_size, self.h_dim]) 82 | b = tf.get_variable("b", [self.reader.vocab_size]) 83 | 84 | x_i = tf.diag([1.]*self.reader.vocab_size) 85 | 86 | e = -tf.matmul(tf.matmul(self.h, R, transpose_b=True), x_i) + b 87 | self.p_x_i = tf.squeeze(tf.nn.softmax(e)) 88 | 89 | def train(self, config): 90 | start_time = time.time() 91 | 92 | merged_sum = tf.merge_all_summaries() 93 | writer = tf.train.SummaryWriter("./logs", self.sess.graph_def) 94 | 95 | tf.initialize_all_variables().run() 96 | self.load(self.checkpoint_dir) 97 | 98 | for epoch in range(self.epoch): 99 | epoch_loss = 0. 100 | 101 | for idx, x in enumerate(self.reader.next_batch()): 102 | _, loss, e_loss, g_loss, summary_str = self.sess.run( 103 | [self.optim, self.loss, self.e_loss, self.g_loss, merged_sum], feed_dict={self.x: x}) 104 | 105 | epoch_loss += loss 106 | if idx % 10 == 0: 107 | print("Epoch: [%2d] [%4d/%4d] time: %4.4f, loss: %.8f, e_loss: %.8f, g_loss: %.8f" \ 108 | % (epoch, idx, self.reader.batch_cnt, time.time() - start_time, loss, e_loss, g_loss)) 109 | 110 | if idx % 2 == 0: 111 | writer.add_summary(summary_str, step) 112 | 113 | if idx != 0 and idx % 1000 == 0: 114 | self.save(self.checkpoint_dir, step) 115 | 116 | def sample(self, sample_size=10): 117 | """Sample the documents.""" 118 | sample_h = np.random.uniform(-1, 1, size=(self.sample_size , self.h_dim)) 119 | 120 | _, = self.sess.run([self.p_x_i], feed_dict={self.x: x}) 121 | -------------------------------------------------------------------------------- /models/nvdm.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | from base import Model 6 | 7 | try: 8 | linear = tf.nn.rnn_cell.linear 9 | except: 10 | from tensorflow.python.ops.rnn_cell import _linear as linear 11 | 12 | class NVDM(Model): 13 | """Neural Varational Document Model""" 14 | 15 | def __init__(self, sess, reader, dataset="ptb", 16 | decay_rate=0.96, decay_step=10000, embed_dim=500, 17 | h_dim=50, learning_rate=0.001, max_iter=450000, 18 | checkpoint_dir="checkpoint"): 19 | """Initialize Neural Varational Document Model. 20 | 21 | params: 22 | sess: TensorFlow Session object. 23 | reader: TextReader object for training and test. 24 | dataset: The name of dataset to use. 25 | h_dim: The dimension of document representations (h). [50, 200] 26 | """ 27 | self.sess = sess 28 | self.reader = reader 29 | 30 | self.h_dim = h_dim 31 | self.embed_dim = embed_dim 32 | 33 | self.max_iter = max_iter 34 | self.decay_rate = decay_rate 35 | self.decay_step = decay_step 36 | self.checkpoint_dir = checkpoint_dir 37 | self.step = tf.Variable(0, trainable=False) 38 | self.lr = tf.train.exponential_decay( 39 | learning_rate, self.step, 10000, decay_rate, staircase=True, name="lr") 40 | 41 | _ = tf.scalar_summary("learning rate", self.lr) 42 | 43 | self.dataset = dataset 44 | self._attrs = ["h_dim", "embed_dim", "max_iter", "dataset", 45 | "learning_rate", "decay_rate", "decay_step"] 46 | 47 | self.build_model() 48 | 49 | def build_model(self): 50 | self.x = tf.placeholder(tf.float32, [self.reader.vocab_size], name="input") 51 | self.x_idx = tf.placeholder(tf.int32, [None], name="x_idx") 52 | 53 | self.build_encoder() 54 | self.build_generator() 55 | 56 | # Kullback Leibler divergence 57 | self.e_loss = -0.5 * tf.reduce_sum(1 + self.log_sigma_sq - tf.square(self.mu) - tf.exp(self.log_sigma_sq)) 58 | 59 | # Log likelihood 60 | self.g_loss = -tf.reduce_sum(tf.log(tf.gather(self.p_x_i, self.x_idx) + 1e-10)) 61 | 62 | self.loss = self.e_loss + self.g_loss 63 | 64 | self.encoder_var_list, self.generator_var_list = [], [] 65 | for var in tf.trainable_variables(): 66 | if "encoder" in var.name: 67 | self.encoder_var_list.append(var) 68 | elif "generator" in var.name: 69 | self.generator_var_list.append(var) 70 | 71 | # optimizer for alternative update 72 | self.optim_e = tf.train.AdamOptimizer(learning_rate=self.lr) \ 73 | .minimize(self.e_loss, global_step=self.step, var_list=self.encoder_var_list) 74 | self.optim_g = tf.train.AdamOptimizer(learning_rate=self.lr) \ 75 | .minimize(self.g_loss, global_step=self.step, var_list=self.generator_var_list) 76 | 77 | # optimizer for one shot update 78 | self.optim = tf.train.AdamOptimizer(learning_rate=self.lr) \ 79 | .minimize(self.loss, global_step=self.step) 80 | 81 | _ = tf.scalar_summary("encoder loss", self.e_loss) 82 | _ = tf.scalar_summary("generator loss", self.g_loss) 83 | _ = tf.scalar_summary("total loss", self.loss) 84 | 85 | def build_encoder(self): 86 | """Inference Network. q(h|X)""" 87 | with tf.variable_scope("encoder"): 88 | self.l1_lin = linear(tf.expand_dims(self.x, 0), self.embed_dim, bias=True, scope="l1") 89 | self.l1 = tf.nn.relu(self.l1_lin) 90 | 91 | self.l2_lin = linear(self.l1, self.embed_dim, bias=True, scope="l2") 92 | self.l2 = tf.nn.relu(self.l2_lin) 93 | 94 | self.mu = linear(self.l2, self.h_dim, bias=True, scope="mu") 95 | self.log_sigma_sq = linear(self.l2, self.h_dim, bias=True, scope="log_sigma_sq") 96 | 97 | self.eps = tf.random_normal((1, self.h_dim), 0, 1, dtype=tf.float32) 98 | self.sigma = tf.sqrt(tf.exp(self.log_sigma_sq)) 99 | 100 | self.h = tf.add(self.mu, tf.mul(self.sigma, self.eps)) 101 | 102 | _ = tf.histogram_summary("mu", self.mu) 103 | _ = tf.histogram_summary("sigma", self.sigma) 104 | _ = tf.histogram_summary("h", self.h) 105 | _ = tf.histogram_summary("mu + sigma", self.mu + self.sigma) 106 | 107 | def build_generator(self): 108 | """Inference Network. p(X|h)""" 109 | with tf.variable_scope("generator"): 110 | self.R = tf.get_variable("R", [self.reader.vocab_size, self.h_dim]) 111 | self.b = tf.get_variable("b", [self.reader.vocab_size]) 112 | 113 | self.e = -tf.matmul(self.h, self.R, transpose_b=True) + self.b 114 | self.p_x_i = tf.squeeze(tf.nn.softmax(self.e)) 115 | 116 | def train(self, config): 117 | merged_sum = tf.merge_all_summaries() 118 | writer = tf.train.SummaryWriter("./logs/%s" % self.get_model_dir(), self.sess.graph_def) 119 | 120 | tf.initialize_all_variables().run() 121 | self.load(self.checkpoint_dir) 122 | 123 | start_time = time.time() 124 | start_iter = self.step.eval() 125 | 126 | iterator = self.reader.iterator() 127 | for step in xrange(start_iter, start_iter + self.max_iter): 128 | x, x_idx = iterator.next() 129 | 130 | """The paper update the parameters alternatively but in this repo I used oneshot update. 131 | 132 | _, e_loss, mu, sigma, h = self.sess.run( 133 | [self.optim_e, self.e_loss, self.mu, self.sigma, self.h], feed_dict={self.x: x}) 134 | 135 | _, g_loss, summary_str = self.sess.run( 136 | [self.optim_g, self.g_loss, merged_sum], feed_dict={self.h: h, 137 | self.mu: mu, 138 | self.sigma: sigma, 139 | self.e_loss: e_loss, 140 | self.x_idx: x_idx}) 141 | """ 142 | 143 | _, loss, mu, sigma, h, summary_str = self.sess.run( 144 | [self.optim, self.loss, self.mu, self.sigma, self.h, merged_sum], 145 | feed_dict={self.x: x, self.x_idx: x_idx}) 146 | 147 | if step % 2 == 0: 148 | writer.add_summary(summary_str, step) 149 | 150 | if step % 10 == 0: 151 | print("Step: [%4d/%4d] time: %4.4f, loss: %.8f" \ 152 | % (step, self.max_iter, time.time() - start_time, loss)) 153 | #print("Step: [%4d/%4d] time: %4.4f, loss: %.8f, e_loss: %.8f, g_loss: %.8f" \ 154 | # % (step, self.max_iter, time.time() - start_time, e_loss + g_loss, e_loss, g_loss)) 155 | 156 | if step % 500 == 0: 157 | self.save(self.checkpoint_dir, step) 158 | 159 | if self.dataset == "ptb": 160 | self.sample(3, "costs") 161 | self.sample(3, "chemical company") 162 | self.sample(3, "government violated") 163 | elif self.dataset == "toy": 164 | self.sample(3, "a") 165 | self.sample(3, "g") 166 | self.sample(3, "k") 167 | 168 | def sample(self, sample_size=20, text=None): 169 | """Sample the documents.""" 170 | p = 1 171 | 172 | if text != None: 173 | try: 174 | x, word_idxs = self.reader.get(text) 175 | except Exception as e: 176 | print(e) 177 | return 178 | else: 179 | x, word_idxs = self.reader.random() 180 | print(" [*] Text: %s" % " ".join([self.reader.idx2word[word_idx] for word_idx in word_idxs])) 181 | 182 | cur_ps = self.sess.run(self.p_x_i, feed_dict={self.x: x}) 183 | word_idxs = np.array(cur_ps).argsort()[-sample_size:][::-1] 184 | ps = cur_ps[word_idxs] 185 | 186 | for idx, (cur_p, word_idx) in enumerate(zip(ps, word_idxs)): 187 | print(" [%d] %-20s: %.8f" % (idx+1, self.reader.idx2word[word_idx], cur_p)) 188 | p *= cur_p 189 | 190 | print(" [*] perp : %8.f" % -np.log(p)) 191 | -------------------------------------------------------------------------------- /reader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import itertools 3 | import numpy as np 4 | import tensorflow as tf 5 | 6 | from utils import * 7 | from collections import Counter 8 | from nltk.tokenize import TreebankWordTokenizer 9 | 10 | EOS_TOKEN = "_eos_" 11 | 12 | class TextReader(object): 13 | def __init__(self, data_path): 14 | train_path = os.path.join(data_path, "train.txt") 15 | valid_path = os.path.join(data_path, "valid.txt") 16 | test_path = os.path.join(data_path, "test.txt") 17 | vocab_path = os.path.join(data_path, "vocab.pkl") 18 | 19 | if os.path.exists(vocab_path): 20 | self._load(vocab_path, train_path, valid_path, test_path) 21 | else: 22 | self._build_vocab(train_path, vocab_path) 23 | self.train_data = self._file_to_data(train_path) 24 | self.valid_data = self._file_to_data(valid_path) 25 | self.test_data = self._file_to_data(test_path) 26 | 27 | self.idx2word = {v:k for k, v in self.vocab.items()} 28 | self.vocab_size = len(self.vocab) 29 | 30 | def _read_text(self, file_path): 31 | with open(file_path) as f: 32 | return f.read().replace("\n", " %s " % EOS_TOKEN) 33 | 34 | def _build_vocab(self, file_path, vocab_path): 35 | counter = Counter(self._read_text(file_path).split()) 36 | 37 | count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0])) 38 | words, _ = list(zip(*count_pairs)) 39 | self.vocab = dict(zip(words, range(len(words)))) 40 | 41 | save_pkl(vocab_path, self.vocab) 42 | 43 | def _file_to_data(self, file_path): 44 | texts = self._read_text(file_path).split(EOS_TOKEN) 45 | data = [] 46 | for text in texts: 47 | data.append(np.array(map(self.vocab.get, text.split()))) 48 | 49 | save_npy(file_path + ".npy", data) 50 | return data 51 | 52 | def _load(self, vocab_path, train_path, valid_path, test_path): 53 | self.vocab = load_pkl(vocab_path) 54 | 55 | self.train_data = load_npy(train_path + ".npy") 56 | self.valid_data = load_npy(valid_path + ".npy") 57 | self.test_data = load_npy(test_path + ".npy") 58 | 59 | def get_data_from_type(self, data_type): 60 | if data_type == "train": 61 | raw_data = self.train_data 62 | elif data_type == "valid": 63 | raw_data = self.valid_data 64 | elif data_type == "test": 65 | raw_data = self.test_data 66 | else: 67 | raise Exception(" [!] Unkown data type %s: %s" % data_type) 68 | 69 | return raw_data 70 | 71 | def onehot(self, data, min_length=None): 72 | if min_length == None: 73 | min_length = self.vocab_size 74 | return np.bincount(data, minlength=min_length) 75 | 76 | def iterator(self, data_type="train"): 77 | raw_data = self.get_data_from_type(data_type) 78 | return itertools.cycle(([self.onehot(data), data] for data in raw_data if data != [])) 79 | 80 | def get(self, text=["medical"]): 81 | if type(text) == str: 82 | text = text.lower() 83 | text = TreebankWordTokenizer().tokenize(text) 84 | 85 | try: 86 | data = np.array(map(self.vocab.get, text)) 87 | return self.onehot(data), data 88 | except: 89 | unknowns = [] 90 | for word in text: 91 | if self.vocab.get(word) == None: 92 | unknowns.append(word) 93 | raise Exception(" [!] unknown words: %s" % ",".join(unknowns)) 94 | 95 | def random(self, data_type="train"): 96 | raw_data = self.get_data_from_type(data_type) 97 | idx = np.random.randint(len(raw_data)) 98 | 99 | data = raw_data[idx] 100 | return self.onehot(data), data 101 | -------------------------------------------------------------------------------- /toy_generator.py: -------------------------------------------------------------------------------- 1 | import os 2 | import string 3 | import random 4 | import numpy as np 5 | 6 | data_dir = "data" 7 | dataset = "toy" 8 | 9 | if not os.path.exists(os.path.join(data_dir, dataset)): 10 | os.makedirs(os.path.join(data_dir, dataset)) 11 | train_path = os.path.join(data_dir, dataset, "train.txt") 12 | test_path = os.path.join(data_dir, dataset, "test.txt") 13 | valid_path = os.path.join(data_dir, dataset, "valid.txt") 14 | 15 | words = list(string.ascii_lowercase + "0123456789") 16 | 17 | def get_neighbors(words, word, window_size=2): 18 | if type(word) == str: 19 | idx = words.index(word) 20 | elif type(word) == int: 21 | idx = word 22 | else: 23 | raise Exception(" [!] Invalid type for word: %s" % type(word)) 24 | 25 | if idx < window_size: 26 | ans = words[-(window_size - idx):] + words[:idx + window_size + 1] 27 | elif idx >= len(words) - window_size: 28 | ans = words[idx-window_size:] + words[:window_size + idx - len(words) + 1] 29 | else: 30 | ans = words[idx-window_size:idx+window_size+1] 31 | 32 | for _ in xrange(15): 33 | if random.random() < 0.1: 34 | ans.append(random.choice(ans)) 35 | 36 | random.shuffle(ans) 37 | return ans 38 | 39 | for path, size in [(train_path, 10000), (test_path, 2000), (valid_path, 2000)]: 40 | with open(path, "w") as f: 41 | for idx in xrange(size): 42 | pivot = np.random.randint(len(words)) 43 | window_size = np.random.randint(1, 4) 44 | 45 | f.write(" ".join(get_neighbors(words, pivot, window_size)) + "\n") 46 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import pprint 2 | import cPickle 3 | import numpy as np 4 | 5 | pp = pprint.PrettyPrinter() 6 | 7 | def save_pkl(path, obj): 8 | with open(path, 'w') as f: 9 | cPickle.dump(obj, f) 10 | print(" [*] save %s" % path) 11 | 12 | def load_pkl(path): 13 | with open(path) as f: 14 | obj = cPickle.load(f) 15 | print(" [*] load %s" % path) 16 | return obj 17 | 18 | def save_npy(path, obj): 19 | np.save(path, obj) 20 | print(" [*] save %s" % path) 21 | 22 | def load_npy(path): 23 | obj = np.load(path) 24 | print(" [*] load %s" % path) 25 | return obj 26 | --------------------------------------------------------------------------------