├── .gitignore ├── sous_le_pont_Mirabeau.png ├── sous_le_pont_Mirabeau_2.png ├── sample_model.py ├── README.md ├── main_uncond.py ├── data.py ├── data_raw2hdf5.py ├── utilities.py ├── main_cond.py ├── raccoon_extensions.py ├── model.py └── char_dict.pkl /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.pyc 3 | *.xml 4 | -------------------------------------------------------------------------------- /sous_le_pont_Mirabeau.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adbrebs/handwriting/HEAD/sous_le_pont_Mirabeau.png -------------------------------------------------------------------------------- /sous_le_pont_Mirabeau_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adbrebs/handwriting/HEAD/sous_le_pont_Mirabeau_2.png -------------------------------------------------------------------------------- /sample_model.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import theano 3 | 4 | 5 | import argparse 6 | import cPickle 7 | import time 8 | 9 | from data import char2int 10 | from utilities import plot_generated_sequences 11 | 12 | floatX = theano.config.floatX 13 | 14 | 15 | if __name__ == '__main__': 16 | # Parser 17 | parser = argparse.ArgumentParser(description='Sample handriting model') 18 | parser.add_argument('-f', '--f_sampling_path') 19 | parser.add_argument('-s', '--sample_string', default='Hello world') 20 | parser.add_argument('-b', '--bias', type=float, default=0.5) 21 | parser.add_argument('-n', '--n_times', type=int, default=4) 22 | 23 | options = parser.parse_args() 24 | 25 | print 'Unpickling... ', 26 | model, f_sampling, dict_char2int = \ 27 | cPickle.load(open(options.f_sampling_path, 'r')) 28 | print 'done.' 29 | 30 | n = options.n_times 31 | sample_string = [options.sample_string + ' '] * n 32 | 33 | cond, cond_mask = char2int(sample_string, dict_char2int) 34 | 35 | n_samples = len(sample_string) 36 | pt_ini_mat = np.zeros((n_samples, 3), floatX) 37 | h_ini_mat = np.zeros((n_samples, model.n_hidden), floatX) 38 | k_ini_mat = np.zeros((n_samples, model.n_mixt_attention), floatX) 39 | w_ini_mat = np.zeros((n_samples, model.n_chars), floatX) 40 | 41 | print 'Generating... ', 42 | beg = time.time() 43 | pt_gen, a_gen, k_gen, p_gen, w_gen, mask_gen = f_sampling( 44 | pt_ini_mat, cond, cond_mask, 45 | h_ini_mat, k_ini_mat, w_ini_mat, options.bias) 46 | print 'done in {} seconds'.format(time.time()-beg) 47 | p_gen = np.swapaxes(p_gen, 1, 2) 48 | mats = [(a_gen, 'alpha'), (k_gen, 'kapa'), (p_gen, 'phi'), 49 | (w_gen, 'omega')] 50 | print 'Printing...', 51 | beg = time.clock() 52 | plot_generated_sequences( 53 | pt_gen, mats, 54 | mask_gen, folder_path='./', 55 | file_name='a_just_now') 56 | print 'done in {} seconds'.format(time.time() - beg) 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Conditional handwriting generation 2 | 3 | ![img](https://raw.githubusercontent.com/adbrebs/handwriting/master/sous_le_pont_Mirabeau.png?token=AGnjokequVSx2LtbQW_UcGMmqoNg9kzHks5XGAtMwA%3D%3D "Guillaume Apollinaire") 4 | 5 | Reproduces Alex Graves paper [Generating Sequences With Recurrent Neural Networks](http://arxiv.org/abs/1308.0850). 6 | Thanks to [Jose Sotelo](https://github.com/sotelo/) and [Kyle Kastner](https://github.com/kastnerkyle) for helpful discussions. 7 | 8 | ### Requirements 9 | 10 | - Theano 11 | - Lasagne (just for the Adam optimizer) 12 | - [Raccoon](https://github.com/adbrebs/raccoon): version 98b42b21e6df0ce09eaa7b9ad2dd10fcc993dd85 13 | 14 | ### Generate the data 15 | Download the following files and unpack them in a directory named 'handwriting': 16 | 17 | - http://www.fki.inf.unibe.ch/databases/iam-on-line-handwriting-database/download-the-iam-on-line-handwriting-database: the files lineStrokes-all.tar.gz and ascii-all.tar.gz 18 | - https://raw.githubusercontent.com/szcom/rnnlib/master/examples/online_prediction/validation_set.txt 19 | - https://github.com/szcom/rnnlib/blob/master/examples/online_prediction/training_set.txt 20 | 21 | Add an environment variable $DATA_PATH containing the parent directory of 'handwriting'. 22 | 23 | ### Train the model 24 | Set an environment variable $TMP_PATH to a folder where the intermediate results, parameters, plots will be saved during training. 25 | 26 | If you want to change the training configuration, modify the beginning of main_cond.py. 27 | Run main_cond.py. By default, it is a one-layer GRU network with 400 hidden units. 28 | There is no ending condition, so you will have to stop training by doing ctrl+c. 29 | 30 | It takes a few hours to train the model on a high-end GPU. 31 | 32 | ### Generate sequences 33 | Run 34 | python sample_model.py -f $TMP_PATH/handwriting/EXP_ID/f_sampling.pkl -s 'Sous le pont Mirabeau coule la Seine.' -b 0.7 -n 2 35 | where EXP_ID is the generated id of the experiment you launched. 36 | 37 | ![img](https://raw.githubusercontent.com/adbrebs/handwriting/master/sous_le_pont_Mirabeau_2.png "Guillaume Apollinaire") 38 | -------------------------------------------------------------------------------- /main_uncond.py: -------------------------------------------------------------------------------- 1 | """ 2 | Outdated code. 3 | """ 4 | 5 | import os 6 | import sys 7 | 8 | from lasagne.updates import adam 9 | import numpy as np 10 | import theano 11 | import theano.tensor as T 12 | 13 | from raccoon.trainer import Trainer 14 | from raccoon.extensions import TrainMonitor 15 | from raccoon.archi.utils import clip_norm_gradients 16 | 17 | from data import create_generator, load_data, extract_sequence 18 | from model import UnconditionedModel 19 | from extensions import Sampler 20 | 21 | from utilities import plot_seq, plot_seq_pt 22 | 23 | floatX = theano.config.floatX = 'float32' 24 | # theano.config.optimizer = 'None' 25 | # np.random.seed(42) 26 | 27 | # CONFIG 28 | learning_rate = 0.1 29 | n_hidden = 400 30 | n_mixtures = 20 31 | gain = 0.01 32 | batch_size = 100 33 | chunk = None 34 | every = 1000 35 | tmp_path = os.environ.get('TMP_PATH') 36 | dump_path = os.path.join(tmp_path, 'handwriting', 37 | str(np.random.randint(0, 100000000, 1)[0])) 38 | if not os.path.exists(dump_path): 39 | os.makedirs(dump_path) 40 | 41 | # DATA 42 | tr_coord_seq, tr_coord_idx, tr_strings_seq, tr_strings_idx = \ 43 | load_data('hand_training.hdf5') 44 | # pt_batch, pt_mask_batch, str_batch = \ 45 | # extract_sequence(slice(0, 4), 46 | # tr_coord_seq, tr_coord_idx, tr_strings_seq, tr_strings_idx) 47 | # plot_seq_pt(pt_batch, pt_mask_batch, use_mask=True, show=True) 48 | batch_gen = create_generator( 49 | True, batch_size, 50 | tr_coord_seq, tr_coord_idx, 51 | tr_strings_seq, tr_strings_idx, chunk=chunk) 52 | 53 | 54 | # MODEL CREATION 55 | # shape (seq, element_id, features) 56 | seq_coord = T.tensor3('input', floatX) 57 | seq_tg = T.tensor3('tg', floatX) 58 | seq_mask = T.matrix('mask', floatX) 59 | h_ini = theano.shared(np.zeros((batch_size, n_hidden), floatX), 'hidden_state') 60 | 61 | model = UnconditionedModel(gain, n_hidden, n_mixtures) 62 | loss, updates, monitoring = model.apply(seq_coord, seq_mask, seq_tg, h_ini) 63 | loss.name = 'negll' 64 | 65 | 66 | # GRADIENT AND UPDATES 67 | params = model.params 68 | grads = T.grad(loss, params) 69 | grads = clip_norm_gradients(grads) 70 | 71 | # updates_params = adam(grads, params, 0.0003) 72 | 73 | updates_params = [] 74 | for p, g in zip(params, grads): 75 | updates_params.append((p, p - learning_rate * g)) 76 | 77 | updates_all = updates + updates_params 78 | 79 | coord_ini = T.matrix('coord', floatX) 80 | h_ini_pred = T.matrix('h_ini_pred', floatX) 81 | gen_coord, updates_pred = model.prediction(coord_ini, h_ini_pred) 82 | f_sampling = theano.function([coord_ini, h_ini_pred], gen_coord, 83 | updates=updates_pred) 84 | 85 | # MONITORING 86 | train_monitor = TrainMonitor(every, [seq_coord, seq_tg, seq_mask], 87 | [loss] + monitoring, updates_all) 88 | 89 | sampler = Sampler('sampler', every, dump_path, 'essai', 90 | f_sampling, n_hidden) 91 | 92 | train_m = Trainer(train_monitor, [sampler], []) 93 | 94 | 95 | it = 0 96 | epoch = 0 97 | h_ini.set_value(np.zeros((batch_size, n_hidden), dtype=floatX)) 98 | try: 99 | while True: 100 | epoch += 1 101 | for (pt_in, pt_tg, pt_mask, str, str_mask), next_seq in batch_gen(): 102 | res = train_m.process_batch(epoch, it, 103 | pt_in, pt_tg, 104 | pt_mask) 105 | if next_seq: 106 | h_ini.set_value(np.zeros((batch_size, n_hidden), dtype=floatX)) 107 | it += 1 108 | if res: 109 | train_m.finish(it) 110 | sys.exit() 111 | except KeyboardInterrupt: 112 | print 'Training interrupted by user.' 113 | train_m.finish(it) 114 | -------------------------------------------------------------------------------- /data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import h5py 3 | import numpy as np 4 | import theano 5 | 6 | 7 | theano.config.floatX = 'float32' 8 | floatX = theano.config.floatX 9 | 10 | 11 | def load_data(filename='hand_training.hdf5'): 12 | data_folder = os.path.join(os.environ['DATA_PATH'], 'handwriting') 13 | training_data_file = os.path.join(data_folder, filename) 14 | train_data = h5py.File(training_data_file, 'r') 15 | 16 | pt_seq = train_data['pt_seq'][:] 17 | pt_idx = train_data['pt_idx'][:] 18 | strings_seq = train_data['str_seq'][:] 19 | strings_idx = train_data['str_idx'][:] 20 | 21 | train_data.close() 22 | return pt_seq, pt_idx, strings_seq, strings_idx 23 | 24 | 25 | def create_generator(shuffle, batch_size, seq_pt, pt_idx, 26 | seq_strings, strings_idx, chunk=None): 27 | n_seq = pt_idx.shape[0] 28 | 29 | if shuffle: 30 | idx = np.arange(n_seq) 31 | np.random.shuffle(idx) 32 | pt_idx = pt_idx[idx] 33 | strings_idx = strings_idx[idx] 34 | 35 | def generator(): 36 | for i in range(0, n_seq-batch_size, batch_size): 37 | pt, pt_mask, str, str_mask = \ 38 | extract_sequence(slice(i, i + batch_size), 39 | seq_pt, pt_idx, seq_strings, strings_idx) 40 | 41 | pt_input = pt[:-1] 42 | pt_tg = pt[1:] 43 | pt_mask = pt_mask[1:] 44 | 45 | if not chunk: 46 | yield (pt_input, pt_tg, pt_mask, str, str_mask), True 47 | continue 48 | 49 | l_seq = pt_input.shape[0] 50 | for j in range(0, l_seq-chunk-1, chunk): 51 | s = slice(j, j+chunk) 52 | yield (pt_input[s], pt_tg[s], pt_mask[s], str, str_mask), False 53 | s = slice(j + chunk, None) 54 | yield (pt_input[s], pt_tg[s], pt_mask[s], str, str_mask), True 55 | 56 | return generator 57 | 58 | 59 | def extract_sequence(s, pt, pt_idx, strings, str_idx, M=None): 60 | """ 61 | the slice s represents the minibatch 62 | - pt: shape (number points, 3) 63 | - pt_idx: shape (number of sequences, 2): the starting and end points of 64 | each sequence 65 | """ 66 | if not M: 67 | M = 1500 68 | 69 | pt_idxs = pt_idx[s] 70 | str_idxs = str_idx[s] 71 | 72 | longuest_pt_seq = max([b - a for a, b in pt_idxs]) 73 | longuest_pt_seq = min(M, longuest_pt_seq) 74 | pt_batch = np.zeros((longuest_pt_seq, len(pt_idxs), 3), dtype=floatX) 75 | pt_mask_batch = np.zeros((longuest_pt_seq, len(pt_idxs)), dtype=floatX) 76 | 77 | longuest_str_seq = max([b - a for a, b in str_idxs]) 78 | str_batch = np.zeros((longuest_str_seq, len(str_idxs)), dtype='int32') 79 | str_mask_batch = np.zeros((longuest_str_seq, len(str_idxs)), dtype=floatX) 80 | 81 | for i, (pt_seq, str_seq) in enumerate(zip(pt_idxs, str_idxs)): 82 | pts = pt[pt_seq[0]:pt_seq[1]] 83 | limit2 = min(pts.shape[0], longuest_pt_seq) 84 | pt_batch[:limit2, i] = pts[:limit2] 85 | pt_mask_batch[:limit2, i] = 1 86 | 87 | strs = strings[str_seq[0]:str_seq[1]] 88 | limit2 = min(strs.shape[0], longuest_pt_seq) 89 | str_batch[:limit2, i] = strs[:limit2] 90 | str_mask_batch[:limit2, i] = 1 91 | 92 | return pt_batch, pt_mask_batch, str_batch, str_mask_batch 93 | 94 | 95 | def char2int(ls_str, dict_char2int): 96 | 97 | longuest_str = 0 98 | for s in ls_str: 99 | if len(s) > longuest_str: 100 | longuest_str = len(s) 101 | 102 | res = np.zeros((longuest_str, len(ls_str)), dtype='int32') 103 | res_mask = np.zeros((longuest_str, len(ls_str)), dtype=floatX) 104 | 105 | for i, s in enumerate(ls_str): 106 | for j, c in enumerate(s): 107 | res[j, i] = dict_char2int[ls_str[i][j]] 108 | res_mask[j, i] = 1 109 | 110 | return res, res_mask 111 | 112 | -------------------------------------------------------------------------------- /data_raw2hdf5.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Generate .hdf5 training and validation datasets from the raw files. 5 | Inspired from Alex Graves' pipeline. 6 | """ 7 | 8 | import time 9 | import os 10 | from os.path import join 11 | import re 12 | from xml.dom.minidom import parse 13 | import numpy as np 14 | import h5py 15 | import cPickle 16 | 17 | # Config 18 | dataset = 'training' 19 | 20 | # mean and std used for standardization of the data 21 | M_x = 8.18586 22 | M_y = 0.11457 23 | s_x = 40.3719 24 | s_y = 37.0466 25 | 26 | data_folder = join(os.environ['DATA_PATH'], 'handwriting') 27 | strokes_filename = join(data_folder, dataset + '_set.txt') 28 | h5_filename = join(data_folder, 'handwriting_' + dataset + '.hdf5') 29 | 30 | char_dic, _ = cPickle.load(open('char_dict.pkl', 'r')) 31 | 32 | 33 | def get_target_string(stroke_filename): 34 | 35 | ascii_filename = re.sub('lineStrokes', 'ascii', stroke_filename) 36 | ascii_filename = re.sub('-[0-9]+\.xml', '.txt', ascii_filename) 37 | try: 38 | line_nr = int(re.search('-([0-9]+)\.xml', stroke_filename).group(1)) 39 | lines = [line.strip() for line in open(ascii_filename)] 40 | return lines[line_nr+lines.index('CSR:') + 1] 41 | except (AttributeError, IndexError): 42 | raise SystemExit 43 | 44 | start = time.time() 45 | 46 | pt_seq = [] # all sequences concatenated 47 | pt_idx = [] # dim (n_seq, 2) beginning and end of each sequence 48 | str_seq = [] # all sequences concatenated 49 | str_idx = [] # dim (n_seq, 2) beginning and end of each sequence 50 | 51 | old_string = 0 52 | old_point = 0 53 | 54 | 55 | def read_file(file_path): 56 | pts = [] 57 | pre_pt = np.array([]) 58 | for trace in parse(file_path).getElementsByTagName('Stroke'): 59 | for pts in trace.getElementsByTagName('Point'): 60 | pt = np.array([pts.getAttribute('x').strip(), 61 | pts.getAttribute('y').strip(), 0], 62 | dtype='float32') 63 | if not len(pre_pt): 64 | pre_pt = pt 65 | diff = pt - pre_pt 66 | if np.any(diff >= 1000) or np.any(diff <= -1000): 67 | return False 68 | diff[-1] = 0 69 | pre_pt = pt 70 | pts.append(diff) 71 | pts[-1][-1] = 1 72 | 73 | return pts, get_target_string(file_path) 74 | 75 | last_pt_idx = 0 76 | last_str_idx = 0 77 | for ii, l in enumerate(file(strokes_filename).readlines()): 78 | # if ii == 10: 79 | # break 80 | file_path = l.strip() 81 | file_path = join(data_folder, file_path) 82 | 83 | if not len(file_path): 84 | continue 85 | 86 | res = read_file(file_path) 87 | if not res: 88 | continue 89 | 90 | pts, str = res 91 | pt_seq.extend(pts) 92 | str_seq.extend([char_dic.get(c, char_dic[' ']) for c in str]) 93 | 94 | next_pt_idx = last_pt_idx + len(pts) 95 | pt_idx.append((last_pt_idx, next_pt_idx)) 96 | last_pt_idx = next_pt_idx 97 | 98 | next_str_idx = last_str_idx + len(str) 99 | str_idx.append((last_str_idx, next_str_idx)) 100 | last_str_idx = next_str_idx 101 | 102 | str_seq = np.array(str_seq, dtype='int32') 103 | pt_seq = np.array(pt_seq, dtype='float32') 104 | 105 | print time.time() - start 106 | 107 | 108 | # Normalize 109 | pt_seq[:, 0] = (pt_seq[:, 0] - M_x) / s_x 110 | pt_seq[:, 1] = (pt_seq[:, 1] - M_y) / s_y 111 | 112 | 113 | # Write the dataset 114 | f = h5py.File(h5_filename, 'w') 115 | 116 | ds_points = f.create_dataset( 117 | "pt_seq", 118 | (len(pt_seq), 3), 119 | dtype='float32', 120 | data=pt_seq) 121 | ds_points_length = f.create_dataset( 122 | "pt_idx", 123 | (len(pt_idx), 2), 124 | dtype='int32', 125 | data=pt_idx) 126 | ds_strings = f.create_dataset( 127 | "str_seq", 128 | (len(str_seq),), 129 | dtype='int32', 130 | data=str_seq) 131 | ds_strings_length = f.create_dataset( 132 | "str_idx", 133 | (len(str_idx), 2), 134 | dtype='int32', 135 | data=str_idx) 136 | 137 | # chars = np.unique(tr_strings_seq) 138 | # d = {} 139 | # for i, c in enumerate(chars): 140 | # d[c] = np.int32(i) 141 | # d_inv = {} 142 | # for c, i in d.iteritems(): 143 | # d_inv[np.int32(i)] = c 144 | # import cPickle 145 | # cPickle.dump((d, d_inv), open('char_dict.pkl', 'w')) 146 | -------------------------------------------------------------------------------- /utilities.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | import matplotlib 5 | matplotlib.use('Agg') 6 | import matplotlib.pyplot as plt 7 | from mpl_toolkits.axes_grid1 import make_axes_locatable 8 | 9 | import theano 10 | 11 | floatX = theano.config.floatX 12 | 13 | M_x = 8.18586 14 | M_y = 0.11457 15 | s_x = 40.3719 16 | s_y = 37.0466 17 | 18 | 19 | def plot_generated_sequences(pt_batch, other_mats=None, pt_mask=None, 20 | show=False, folder_path=None, file_name=None): 21 | """ 22 | Plot sequences of (x, y, penup) points and other information 23 | 24 | Parameters: 25 | ----------- 26 | pt_batch: numpy array (seq_length, batch_size, 3) 27 | other_mats: list of numpy arrays of shapes (seq_length, batch_size) 28 | pt_mask: numpy array (seq_length, batch_size) 29 | """ 30 | if not other_mats: 31 | other_mats = [] 32 | 33 | # Renorm batch 34 | batch_normed = np.zeros_like(pt_batch, dtype=pt_batch.dtype) 35 | batch_normed[:] = pt_batch 36 | batch_normed[:, :, 0] = s_x * batch_normed[:, :, 0] + M_x 37 | batch_normed[:, :, 1] = s_y * batch_normed[:, :, 1] + M_y 38 | 39 | n_samples = batch_normed.shape[1] 40 | n_mats = len(other_mats) 41 | fig = plt.figure(figsize=(10*n_samples, (1 + n_mats) * 3)) 42 | 43 | for i in range(n_samples): 44 | mask_term = pt_mask[:, i].astype(bool) 45 | splot_pt = plt.subplot2grid((1 + n_mats, n_samples), (0, i)) 46 | plot_seq(splot_pt, batch_normed[:, i], mask_term) 47 | 48 | for j, (mat, title) in enumerate(other_mats): 49 | splot_pt = plt.subplot2grid((1 + n_mats, n_samples), (j+1, i)) 50 | plot_matrix(splot_pt, mat[:, i], mask_term, title) 51 | 52 | if folder_path and file_name: 53 | fig.savefig(os.path.join(folder_path, file_name + '.png'), 54 | bbox_inches='tight', dpi=200) 55 | if show: 56 | plt.show() 57 | plt.close() 58 | 59 | 60 | def plot_seq_pt(pt_batch, pt_mask_batch=None, use_mask=False, show=False, 61 | folder_path=None, file_name=None): 62 | batch_normed = np.zeros_like(pt_batch, dtype=pt_batch.dtype) 63 | batch_normed[:] = pt_batch 64 | batch_normed[:, :, 0] = s_x * batch_normed[:, :, 0] + M_x 65 | batch_normed[:, :, 1] = s_y * batch_normed[:, :, 1] + M_y 66 | 67 | # print 'mean x: {}'.format(batch_normed[:, :, 0].mean()) 68 | # print 'std x: {}'.format(batch_normed[:, :, 0].std()) 69 | # print 'mean y: {}'.format(batch_normed[:, :, 1].mean()) 70 | # print 'std y: {}'.format(batch_normed[:, :, 1].std()) 71 | 72 | n_samples = batch_normed.shape[1] 73 | fig = plt.figure(figsize=(10, n_samples*3)) 74 | 75 | for i in range(n_samples): 76 | mask_term = np.array([]) 77 | if use_mask: 78 | mask_term = pt_mask_batch[:, i].astype(bool) 79 | subplot = fig.add_subplot(n_samples, 1, i + 1) 80 | plot_seq(subplot, batch_normed[:, i], mask_term) 81 | 82 | if folder_path and file_name: 83 | fig.savefig(os.path.join(folder_path, file_name + '.png'), 84 | bbox_inches='tight', dpi=200) 85 | if show: 86 | plt.show() 87 | plt.close() 88 | 89 | 90 | def plot_seq(subplot, seq_pt, seq_mask=np.array([]), norm=False): 91 | if norm: 92 | seq_normed = np.zeros_like(seq_pt, dtype=seq_pt.dtype) 93 | seq_normed[:] = seq_pt 94 | seq_normed[:, 0] = s_x * seq_normed[:, 0] + M_x 95 | seq_normed[:, 1] = s_y * seq_normed[:, 1] + M_y 96 | else: 97 | seq_normed = seq_pt 98 | 99 | if seq_mask.size: 100 | seq_normed = seq_normed[seq_mask.astype(bool)] 101 | 102 | pt = seq_normed[:, 0:2] 103 | penup = seq_normed[:, 2].astype(bool) 104 | 105 | pos = np.where(penup)[0]+1 106 | 107 | pt = np.cumsum(pt, axis=0) 108 | pt = np.insert(pt, pos, [np.nan, np.nan], axis=0) 109 | 110 | subplot.plot(pt[:, 0], -pt[:, 1]) 111 | subplot.axis('equal') 112 | 113 | 114 | def plot_matrix(subplot, mat, seq_mask=np.array([]), title=''): 115 | if seq_mask.size: 116 | mat = mat[seq_mask.astype(bool)] 117 | 118 | im = subplot.matshow(mat.T) 119 | subplot.set_aspect('auto') 120 | subplot.set_title(title) 121 | cax = make_axes_locatable(subplot).append_axes("right", size="1%", 122 | pad=0.05) 123 | plt.colorbar(im, cax=cax) 124 | 125 | 126 | def create_train_tag_values(seq_pt, seq_str, seq_tg, seq_pt_mask, 127 | seq_str_mask, batch_size): 128 | f_s_pt = 6 129 | f_s_str = 7 130 | seq_pt.tag.test_value = np.zeros((f_s_pt, batch_size, 3), dtype=floatX) 131 | seq_str.tag.test_value = np.zeros((f_s_str, batch_size), dtype='int32') 132 | seq_tg.tag.test_value = np.ones((f_s_pt, batch_size, 3), dtype=floatX) 133 | seq_pt_mask.tag.test_value = np.ones((f_s_pt, batch_size), dtype=floatX) 134 | seq_str_mask.tag.test_value = np.ones((f_s_str, batch_size), dtype=floatX) 135 | 136 | 137 | def create_gen_tag_values(model, pt_ini, h_ini_pred, k_ini_pred, w_ini_pred, 138 | bias, seq_str, seq_str_mask): 139 | f_s_str = 7 140 | n_samples = 3 141 | n_hidden, n_chars = model.n_hidden, model.n_chars 142 | n_mixt_attention = model.n_mixt_attention 143 | pt_ini.tag.test_value = np.zeros((n_samples, 3), floatX) 144 | h_ini_pred.tag.test_value = np.zeros((n_samples, n_hidden), floatX) 145 | k_ini_pred.tag.test_value = np.zeros((n_samples, n_mixt_attention), floatX) 146 | w_ini_pred.tag.test_value = np.zeros((n_samples, n_chars), floatX) 147 | bias.tag.test_value = np.float32(0.0) 148 | seq_str.tag.test_value = np.zeros((f_s_str, n_samples), dtype='int32') 149 | seq_str_mask.tag.test_value = np.ones((f_s_str, n_samples), dtype=floatX) 150 | -------------------------------------------------------------------------------- /main_cond.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import cPickle 4 | 5 | from lasagne.updates import adam 6 | import numpy as np 7 | import theano 8 | import theano.tensor as T 9 | 10 | from raccoon.trainer import Trainer 11 | from raccoon.extensions import TrainMonitor 12 | from raccoon.layers.utils import clip_norm_gradients 13 | 14 | from data import create_generator, load_data, extract_sequence 15 | from model import ConditionedModel 16 | from raccoon_extensions import (SamplerCond, SamplingFunctionSaver, 17 | ValMonitorHandwriting) 18 | from utilities import create_train_tag_values, create_gen_tag_values 19 | 20 | floatX = theano.config.floatX = 'float32' 21 | # theano.config.optimizer = 'None' 22 | # theano.config.compute_test_value = 'raise' 23 | # np.random.seed(42) 24 | 25 | ########## 26 | # CONFIG # 27 | ########## 28 | learning_rate = 0.1 29 | n_hidden = 400 30 | n_chars = 81 31 | n_mixt_attention = 10 32 | n_mixt_output = 20 33 | gain = 0.01 34 | batch_size = 50 35 | chunk = None 36 | train_freq_print = 100 37 | valid_freq_print = 1000 38 | sample_strings = ['Sous le pont Mirabeau coule la Seine.']*4 39 | algo = 'adam' # adam, sgd 40 | 41 | dump_path = os.path.join(os.environ.get('TMP_PATH'), 'handwriting', 42 | str(np.random.randint(0, 100000000, 1)[0])) 43 | if not os.path.exists(dump_path): 44 | os.makedirs(dump_path) 45 | 46 | ######## 47 | # DATA # 48 | ######## 49 | char_dict, inv_char_dict = cPickle.load(open('char_dict.pkl', 'r')) 50 | 51 | # All the data is loaded in memory 52 | train_pt_seq, train_pt_idx, train_str_seq, train_str_idx = \ 53 | load_data('hand_training.hdf5') 54 | train_batch_gen = create_generator( 55 | True, batch_size, 56 | train_pt_seq, train_pt_idx, train_str_seq, train_str_idx, chunk=chunk) 57 | valid_pt_seq, valid_pt_idx, valid_str_seq, valid_str_idx = \ 58 | load_data('hand_training.hdf5') 59 | valid_batch_gen = create_generator( 60 | True, batch_size, 61 | valid_pt_seq, valid_pt_idx, valid_str_seq, valid_str_idx, chunk=chunk) 62 | 63 | ################## 64 | # MODEL CREATION # 65 | ################## 66 | # shape (seq, element_id, features) 67 | seq_pt = T.tensor3('input', floatX) 68 | seq_str = T.matrix('str_input', 'int32') 69 | seq_tg = T.tensor3('tg', floatX) 70 | seq_pt_mask = T.matrix('pt_mask', floatX) 71 | seq_str_mask = T.matrix('str_mask', floatX) 72 | create_train_tag_values(seq_pt, seq_str, seq_tg, seq_pt_mask, 73 | seq_str_mask, batch_size) # for debug 74 | 75 | 76 | model = ConditionedModel(gain, n_hidden, n_chars, n_mixt_attention, 77 | n_mixt_output) 78 | # Initial values of the variables that are transmitted through the recursion 79 | h_ini, k_ini, w_ini = model.create_shared_init_states(batch_size) 80 | loss, updates_ini, monitoring = model.apply(seq_pt, seq_pt_mask, seq_tg, 81 | seq_str, seq_str_mask, 82 | h_ini, k_ini, w_ini) 83 | 84 | 85 | ######################## 86 | # GRADIENT AND UPDATES # 87 | ######################## 88 | params = model.params 89 | grads = T.grad(loss, params) 90 | grads = clip_norm_gradients(grads) 91 | 92 | if algo == 'adam': 93 | updates_params = adam(grads, params, 0.0003) 94 | elif algo == 'sgd': 95 | updates_params = [] 96 | for p, g in zip(params, grads): 97 | updates_params.append((p, p - learning_rate * g)) 98 | else: 99 | raise ValueError('Specified algo does not exist') 100 | 101 | updates_all = updates_ini + updates_params 102 | 103 | 104 | ##################### 105 | # SAMPLING FUNCTION # 106 | ##################### 107 | pt_ini, h_ini_pred, k_ini_pred, w_ini_pred, bias = \ 108 | model.create_sym_init_states() 109 | create_gen_tag_values(model, pt_ini, h_ini_pred, k_ini_pred, w_ini_pred, 110 | bias, seq_str, seq_str_mask) # for debug 111 | 112 | (pt_gen, a_gen, k_gen, p_gen, w_gen, mask_gen), updates_pred = \ 113 | model.prediction(pt_ini, seq_str, seq_str_mask, 114 | h_ini_pred, k_ini_pred, w_ini_pred, bias=bias) 115 | 116 | f_sampling = theano.function( 117 | [pt_ini, seq_str, seq_str_mask, h_ini_pred, k_ini_pred, w_ini_pred, 118 | bias], [pt_gen, a_gen, k_gen, p_gen, w_gen, mask_gen], 119 | updates=updates_pred) 120 | 121 | ############## 122 | # MONITORING # 123 | ############## 124 | train_monitor = TrainMonitor( 125 | train_freq_print, [seq_pt, seq_tg, seq_pt_mask, seq_str, seq_str_mask], 126 | monitoring, updates_all) 127 | 128 | valid_monitor = ValMonitorHandwriting( 129 | 'Validation', valid_freq_print, [seq_pt, seq_tg, seq_pt_mask, seq_str, 130 | seq_str_mask], monitoring, 131 | valid_batch_gen, updates_ini, model, h_ini, k_ini, w_ini, batch_size, 132 | apply_at_the_start=False) 133 | 134 | sampler = SamplerCond('sampler', train_freq_print, dump_path, 'essai', 135 | model, f_sampling, sample_strings, 136 | dict_char2int=char_dict, bias_value=0.5) 137 | 138 | sampling_saver = SamplingFunctionSaver( 139 | valid_monitor, loss, valid_freq_print, dump_path, 'f_sampling', 140 | model, f_sampling, char_dict, apply_at_the_start=True) 141 | 142 | 143 | def custom_process_fun(generator_output): 144 | inputs, next_seq = generator_output 145 | 146 | res = train_m.train_monitor.train(*inputs) 147 | 148 | if next_seq: 149 | model.reset_shared_init_states(h_ini, k_ini, w_ini, batch_size) 150 | 151 | return res 152 | 153 | 154 | train_m = Trainer(train_monitor, train_batch_gen, 155 | [valid_monitor, sampler, sampling_saver], [], 156 | custom_process_fun=custom_process_fun) 157 | 158 | 159 | ############ 160 | # TRAINING # 161 | ############ 162 | 163 | model.reset_shared_init_states(h_ini, k_ini, w_ini, batch_size) 164 | train_m.train() -------------------------------------------------------------------------------- /raccoon_extensions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import theano 4 | import theano.tensor as T 5 | import numpy as np 6 | 7 | from raccoon import Extension 8 | from raccoon.extensions import Saver, ValidationMonitor 9 | 10 | from data import char2int 11 | from utilities import plot_seq_pt, plot_generated_sequences 12 | 13 | floatX = theano.config.floatX 14 | 15 | 16 | class Sampler(Extension): 17 | """Extension to pickle objects. 18 | 19 | Only the compute_object method should be overwritten. 20 | """ 21 | def __init__(self, name_extension, freq, folder_path, file_name, 22 | fun_pred, n_hidden, apply_at_the_start=True, 23 | apply_at_the_end=True, n_samples=4): 24 | super(Sampler, self).__init__(name_extension, freq, 25 | apply_at_the_end=apply_at_the_end, 26 | apply_at_the_start=apply_at_the_start) 27 | self.folder_path = folder_path 28 | self.file_name = file_name 29 | self.fun_pred = fun_pred 30 | self.n_samples = n_samples 31 | self.n_hidden = n_hidden 32 | 33 | def execute_virtual(self, batch_id, epoch_id=None): 34 | sample = self.fun_pred(np.zeros((self.n_samples, 3), floatX), 35 | np.zeros((self.n_samples, self.n_hidden), floatX)) 36 | 37 | plot_seq_pt(sample, 38 | folder_path=self.folder_path, 39 | file_name='{}_'.format(batch_id) + self.file_name) 40 | 41 | return ['executed'] 42 | 43 | 44 | class SamplerCond(Extension): 45 | """Extension to pickle objects. 46 | 47 | Only the compute_object method should be overwritten. 48 | """ 49 | def __init__(self, name_extension, freq, folder_path, file_name, 50 | model, f_sampling, sample_strings, dict_char2int, 51 | bias_value=0.5, 52 | apply_at_the_start=True, apply_at_the_end=True): 53 | super(SamplerCond, self).__init__(name_extension, freq, 54 | apply_at_the_end=apply_at_the_end, 55 | apply_at_the_start=apply_at_the_start) 56 | self.folder_path = folder_path 57 | self.file_name = file_name 58 | 59 | self.sample_strings = [s + ' ' for s in sample_strings] 60 | n_samples = len(sample_strings) 61 | self.dict_char2int = dict_char2int 62 | self.f_sampling = f_sampling 63 | self.bias_value = bias_value 64 | 65 | # Initial values 66 | self.pt_ini_mat = np.zeros((n_samples, 3), floatX) 67 | self.h_ini_mat = np.zeros((n_samples, model.n_hidden), floatX) 68 | self.k_ini_mat = np.zeros((n_samples, model.n_mixt_attention), floatX) 69 | self.w_ini_mat = np.zeros((n_samples, model.n_chars), floatX) 70 | 71 | def execute_virtual(self, batch_id, epoch_id=None): 72 | 73 | cond, cond_mask = char2int(self.sample_strings, self.dict_char2int) 74 | 75 | pt_gen, a_gen, k_gen, p_gen, w_gen, mask_gen = self.f_sampling( 76 | self.pt_ini_mat, cond, cond_mask, 77 | self.h_ini_mat, self.k_ini_mat, self.w_ini_mat, self.bias_value) 78 | 79 | # plot_seq_pt(pt_gen, 80 | # folder_path=self.folder_path, 81 | # file_name='{}_'.format(batch_id) + self.file_name) 82 | p_gen = np.swapaxes(p_gen, 1, 2) 83 | mats = [(a_gen, 'alpha'), (k_gen, 'kapa'), (p_gen, 'phi'), 84 | (w_gen, 'omega')] 85 | plot_generated_sequences( 86 | pt_gen, mats, 87 | mask_gen, folder_path=self.folder_path, 88 | file_name='{}_'.format(batch_id) + self.file_name) 89 | 90 | return ['executed'] 91 | 92 | 93 | class SamplingFunctionSaver(Saver): 94 | def __init__(self, monitor, var, freq, folder_path, file_name, 95 | model, f_sampling, dict_char2int, **kwargs): 96 | Saver.__init__(self, 'Sampling function saver', freq, folder_path, 97 | file_name, apply_at_the_end=False, **kwargs) 98 | 99 | self.val_monitor = monitor 100 | # Index of the variable to check in the monitoring extension 101 | self.var_idx = monitor.output_links[var][0] 102 | self.best_value = np.inf 103 | 104 | self.model = model 105 | self.f_sampling = f_sampling 106 | self.dict_char2int = dict_char2int 107 | 108 | def condition(self, batch_id, epoch_id): 109 | return True 110 | # if not self.val_monitor.history: 111 | # return False 112 | # current_value = self.val_monitor.history[-1][self.var_idx] 113 | # if current_value < self.best_value: 114 | # self.best_value = current_value 115 | # return True 116 | # return False 117 | 118 | def compute_object(self): 119 | return (self.model, self.f_sampling, self.dict_char2int), \ 120 | ['extension executed'] 121 | 122 | def finish(self, bath_id, epoch_id): 123 | return -1, ['not executed at the end'] 124 | 125 | 126 | class ValMonitorHandwriting(ValidationMonitor): 127 | """ 128 | Extension to monitor tensor variables and MonitoredQuantity objects on an 129 | external fuel stream. 130 | """ 131 | def __init__(self, name_extension, freq, inputs, monitored_variables, 132 | stream, updates, model, h_ini, k_ini, w_ini, batch_size, 133 | **kwargs): 134 | ValidationMonitor.__init__(self, name_extension, freq, inputs, 135 | monitored_variables, stream, 136 | updates=updates, **kwargs) 137 | self.stream = stream 138 | self.model = model 139 | 140 | self.h_ini = h_ini 141 | self.k_ini = k_ini 142 | self.w_ini = w_ini 143 | self.var = [h_ini, k_ini, w_ini] 144 | self.batch_size = batch_size 145 | 146 | def compute_metrics(self): 147 | 148 | # Save current state 149 | previous_states = [v.get_value() for v in self.var] 150 | 151 | self.model.reset_shared_init_states( 152 | self.h_ini, self.k_ini, self.w_ini, self.batch_size) 153 | 154 | metric_values = np.zeros(self.n_outputs, dtype=floatX) 155 | counter_values = np.zeros(self.n_outputs, dtype=floatX) 156 | 157 | for inputs, signal in self.stream(): 158 | m_values, c_values = self.compute_metrics_minibatch(*inputs) 159 | metric_values += m_values 160 | counter_values += c_values 161 | 162 | if signal: 163 | self.model.reset_shared_init_states( 164 | self.h_ini, self.k_ini, self.w_ini, self.batch_size) 165 | 166 | metric_values /= counter_values 167 | 168 | # restore states 169 | for s, v in zip(previous_states, self.var): 170 | v.set_value(s) 171 | 172 | return metric_values -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- 1 | import theano 2 | from theano import shared 3 | import theano.tensor as T 4 | from theano.tensor.shared_randomstreams import RandomStreams 5 | 6 | import numpy as np 7 | from lasagne.init import GlorotNormal 8 | 9 | from raccoon.layers.attention import PositionAttentionLayer 10 | from raccoon.layers.reccurrent import GRULayer 11 | from raccoon.layers.utils import create_uneven_weight 12 | 13 | theano.config.floatX = 'float32' 14 | floatX = theano.config.floatX 15 | 16 | 17 | def logsumexp(x, axis=None): 18 | """ 19 | Efficient log of a sum of exponentials 20 | """ 21 | x_max = T.max(x, axis=axis, keepdims=True) 22 | z = T.log(T.sum(T.exp(x - x_max), axis=axis, keepdims=True)) + x_max 23 | return z.sum(axis=axis) 24 | 25 | 26 | class MixtureGaussians2D: 27 | def __init__(self, ls_n_in, n_mixtures, initializer, eps=1e-5): 28 | if not isinstance(ls_n_in, (tuple, list)): 29 | ls_n_in = [ls_n_in] 30 | 31 | self.n_in = sum(ls_n_in) 32 | self.n_mixtures = n_mixtures 33 | self.eps = eps 34 | 35 | self.n_out = (n_mixtures + # proportions 36 | n_mixtures * 2 + # means 37 | n_mixtures * 2 + # stds 38 | n_mixtures + # correlations 39 | 1) # bernoulli 40 | 41 | w_in_mat = create_uneven_weight(ls_n_in, self.n_out, initializer) 42 | self.w = shared(w_in_mat, 'w_mixt') 43 | self.b = shared(np.random.normal( 44 | 0, 0.001, size=(self.n_out,)).astype(floatX), 'b_mixt') 45 | 46 | self.params = [self.w, self.b] 47 | 48 | def compute_parameters(self, h, bias): 49 | """ 50 | h: (batch or batch*seq, features) 51 | """ 52 | n = self.n_mixtures 53 | out = T.dot(h, self.w) + self.b 54 | prop = T.nnet.softmax(out[:, :n]*(1 + bias)) 55 | mean_x = out[:, n:n*2] 56 | mean_y = out[:, n*2:n*3] 57 | std_x = T.exp(out[:, n*3:n*4] - bias) + self.eps 58 | std_y = T.exp(out[:, n*4:n*5] - bias) + self.eps 59 | rho = T.tanh(out[:, n*5:n*6]) 60 | rho = (1+rho + self.eps) / (2 + 2*self.eps) - 1 61 | bernoulli = T.nnet.sigmoid(out[:, -1]) 62 | bernoulli = (bernoulli + self.eps) / (1 + 2*self.eps) 63 | 64 | return prop, mean_x, mean_y, std_x, std_y, rho, bernoulli 65 | 66 | def prediction(self, h, bias): 67 | srng = RandomStreams(seed=42) 68 | 69 | prop, mean_x, mean_y, std_x, std_y, rho, bernoulli = \ 70 | self.compute_parameters(h, bias) 71 | 72 | mode = T.argmax(srng.multinomial(pvals=prop, dtype=prop.dtype), axis=1) 73 | 74 | v = T.arange(0, mean_x.shape[0]) 75 | m_x = mean_x[v, mode] 76 | m_y = mean_y[v, mode] 77 | s_x = std_x[v, mode] 78 | s_y = std_y[v, mode] 79 | r = rho[v, mode] 80 | # cov = r * (s_x * s_y) 81 | 82 | normal = srng.normal((h.shape[0], 2)) 83 | x = normal[:, 0] 84 | y = normal[:, 1] 85 | 86 | # x_n = T.shape_padright(s_x * x + cov * y + m_x) 87 | # y_n = T.shape_padright(s_y * y + cov * x + m_y) 88 | 89 | x_n = T.shape_padright(m_x + s_x * x) 90 | y_n = T.shape_padright(m_y + s_y * (x * r + y * T.sqrt(1.-r**2))) 91 | 92 | uniform = srng.uniform((h.shape[0],)) 93 | pin = T.shape_padright(T.cast(bernoulli > uniform, floatX)) 94 | 95 | return T.concatenate([x_n, y_n, pin], axis=1) 96 | 97 | def apply(self, h_seq, mask_seq, tg_seq): 98 | """ 99 | h_seq: (seq, batch, features) 100 | mask_seq: (seq, batch) 101 | tg_seq: (seq, batch, features=3) 102 | """ 103 | h_seq = T.reshape(h_seq, (-1, h_seq.shape[-1])) 104 | tg_seq = T.reshape(tg_seq, (-1, tg_seq.shape[-1])) 105 | mask_seq = T.reshape(mask_seq, (-1,)) 106 | 107 | prop, mean_x, mean_y, std_x, std_y, rho, bernoulli = \ 108 | self.compute_parameters(h_seq, .0) 109 | 110 | tg_x = T.addbroadcast(tg_seq[:, 0:1], 1) 111 | tg_y = T.addbroadcast(tg_seq[:, 1:2], 1) 112 | tg_pin = tg_seq[:, 2] 113 | 114 | tg_x_s = (tg_x - mean_x) / std_x 115 | tg_y_s = (tg_y - mean_y) / std_y 116 | 117 | z = tg_x_s**2 + tg_y_s**2 - 2*rho*tg_x_s*tg_y_s 118 | buff = 1-rho**2 119 | 120 | tmp = (-z / (2 * buff) - 121 | T.log(2*np.pi) - T.log(std_x) - T.log(std_y) - 0.5*T.log(buff) + 122 | T.log(prop)) 123 | 124 | c = (-logsumexp(tmp, axis=1) - 125 | tg_pin * T.log(bernoulli) - 126 | (1-tg_pin) * T.log(1 - bernoulli)) 127 | 128 | c = T.sum(c * mask_seq) / T.sum(mask_seq) 129 | c.name = 'negll' 130 | 131 | max_prop = T.argmax(prop, axis=1).mean() 132 | max_prop.name = 'max_prop' 133 | 134 | std_max_prop = T.argmax(prop, axis=1).std() 135 | std_max_prop.name = 'std_max_prop' 136 | 137 | return c, [c, max_prop, std_max_prop] 138 | 139 | 140 | class UnconditionedModel: 141 | def __init__(self, gain_ini, n_hidden, n_mixtures): 142 | ini = GlorotNormal(gain_ini) 143 | 144 | self.gru_layer = GRULayer(3, n_hidden, ini) 145 | 146 | self.mixture = MixtureGaussians2D(n_hidden, n_mixtures, ini) 147 | 148 | self.params = self.gru_layer.params + self.mixture.params 149 | 150 | def apply(self, seq_pt, seq_mask, seq_tg, h_ini): 151 | 152 | seq_h, scan_updates = self.gru_layer.apply(seq_pt, seq_mask, h_ini) 153 | loss, monitoring = self.mixture.apply(seq_h, seq_mask, seq_tg) 154 | 155 | h_mean = seq_h.mean() 156 | h_mean.name = 'h_mean' 157 | monitoring.append(h_mean) 158 | 159 | return loss, [(h_ini, seq_h[-1])] + scan_updates, monitoring 160 | 161 | def prediction(self, pt_ini, h_ini, bias=.0, n_steps=500): 162 | 163 | def gru_step(pt_pre, h_pre): 164 | 165 | h = self.gru_layer.step(pt_pre, h_pre, 166 | mask=None, process_inputs=True) 167 | 168 | pt = self.mixture.prediction(h, bias) 169 | 170 | return pt, h 171 | 172 | res, scan_updates = theano.scan( 173 | fn=gru_step, 174 | outputs_info=[pt_ini, h_ini], 175 | n_steps=n_steps) 176 | 177 | return res[0], scan_updates 178 | 179 | 180 | class ConditionedModel: 181 | def __init__(self, gain_ini, n_hidden, n_chars, n_mixt_attention, 182 | n_mixtures): 183 | """ 184 | Parameters 185 | ---------- 186 | n_mixt_attention: int 187 | Number of mixtures used by the attention mechanism 188 | n_chars: int 189 | Number of different characters 190 | n_mixtures: int 191 | Number of mixtures in the Gaussian Mixture model 192 | """ 193 | self.n_hidden = n_hidden 194 | self.n_chars = n_chars 195 | self.n_mixt_attention = n_mixt_attention 196 | self.n_mixtures = n_mixtures 197 | 198 | ini = GlorotNormal(gain_ini) 199 | 200 | self.pos_layer = PositionAttentionLayer( 201 | GRULayer([3, self.n_chars], n_hidden, ini), 202 | self.n_chars, 203 | self.n_mixt_attention, ini) 204 | 205 | self.mixture = MixtureGaussians2D([n_hidden, self.n_chars], 206 | n_mixtures, ini) 207 | 208 | self.params = self.pos_layer.params + self.mixture.params 209 | 210 | def apply(self, seq_pt, seq_mask, seq_tg, seq_str, seq_str_mask, 211 | h_ini, k_ini, w_ini): 212 | """ 213 | Parameters 214 | ---------- 215 | seq_pt: (length_pt_seq, batch_size, 3) 216 | seq_mask: (length_pt_seq, batch_size) 217 | seq_tg: (length_pt_seq, batch_size, 3) 218 | seq_str: (length_str_seq, batch_size) 219 | Each character is represented by an integer 220 | seq_str_mask: (length_str_seq, batch_size) 221 | 222 | h_ini: (batch_size, n_hidden) 223 | k_ini: (batch_size, n_mixture_attention) 224 | w_ini: (batch_size, n_chars) 225 | """ 226 | 227 | # Convert the integers representing chars into one-hot encodings 228 | # seq_str will have shape (seq_length, batch_size, n_chars) 229 | seq_str = T.eye(self.n_chars, dtype=floatX)[seq_str] 230 | 231 | (seq_h, seq_k, seq_w), scan_updates = self.pos_layer.apply( 232 | seq_pt, seq_mask, seq_str, seq_str_mask, 233 | h_ini, k_ini, w_ini) 234 | 235 | seq_h_conc = T.concatenate([seq_h, seq_w], axis=-1) 236 | 237 | loss, monitoring = self.mixture.apply(seq_h_conc, seq_mask, seq_tg) 238 | 239 | updates = [(h_ini, seq_h[-1]), (k_ini, seq_k[-1]), (w_ini, seq_w[-1])] 240 | 241 | # Monitoring variables 242 | monitoring.extend( 243 | self.create_monitoring_variables(seq_h, seq_k, seq_w, seq_mask)) 244 | 245 | return loss, updates + scan_updates, monitoring 246 | 247 | def prediction(self, pt_ini, seq_str, seq_str_mask, 248 | h_ini, k_ini, w_ini, bias=.0, n_steps=10000): 249 | """ 250 | Parameters 251 | ---------- 252 | pt_ini: (batch_size, 3) 253 | seq_str: (length_str_seq, batch_size) 254 | seq_str_mask: (length_str_seq, batch_size) 255 | 256 | h_ini: (batch_size, n_hidden) 257 | k_ini: (batch_size, n_mixture_attention) 258 | w_ini: (batch_size, n_chars) 259 | 260 | bias: float 261 | The bias that controls the variance of the generation 262 | n_steps: int 263 | The maximal number of generation steps. 264 | """ 265 | 266 | # Convert the integers representing chars into one-hot encodings 267 | # seq_str will have shape (seq_length, batch_size, n_chars) 268 | seq_str = T.eye(self.n_chars, dtype=floatX)[seq_str] 269 | batch_size = pt_ini.shape[0] 270 | 271 | def scan_step(pt_pre, h_pre, k_pre, w_pre, mask, 272 | seq_str, seq_str_mask, bias): 273 | 274 | h, a, k, p, w = self.pos_layer.step( 275 | pt_pre, h_pre, k_pre, w_pre, 276 | seq_str, seq_str_mask, mask=mask) 277 | 278 | h_conc = T.concatenate([h, w], axis=-1) 279 | 280 | pt = self.mixture.prediction(h_conc, bias) 281 | 282 | # ending condition 283 | last_char = T.cast(T.sum(seq_str_mask, axis=0)-1, 'int32') 284 | last_phi = p[last_char, T.arange(last_char.shape[0])] 285 | max_phi = T.max(p, axis=0) 286 | condition = last_phi >= 0.95*max_phi 287 | mask = T.switch(condition, .0, mask) 288 | 289 | return ((pt, h, a, k, p, w, mask), 290 | theano.scan_module.until(T.all(mask < 1.))) 291 | 292 | (seq_pt, _, seq_a, seq_k, seq_p, seq_w, seq_mask), scan_updates = \ 293 | theano.scan( 294 | fn=scan_step, 295 | outputs_info=[pt_ini, h_ini, None, k_ini, None, w_ini, 296 | T.alloc(1., batch_size)], 297 | non_sequences=[seq_str, seq_str_mask, bias], 298 | n_steps=n_steps) 299 | 300 | return (seq_pt, seq_a, seq_k, seq_p, seq_w, seq_mask), scan_updates 301 | 302 | def create_monitoring_variables(self, seq_h, seq_k, seq_w, seq_mask): 303 | """ 304 | seq_h: (length_pt_seq, batch_size, n_hidden) 305 | seq_k: (length_pt_seq, batch_size, n_mixt) 306 | """ 307 | 308 | seq_h = seq_h * seq_mask[:, :, None] 309 | seq_k = seq_k * seq_mask[:, :, None] 310 | seq_w = seq_w * seq_mask[:, :, None] 311 | 312 | n = seq_mask[:, :, None].sum() 313 | 314 | seq_h_mean = T.sum(seq_h.mean(axis=-1)) / n 315 | seq_h_mean.name = 'seq_h_mean' 316 | 317 | seq_k_mean = T.sum(seq_k.mean(axis=-1)) / n 318 | seq_k_mean.name = 'seq_k_mean' 319 | 320 | seq_w_mean = T.sum(seq_w.mean(axis=-1)) / n 321 | seq_w_mean.name = 'seq_w_mean' 322 | 323 | return [seq_h_mean, seq_k_mean, seq_w_mean] 324 | 325 | def create_shared_init_states(self, batch_size): 326 | def create_shared(size, name): 327 | return theano.shared(np.zeros(size, floatX), name) 328 | 329 | h_ini = create_shared((batch_size, self.n_hidden), 'h_ini') 330 | k_ini = create_shared((batch_size, self.n_mixt_attention), 'k_ini') 331 | w_ini = create_shared((batch_size, self.n_chars), 'w_ini') 332 | 333 | return h_ini, k_ini, w_ini 334 | 335 | def reset_shared_init_states(self, h_ini, k_ini, w_ini, batch_size): 336 | def set_value(var, size): 337 | var.set_value(np.zeros(size, dtype=floatX)) 338 | 339 | set_value(h_ini, (batch_size, self.n_hidden)) 340 | set_value(k_ini, (batch_size, self.n_mixt_attention)) 341 | set_value(w_ini, (batch_size, self.n_chars)) 342 | 343 | def create_sym_init_states(self): 344 | pt_ini = T.matrix('pt_pred', floatX) 345 | h_ini_pred = T.matrix('h_ini_pred', floatX) 346 | k_ini_pred = T.matrix('k_ini_pred', floatX) 347 | w_ini_pred = T.matrix('w_ini_pred', floatX) 348 | bias = T.scalar('bias_generation_pred', floatX) 349 | return pt_ini, h_ini_pred, k_ini_pred, w_ini_pred, bias 350 | -------------------------------------------------------------------------------- /char_dict.pkl: -------------------------------------------------------------------------------- 1 | ((dp1 2 | cnumpy.core.multiarray 3 | scalar 4 | p2 5 | (cnumpy 6 | dtype 7 | p3 8 | (S'S1' 9 | I0 10 | I1 11 | tRp4 12 | (I3 13 | S'|' 14 | NNNI1 15 | I1 16 | I0 17 | tbS'!' 18 | tRp5 19 | g2 20 | (g3 21 | (S'i4' 22 | I0 23 | I1 24 | tRp6 25 | (I3 26 | S'<' 27 | NNNI-1 28 | I-1 29 | I0 30 | tbS'\x01\x00\x00\x00' 31 | tRp7 32 | sg2 33 | (g3 34 | (S'S1' 35 | I0 36 | I1 37 | tRp8 38 | (I3 39 | S'|' 40 | NNNI1 41 | I1 42 | I0 43 | tbS' ' 44 | tRp9 45 | g2 46 | (g6 47 | S'\x00\x00\x00\x00' 48 | tRp10 49 | sg2 50 | (g3 51 | (S'S1' 52 | I0 53 | I1 54 | tRp11 55 | (I3 56 | S'|' 57 | NNNI1 58 | I1 59 | I0 60 | tbS'#' 61 | tRp12 62 | g2 63 | (g6 64 | S'\x03\x00\x00\x00' 65 | tRp13 66 | sg2 67 | (g3 68 | (S'S1' 69 | I0 70 | I1 71 | tRp14 72 | (I3 73 | S'|' 74 | NNNI1 75 | I1 76 | I0 77 | tbS'"' 78 | tRp15 79 | g2 80 | (g6 81 | S'\x02\x00\x00\x00' 82 | tRp16 83 | sg2 84 | (g3 85 | (S'S1' 86 | I0 87 | I1 88 | tRp17 89 | (I3 90 | S'|' 91 | NNNI1 92 | I1 93 | I0 94 | tbS'%' 95 | tRp18 96 | g2 97 | (g6 98 | S'\x04\x00\x00\x00' 99 | tRp19 100 | sg2 101 | (g3 102 | (S'S1' 103 | I0 104 | I1 105 | tRp20 106 | (I3 107 | S'|' 108 | NNNI1 109 | I1 110 | I0 111 | tbS"'" 112 | tRp21 113 | g2 114 | (g6 115 | S'\x06\x00\x00\x00' 116 | tRp22 117 | sg2 118 | (g3 119 | (S'S1' 120 | I0 121 | I1 122 | tRp23 123 | (I3 124 | S'|' 125 | NNNI1 126 | I1 127 | I0 128 | tbS'&' 129 | tRp24 130 | g2 131 | (g6 132 | S'\x05\x00\x00\x00' 133 | tRp25 134 | sg2 135 | (g3 136 | (S'S1' 137 | I0 138 | I1 139 | tRp26 140 | (I3 141 | S'|' 142 | NNNI1 143 | I1 144 | I0 145 | tbS')' 146 | tRp27 147 | g2 148 | (g6 149 | S'\x08\x00\x00\x00' 150 | tRp28 151 | sg2 152 | (g3 153 | (S'S1' 154 | I0 155 | I1 156 | tRp29 157 | (I3 158 | S'|' 159 | NNNI1 160 | I1 161 | I0 162 | tbS'(' 163 | tRp30 164 | g2 165 | (g6 166 | S'\x07\x00\x00\x00' 167 | tRp31 168 | sg2 169 | (g3 170 | (S'S1' 171 | I0 172 | I1 173 | tRp32 174 | (I3 175 | S'|' 176 | NNNI1 177 | I1 178 | I0 179 | tbS'+' 180 | tRp33 181 | g2 182 | (g6 183 | S'\t\x00\x00\x00' 184 | tRp34 185 | sg2 186 | (g3 187 | (S'S1' 188 | I0 189 | I1 190 | tRp35 191 | (I3 192 | S'|' 193 | NNNI1 194 | I1 195 | I0 196 | tbS'-' 197 | tRp36 198 | g2 199 | (g6 200 | S'\x0b\x00\x00\x00' 201 | tRp37 202 | sg2 203 | (g3 204 | (S'S1' 205 | I0 206 | I1 207 | tRp38 208 | (I3 209 | S'|' 210 | NNNI1 211 | I1 212 | I0 213 | tbS',' 214 | tRp39 215 | g2 216 | (g6 217 | S'\n\x00\x00\x00' 218 | tRp40 219 | sg2 220 | (g3 221 | (S'S1' 222 | I0 223 | I1 224 | tRp41 225 | (I3 226 | S'|' 227 | NNNI1 228 | I1 229 | I0 230 | tbS'/' 231 | tRp42 232 | g2 233 | (g6 234 | S'\r\x00\x00\x00' 235 | tRp43 236 | sg2 237 | (g3 238 | (S'S1' 239 | I0 240 | I1 241 | tRp44 242 | (I3 243 | S'|' 244 | NNNI1 245 | I1 246 | I0 247 | tbS'.' 248 | tRp45 249 | g2 250 | (g6 251 | S'\x0c\x00\x00\x00' 252 | tRp46 253 | sg2 254 | (g3 255 | (S'S1' 256 | I0 257 | I1 258 | tRp47 259 | (I3 260 | S'|' 261 | NNNI1 262 | I1 263 | I0 264 | tbS'1' 265 | tRp48 266 | g2 267 | (g6 268 | S'\x0f\x00\x00\x00' 269 | tRp49 270 | sg2 271 | (g3 272 | (S'S1' 273 | I0 274 | I1 275 | tRp50 276 | (I3 277 | S'|' 278 | NNNI1 279 | I1 280 | I0 281 | tbS'0' 282 | tRp51 283 | g2 284 | (g6 285 | S'\x0e\x00\x00\x00' 286 | tRp52 287 | sg2 288 | (g3 289 | (S'S1' 290 | I0 291 | I1 292 | tRp53 293 | (I3 294 | S'|' 295 | NNNI1 296 | I1 297 | I0 298 | tbS'3' 299 | tRp54 300 | g2 301 | (g6 302 | S'\x11\x00\x00\x00' 303 | tRp55 304 | sg2 305 | (g3 306 | (S'S1' 307 | I0 308 | I1 309 | tRp56 310 | (I3 311 | S'|' 312 | NNNI1 313 | I1 314 | I0 315 | tbS'2' 316 | tRp57 317 | g2 318 | (g6 319 | S'\x10\x00\x00\x00' 320 | tRp58 321 | sg2 322 | (g3 323 | (S'S1' 324 | I0 325 | I1 326 | tRp59 327 | (I3 328 | S'|' 329 | NNNI1 330 | I1 331 | I0 332 | tbS'5' 333 | tRp60 334 | g2 335 | (g6 336 | S'\x13\x00\x00\x00' 337 | tRp61 338 | sg2 339 | (g3 340 | (S'S1' 341 | I0 342 | I1 343 | tRp62 344 | (I3 345 | S'|' 346 | NNNI1 347 | I1 348 | I0 349 | tbS'4' 350 | tRp63 351 | g2 352 | (g6 353 | S'\x12\x00\x00\x00' 354 | tRp64 355 | sg2 356 | (g3 357 | (S'S1' 358 | I0 359 | I1 360 | tRp65 361 | (I3 362 | S'|' 363 | NNNI1 364 | I1 365 | I0 366 | tbS'7' 367 | tRp66 368 | g2 369 | (g6 370 | S'\x15\x00\x00\x00' 371 | tRp67 372 | sg2 373 | (g3 374 | (S'S1' 375 | I0 376 | I1 377 | tRp68 378 | (I3 379 | S'|' 380 | NNNI1 381 | I1 382 | I0 383 | tbS'6' 384 | tRp69 385 | g2 386 | (g6 387 | S'\x14\x00\x00\x00' 388 | tRp70 389 | sg2 390 | (g3 391 | (S'S1' 392 | I0 393 | I1 394 | tRp71 395 | (I3 396 | S'|' 397 | NNNI1 398 | I1 399 | I0 400 | tbS'9' 401 | tRp72 402 | g2 403 | (g6 404 | S'\x17\x00\x00\x00' 405 | tRp73 406 | sg2 407 | (g3 408 | (S'S1' 409 | I0 410 | I1 411 | tRp74 412 | (I3 413 | S'|' 414 | NNNI1 415 | I1 416 | I0 417 | tbS'8' 418 | tRp75 419 | g2 420 | (g6 421 | S'\x16\x00\x00\x00' 422 | tRp76 423 | sg2 424 | (g3 425 | (S'S1' 426 | I0 427 | I1 428 | tRp77 429 | (I3 430 | S'|' 431 | NNNI1 432 | I1 433 | I0 434 | tbS';' 435 | tRp78 436 | g2 437 | (g6 438 | S'\x19\x00\x00\x00' 439 | tRp79 440 | sg2 441 | (g3 442 | (S'S1' 443 | I0 444 | I1 445 | tRp80 446 | (I3 447 | S'|' 448 | NNNI1 449 | I1 450 | I0 451 | tbS':' 452 | tRp81 453 | g2 454 | (g6 455 | S'\x18\x00\x00\x00' 456 | tRp82 457 | sg2 458 | (g3 459 | (S'S1' 460 | I0 461 | I1 462 | tRp83 463 | (I3 464 | S'|' 465 | NNNI1 466 | I1 467 | I0 468 | tbS'?' 469 | tRp84 470 | g2 471 | (g6 472 | S'\x1a\x00\x00\x00' 473 | tRp85 474 | sg2 475 | (g3 476 | (S'S1' 477 | I0 478 | I1 479 | tRp86 480 | (I3 481 | S'|' 482 | NNNI1 483 | I1 484 | I0 485 | tbS'A' 486 | tRp87 487 | g2 488 | (g6 489 | S'\x1b\x00\x00\x00' 490 | tRp88 491 | sg2 492 | (g3 493 | (S'S1' 494 | I0 495 | I1 496 | tRp89 497 | (I3 498 | S'|' 499 | NNNI1 500 | I1 501 | I0 502 | tbS'C' 503 | tRp90 504 | g2 505 | (g6 506 | S'\x1d\x00\x00\x00' 507 | tRp91 508 | sg2 509 | (g3 510 | (S'S1' 511 | I0 512 | I1 513 | tRp92 514 | (I3 515 | S'|' 516 | NNNI1 517 | I1 518 | I0 519 | tbS'B' 520 | tRp93 521 | g2 522 | (g6 523 | S'\x1c\x00\x00\x00' 524 | tRp94 525 | sg2 526 | (g3 527 | (S'S1' 528 | I0 529 | I1 530 | tRp95 531 | (I3 532 | S'|' 533 | NNNI1 534 | I1 535 | I0 536 | tbS'E' 537 | tRp96 538 | g2 539 | (g6 540 | S'\x1f\x00\x00\x00' 541 | tRp97 542 | sg2 543 | (g3 544 | (S'S1' 545 | I0 546 | I1 547 | tRp98 548 | (I3 549 | S'|' 550 | NNNI1 551 | I1 552 | I0 553 | tbS'D' 554 | tRp99 555 | g2 556 | (g6 557 | S'\x1e\x00\x00\x00' 558 | tRp100 559 | sg2 560 | (g3 561 | (S'S1' 562 | I0 563 | I1 564 | tRp101 565 | (I3 566 | S'|' 567 | NNNI1 568 | I1 569 | I0 570 | tbS'G' 571 | tRp102 572 | g2 573 | (g6 574 | S'!\x00\x00\x00' 575 | tRp103 576 | sg2 577 | (g3 578 | (S'S1' 579 | I0 580 | I1 581 | tRp104 582 | (I3 583 | S'|' 584 | NNNI1 585 | I1 586 | I0 587 | tbS'F' 588 | tRp105 589 | g2 590 | (g6 591 | S' \x00\x00\x00' 592 | tRp106 593 | sg2 594 | (g3 595 | (S'S1' 596 | I0 597 | I1 598 | tRp107 599 | (I3 600 | S'|' 601 | NNNI1 602 | I1 603 | I0 604 | tbS'I' 605 | tRp108 606 | g2 607 | (g6 608 | S'#\x00\x00\x00' 609 | tRp109 610 | sg2 611 | (g3 612 | (S'S1' 613 | I0 614 | I1 615 | tRp110 616 | (I3 617 | S'|' 618 | NNNI1 619 | I1 620 | I0 621 | tbS'H' 622 | tRp111 623 | g2 624 | (g6 625 | S'"\x00\x00\x00' 626 | tRp112 627 | sg2 628 | (g3 629 | (S'S1' 630 | I0 631 | I1 632 | tRp113 633 | (I3 634 | S'|' 635 | NNNI1 636 | I1 637 | I0 638 | tbS'K' 639 | tRp114 640 | g2 641 | (g6 642 | S'%\x00\x00\x00' 643 | tRp115 644 | sg2 645 | (g3 646 | (S'S1' 647 | I0 648 | I1 649 | tRp116 650 | (I3 651 | S'|' 652 | NNNI1 653 | I1 654 | I0 655 | tbS'J' 656 | tRp117 657 | g2 658 | (g6 659 | S'$\x00\x00\x00' 660 | tRp118 661 | sg2 662 | (g3 663 | (S'S1' 664 | I0 665 | I1 666 | tRp119 667 | (I3 668 | S'|' 669 | NNNI1 670 | I1 671 | I0 672 | tbS'M' 673 | tRp120 674 | g2 675 | (g6 676 | S"'\x00\x00\x00" 677 | tRp121 678 | sg2 679 | (g3 680 | (S'S1' 681 | I0 682 | I1 683 | tRp122 684 | (I3 685 | S'|' 686 | NNNI1 687 | I1 688 | I0 689 | tbS'L' 690 | tRp123 691 | g2 692 | (g6 693 | S'&\x00\x00\x00' 694 | tRp124 695 | sg2 696 | (g3 697 | (S'S1' 698 | I0 699 | I1 700 | tRp125 701 | (I3 702 | S'|' 703 | NNNI1 704 | I1 705 | I0 706 | tbS'O' 707 | tRp126 708 | g2 709 | (g6 710 | S')\x00\x00\x00' 711 | tRp127 712 | sg2 713 | (g3 714 | (S'S1' 715 | I0 716 | I1 717 | tRp128 718 | (I3 719 | S'|' 720 | NNNI1 721 | I1 722 | I0 723 | tbS'N' 724 | tRp129 725 | g2 726 | (g6 727 | S'(\x00\x00\x00' 728 | tRp130 729 | sg2 730 | (g3 731 | (S'S1' 732 | I0 733 | I1 734 | tRp131 735 | (I3 736 | S'|' 737 | NNNI1 738 | I1 739 | I0 740 | tbS'Q' 741 | tRp132 742 | g2 743 | (g6 744 | S'+\x00\x00\x00' 745 | tRp133 746 | sg2 747 | (g3 748 | (S'S1' 749 | I0 750 | I1 751 | tRp134 752 | (I3 753 | S'|' 754 | NNNI1 755 | I1 756 | I0 757 | tbS'P' 758 | tRp135 759 | g2 760 | (g6 761 | S'*\x00\x00\x00' 762 | tRp136 763 | sg2 764 | (g3 765 | (S'S1' 766 | I0 767 | I1 768 | tRp137 769 | (I3 770 | S'|' 771 | NNNI1 772 | I1 773 | I0 774 | tbS'S' 775 | tRp138 776 | g2 777 | (g6 778 | S'-\x00\x00\x00' 779 | tRp139 780 | sg2 781 | (g3 782 | (S'S1' 783 | I0 784 | I1 785 | tRp140 786 | (I3 787 | S'|' 788 | NNNI1 789 | I1 790 | I0 791 | tbS'R' 792 | tRp141 793 | g2 794 | (g6 795 | S',\x00\x00\x00' 796 | tRp142 797 | sg2 798 | (g3 799 | (S'S1' 800 | I0 801 | I1 802 | tRp143 803 | (I3 804 | S'|' 805 | NNNI1 806 | I1 807 | I0 808 | tbS'U' 809 | tRp144 810 | g2 811 | (g6 812 | S'/\x00\x00\x00' 813 | tRp145 814 | sg2 815 | (g3 816 | (S'S1' 817 | I0 818 | I1 819 | tRp146 820 | (I3 821 | S'|' 822 | NNNI1 823 | I1 824 | I0 825 | tbS'T' 826 | tRp147 827 | g2 828 | (g6 829 | S'.\x00\x00\x00' 830 | tRp148 831 | sg2 832 | (g3 833 | (S'S1' 834 | I0 835 | I1 836 | tRp149 837 | (I3 838 | S'|' 839 | NNNI1 840 | I1 841 | I0 842 | tbS'W' 843 | tRp150 844 | g2 845 | (g6 846 | S'1\x00\x00\x00' 847 | tRp151 848 | sg2 849 | (g3 850 | (S'S1' 851 | I0 852 | I1 853 | tRp152 854 | (I3 855 | S'|' 856 | NNNI1 857 | I1 858 | I0 859 | tbS'V' 860 | tRp153 861 | g2 862 | (g6 863 | S'0\x00\x00\x00' 864 | tRp154 865 | sg2 866 | (g3 867 | (S'S1' 868 | I0 869 | I1 870 | tRp155 871 | (I3 872 | S'|' 873 | NNNI1 874 | I1 875 | I0 876 | tbS'Y' 877 | tRp156 878 | g2 879 | (g6 880 | S'3\x00\x00\x00' 881 | tRp157 882 | sg2 883 | (g3 884 | (S'S1' 885 | I0 886 | I1 887 | tRp158 888 | (I3 889 | S'|' 890 | NNNI1 891 | I1 892 | I0 893 | tbS'X' 894 | tRp159 895 | g2 896 | (g6 897 | S'2\x00\x00\x00' 898 | tRp160 899 | sg2 900 | (g3 901 | (S'S1' 902 | I0 903 | I1 904 | tRp161 905 | (I3 906 | S'|' 907 | NNNI1 908 | I1 909 | I0 910 | tbS'[' 911 | tRp162 912 | g2 913 | (g6 914 | S'5\x00\x00\x00' 915 | tRp163 916 | sg2 917 | (g3 918 | (S'S1' 919 | I0 920 | I1 921 | tRp164 922 | (I3 923 | S'|' 924 | NNNI1 925 | I1 926 | I0 927 | tbS'Z' 928 | tRp165 929 | g2 930 | (g6 931 | S'4\x00\x00\x00' 932 | tRp166 933 | sg2 934 | (g3 935 | (S'S1' 936 | I0 937 | I1 938 | tRp167 939 | (I3 940 | S'|' 941 | NNNI1 942 | I1 943 | I0 944 | tbS']' 945 | tRp168 946 | g2 947 | (g6 948 | S'6\x00\x00\x00' 949 | tRp169 950 | sg2 951 | (g3 952 | (S'S1' 953 | I0 954 | I1 955 | tRp170 956 | (I3 957 | S'|' 958 | NNNI1 959 | I1 960 | I0 961 | tbS'a' 962 | tRp171 963 | g2 964 | (g6 965 | S'7\x00\x00\x00' 966 | tRp172 967 | sg2 968 | (g3 969 | (S'S1' 970 | I0 971 | I1 972 | tRp173 973 | (I3 974 | S'|' 975 | NNNI1 976 | I1 977 | I0 978 | tbS'c' 979 | tRp174 980 | g2 981 | (g6 982 | S'9\x00\x00\x00' 983 | tRp175 984 | sg2 985 | (g3 986 | (S'S1' 987 | I0 988 | I1 989 | tRp176 990 | (I3 991 | S'|' 992 | NNNI1 993 | I1 994 | I0 995 | tbS'b' 996 | tRp177 997 | g2 998 | (g6 999 | S'8\x00\x00\x00' 1000 | tRp178 1001 | sg2 1002 | (g3 1003 | (S'S1' 1004 | I0 1005 | I1 1006 | tRp179 1007 | (I3 1008 | S'|' 1009 | NNNI1 1010 | I1 1011 | I0 1012 | tbS'e' 1013 | tRp180 1014 | g2 1015 | (g6 1016 | S';\x00\x00\x00' 1017 | tRp181 1018 | sg2 1019 | (g3 1020 | (S'S1' 1021 | I0 1022 | I1 1023 | tRp182 1024 | (I3 1025 | S'|' 1026 | NNNI1 1027 | I1 1028 | I0 1029 | tbS'd' 1030 | tRp183 1031 | g2 1032 | (g6 1033 | S':\x00\x00\x00' 1034 | tRp184 1035 | sg2 1036 | (g3 1037 | (S'S1' 1038 | I0 1039 | I1 1040 | tRp185 1041 | (I3 1042 | S'|' 1043 | NNNI1 1044 | I1 1045 | I0 1046 | tbS'g' 1047 | tRp186 1048 | g2 1049 | (g6 1050 | S'=\x00\x00\x00' 1051 | tRp187 1052 | sg2 1053 | (g3 1054 | (S'S1' 1055 | I0 1056 | I1 1057 | tRp188 1058 | (I3 1059 | S'|' 1060 | NNNI1 1061 | I1 1062 | I0 1063 | tbS'f' 1064 | tRp189 1065 | g2 1066 | (g6 1067 | S'<\x00\x00\x00' 1068 | tRp190 1069 | sg2 1070 | (g3 1071 | (S'S1' 1072 | I0 1073 | I1 1074 | tRp191 1075 | (I3 1076 | S'|' 1077 | NNNI1 1078 | I1 1079 | I0 1080 | tbS'i' 1081 | tRp192 1082 | g2 1083 | (g6 1084 | S'?\x00\x00\x00' 1085 | tRp193 1086 | sg2 1087 | (g3 1088 | (S'S1' 1089 | I0 1090 | I1 1091 | tRp194 1092 | (I3 1093 | S'|' 1094 | NNNI1 1095 | I1 1096 | I0 1097 | tbS'h' 1098 | tRp195 1099 | g2 1100 | (g6 1101 | S'>\x00\x00\x00' 1102 | tRp196 1103 | sg2 1104 | (g3 1105 | (S'S1' 1106 | I0 1107 | I1 1108 | tRp197 1109 | (I3 1110 | S'|' 1111 | NNNI1 1112 | I1 1113 | I0 1114 | tbS'k' 1115 | tRp198 1116 | g2 1117 | (g6 1118 | S'A\x00\x00\x00' 1119 | tRp199 1120 | sg2 1121 | (g3 1122 | (S'S1' 1123 | I0 1124 | I1 1125 | tRp200 1126 | (I3 1127 | S'|' 1128 | NNNI1 1129 | I1 1130 | I0 1131 | tbS'j' 1132 | tRp201 1133 | g2 1134 | (g6 1135 | S'@\x00\x00\x00' 1136 | tRp202 1137 | sg2 1138 | (g3 1139 | (S'S1' 1140 | I0 1141 | I1 1142 | tRp203 1143 | (I3 1144 | S'|' 1145 | NNNI1 1146 | I1 1147 | I0 1148 | tbS'm' 1149 | tRp204 1150 | g2 1151 | (g6 1152 | S'C\x00\x00\x00' 1153 | tRp205 1154 | sg2 1155 | (g3 1156 | (S'S1' 1157 | I0 1158 | I1 1159 | tRp206 1160 | (I3 1161 | S'|' 1162 | NNNI1 1163 | I1 1164 | I0 1165 | tbS'l' 1166 | tRp207 1167 | g2 1168 | (g6 1169 | S'B\x00\x00\x00' 1170 | tRp208 1171 | sg2 1172 | (g3 1173 | (S'S1' 1174 | I0 1175 | I1 1176 | tRp209 1177 | (I3 1178 | S'|' 1179 | NNNI1 1180 | I1 1181 | I0 1182 | tbS'o' 1183 | tRp210 1184 | g2 1185 | (g6 1186 | S'E\x00\x00\x00' 1187 | tRp211 1188 | sg2 1189 | (g3 1190 | (S'S1' 1191 | I0 1192 | I1 1193 | tRp212 1194 | (I3 1195 | S'|' 1196 | NNNI1 1197 | I1 1198 | I0 1199 | tbS'n' 1200 | tRp213 1201 | g2 1202 | (g6 1203 | S'D\x00\x00\x00' 1204 | tRp214 1205 | sg2 1206 | (g3 1207 | (S'S1' 1208 | I0 1209 | I1 1210 | tRp215 1211 | (I3 1212 | S'|' 1213 | NNNI1 1214 | I1 1215 | I0 1216 | tbS'q' 1217 | tRp216 1218 | g2 1219 | (g6 1220 | S'G\x00\x00\x00' 1221 | tRp217 1222 | sg2 1223 | (g3 1224 | (S'S1' 1225 | I0 1226 | I1 1227 | tRp218 1228 | (I3 1229 | S'|' 1230 | NNNI1 1231 | I1 1232 | I0 1233 | tbS'p' 1234 | tRp219 1235 | g2 1236 | (g6 1237 | S'F\x00\x00\x00' 1238 | tRp220 1239 | sg2 1240 | (g3 1241 | (S'S1' 1242 | I0 1243 | I1 1244 | tRp221 1245 | (I3 1246 | S'|' 1247 | NNNI1 1248 | I1 1249 | I0 1250 | tbS's' 1251 | tRp222 1252 | g2 1253 | (g6 1254 | S'I\x00\x00\x00' 1255 | tRp223 1256 | sg2 1257 | (g3 1258 | (S'S1' 1259 | I0 1260 | I1 1261 | tRp224 1262 | (I3 1263 | S'|' 1264 | NNNI1 1265 | I1 1266 | I0 1267 | tbS'r' 1268 | tRp225 1269 | g2 1270 | (g6 1271 | S'H\x00\x00\x00' 1272 | tRp226 1273 | sg2 1274 | (g3 1275 | (S'S1' 1276 | I0 1277 | I1 1278 | tRp227 1279 | (I3 1280 | S'|' 1281 | NNNI1 1282 | I1 1283 | I0 1284 | tbS'u' 1285 | tRp228 1286 | g2 1287 | (g6 1288 | S'K\x00\x00\x00' 1289 | tRp229 1290 | sg2 1291 | (g3 1292 | (S'S1' 1293 | I0 1294 | I1 1295 | tRp230 1296 | (I3 1297 | S'|' 1298 | NNNI1 1299 | I1 1300 | I0 1301 | tbS't' 1302 | tRp231 1303 | g2 1304 | (g6 1305 | S'J\x00\x00\x00' 1306 | tRp232 1307 | sg2 1308 | (g3 1309 | (S'S1' 1310 | I0 1311 | I1 1312 | tRp233 1313 | (I3 1314 | S'|' 1315 | NNNI1 1316 | I1 1317 | I0 1318 | tbS'w' 1319 | tRp234 1320 | g2 1321 | (g6 1322 | S'M\x00\x00\x00' 1323 | tRp235 1324 | sg2 1325 | (g3 1326 | (S'S1' 1327 | I0 1328 | I1 1329 | tRp236 1330 | (I3 1331 | S'|' 1332 | NNNI1 1333 | I1 1334 | I0 1335 | tbS'v' 1336 | tRp237 1337 | g2 1338 | (g6 1339 | S'L\x00\x00\x00' 1340 | tRp238 1341 | sg2 1342 | (g3 1343 | (S'S1' 1344 | I0 1345 | I1 1346 | tRp239 1347 | (I3 1348 | S'|' 1349 | NNNI1 1350 | I1 1351 | I0 1352 | tbS'y' 1353 | tRp240 1354 | g2 1355 | (g6 1356 | S'O\x00\x00\x00' 1357 | tRp241 1358 | sg2 1359 | (g3 1360 | (S'S1' 1361 | I0 1362 | I1 1363 | tRp242 1364 | (I3 1365 | S'|' 1366 | NNNI1 1367 | I1 1368 | I0 1369 | tbS'x' 1370 | tRp243 1371 | g2 1372 | (g6 1373 | S'N\x00\x00\x00' 1374 | tRp244 1375 | sg2 1376 | (g3 1377 | (S'S1' 1378 | I0 1379 | I1 1380 | tRp245 1381 | (I3 1382 | S'|' 1383 | NNNI1 1384 | I1 1385 | I0 1386 | tbS'z' 1387 | tRp246 1388 | g2 1389 | (g6 1390 | S'P\x00\x00\x00' 1391 | tRp247 1392 | s(dp248 1393 | g2 1394 | (g6 1395 | S'\x00\x00\x00\x00' 1396 | tRp249 1397 | g9 1398 | sg2 1399 | (g6 1400 | S'\x01\x00\x00\x00' 1401 | tRp250 1402 | g5 1403 | sg2 1404 | (g6 1405 | S'\x02\x00\x00\x00' 1406 | tRp251 1407 | g15 1408 | sg2 1409 | (g6 1410 | S'\x03\x00\x00\x00' 1411 | tRp252 1412 | g12 1413 | sg2 1414 | (g6 1415 | S'\x04\x00\x00\x00' 1416 | tRp253 1417 | g18 1418 | sg2 1419 | (g6 1420 | S'\x05\x00\x00\x00' 1421 | tRp254 1422 | g24 1423 | sg2 1424 | (g6 1425 | S'\x06\x00\x00\x00' 1426 | tRp255 1427 | g21 1428 | sg2 1429 | (g6 1430 | S'\x07\x00\x00\x00' 1431 | tRp256 1432 | g30 1433 | sg2 1434 | (g6 1435 | S'\x08\x00\x00\x00' 1436 | tRp257 1437 | g27 1438 | sg2 1439 | (g6 1440 | S'\t\x00\x00\x00' 1441 | tRp258 1442 | g33 1443 | sg2 1444 | (g6 1445 | S'\n\x00\x00\x00' 1446 | tRp259 1447 | g39 1448 | sg2 1449 | (g6 1450 | S'\x0b\x00\x00\x00' 1451 | tRp260 1452 | g36 1453 | sg2 1454 | (g6 1455 | S'\x0c\x00\x00\x00' 1456 | tRp261 1457 | g45 1458 | sg2 1459 | (g6 1460 | S'\r\x00\x00\x00' 1461 | tRp262 1462 | g42 1463 | sg2 1464 | (g6 1465 | S'\x0e\x00\x00\x00' 1466 | tRp263 1467 | g51 1468 | sg2 1469 | (g6 1470 | S'\x0f\x00\x00\x00' 1471 | tRp264 1472 | g48 1473 | sg2 1474 | (g6 1475 | S'\x10\x00\x00\x00' 1476 | tRp265 1477 | g57 1478 | sg2 1479 | (g6 1480 | S'\x11\x00\x00\x00' 1481 | tRp266 1482 | g54 1483 | sg2 1484 | (g6 1485 | S'\x12\x00\x00\x00' 1486 | tRp267 1487 | g63 1488 | sg2 1489 | (g6 1490 | S'\x13\x00\x00\x00' 1491 | tRp268 1492 | g60 1493 | sg2 1494 | (g6 1495 | S'\x14\x00\x00\x00' 1496 | tRp269 1497 | g69 1498 | sg2 1499 | (g6 1500 | S'\x15\x00\x00\x00' 1501 | tRp270 1502 | g66 1503 | sg2 1504 | (g6 1505 | S'\x16\x00\x00\x00' 1506 | tRp271 1507 | g75 1508 | sg2 1509 | (g6 1510 | S'\x17\x00\x00\x00' 1511 | tRp272 1512 | g72 1513 | sg2 1514 | (g6 1515 | S'\x18\x00\x00\x00' 1516 | tRp273 1517 | g81 1518 | sg2 1519 | (g6 1520 | S'\x19\x00\x00\x00' 1521 | tRp274 1522 | g78 1523 | sg2 1524 | (g6 1525 | S'\x1a\x00\x00\x00' 1526 | tRp275 1527 | g84 1528 | sg2 1529 | (g6 1530 | S'\x1b\x00\x00\x00' 1531 | tRp276 1532 | g87 1533 | sg2 1534 | (g6 1535 | S'\x1c\x00\x00\x00' 1536 | tRp277 1537 | g93 1538 | sg2 1539 | (g6 1540 | S'\x1d\x00\x00\x00' 1541 | tRp278 1542 | g90 1543 | sg2 1544 | (g6 1545 | S'\x1e\x00\x00\x00' 1546 | tRp279 1547 | g99 1548 | sg2 1549 | (g6 1550 | S'\x1f\x00\x00\x00' 1551 | tRp280 1552 | g96 1553 | sg2 1554 | (g6 1555 | S' \x00\x00\x00' 1556 | tRp281 1557 | g105 1558 | sg2 1559 | (g6 1560 | S'!\x00\x00\x00' 1561 | tRp282 1562 | g102 1563 | sg2 1564 | (g6 1565 | S'"\x00\x00\x00' 1566 | tRp283 1567 | g111 1568 | sg2 1569 | (g6 1570 | S'#\x00\x00\x00' 1571 | tRp284 1572 | g108 1573 | sg2 1574 | (g6 1575 | S'$\x00\x00\x00' 1576 | tRp285 1577 | g117 1578 | sg2 1579 | (g6 1580 | S'%\x00\x00\x00' 1581 | tRp286 1582 | g114 1583 | sg2 1584 | (g6 1585 | S'&\x00\x00\x00' 1586 | tRp287 1587 | g123 1588 | sg2 1589 | (g6 1590 | S"'\x00\x00\x00" 1591 | tRp288 1592 | g120 1593 | sg2 1594 | (g6 1595 | S'(\x00\x00\x00' 1596 | tRp289 1597 | g129 1598 | sg2 1599 | (g6 1600 | S')\x00\x00\x00' 1601 | tRp290 1602 | g126 1603 | sg2 1604 | (g6 1605 | S'*\x00\x00\x00' 1606 | tRp291 1607 | g135 1608 | sg2 1609 | (g6 1610 | S'+\x00\x00\x00' 1611 | tRp292 1612 | g132 1613 | sg2 1614 | (g6 1615 | S',\x00\x00\x00' 1616 | tRp293 1617 | g141 1618 | sg2 1619 | (g6 1620 | S'-\x00\x00\x00' 1621 | tRp294 1622 | g138 1623 | sg2 1624 | (g6 1625 | S'.\x00\x00\x00' 1626 | tRp295 1627 | g147 1628 | sg2 1629 | (g6 1630 | S'/\x00\x00\x00' 1631 | tRp296 1632 | g144 1633 | sg2 1634 | (g6 1635 | S'0\x00\x00\x00' 1636 | tRp297 1637 | g153 1638 | sg2 1639 | (g6 1640 | S'1\x00\x00\x00' 1641 | tRp298 1642 | g150 1643 | sg2 1644 | (g6 1645 | S'2\x00\x00\x00' 1646 | tRp299 1647 | g159 1648 | sg2 1649 | (g6 1650 | S'3\x00\x00\x00' 1651 | tRp300 1652 | g156 1653 | sg2 1654 | (g6 1655 | S'4\x00\x00\x00' 1656 | tRp301 1657 | g165 1658 | sg2 1659 | (g6 1660 | S'5\x00\x00\x00' 1661 | tRp302 1662 | g162 1663 | sg2 1664 | (g6 1665 | S'6\x00\x00\x00' 1666 | tRp303 1667 | g168 1668 | sg2 1669 | (g6 1670 | S'7\x00\x00\x00' 1671 | tRp304 1672 | g171 1673 | sg2 1674 | (g6 1675 | S'8\x00\x00\x00' 1676 | tRp305 1677 | g177 1678 | sg2 1679 | (g6 1680 | S'9\x00\x00\x00' 1681 | tRp306 1682 | g174 1683 | sg2 1684 | (g6 1685 | S':\x00\x00\x00' 1686 | tRp307 1687 | g183 1688 | sg2 1689 | (g6 1690 | S';\x00\x00\x00' 1691 | tRp308 1692 | g180 1693 | sg2 1694 | (g6 1695 | S'<\x00\x00\x00' 1696 | tRp309 1697 | g189 1698 | sg2 1699 | (g6 1700 | S'=\x00\x00\x00' 1701 | tRp310 1702 | g186 1703 | sg2 1704 | (g6 1705 | S'>\x00\x00\x00' 1706 | tRp311 1707 | g195 1708 | sg2 1709 | (g6 1710 | S'?\x00\x00\x00' 1711 | tRp312 1712 | g192 1713 | sg2 1714 | (g6 1715 | S'@\x00\x00\x00' 1716 | tRp313 1717 | g201 1718 | sg2 1719 | (g6 1720 | S'A\x00\x00\x00' 1721 | tRp314 1722 | g198 1723 | sg2 1724 | (g6 1725 | S'B\x00\x00\x00' 1726 | tRp315 1727 | g207 1728 | sg2 1729 | (g6 1730 | S'C\x00\x00\x00' 1731 | tRp316 1732 | g204 1733 | sg2 1734 | (g6 1735 | S'D\x00\x00\x00' 1736 | tRp317 1737 | g213 1738 | sg2 1739 | (g6 1740 | S'E\x00\x00\x00' 1741 | tRp318 1742 | g210 1743 | sg2 1744 | (g6 1745 | S'F\x00\x00\x00' 1746 | tRp319 1747 | g219 1748 | sg2 1749 | (g6 1750 | S'G\x00\x00\x00' 1751 | tRp320 1752 | g216 1753 | sg2 1754 | (g6 1755 | S'H\x00\x00\x00' 1756 | tRp321 1757 | g225 1758 | sg2 1759 | (g6 1760 | S'I\x00\x00\x00' 1761 | tRp322 1762 | g222 1763 | sg2 1764 | (g6 1765 | S'J\x00\x00\x00' 1766 | tRp323 1767 | g231 1768 | sg2 1769 | (g6 1770 | S'K\x00\x00\x00' 1771 | tRp324 1772 | g228 1773 | sg2 1774 | (g6 1775 | S'L\x00\x00\x00' 1776 | tRp325 1777 | g237 1778 | sg2 1779 | (g6 1780 | S'M\x00\x00\x00' 1781 | tRp326 1782 | g234 1783 | sg2 1784 | (g6 1785 | S'N\x00\x00\x00' 1786 | tRp327 1787 | g243 1788 | sg2 1789 | (g6 1790 | S'O\x00\x00\x00' 1791 | tRp328 1792 | g240 1793 | sg2 1794 | (g6 1795 | S'P\x00\x00\x00' 1796 | tRp329 1797 | g246 1798 | st. --------------------------------------------------------------------------------