├── __init__.py ├── writeup.pdf ├── .gitignore ├── README.md ├── restore.py ├── reconstructions.py ├── nn_utils.py ├── evaluate.py ├── loss.py ├── datasets.py ├── original_impl ├── vanilla_vae_class.py ├── vae_nf.py ├── vae_iaf.py └── approximating_distributions.ipynb ├── neural_networks.py ├── models.py └── train.py /__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /writeup.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wellecks/vaes/HEAD/writeup.pdf -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .ipynb_checkpoints 3 | *.pyc 4 | data 5 | results 6 | results_old 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | To train: 2 | ```bash 3 | python train.py [-h] (--basic | --nf | --iaf | --hf | --liaf) [--flow FLOW] 4 | 5 | # E.g. 6 | python train.py --basic 7 | python train.py --nf --flow 10 8 | ``` 9 | 10 | #### Notes 11 | - All the models (IAF, NF, VAE etc) are in `models.py`. 12 | 13 | - All the neural net functions are in `neural_networks.py`. 14 | 15 | - All the loss functions are in `loss.py` 16 | 17 | - The train function accepts an encoder and a decoder. This is where you get to specify the type of encoder/decoder 18 | 19 | - An encoder takes in (x, e) and spits out z. 20 | 21 | - A decoder takes in z and spits out x 22 | 23 | - To implement a new encoder F with hyperparameters W, we define a hidden function _F_encoder(x, e, W) and then set F_encoder(W) = lambda x, e: _F_encoder(x, e, W). Usually W comprises a neural network, flow lengths and so on. The method for decoders is identical. This allows us to define completely generic encoders and decoders with arbitrary structures and hyperparameters. 24 | -------------------------------------------------------------------------------- /restore.py: -------------------------------------------------------------------------------- 1 | """Functions for restoring variables""" 2 | 3 | import os 4 | import tensorflow as tf 5 | 6 | def get_saved_variable_values(checkpoint_directory, metagraph_name): 7 | """Return mapping of variable names to numpy arrays""" 8 | # HACK: create a temporary session, and store the raw values of the variables 9 | sess = tf.Session() 10 | saver = tf.train.import_meta_graph(os.path.join(checkpoint_directory, metagraph_name)) 11 | saver.restore(sess, tf.train.latest_checkpoint(checkpoint_directory)) 12 | 13 | variables = {} 14 | for v in tf.trainable_variables(): 15 | variables[v.name] = v.eval(session=sess) 16 | 17 | # Clear the loaded variables from the graph 18 | tf.reset_default_graph() 19 | sess.close() 20 | return variables 21 | 22 | def set_variables(sess, variable_values): 23 | """Set trainable variables in `sess` to the values contained in `variable_values`.""" 24 | current_variables = {v.name: v for v in tf.trainable_variables()} 25 | for name, v in variable_values.iteritems(): 26 | if name in current_variables: 27 | sess.run(current_variables[name].assign(v)) 28 | del variable_values 29 | print('Set pre-trained variables') 30 | -------------------------------------------------------------------------------- /reconstructions.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | 5 | def reconstruct(sess, input_data, out_op, x): 6 | x_rec = sess.run([out_op], feed_dict={x: input_data}) 7 | return x_rec 8 | 9 | def show_reconstruction(actual, recons, image_width): 10 | fig, axs = plt.subplots(1, len(recons) + 1) 11 | axs[0].set_title('actual') 12 | axs[0].imshow(actual.reshape(image_width, image_width), cmap='gray') 13 | for i, recon in enumerate(recons): 14 | axs[i+1].imshow(recon[0].reshape(image_width, image_width), cmap='gray') 15 | axs[i+1].set_title('reconstructed, iteration {}'.format(recon[1])) 16 | plt.show() 17 | 18 | def vec2im(x, batch_size, image_width): 19 | shape = (batch_size, image_width, image_width, 1) 20 | return tf.reshape(x, shape) # reshape to 4D 21 | 22 | def plot_images_together(images, image_width=28): 23 | """ Plot a single image containing all six MNIST images, one after 24 | the other. Note that we crop the sides of the images so that they 25 | appear reasonably close together.""" 26 | fig = plt.figure() 27 | image = np.concatenate(images, axis=1) 28 | ax = fig.add_subplot(1, 1, 1) 29 | ax.matshow(image, cmap = 'gray') 30 | plt.xticks(np.array([])) 31 | plt.yticks(np.array([])) 32 | plt.show() 33 | -------------------------------------------------------------------------------- /nn_utils.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | def conv2d(x, W, b, strides=1): 5 | """Conv2D wrapper, with bias and relu activation""" 6 | x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='SAME') 7 | x = tf.nn.bias_add(x, b) 8 | return tf.nn.relu(x) 9 | 10 | def maxpool2d(x, k=2): 11 | return tf.nn.max_pool(x, ksize=[1, k, k, 1], strides=[1, k, k, 1], 12 | padding='SAME') 13 | 14 | def dtanh(tensor): 15 | return 1.0 - tf.square(tf.nn.tanh(tensor)) 16 | 17 | def weight_variable(shape): 18 | """Create a weight variable with appropriate initialization.""" 19 | return tf.get_variable('weights', shape, initializer=tf.contrib.layers.xavier_initializer()) 20 | 21 | def bias_variable(shape): 22 | """Create a bias variable with appropriate initialization.""" 23 | return tf.get_variable('bias', shape, initializer=tf.constant_initializer(0.1)) 24 | 25 | def variable_summaries(var): 26 | """Attach a lot of summaries to a Tensor (for TensorBoard visualization).""" 27 | with tf.variable_scope('summaries'): 28 | mean = tf.reduce_mean(var) 29 | tf.scalar_summary('mean', mean) 30 | stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) 31 | tf.scalar_summary('stddev', stddev) 32 | tf.scalar_summary('max', tf.reduce_max(var)) 33 | tf.scalar_summary('min', tf.reduce_min(var)) 34 | tf.histogram_summary('histogram', var) 35 | 36 | def whiten(data): 37 | mean = np.mean(data, 0, keepdims=True) 38 | std = np.std(data - mean, 0, keepdims=True) 39 | return (data - mean) / (std + 1e-7) 40 | -------------------------------------------------------------------------------- /evaluate.py: -------------------------------------------------------------------------------- 1 | """Evaluate a check-pointed model on MNIST test set. 2 | 3 | E.g. python evaluate.py results/2016-12-14_11-45-25-basic 4 | """ 5 | 6 | import argparse 7 | import numpy as np 8 | import tensorflow as tf 9 | 10 | from loss import elbo_loss 11 | from datasets import binarized_mnist 12 | from models import basic_decoder, basic_encoder, nf_encoder, iaf_encoder, hf_encoder, nn 13 | 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument('checkpoint_dir') 16 | args = parser.parse_args() 17 | 18 | tf.reset_default_graph() 19 | 20 | # Parse model name and flow 21 | toks = args.checkpoint_dir.split('-') 22 | model = toks[5] 23 | flow = int(toks[6]) if len(toks) > 6 else None 24 | batch_size = 100 25 | print('model: %s\tflow:%s' % (model, flow)) 26 | 27 | # Build computation graph and operations 28 | if model == 'basic': 29 | encoder_type = basic_encoder 30 | if model == 'nf': 31 | encoder_type = nf_encoder 32 | if model == 'iaf': 33 | encoder_type = iaf_encoder 34 | if model == 'hf': 35 | encoder_type = hf_encoder 36 | 37 | dim_x, dim_z, enc_dims, dec_dims = 784, 40, [300, 300], [300, 300] # HACK 38 | decoder_net = lambda z: nn(z, dec_dims, name='decoder', act=tf.nn.tanh) 39 | encoder_net = lambda x: nn(x, enc_dims, name='encoder', act=tf.nn.tanh, is_training=False) 40 | encoder = encoder_type(encoder_net, dim_z, flow) 41 | decoder = basic_decoder(decoder_net, dim_x) 42 | x = tf.placeholder(tf.float32, [None, dim_x], 'x') 43 | x_w = tf.placeholder(tf.float32, [None, dim_x], 'x_w') 44 | e = tf.placeholder(tf.float32, (None, dim_z), 'noise') 45 | is_training = tf.placeholder(tf.bool) 46 | z_params, z = encoder(x, e) 47 | x_pred = decoder(z) 48 | loss_ops = elbo_loss(x_pred, x, **z_params) 49 | 50 | # Create a session and restore checkpoint 51 | sess = tf.InteractiveSession() 52 | saver = tf.train.Saver() 53 | ckpt = tf.train.get_checkpoint_state(args.checkpoint_dir) 54 | saver.restore(sess, ckpt.model_checkpoint_path) 55 | 56 | # HACK: assumes specific directory structure and dataset 57 | dataset = binarized_mnist()['test'] 58 | n_batches = dataset.num_examples / batch_size 59 | feed_dict = {} 60 | outputs = {k: [] for k in loss_ops.iterkeys()} 61 | 62 | for _ in range(n_batches): 63 | feed_dict[x], feed_dict[x_w] = dataset.next_batch(batch_size, whitened=False) 64 | feed_dict[e] = np.random.normal(0, 1, (batch_size, dim_z)) 65 | feed_dict[is_training] = False 66 | output = sess.run(loss_ops, feed_dict=feed_dict) 67 | for k, v in output.iteritems(): 68 | outputs[k].append(v) 69 | 70 | for k, vs in outputs.iteritems(): 71 | outputs[k] = np.array(vs) 72 | print('avg. %s: %.3f' % (k, outputs[k].mean())) 73 | -------------------------------------------------------------------------------- /loss.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | 4 | def cross_entropy(obs, actual, offset=1e-7): 5 | """Binary cross-entropy, per training example""" 6 | with tf.name_scope("cross_entropy"): 7 | # bound by clipping to avoid nan 8 | obs_ = tf.clip_by_value(obs, offset, 1 - offset) 9 | return -tf.reduce_sum(actual * tf.log(obs_) + 10 | (1 - actual) * tf.log(1 - obs_), 1) 11 | 12 | def l2_loss(obs, actual): 13 | with tf.name_scope("l2"): 14 | return tf.nn.l2_loss(obs - actual) 15 | 16 | def analytic_kl_divergence(mu, log_std): 17 | """Compute the batched KL divergence between a diagonal Gaussian and a standard Gaussian""" 18 | kl = -(0.5*tf.reduce_sum(1.0 + 2 * log_std - tf.square(mu) - tf.exp(2 * log_std), 1)) 19 | return kl 20 | 21 | def gaussian_log_pdf(mu, log_std, z): 22 | """Compute the log probability density of point z under Gaussian with mean mu and 23 | diagonal standard deviation exp(log_std)""" 24 | return tf.contrib.distributions.MultivariateNormalDiag( 25 | mu=mu, diag_stdev=tf.maximum(tf.exp(log_std), 1e-15)).log_pdf(z) 26 | 27 | def elbo_loss(pred, actual, var_reg=1, kl_weighting=1, rec_err_fn=cross_entropy, **kwargs): 28 | monitor_functions = {} 29 | 30 | mu = kwargs['mu'] 31 | log_std = kwargs['log_std'] 32 | if 'sum_log_detj' not in kwargs: 33 | raw_kl = tf.reduce_mean(analytic_kl_divergence(mu, log_std), name='kl') 34 | rec_err = tf.reduce_mean(rec_err_fn(pred, actual), name='rec_err') 35 | else: 36 | sum_log_detj, z0, zk = kwargs['sum_log_detj'], kwargs['z0'], kwargs['zk'] 37 | sum_log_detj = tf.reduce_mean(sum_log_detj, name='sum_log_detj') 38 | log_q0_z0 = tf.reduce_mean(gaussian_log_pdf(mu, log_std, z0), name='log_q0_z0') 39 | log_qk_zk = log_q0_z0 - sum_log_detj 40 | 41 | log_p_zk = tf.reduce_mean(gaussian_log_pdf(tf.zeros_like(mu), tf.zeros_like(mu), zk), name='log_p_zk') 42 | log_p_x_given_zk = -tf.reduce_mean(rec_err_fn(pred, actual), name='log_p_x_given_zk') 43 | rec_err = -log_p_x_given_zk 44 | raw_kl = log_qk_zk - log_p_zk 45 | monitor_functions.update({ 46 | 'log_p_zk': log_p_zk, 47 | 'log_qk_zk': log_qk_zk, 48 | 'log_p_x_given_zk': log_p_x_given_zk, 49 | 'log_q0_z0': log_q0_z0, 50 | 'sum_log_detj': sum_log_detj 51 | }) 52 | 53 | weighted_kl = kl_weighting * raw_kl 54 | unweighted_elbo = raw_kl + rec_err 55 | weighted_elbo = weighted_kl + rec_err 56 | 57 | train_loss = weighted_elbo 58 | valid_loss = unweighted_elbo 59 | 60 | monitor_functions.update({ 61 | 'weighted_kl': weighted_kl, 62 | 'unweighted_elbo': unweighted_elbo, 63 | 'weighted_elbo': weighted_elbo, 64 | 'raw_kl': raw_kl, 65 | 'rec_err': rec_err, 66 | 'train_loss': train_loss, 67 | 'valid_loss': valid_loss 68 | }) 69 | return monitor_functions 70 | -------------------------------------------------------------------------------- /datasets.py: -------------------------------------------------------------------------------- 1 | import os 2 | import urllib 3 | import numpy as np 4 | 5 | from nn_utils import whiten 6 | 7 | 8 | def create_binarized_mnist(tpe, dataset_dir='data'): 9 | print('Creating binarized %s dataset' % tpe) 10 | file_path = os.path.join(dataset_dir, 'binarized_mnist_{}.amat'.format(tpe)) 11 | 12 | # Download dataset if necessary 13 | if not os.path.isfile(file_path): 14 | if not os.path.exists(dataset_dir): 15 | os.makedirs(dataset_dir) 16 | url = 'http://www.cs.toronto.edu/~larocheh/public/datasets/binarized_mnist/binarized_mnist_{}.amat'.format(tpe) 17 | urllib.urlretrieve(url, file_path) 18 | print('Downloaded %s to %s' % (url, file_path)) 19 | 20 | with open(file_path) as f: 21 | data = [l.strip().split(' ') for l in f.readlines()] 22 | data = np.array(data).astype(int) 23 | np.save(os.path.join(dataset_dir, 'binarized_mnist_{}.npy'.format(tpe)), data) 24 | return data 25 | 26 | 27 | def binarized_mnist(dataset_dir='data'): 28 | # Download and create datasets if necessary 29 | tpes = ['train', 'valid', 'test'] 30 | for tpe in tpes: 31 | if not os.path.isfile(os.path.join(dataset_dir, 'binarized_mnist_{}.npy'.format(tpe))): 32 | create_binarized_mnist(tpe) 33 | 34 | return {tpe: UnlabelledDataSet(np.load(os.path.join(dataset_dir, 'binarized_mnist_{}.npy'.format(tpe)))) 35 | for tpe in tpes} 36 | 37 | 38 | class UnlabelledDataSet(object): 39 | def __init__(self, 40 | images): 41 | self._num_examples = images.shape[0] 42 | self._images = images 43 | self._whitened_images = whiten(images) 44 | self._epochs_completed = 0 45 | self._index_in_epoch = 0 46 | 47 | @property 48 | def images(self): 49 | return self._images 50 | 51 | @property 52 | def whitened_images(self): 53 | return self._whitened_images 54 | 55 | @property 56 | def num_examples(self): 57 | return self._num_examples 58 | 59 | @property 60 | def epochs_completed(self): 61 | return self._epochs_completed 62 | 63 | def next_batch(self, batch_size, whitened=False): 64 | """Return the next `batch_size` examples from this data set.""" 65 | start = self._index_in_epoch 66 | self._index_in_epoch += batch_size 67 | if self._index_in_epoch > self._num_examples: 68 | # Finished epoch 69 | self._epochs_completed += 1 70 | # Shuffle the data 71 | perm = np.arange(self._num_examples) 72 | np.random.shuffle(perm) 73 | self._images = self._images[perm] 74 | # Start next epoch 75 | start = 0 76 | self._index_in_epoch = batch_size 77 | assert batch_size <= self._num_examples 78 | end = self._index_in_epoch 79 | if whitened: 80 | return self.images[start:end], self._whitened_images[start:end] 81 | else: return self._images[start:end], self.images[start:end] 82 | -------------------------------------------------------------------------------- /original_impl/vanilla_vae_class.py: -------------------------------------------------------------------------------- 1 | """Variational Auto-encoder, class-based. 2 | 3 | References 4 | ---------- 5 | https://arxiv.org/pdf/1312.6114v10.pdf 6 | """ 7 | 8 | import numpy as np 9 | import tensorflow as tf 10 | 11 | class VAE(object): 12 | def __init__(self, data_dim, z_dim, enc_h=128, dec_h=128, lr=0.01): 13 | self.initializer = tf.contrib.layers.xavier_initializer 14 | self.z_dim = z_dim 15 | self.x, self.e = self.inputs(data_dim, z_dim) 16 | self.mu, self.log_var, self.z = self.encoder(self.x, self.e, data_dim, enc_h, z_dim) 17 | self.out, self.out_mu, self.out_log_var = self.decoder(self.z, data_dim, dec_h, z_dim) 18 | self.loss_op = self.make_loss(self.out_mu, self.x, self.log_var, self.mu, self.out_log_var) 19 | self.train_op = tf.train.AdamOptimizer(lr).minimize(self.loss_op) 20 | 21 | def inputs(self, D, Z): 22 | x = tf.placeholder(tf.float32, [None, D], 'x') 23 | e = tf.placeholder(tf.float32, [None, Z], 'e') 24 | return x, e 25 | 26 | def encoder(self, x, e, D, H, Z): 27 | with tf.variable_scope('encoder'): 28 | w_h = tf.get_variable('w_h', [D, H], initializer=self.initializer()) 29 | b_h = tf.get_variable('b_h', [H], initializer=self.initializer()) 30 | w_mu = tf.get_variable('w_mu', [H, Z], initializer=self.initializer()) 31 | b_mu = tf.get_variable('b_mu', [Z], initializer=self.initializer()) 32 | w_v = tf.get_variable('w_v', [H, Z], initializer=self.initializer()) 33 | b_v = tf.get_variable('b_v', [Z], initializer=self.initializer()) 34 | 35 | h = tf.nn.tanh(tf.matmul(x, w_h) + b_h) 36 | mu = tf.matmul(h, w_mu) + b_mu 37 | log_var = tf.matmul(h, w_v) + b_v 38 | z = mu + tf.sqrt(tf.exp(log_var)) * e 39 | return mu, log_var, z 40 | 41 | def decoder(self, z, D, H, Z): 42 | with tf.variable_scope('decoder'): 43 | w_h = tf.get_variable('w_h', [Z, H], initializer=self.initializer()) 44 | b_h = tf.get_variable('b_h', [H], initializer=self.initializer()) 45 | w_mu = tf.get_variable('w_mu', [H, D], initializer=self.initializer()) 46 | b_mu = tf.get_variable('b_mu', [D], initializer=self.initializer()) 47 | w_v = tf.get_variable('w_v', [H, 1], initializer=self.initializer()) 48 | b_v = tf.get_variable('b_v', [1], initializer=self.initializer()) 49 | 50 | h = tf.nn.tanh(tf.matmul(z, w_h) + b_h) 51 | out_mu = tf.matmul(h, w_mu) + b_mu 52 | out_log_var = tf.matmul(h, w_v) + b_v 53 | # NOTE(wellecks) Enforce 0, 1 (MNIST-specific) 54 | out = tf.sigmoid(out_mu) 55 | return out, out_mu, out_log_var 56 | 57 | def make_loss(self, pred, actual, log_var, mu, out_log_var): 58 | kl = 0.5 * tf.reduce_sum(1.0 + log_var - tf.square(mu) - tf.exp(log_var), 1) 59 | rec_err = -0.5 * (tf.nn.l2_loss(actual - pred)) 60 | loss = -tf.reduce_mean(kl + rec_err) 61 | return loss 62 | 63 | def train_step(self, sess, input_data): 64 | e_ = np.random.normal(size=(input_data.shape[0], self.z_dim)) 65 | _, loss = sess.run([self.train_op, self.loss_op], {self.x: input_data, self.e: e_}) 66 | return loss 67 | 68 | def reconstruct(self, sess, input_data): 69 | e_ = np.random.normal(size=(input_data.shape[0], self.z_dim)) 70 | x_rec = sess.run([self.out], {self.x: input_data, self.e: e_}) 71 | return x_rec 72 | 73 | def sample_latent(self, sess, input_data): 74 | e_ = np.random.normal(size=(input_data.shape[0], self.z_dim)) 75 | zs = sess.run(self.z, feed_dict={self.x: input_data, self.e: e_}) 76 | return zs 77 | -------------------------------------------------------------------------------- /original_impl/vae_nf.py: -------------------------------------------------------------------------------- 1 | """Variational Auto-encoder with Normalizing Flows 2 | 3 | References 4 | ---------- 5 | https://arxiv.org/pdf/1505.05770v6.pdf 6 | """ 7 | 8 | import matplotlib.pyplot as plt 9 | import numpy as np 10 | import tensorflow as tf 11 | 12 | def inputs(D, Z): 13 | x = tf.placeholder(tf.float32, [None, D], 'x') 14 | e = tf.placeholder(tf.float32, [None, Z], 'e') 15 | return x, e 16 | 17 | def encoder(x, e, D, H, Z, K, initializer=tf.contrib.layers.xavier_initializer): 18 | with tf.variable_scope('encoder'): 19 | w_h = tf.get_variable('w_h', [D, H], initializer=initializer()) 20 | b_h = tf.get_variable('b_h', [H]) 21 | w_mu = tf.get_variable('w_mu', [H, Z], initializer=initializer()) 22 | b_mu = tf.get_variable('b_mu', [Z]) 23 | w_v = tf.get_variable('w_v', [H, Z], initializer=initializer()) 24 | b_v = tf.get_variable('b_v', [Z]) 25 | 26 | # Weights for outputting normalizing flow parameters 27 | w_us = tf.get_variable('w_us', [H, K*Z]) 28 | b_us = tf.get_variable('b_us', [K*Z]) 29 | w_ws = tf.get_variable('w_ws', [H, K*Z]) 30 | b_ws = tf.get_variable('b_ws', [K*Z]) 31 | w_bs = tf.get_variable('w_bs', [H, K]) 32 | b_bs = tf.get_variable('b_bs', [K]) 33 | 34 | h = tf.nn.tanh(tf.matmul(x, w_h) + b_h) 35 | mu = tf.matmul(h, w_mu) + b_mu 36 | log_var = tf.matmul(h, w_v) + b_v 37 | z = mu + tf.sqrt(tf.exp(log_var))*e 38 | 39 | # Normalizing Flow parameters 40 | us = tf.matmul(h, w_us) + b_us 41 | ws = tf.matmul(h, w_ws) + b_ws 42 | bs = tf.matmul(h, w_bs) + b_bs 43 | 44 | lambd = (us, ws, bs) 45 | 46 | return mu, log_var, z, lambd 47 | 48 | def dtanh(tensor): 49 | return 1.0 - tf.square(tf.tanh(tensor)) 50 | 51 | def norm_flow(z, lambd, K, Z): 52 | us, ws, bs = lambd 53 | 54 | log_detjs = [] 55 | for k in range(K): 56 | u, w, b = us[:, k*Z:(k+1)*Z], ws[:, k*Z:(k+1)*Z], bs[:, k] 57 | temp = tf.expand_dims(tf.nn.tanh(tf.reduce_sum(w*z, 1) + b), 1) 58 | temp = tf.tile(temp, [1, u.get_shape()[1].value]) 59 | z = z + tf.mul(u, temp) 60 | 61 | # Eqn. (11) and (12) 62 | temp = tf.expand_dims(dtanh(tf.reduce_sum(w*z, 1) + b), 1) 63 | temp = tf.tile(temp, [1, w.get_shape()[1].value]) 64 | log_detj = tf.abs(1. + tf.reduce_sum(tf.mul(u, temp*w), 1)) 65 | log_detjs.append(log_detj) 66 | 67 | if K != 0: 68 | log_detj = tf.reduce_sum(log_detjs) 69 | else: log_detj = 0 70 | 71 | return z, log_detj 72 | 73 | 74 | def decoder(z, D, H, Z, initializer=tf.contrib.layers.xavier_initializer, out_fn=tf.sigmoid): 75 | with tf.variable_scope('decoder'): 76 | w_h = tf.get_variable('w_h', [Z, H], initializer=initializer()) 77 | b_h = tf.get_variable('b_h', [H]) 78 | w_mu = tf.get_variable('w_mu', [H, D], initializer=initializer()) 79 | b_mu = tf.get_variable('b_mu', [D]) 80 | w_v = tf.get_variable('w_v', [H, 1], initializer=initializer()) 81 | b_v = tf.get_variable('b_v', [1]) 82 | 83 | h = tf.nn.tanh(tf.matmul(z, w_h) + b_h) 84 | out_mu = tf.matmul(h, w_mu) + b_mu 85 | out_log_var = tf.matmul(h, w_v) + b_v 86 | out = out_fn(out_mu) 87 | return out, out_mu, out_log_var 88 | 89 | def make_loss(pred, actual, log_var, mu, log_detj, z0, sigma=1.0): 90 | #q0 = tf.contrib.distributions.Normal(mu=mu, sigma=tf.exp(log_var)).pdf(z0) 91 | #ln_q0 = tf.reduce_sum(tf.log(q0), 1) 92 | kl = -tf.reduce_mean(0.5*tf.reduce_sum(1.0 + log_var - tf.square(mu) - tf.exp(log_var), 1)) 93 | rec_err = 0.5*(tf.nn.l2_loss(actual - pred)) / sigma 94 | #loss = tf.reduce_mean(ln_q0 + rec_err - log_detj) 95 | loss = tf.reduce_mean(kl + rec_err - log_detj) 96 | return loss 97 | 98 | def train_step(sess, input_data, train_op, loss_op, x_op, e_op, Z): 99 | e_ = np.random.normal(size=(input_data.shape[0], Z)) 100 | _, l = sess.run([train_op, loss_op], feed_dict={x_op: input_data, e_op: e_}) 101 | return l 102 | 103 | def reconstruct(sess, input_data, out_op, x_op, e_op, Z): 104 | e_ = np.random.normal(size=(input_data.shape[0], Z)) 105 | x_rec = sess.run([out_op], feed_dict={x_op: input_data, e_op: e_}) 106 | return x_rec 107 | 108 | def show_reconstruction(actual, recon): 109 | fig, axs = plt.subplots(1, 2) 110 | axs[0].imshow(actual.reshape(28, 28), cmap='gray') 111 | axs[1].imshow(recon.reshape(28, 28), cmap='gray') 112 | axs[0].set_title('actual') 113 | axs[1].set_title('reconstructed') 114 | plt.show() 115 | 116 | 117 | if __name__ == '__main__': 118 | from tensorflow.examples.tutorials.mnist import input_data 119 | 120 | data = input_data.read_data_sets('data') 121 | data_dim = data.train.images.shape[1] 122 | enc_h = 128 123 | enc_z = 64 124 | dec_h = 128 125 | max_iters = 10000 126 | batch_size = 128 127 | learning_rate = 0.001 128 | k = 0 129 | 130 | x, e = inputs(data_dim, enc_z) 131 | mu, log_var, z0, lambd = encoder(x, e, data_dim, enc_h, enc_z, k) 132 | 133 | z_k, log_detj = norm_flow(z0, lambd, k, enc_z) 134 | 135 | out_op, out_mu, out_log_var = decoder(z_k, data_dim, dec_h, enc_z) 136 | 137 | loss_op = make_loss(out_op, x, log_var, mu, log_detj, z0) 138 | 139 | train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss_op) 140 | 141 | sess = tf.InteractiveSession() 142 | sess.run(tf.initialize_all_variables()) 143 | x_test, _ = data.test.next_batch(1) 144 | recons = [] 145 | 146 | for i in xrange(max_iters): 147 | x_, y_ = data.train.next_batch(batch_size) 148 | l = train_step(sess, x_, train_op, loss_op, x, e, enc_z) 149 | if i % 1000 == 0: 150 | print('iter: %d\tavg. loss: %.2f' % (i, l/batch_size)) 151 | recons.append(reconstruct(sess, x_test, out_op, x, e, enc_z)[0]) 152 | 153 | for r in recons: 154 | show_reconstruction(x_test[0], r) 155 | 156 | sess.close() 157 | -------------------------------------------------------------------------------- /original_impl/vae_iaf.py: -------------------------------------------------------------------------------- 1 | """Variational Auto-Encoder with Inverse Autoregressive Flow (using MADE) 2 | 3 | References 4 | ---------- 5 | https://arxiv.org/pdf/1606.04934v1.pdf 6 | https://arxiv.org/pdf/1502.03509v2.pdf 7 | """ 8 | 9 | import matplotlib.pyplot as plt 10 | import numpy as np 11 | import tensorflow as tf 12 | from utils import * 13 | 14 | def inputs(D, Z): 15 | x = tf.placeholder(tf.float32, [None, D], 'x') 16 | e = tf.placeholder(tf.float32, [None, Z], 'e') 17 | return x, e 18 | 19 | def encoder(x, e, D, H, Z, K, initializer=tf.contrib.layers.xavier_initializer): 20 | with tf.variable_scope('encoder'): 21 | w_h = tf.get_variable('w_h', [D, H], initializer=initializer()) 22 | b_h = tf.get_variable('b_h', [H]) 23 | w_mu = tf.get_variable('w_mu', [H, Z], initializer=initializer()) 24 | b_mu = tf.get_variable('b_mu', [Z]) 25 | w_v = tf.get_variable('w_v', [H, Z], initializer=initializer()) 26 | b_v = tf.get_variable('b_v', [Z]) 27 | 28 | h = tf.nn.tanh(tf.matmul(x, w_h) + b_h) 29 | mu = tf.matmul(h, w_mu) + b_mu 30 | log_var = tf.matmul(h, w_v) + b_v 31 | z = mu + tf.sqrt(tf.exp(log_var))*e 32 | return mu, log_var, z 33 | 34 | def decoder(z, D, H, Z, initializer=tf.contrib.layers.xavier_initializer, out_fn=tf.sigmoid): 35 | with tf.variable_scope('decoder'): 36 | w_h = tf.get_variable('w_h', [Z, H], initializer=initializer()) 37 | b_h = tf.get_variable('b_h', [H]) 38 | w_mu = tf.get_variable('w_mu', [H, D], initializer=initializer()) 39 | b_mu = tf.get_variable('b_mu', [D]) 40 | w_v = tf.get_variable('w_v', [H, 1], initializer=initializer()) 41 | b_v = tf.get_variable('b_v', [1]) 42 | 43 | h = tf.nn.tanh(tf.matmul(z, w_h) + b_h) 44 | out_mu = tf.matmul(h, w_mu) + b_mu 45 | out_log_var = tf.matmul(h, w_v) + b_v 46 | out = out_fn(out_mu) 47 | return out, out_mu, out_log_var 48 | 49 | def made(input_data, Z, H, name, act_fn): 50 | with tf.variable_scope('made'): 51 | # Follows equations (6) and (7) of [Germain 2015] 52 | W, b, V, c = _init_made_params(Z, H, name) 53 | M_W, M_V = _create_made_masks(Z, H, name) 54 | h = tf.nn.relu(b + tf.matmul(input_data, W*M_W)) 55 | out = act_fn(c + tf.matmul(h, (V*M_V)), name='%s_out' % name) 56 | return out 57 | 58 | def _init_made_params(D, K, name, initializer=tf.contrib.layers.xavier_initializer): 59 | with tf.variable_scope('%s' % name): 60 | W = tf.get_variable('W', [D, K], initializer=initializer()) 61 | b = tf.get_variable('b', [K]) 62 | V = tf.get_variable('V', [K, D], initializer=initializer()) 63 | c = tf.get_variable('c', [D]) 64 | return W, b, V, c 65 | 66 | def _create_made_masks(D, K, name): 67 | with tf.variable_scope('%s' % name): 68 | # See equations (8) and (9) of [Germain 2015] 69 | msh = np.random.randint(1, D, size=K) 70 | M_W_ = (msh[:, np.newaxis] >= (np.tile(range(0, D), [K, 1])+1)).astype(np.float).T 71 | M_V_ = ((np.tile(np.arange(0, D)[:, np.newaxis], [1, K])+1) > msh[np.newaxis, :]).astype(np.float).T 72 | M_W = tf.constant(M_W_, 'float32', M_W_.shape, name='M_W') 73 | M_V = tf.constant(M_V_, 'float32', M_V_.shape, name='M_V') 74 | return M_W, M_V 75 | 76 | def inverse_autoregressive_flow(z, Z, H): 77 | made_mu = made(z, Z, H, 'made_mu', tf.nn.sigmoid) 78 | made_sd = made(z, Z, H, 'made_sd', tf.nn.sigmoid) 79 | 80 | # IAF transformation and log det of Jacobian; eqns (9) and (10) of [Kingma 2016] 81 | z = (z - made_mu) / made_sd 82 | log_detj = -tf.reduce_sum(tf.log(made_sd), 1) 83 | return z, log_detj 84 | 85 | def make_loss(pred, actual, log_var, mu, log_detj, z0, sigma=1.0): 86 | q0 = tf.contrib.distributions.Normal(mu=mu, sigma=tf.exp(tf.maximum(1e-5, log_var))).pdf(z0) 87 | ln_q0 = tf.reduce_sum(tf.log(q0), 1) 88 | rec_err = tf.reduce_mean(crossentropy(pred, actual)) 89 | 90 | loss = -tf.reduce_mean(ln_q0 + rec_err - log_detj) 91 | return loss 92 | 93 | def train_step(sess, input_data, train_op, loss_op, x_op, e_op, Z): 94 | e_ = np.random.normal(size=(input_data.shape[0], Z)) 95 | _, l = sess.run([train_op, loss_op], feed_dict={x_op: input_data, e_op: e_}) 96 | return l 97 | 98 | def reconstruct(sess, input_data, out_op, x_op, e_op, Z): 99 | e_ = np.random.normal(size=(input_data.shape[0], Z)) 100 | x_rec = sess.run([out_op], feed_dict={x_op: input_data, e_op: e_}) 101 | return x_rec 102 | 103 | 104 | def sample_latent(sess, input_data, z_op, x_op, e_op, Z): 105 | e_ = np.random.normal(size=(input_data.shape[0], Z)) 106 | zs = sess.run(z_op, feed_dict={x_op: input_data, e_op: e_}) 107 | return zs 108 | 109 | def show_reconstruction(actual, recon): 110 | fig, axs = plt.subplots(1, 2) 111 | axs[0].imshow(actual.reshape(28, 28), cmap='gray') 112 | axs[1].imshow(recon.reshape(28, 28), cmap='gray') 113 | axs[0].set_title('actual') 114 | axs[1].set_title('reconstructed') 115 | plt.show() 116 | 117 | 118 | if __name__ == '__main__': 119 | from tensorflow.examples.tutorials.mnist import input_data 120 | 121 | data = input_data.read_data_sets('data') 122 | data_dim = data.train.images.shape[1] 123 | enc_h = 128 124 | enc_z = 64 125 | dec_h = 128 126 | max_iters = 10000 127 | batch_size = 100 128 | learning_rate = 0.0005 129 | k = 5 130 | 131 | x, e = inputs(data_dim, enc_z) 132 | mu, log_var, z0 = encoder(x, e, data_dim, enc_h, enc_z, k) 133 | 134 | z_k, log_detj = inverse_autoregressive_flow(z0, enc_z, enc_h) 135 | 136 | out_op, out_mu, out_log_var = decoder(z_k, data_dim, dec_h, enc_z) 137 | 138 | loss_op = make_loss(out_op, x, log_var, mu, log_detj, z0) 139 | 140 | train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss_op) 141 | 142 | sess = tf.InteractiveSession() 143 | sess.run(tf.initialize_all_variables()) 144 | x_test, _ = data.test.next_batch(1) 145 | recons = [] 146 | 147 | for i in xrange(max_iters): 148 | x_, y_ = data.train.next_batch(batch_size) 149 | l = train_step(sess, x_, train_op, loss_op, x, e, enc_z) 150 | if i % 1000 == 0: 151 | print('iter: %d\tavg. loss: %.2f' % (i, l/batch_size)) 152 | recons.append(reconstruct(sess, x_test, out_op, x, e, enc_z)[0]) 153 | 154 | for r in recons: 155 | show_reconstruction(x_test[0], r) 156 | 157 | sess.close() 158 | -------------------------------------------------------------------------------- /neural_networks.py: -------------------------------------------------------------------------------- 1 | """Reusable neural network layers""" 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | 6 | from nn_utils import * 7 | 8 | def fc_layer(input_tensor, output_dim, layer_name, act=tf.nn.relu): 9 | """Reusable code for making a simple neural net layer.""" 10 | input_dim = input_tensor.get_shape()[-1].value 11 | with tf.variable_scope(layer_name): 12 | with tf.variable_scope('weights'): 13 | weights = weight_variable([input_dim, output_dim]) 14 | with tf.variable_scope('bias'): 15 | biases = bias_variable([output_dim]) 16 | with tf.variable_scope('Wx_plus_b'): 17 | preactivate = tf.matmul(input_tensor, weights) + biases 18 | if act is not None: 19 | activations = act(preactivate, name='activation') 20 | else: 21 | activations = preactivate 22 | return activations 23 | 24 | def made_layer(input_tensor, output_dim, layer_name, act=tf.nn.relu): 25 | 26 | def _get_made_masks(dim_in, dim_out): 27 | """See eqns. (8), (9) of Germain 2015. These assume a single hidden layer auto-encoder.""" 28 | # msh[k] is max number of input units that the k'th hidden dimension can be connected to. 29 | msh = np.random.randint(1, dim_in, size=dim_out) 30 | # Eqn (8). An element is 1 when msh[k] >= d, for d in {1 ... dim_in} 31 | mask_in = (msh[:, np.newaxis] >= (np.tile(range(0, dim_in), [dim_out, 1]) + 1)).astype(np.float).T 32 | # Eqn (9). An element is 1 when d > msh[k] 33 | mask_out = ((np.tile(np.arange(0, dim_in)[:, np.newaxis], [1, dim_out])+1) > msh[np.newaxis, :]).astype(np.float).T 34 | return mask_in, mask_out 35 | 36 | input_dim = input_tensor.get_shape()[-1].value 37 | made_hidden_dim = 300 38 | with tf.variable_scope(layer_name): 39 | with tf.variable_scope('weight_in'): 40 | weights_in = weight_variable([input_dim, made_hidden_dim]) 41 | with tf.variable_scope('weight_out'): 42 | weights_out = weight_variable([made_hidden_dim, input_dim]) 43 | with tf.variable_scope('masks'): 44 | mask_in, mask_out = _get_made_masks(input_dim, made_hidden_dim) 45 | with tf.variable_scope('bias_in'): 46 | biases_in = bias_variable([made_hidden_dim]) 47 | with tf.variable_scope('bias_out'): 48 | biases_out = bias_variable([input_dim]) 49 | 50 | with tf.variable_scope('transformations'): 51 | hidden = tf.nn.relu(tf.matmul(input_tensor, weights_in * mask_in) + biases_in) 52 | preactivate = tf.matmul(hidden, weights_out * mask_out) + biases_out 53 | if act is not None: 54 | activations = act(preactivate, name='activation') 55 | else: 56 | activations = preactivate 57 | return activations 58 | 59 | def nf_layer(input_tensor, output_dim, layer_name): 60 | # See equations (10), (11) of Kingma 2016 61 | input_dim = input_tensor.get_shape()[-1].value 62 | with tf.variable_scope(layer_name): 63 | with tf.variable_scope('u'): 64 | u = weight_variable(input_dim) 65 | with tf.variable_scope('w'): 66 | w = weight_variable(input_dim) 67 | with tf.variable_scope('b'): 68 | b = bias_variable(1) 69 | 70 | with tf.variable_scope('transformations'): 71 | z = input_tensor 72 | temp = tf.expand_dims(tf.nn.tanh(tf.reduce_sum(w * z, 1) + b), 1) 73 | temp = tf.tile(temp, [1, output_dim]) 74 | z = z + tf.mul(u, temp) 75 | 76 | temp = tf.expand_dims(dtanh(tf.reduce_sum(w * z, 1) + b), 1) 77 | temp = tf.tile(temp, [1, output_dim]) 78 | log_detj = tf.log(tf.abs(1. + tf.reduce_sum(tf.mul(u, temp * w), 1))) 79 | 80 | return z, log_detj 81 | 82 | def nn(input_tensor, dims_hidden, name, act=tf.nn.relu, is_training=None): 83 | 84 | with tf.variable_scope(name): 85 | dim_out = dims_hidden[0] 86 | h = fc_layer(input_tensor, dim_out, 'layer0', act=None) 87 | if is_training is not None: 88 | h = tf.contrib.layers.batch_norm(inputs=h, decay=0.99, epsilon=1e-7, is_training=is_training) 89 | h = act(h) 90 | for i in range(len(dims_hidden) - 1): 91 | dim_in = dims_hidden[i] 92 | dim_out = dims_hidden[i+1] 93 | h = fc_layer(h, dim_out, 'layer_{}'.format(i+1), act=None) 94 | if is_training is not None: 95 | h = tf.contrib.layers.batch_norm(inputs=h, decay=0.99, epsilon=1e-7, is_training=is_training) 96 | h = act(h) 97 | dim_in = dims_hidden[-1] 98 | return h 99 | 100 | def cnn(input_tensor, in_shape, layer_dict, output_dims_dict, name, act=tf.nn.tanh, final_act=None): 101 | with tf.variable_scope(name): 102 | hidden_channels, strides, filter_sizes = layer_dict['hidden_channels'], layer_dict['strides'], layer_dict['filter_sizes'] 103 | h = conv_layer(input_tensor, in_shape[-1], hidden_channels[0], filter_sizes[0], strides[0], 'layer0', act) 104 | for i in range(len(hidden_channels) - 1): 105 | in_channels = hidden_channels[i] 106 | out_channels = hidden_channels[i+1] 107 | size = filter_sizes[i] 108 | stride = strides[i] 109 | h = conv_layer(h, in_channels, out_channels, size, stride, 'conv_layer_{}'.format(i+1), act) 110 | dim_flattened = np.prod(in_shape) 111 | #print dim_flattened 112 | h = tf.reshape(h, (-1, dim_flattened)) 113 | #print h.get_shape() 114 | outputs = {} 115 | for key in output_dims_dict: 116 | outputs[key] = fc_layer(h, dim_flattened, output_dims_dict[key], layer_name=key, act=None) 117 | print [outputs[k].get_shape() for k in output_dims_dict] 118 | return outputs 119 | 120 | 121 | # Create model 122 | def conv_net(x, layer_dict): 123 | # Reshape input picture 124 | x = tf.reshape(x, shape=[-1, 28, 28, 1]) 125 | 126 | # Convolution Layer 127 | with tf.variable_scope('Conv1'): 128 | shape = layer_dict['wc1'] 129 | weights, biases = weight_variable(shape), bias_variable(shape[-1]) 130 | conv1 = conv2d(x, weights, biases) 131 | # Max Pooling (down-sampling) 132 | conv1 = maxpool2d(conv1, k=2) 133 | 134 | # Convolution Layer 135 | with tf.variable_scope('Conv2'): 136 | shape = layer_dict['wc2'] 137 | weights, biases = weight_variable(shape), bias_variable(shape[-1]) 138 | conv2 = conv2d(conv1, weights, biases) 139 | # Max Pooling (down-sampling) 140 | conv2 = maxpool2d(conv2, k=2) 141 | 142 | # Fully connected layer 143 | with tf.variable_scope('FC1'): 144 | shape = layer_dict['wd1'] 145 | weights, biases = weight_variable(shape), bias_variable(shape[-1]) 146 | # Reshape conv2 output to fit fully connected layer input 147 | fc1 = tf.contrib.layers.flatten(conv2) 148 | fc1 = tf.add(tf.matmul(fc1, weights), biases) 149 | fc1 = tf.nn.relu(fc1) 150 | 151 | # Output, class prediction 152 | with tf.variable_scope('Out'): 153 | shape = layer_dict['out'] 154 | weights, biases = weight_variable(shape), bias_variable(shape[-1]) 155 | out = tf.add(tf.matmul(fc1, weights), biases) 156 | return out 157 | -------------------------------------------------------------------------------- /models.py: -------------------------------------------------------------------------------- 1 | """Encoder and Decoder implementations""" 2 | 3 | import numpy as np 4 | from neural_networks import * 5 | 6 | def basic_encoder(neural_net, dim_z, *args): 7 | def _basic_encoder(x, e, neural_net, dim_z): 8 | output_dims_dict = {'mu': dim_z, 'log_std': dim_z} 9 | last_hidden = neural_net(x) 10 | outputs = {} 11 | for key in output_dims_dict: 12 | outputs[key] = fc_layer(last_hidden, output_dims_dict[key], layer_name=key, act=None) 13 | mu, log_std = outputs['mu'], outputs['log_std'] 14 | z = mu + tf.exp(log_std) * e 15 | return outputs, z 16 | return lambda x, e: _basic_encoder(x, e, neural_net, dim_z) 17 | 18 | def nf_encoder(neural_net, dim_z, flow, use_c=True): 19 | def _nf_encoder(x, e, neural_net, dim_z, flow, use_c): 20 | 21 | def norm_flow_one_step(z, u, w, b): 22 | temp = tf.nn.tanh(tf.reduce_sum(w * z, 1, keep_dims=True) + b) 23 | z = z + tf.mul(u, temp) 24 | 25 | # Eqn. (11) and (12) 26 | temp = dtanh(tf.reduce_sum(w * z, 1, keep_dims=True) + b) 27 | psi = temp * w 28 | log_detj = tf.log(tf.abs(1. + tf.reduce_sum(tf.mul(u, psi), 1))) 29 | return z, log_detj 30 | 31 | def norm_flow(z, us, ws, bs): 32 | d_z = z.get_shape()[-1].value # Get dimension of z 33 | K = us.get_shape()[-1].value / d_z # Find length of flow from given parameters 34 | sum_log_detj = 0.0 35 | for k in range(K): 36 | u, w, b = us[:, k*dim_z:(k+1)*d_z], ws[:, k*d_z:(k+1)*d_z], bs[:, k] 37 | z, log_detj = norm_flow_one_step(z, u, w, b) 38 | sum_log_detj += log_detj 39 | return z, sum_log_detj 40 | 41 | def get_norm_flow_params(c, use_c): 42 | #shape = z.get_shape()[0].value 43 | #v = tf.get_variable('constant_term', shape, initializer=tf.constant_initializer(0.0), trainable=False) 44 | v = tf.ones_like(c) 45 | if use_c: 46 | v = tf.concat(1, (v, c)) 47 | 48 | u = fc_layer(v, dim_z, layer_name='u', act=None) 49 | w = fc_layer(v, dim_z, layer_name='w', act=None) 50 | b = fc_layer(v, 1, layer_name='b', act=None) 51 | return u, w, b 52 | 53 | def nf(z, c, use_c, flow_length): 54 | z = z0 55 | sum_log_detj = 0.0 56 | for i in range(flow_length): 57 | with tf.variable_scope('flow_{}'.format(i)): 58 | u, w, b = get_norm_flow_params(c, use_c) 59 | z, log_detj = norm_flow_one_step(z, u, w, b) 60 | sum_log_detj += log_detj 61 | 62 | return z, sum_log_detj 63 | 64 | #output_dims_dict = {'mu': dim_z, 'log_std': dim_z, 'us': dim_z * flow, 'ws': dim_z * flow, 'bs': dim_z * flow} 65 | output_dims_dict = {'mu': dim_z, 'log_std': dim_z} 66 | 67 | last_hidden = neural_net(x) 68 | outputs = {} 69 | for key in ['mu', 'log_std']: 70 | outputs[key] = fc_layer(last_hidden, output_dims_dict[key], layer_name=key, act=None) 71 | 72 | #mu, log_std, us, ws, bs = outputs['mu'], outputs['log_std'], outputs['us'], outputs['ws'], outputs['bs'] 73 | mu, log_std = outputs['mu'], outputs['log_std'] 74 | 75 | z0 = mu + tf.exp(log_std) * e 76 | 77 | #zk, sum_log_detj = norm_flow(z0, us, ws, bs) 78 | zk, sum_log_detj = nf(z0, last_hidden, use_c, flow_length=flow) 79 | 80 | outputs['sum_log_detj'] = sum_log_detj 81 | outputs['z0'] = z0 82 | outputs['zk'] = zk 83 | 84 | return outputs, zk 85 | 86 | return lambda x, e: _nf_encoder(x, e, neural_net, dim_z, flow, use_c) 87 | 88 | def iaf_encoder(neural_net, dim_z, flow): 89 | def _iaf_encoder(x, e, neural_net, dim_z, flow): 90 | output_dims_dict = {'mu': dim_z, 'log_std': dim_z} 91 | flow_dims_dict = {'mu': dim_z * flow, 'log_std': dim_z * flow} 92 | 93 | last_hidden = neural_net(x) 94 | outputs = {} 95 | for key in ['mu', 'log_std']: 96 | outputs[key] = fc_layer(last_hidden, output_dims_dict[key], layer_name=key, act=None) 97 | 98 | mu, log_std = outputs['mu'], outputs['log_std'] 99 | z0 = mu + tf.exp(log_std) * e # preflow 100 | zk, sum_log_detj = iaf(z0, flow, flow_dims_dict['mu'], flow_dims_dict['log_std']) 101 | 102 | outputs['sum_log_detj'] = sum_log_detj 103 | outputs['z0'] = z0 104 | outputs['zk'] = zk 105 | 106 | return outputs, zk 107 | 108 | def iaf(z0, K, mu_dim, log_std_dim): 109 | z = z0 110 | log_detj = 0.0 111 | for k in range(K): 112 | # See equations (10), (11) of Kingma 2016 113 | mu = made_layer(z, mu_dim, 'flow_mu_%d' % k) 114 | log_std = made_layer(z, log_std_dim, 'flow_log_std_%d' % k) 115 | z = (z - mu) / tf.exp(log_std) 116 | log_detj += -tf.reduce_sum(log_std, 1) 117 | return z, log_detj 118 | 119 | return lambda x, e: _iaf_encoder(x, e, neural_net, dim_z, flow) 120 | 121 | def hf_encoder(neural_net, dim_z, flow): 122 | def _hf_encoder(x, e, neural_net, dim_z, flow): 123 | output_dims_dict = {'mu': dim_z, 'log_std': dim_z, 'flow_vs': dim_z * flow} 124 | last_hidden = neural_net(x) 125 | outputs = {} 126 | for key in ['mu', 'log_std']: 127 | outputs[key] = fc_layer(last_hidden, output_dims_dict[key], layer_name=key, act=None) 128 | if output_dims_dict['flow_vs'] != 0: 129 | outputs['flow_vs'] = fc_layer(last_hidden, output_dims_dict['flow_vs'], layer_name='flow_vs', act=None) 130 | else: outputs['flow_vs'] = None 131 | mu, log_std, flow_vs = outputs['mu'], outputs['log_std'], outputs['flow_vs'] 132 | 133 | z0 = mu + tf.exp(log_std) * e # preflow 134 | 135 | zk, sum_log_detj = householder_flow(z0, flow_vs) # apply the IAF 136 | outputs['sum_log_detj'] = sum_log_detj 137 | outputs['z0'] = z0 138 | outputs['zk'] = zk 139 | 140 | return outputs, zk 141 | 142 | def householder_flow(z, vs): 143 | if vs is not None: 144 | d_z = z.get_shape()[-1].value # Get dimension of z 145 | K = vs.get_shape()[-1].value / d_z # Find length of flow from given parameters 146 | for k in range(K): 147 | v = vs[:, k*d_z:(k+1)*d_z] 148 | z = householder(z, v) 149 | sum_log_detj = 0.0 150 | return z, sum_log_detj 151 | 152 | def householder(z, v): 153 | norm_squared_v = tf.expand_dims(tf.reduce_sum(tf.pow(v, 2), 1, keep_dims=True), 1) # HACK 154 | 155 | I = tf.constant(np.identity(dim_z, dtype=np.float32)) 156 | H = I - 2 * tf.mul(tf.expand_dims(v, 1), tf.expand_dims(v, 2)) / norm_squared_v 157 | return tf.reduce_sum(tf.mul(H, tf.expand_dims(z, 1)), 2) 158 | 159 | return lambda x, e: _hf_encoder(x, e, neural_net, dim_z, flow) 160 | 161 | def linear_iaf_encoder(neural_net, dim_z, *args): 162 | def _linear_iaf_encoder(x, e, neural_net, dim_z): 163 | output_dims_dict = {'mu': dim_z, 'log_std': dim_z, 'L':dim_z * dim_z} 164 | last_hidden = neural_net(x) 165 | outputs = {} 166 | for key in output_dims_dict: 167 | outputs[key] = fc_layer(last_hidden, output_dims_dict[key], layer_name=key, act=None) 168 | mu, log_std, L = outputs['mu'], outputs['log_std'], outputs['L'] 169 | mask = tf.expand_dims(tf.constant(np.triu(np.zeros((dim_z, dim_z))), dtype=tf.float32), 0) 170 | L = tf.reshape(L, (-1, dim_z, dim_z)) 171 | temp = mask * L 172 | ones = tf.expand_dims(tf.constant(np.eye(dim_z), dtype=tf.float32), 0) 173 | L = temp + ones 174 | z0 = mu + tf.exp(log_std) * e 175 | 176 | zk = tf.reduce_sum(tf.mul(L, tf.expand_dims(z0, 1)), 2) 177 | outputs['sum_log_detj'] = 0.0 178 | outputs['z0'] = z0 179 | outputs['zk'] = zk 180 | return outputs, zk 181 | return lambda x, e: _linear_iaf_encoder(x, e, neural_net, dim_z) 182 | 183 | # DECODERS 184 | def basic_decoder(neural_net, dim_x, act=tf.sigmoid): 185 | def _basic_decoder(z, neural_net, dim_x): 186 | with tf.variable_scope('decoder'): 187 | last_hidden = neural_net(z) 188 | x_pred = fc_layer(last_hidden, dim_x, 'mu', act=act) 189 | return x_pred 190 | return lambda z: _basic_decoder(z, neural_net, dim_x) 191 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | """Train various variational auto-encoder models. 2 | 3 | References 4 | ---------- 5 | https://arxiv.org/pdf/1312.6114v10.pdf 6 | """ 7 | 8 | import argparse 9 | import datetime 10 | import inspect 11 | import os 12 | import time 13 | 14 | import numpy as np 15 | import tensorflow as tf 16 | try: 17 | from tensorflow.python import control_flow_ops 18 | except ImportError: 19 | from tensorflow.python.ops import control_flow_ops 20 | 21 | import restore 22 | from models import * 23 | from reconstructions import * 24 | from loss import * 25 | from datasets import binarized_mnist 26 | 27 | def train( 28 | image_width, 29 | dim_x, 30 | dim_z, 31 | encoder_type, 32 | decoder, 33 | dataset, 34 | learning_rate=0.0001, 35 | optimizer=tf.train.AdamOptimizer, 36 | loss=elbo_loss, 37 | batch_size=100, 38 | results_dir='results', 39 | max_epochs=10, 40 | n_view=10, 41 | results_file=None, 42 | bn=False, 43 | **kwargs 44 | ): 45 | saved_variables = kwargs.pop('saved_variables', None) 46 | anneal_lr = kwargs.pop('anneal_lr', False) 47 | learning_rate_temperature = kwargs.pop('learning_rate_temperature', None) 48 | global_step = tf.Variable(0, trainable=False) # for checkpoint saving 49 | on_epoch = tf.placeholder(tf.float32, name='on_epoch') 50 | dt = datetime.datetime.now() 51 | results_file = results_file if results_file is not None else '/{}_{:02d}-{:02d}-{:02d}'.format(dt.date(), dt.hour, dt.minute, dt.second) 52 | results_dir += results_file 53 | os.mkdir(results_dir) 54 | 55 | # Get all the settings and save them. 56 | with open(results_dir + '/settings.txt', 'w') as f: 57 | args = inspect.getargspec(train).args 58 | settings = [locals()[arg] for arg in args] 59 | for s, arg in zip(settings, args): 60 | setting = '{}: {}'.format(arg, s) 61 | f.write('{}\n'.format(setting)) 62 | print(setting) 63 | settings = locals()[inspect.getargspec(train).keywords] 64 | for kw, val in settings.items(): 65 | setting = '{}: {}'.format(kw, val) 66 | f.write('{}\n'.format(setting)) 67 | print(setting) 68 | 69 | # Make the neural neural_networks 70 | is_training = tf.placeholder(tf.bool) 71 | if bn: 72 | encoder_net = lambda x: nn(x, enc_dims, name='encoder', act=tf.nn.tanh, is_training=is_training) 73 | else: 74 | encoder_net = lambda x: nn(x, enc_dims, name='encoder', act=tf.nn.tanh, is_training=None) 75 | encoder = encoder_type(encoder_net, dim_z, flow) 76 | 77 | # Build computation graph and operations 78 | x = tf.placeholder(tf.float32, [None, dim_x], 'x') 79 | x_w = tf.placeholder(tf.float32, [None, dim_x], 'x_w') 80 | e = tf.placeholder(tf.float32, (None, dim_z), 'noise') 81 | 82 | z_params, z = encoder(x_w, e) 83 | x_pred = decoder(z) 84 | kl_weighting = 1.0 - tf.exp(-on_epoch / kl_annealing_rate) if kl_annealing_rate is not None else 1 85 | monitor_functions = loss(x_pred, x, kl_weighting=kl_weighting, **z_params) 86 | monitor_functions_sorted = sorted(monitor_functions.iteritems(), key=lambda x: x[0]) 87 | monitor_output_train = {name: [] for name in monitor_functions.iterkeys()} 88 | monitor_output_valid = {name: [] for name in monitor_functions.iterkeys()} 89 | monitor_function_names = [p[0] for p in monitor_functions_sorted] 90 | monitor_function_list = [p[1] for p in monitor_functions_sorted] 91 | 92 | train_loss, valid_loss = monitor_functions['train_loss'], monitor_functions['valid_loss'] 93 | 94 | out_op = x_pred 95 | 96 | # Batch normalization stuff 97 | update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 98 | if update_ops: 99 | updates = tf.group(*update_ops) 100 | train_loss = control_flow_ops.with_dependencies([updates], train_loss) 101 | 102 | # Optimizer with gradient clipping 103 | lr = tf.Variable(learning_rate) 104 | optimizer = optimizer(lr) 105 | gvs = optimizer.compute_gradients(train_loss) 106 | capped_gvs = [(tf.clip_by_norm(grad, 1), var) if grad is not None else (grad, var) 107 | for grad, var in gvs] 108 | train_op = optimizer.apply_gradients(capped_gvs) 109 | 110 | # Make training and validation sets 111 | training_data, validation_data = dataset['train'], dataset['valid'] 112 | n_train_batches, n_valid_batches = training_data.images.shape[0] / batch_size, validation_data.images.shape[0] / batch_size, 113 | print 'Loaded training and validation data' 114 | visualized, e_visualized = validation_data.images[:n_view], np.random.normal(0, 1, (n_view, dim_z)) 115 | 116 | # Make summaries 117 | rec_summary = tf.image_summary("rec", vec2im(out_op, batch_size, image_width), max_images=10) 118 | for fn_name, fn in monitor_functions.items(): 119 | tf.scalar_summary(fn_name, fn) 120 | summary_op = tf.merge_all_summaries() 121 | 122 | # Create a saver. 123 | saver = tf.train.Saver(tf.all_variables()) 124 | 125 | # Create a session 126 | sess = tf.InteractiveSession() 127 | sess.run(tf.initialize_all_variables()) 128 | 129 | # Use pre-trained weight values 130 | if saved_variables is not None: 131 | restore.set_variables(sess, saved_variables) 132 | 133 | summary_writer = tf.train.SummaryWriter(results_dir, sess.graph) 134 | samples_list = [] 135 | batch_counter = 0 136 | best_validation_loss = 1e100 137 | number_of_validation_failures = 0 138 | feed_dict = {} 139 | validation_losses, training_losses = [], [] 140 | for epoch in range(max_epochs): 141 | feed_dict[on_epoch] = epoch 142 | start_time = time.time() 143 | l_t = 0 144 | monitor_output_epoch = {name: 0 for name in monitor_function_names} 145 | for _ in xrange(n_train_batches): 146 | batch_counter += 1 147 | feed_dict[x], feed_dict[x_w] = training_data.next_batch(batch_size, whitened=False) 148 | feed_dict[e] = np.random.normal(0, 1, (batch_size, dim_z)) 149 | feed_dict[is_training] = True 150 | output = sess.run([train_op, train_loss] + monitor_function_list, feed_dict=feed_dict) 151 | l, monitor_output_batch = output[1], output[2:] 152 | 153 | for name, out in zip(monitor_function_names, monitor_output_batch): 154 | monitor_output_epoch[name] += out 155 | 156 | if batch_counter % 100 == 0: 157 | summary_str = sess.run(summary_op, feed_dict=feed_dict) 158 | summary_writer.add_summary(summary_str, batch_counter) 159 | 160 | # Save the model checkpoint periodically. 161 | if batch_counter % 1000 == 0 or epoch == max_epochs: 162 | checkpoint_path = os.path.join(results_dir, 'model.ckpt') 163 | saver.save(sess, checkpoint_path, global_step=global_step) 164 | l_t += l 165 | l_t /= n_train_batches 166 | 167 | for name in monitor_function_names: 168 | monitor_output_train[name].append(monitor_output_epoch[name] / n_train_batches) 169 | 170 | training_losses.append(l_t) 171 | 172 | # Validation loop 173 | l_v = 0 174 | monitor_output_epoch = {name: 0 for name in monitor_function_names} 175 | for _ in range(n_valid_batches): 176 | feed_dict[x], feed_dict[x_w] = validation_data.next_batch(batch_size, whitened=False) 177 | feed_dict[e] = np.random.normal(0, 1, (batch_size, dim_z)) 178 | feed_dict[is_training] = False 179 | output = sess.run([valid_loss] + monitor_function_list, feed_dict=feed_dict) 180 | l_v_batched, monitor_output_batch = output[0], output[1:] 181 | for name, out in zip(monitor_function_names, monitor_output_batch): 182 | monitor_output_epoch[name] += out 183 | l_v += l_v_batched 184 | 185 | l_v /= n_valid_batches 186 | for name in monitor_function_names: 187 | monitor_output_valid[name].append(monitor_output_epoch[name] / n_valid_batches) 188 | 189 | validation_losses.append(l_v) 190 | duration = time.time() - start_time 191 | examples_per_sec = (n_valid_batches + n_train_batches) * batch_size * 1.0 / duration 192 | print('Epoch: {:d}\t Weighted training loss: {:.2f}, Validation loss {:.2f} ({:.1f} examples/sec, {:.1f} sec/epoch)'.format(epoch, l, l_v, examples_per_sec, duration)) 193 | 194 | samples = sess.run([out_op], feed_dict={x: visualized, x_w: visualized, e: e_visualized, is_training: False}) 195 | samples = np.reshape(samples, (n_view, image_width, image_width)) 196 | samples_list.append(samples) 197 | # show_samples(samples, image_width) 198 | 199 | # Learning rate annealing 200 | lr = lr / (1.0 + epoch * 1.0 / learning_rate_temperature) if learning_rate_temperature is not None else lr 201 | 202 | if epoch % 100 == 0: 203 | np.save(results_dir + '/validation_losses_{}.npy'.format(epoch), validation_losses) 204 | np.save(results_dir + '/training_losses_{}.npy'.format(epoch), training_losses) 205 | np.save(results_dir + '/sample_visualizations_{}.npy'.format(epoch), np.array(samples_list)) 206 | np.save(results_dir + '/real_visualizations_{}.npy'.format(epoch), np.reshape(visualized, (n_view,image_width, image_width))) 207 | for name in monitor_function_names: 208 | np.save(results_dir + '/{}_valid_{}.npy'.format(name, epoch), monitor_output_valid[name]) 209 | np.save(results_dir + '/{}_train_{}.npy'.format(name, epoch), monitor_output_train[name]) 210 | 211 | np.save(results_dir + '/validation_losses.npy', validation_losses) 212 | np.save(results_dir + '/training_losses.npy', training_losses) 213 | np.save(results_dir + '/sample_visualizations.npy', np.array(samples_list)) 214 | np.save(results_dir + '/real_visualizations.npy', np.reshape(visualized, (n_view,image_width, image_width))) 215 | for name in monitor_function_names: 216 | np.save(results_dir + '/{}_valid.npy'.format(name), monitor_output_valid[name]) 217 | np.save(results_dir + '/{}_train.npy'.format(name), monitor_output_train[name]) 218 | 219 | visualize = False 220 | if visualize: 221 | for samples in samples_list: 222 | together = np.hstack((np.reshape(visualized, (n_view,image_width, image_width)), samples > 0.5)) 223 | plot_images_together(together) 224 | 225 | sess.close() 226 | 227 | 228 | def train_simple( 229 | dim_x, 230 | dim_z, 231 | encoder, 232 | decoder, 233 | training_dataset, 234 | validation_dataset=None, 235 | learning_rate=0.0001, 236 | optimizer=tf.train.AdamOptimizer, 237 | batch_size=100, 238 | max_epochs=10, 239 | **kwargs): 240 | print_every = kwargs.pop('print_every', 10) 241 | # Set random seeds 242 | seed = kwargs.pop('seed', 0) 243 | np.random.seed(seed) 244 | tf.set_random_seed(seed) 245 | 246 | rec_err_fn = l2_loss if kwargs.pop('rec_err_type', '') == 'l2_loss' else cross_entropy 247 | anneal_lr = kwargs.pop('anneal_lr', False) 248 | 249 | # Build computation graph and operations 250 | x = tf.placeholder(tf.float32, [None, dim_x], 'x') 251 | e = tf.placeholder(tf.float32, (None, dim_z), 'noise') 252 | z_params, z = encoder(x, e) 253 | x_pred = decoder(z) 254 | 255 | kl_weighting = 1 256 | loss_op = elbo_loss(x_pred, x, kl_weighting=kl_weighting, rec_err_fn=rec_err_fn, **z_params) 257 | out_op = x_pred 258 | lr = tf.Variable(learning_rate) 259 | train_op = optimizer(lr).minimize(loss_op) 260 | 261 | # Make training and validation sets 262 | n_train_batches = max(training_dataset.num_examples / batch_size, 1) 263 | n_valid_batches = validation_dataset.num_examples / batch_size if validation_dataset is not None else 0 264 | 265 | # Create a session 266 | sess = tf.InteractiveSession() 267 | sess.run(tf.initialize_all_variables()) 268 | batch_counter = 0 269 | best_validation_loss = 1e100 270 | for epoch in range(max_epochs): 271 | for _ in xrange(n_train_batches): 272 | batch_counter += 1 273 | 274 | x_ = training_dataset.next_batch(batch_size) 275 | e_ = np.random.normal(0, 1, (x_.shape[0], dim_z)) 276 | feed_dict = {x: x_, e: e_} 277 | _, l = sess.run([train_op, loss_op], feed_dict) 278 | 279 | l_v = 0.0 280 | if validation_dataset is not None: 281 | for _ in range(n_valid_batches): 282 | x_valid = validation_dataset.next_batch(batch_size) 283 | e_valid = np.random.normal(0, 1, (batch_size, dim_z)) 284 | l_v_batched = sess.run(loss_op, feed_dict={x: x_valid, e: e_valid}) 285 | l_v += l_v_batched 286 | l_v /= n_valid_batches 287 | 288 | if l_v > best_validation_loss: 289 | if anneal_lr: 290 | lr /= 2 291 | learning_rate /= 2 292 | print "Annealing learning rate to {}".format(learning_rate) 293 | else: best_validation_loss = l_v 294 | 295 | if (epoch + 1) % print_every == 0: 296 | print('Epoch: {:d}\t Training loss: {:.2f}'.format(epoch+1, l)) 297 | 298 | ops = { 299 | 'z': z, 300 | 'out': out_op, 301 | 'x': x, 302 | 'e': e 303 | } 304 | return ops, sess 305 | 306 | if __name__ == '__main__': 307 | parser = argparse.ArgumentParser() 308 | group = parser.add_mutually_exclusive_group(required=True) 309 | group.add_argument('--basic', action='store_true') 310 | group.add_argument('--nf', action='store_true') 311 | group.add_argument('--iaf', action='store_true') 312 | group.add_argument('--hf', action='store_true') 313 | group.add_argument('--liaf', action='store_true') 314 | 315 | parser.add_argument('--epochs', type=int, default=2000) 316 | parser.add_argument('--anneal-lr', action='store_true') 317 | parser.add_argument('--flow', type=int, default=1) 318 | parser.add_argument('--lrt', type=int, default=100) 319 | parser.add_argument('--seed', type=int, default=0) 320 | parser.add_argument('--pretrained-metagraph', default=None) 321 | args = parser.parse_args() 322 | 323 | # Load pretrained variables 324 | if args.pretrained_metagraph is not None: 325 | s = args.pretrained_metagraph 326 | checkpoint_dir, metagraph_name = '/'.join(s.split('/')[:-1]), s.split('/')[-1] 327 | saved_variables = restore.get_saved_variable_values(checkpoint_dir, metagraph_name) 328 | else: 329 | saved_variables = None 330 | 331 | # Set random seeds 332 | np.random.seed(args.seed) 333 | tf.set_random_seed(args.seed) 334 | 335 | # Results file 336 | dt = datetime.datetime.now() 337 | results_file = '/{}_{:02d}-{:02d}-{:02d}'.format(dt.date(), dt.hour, dt.minute, dt.second) 338 | 339 | # TRAINING SETTINGS 340 | dim_x, dim_z, enc_dims, dec_dims = 784, 40, [300, 300], [300, 300] 341 | decoder_net = lambda z: nn(z, dec_dims, name='decoder', act=tf.nn.tanh) 342 | flow = args.flow 343 | bn = True 344 | 345 | # ENCODER 346 | if args.basic: 347 | encoder_type = basic_encoder 348 | results_file += '-basic' 349 | if args.nf: 350 | encoder_type = nf_encoder 351 | results_file += '-NF-{}'.format(flow) 352 | if args.iaf: 353 | encoder_type = iaf_encoder 354 | results_file += '-IAF-{}'.format(flow) 355 | if args.hf: 356 | encoder_type = hf_encoder 357 | results_file += '-HF-{}'.format(flow) 358 | if args.liaf: 359 | encoder_type = linear_iaf_encoder 360 | results_file += '-linIAF' 361 | 362 | if args.pretrained_metagraph is not None: 363 | results_file += '_pretrained' 364 | 365 | decoder = basic_decoder(decoder_net, dim_x) 366 | 367 | kl_annealing_rate = None 368 | extra_settings = { 369 | 'flow': flow, 370 | 'kl annealing rate': kl_annealing_rate, 371 | 'anneal_lr': args.anneal_lr, 372 | 'bn': bn, 373 | 'enc_dims': enc_dims, 374 | 'learning_rate_temperature': args.lrt 375 | } 376 | 377 | # TRAINING 378 | train( 379 | image_width=28, 380 | dim_x=dim_x, 381 | dim_z=dim_z, 382 | encoder_type=encoder_type, 383 | decoder=decoder, 384 | dataset=binarized_mnist(), 385 | learning_rate=0.0002, 386 | optimizer=tf.train.AdamOptimizer, 387 | loss=elbo_loss, 388 | batch_size=100, 389 | results_dir='results', 390 | results_file=results_file, 391 | max_epochs=args.epochs, 392 | saved_variables=saved_variables, 393 | **extra_settings 394 | ) 395 | -------------------------------------------------------------------------------- /original_impl/approximating_distributions.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### Approximating non-Gaussian distributions" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "#### Test Energy Functions from [Variational Inference with Normalizing Flows](https://arxiv.org/pdf/1505.05770v6.pdf)\n", 15 | "\n", 16 | "$$ p(z) \\propto \\exp^{-U(z)} $$\n", 17 | "\n", 18 | "with `U(z)`'s from Table 1 " 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 46, 24 | "metadata": { 25 | "collapsed": false 26 | }, 27 | "outputs": [ 28 | { 29 | "name": "stdout", 30 | "output_type": "stream", 31 | "text": [ 32 | "Populating the interactive namespace from numpy and matplotlib\n" 33 | ] 34 | } 35 | ], 36 | "source": [ 37 | "%pylab inline\n", 38 | "import seaborn as sns\n", 39 | "sns.set(rc={'image.cmap': 'jet'})" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 2, 45 | "metadata": { 46 | "collapsed": true 47 | }, 48 | "outputs": [], 49 | "source": [ 50 | "sigmoid = lambda x_: 1./(1+np.exp(-x_))\n", 51 | "w1 = lambda z1: np.sin(2.*np.pi*z1/4.)\n", 52 | "w2 = lambda z1: 3.*np.exp(-0.5*((z1-1.)/0.6)**2)\n", 53 | "w3 = lambda z1: 3.*sigmoid((z1-1.)/0.3)" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 3, 59 | "metadata": { 60 | "collapsed": true 61 | }, 62 | "outputs": [], 63 | "source": [ 64 | "p1 = lambda z: np.exp(-0.5*((np.linalg.norm(z, axis=0)-2.)/0.4)**2 +\n", 65 | " np.log(np.exp(-0.5*((z[0]-2)/0.6)**2)+\n", 66 | " np.exp(-0.5*((z[0]+2)/0.6)**2)))\n", 67 | "\n", 68 | "p2 = lambda z1, z2: np.exp(-0.5*((z2-w1(z1))/0.4)**2)\n", 69 | "\n", 70 | "p3 = lambda z: (np.exp(-0.5*((z[1]-w1(z[0]))/0.35)**2) +\n", 71 | " np.exp(-0.5*((z[1]-w1(z[0])+w2(z[0]))/0.35)**2))\n", 72 | "\n", 73 | "p4 = lambda z: (np.exp(-0.5*((z[1]-w1(z[0]))/0.4)**2) +\n", 74 | " np.exp(-0.5*((z[1]-w1(z[0])+w3(z[0]))/0.35)**2))" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "Visualize each density function" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 4, 87 | "metadata": { 88 | "collapsed": false 89 | }, 90 | "outputs": [], 91 | "source": [ 92 | "x = np.arange(-4., 4., 0.01)\n", 93 | "y = np.arange(-4., 4., 0.01)\n", 94 | "X, Y = np.meshgrid(x, y)\n", 95 | "\n", 96 | "XY = np.rollaxis(np.dstack([X,Y]), axis=2)" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 36, 102 | "metadata": { 103 | "collapsed": false 104 | }, 105 | "outputs": [ 106 | { 107 | "data": { 108 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4cAAAKeCAYAAAAWQ25xAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3X90VPWd//FXME2ClGwIBC3KHtYfa3BRQUQOFrQq4qpr\npbvqVlegrVS3XcHdiEv5oYnkoCCpP1pabZFSC63SWqu24moR127trgIKgUKsxtNVdjWSCgIaEr5k\nvn+wEyeTOzP3x+f+mvt8nJOjmUzuvWQ+M+/P634+n3tLUqlUSgAAAACAROsX9gEAAAAAAMJHOAQA\nAAAAEA4BAAAAAIRDAAAAAIAIhwAAAAAAEQ4BAAAAACIcAgAAAABEOAQAAAAAiHAIAAAAAJCP4bCh\noUHTpk3za/MAgIShrgAAillXV5cuv/xybdy4MedzduzYoauvvlqjR4/WVVddpd///ve9fn7WWWdp\n5MiRqq2tVW1trUaOHKmOjg7bx+BLOHz11Ve1du1alZSU+LF5AEDCUFcAAMWsq6tLdXV1evPNN3M+\np6OjQzfccIPGjRunxx9/XKNHj9aNN96ogwcPSpLa2tr00Ucfaf369XrppZf00ksv6be//a369+9v\n+ziMh8NDhw6pvr5eY8aMMb1pAEACUVcAAMWstbVVV199tXbt2pX3eU8//bT69++vW2+9VSeccIIW\nLFigAQMG6N/+7d8kSW+99ZZqamp03HHHafDgwT1fThgPh9/73vd0yimn6JxzzjG9aQBAAlFXAADF\n7JVXXtGECRO0du1apVKpnM9rbm7W2LFjez125pln6rXXXpMkvfnmmxoxYoSnYzEaDltbW/Xoo49q\n/vz5JjcLAEgo6goAoNhdc801mjt3rsrLy/M+7/3339fQoUN7PTZ48GC1tbVJOlIzOzo6NG3aNE2c\nOFE33HCD/vjHPzo6llInT+7s7OzZebaamhrV19fr5ptvVnV1taODAAAkE3UFAAB7Dh48qLKysl6P\nlZWVqaurS9KRaaX79u3TLbfcogEDBmjFihX60pe+pHXr1unoo4+2tQ9H4XDr1q2aPn265QUB6urq\n1N3drauuusrJJvsoKWn29PsAkHSp1OlhH4JtftcVagoAeBenupKpOcSLmJ2eZ3qoW+Xl5T1BMK2r\nq0sVFRWSpJUrV+r//b//13MBmqamJp133nl64YUXdNlll9nah6NwePbZZ6ulpcXyZ9OnT9f27dt7\nLhhw6NAhdXd368wzz9S6det07LHHOtkVACABqCsAANhzzDHHaPfu3b0ea29vV01NjSTpU5/6lD71\nqU/1/KysrEzHH398zhk6VhyFw3yamprU2dnZ8/3DDz+sbdu2qampqc/cWAAACqGuAADwiTPOOEMr\nVqzo9dhrr72mr33ta5Kkiy66SP/0T/+kqVOnSpI+/vhj/fd//7dOOOEE2/swFg6zC3VVVZXKy8s1\nfPhwU7sAACQIdQUAkHTt7e0aOHCgysvLdfHFF+uee+7RnXfeqb//+7/XI488oo8//lh//dd/LUk6\n77zz9K1vfUvDhg3ToEGDdP/99+szn/mMzjvvPNv7M34rCwAAAACAc9lr8CdOnKhnnnlGkvTpT39a\nDz74oDZt2qS/+7u/07Zt27RixYqeNYf/+q//qosvvlhz5szR1Vdfre7ubn3/+9+3XNefc/+pfDfT\nCAEXDwAAb+J64QA/UFMAwLu41pViuyBNEBg5BAAAAAAQDgEAAAAAhEMAAAAAgAiHAAAAAAARDgEA\nAAAAIhwCAAAAAEQ4BAAAAACIcAgAAAAAEOEQAAAAACDCIQAAAABAhEMAAAAAgAiHAAAAAAARDgEA\nAAAAIhwCAAAAAEQ4BAAAAACIcAgAAAAAEOEQAAAAACDCIQAAAABAhEMAAAAAgAiHAAAAAAARDgEA\nAAAAIhwCAAAAAEQ4BAAAAADIcDj84IMPNHv2bJ111lmaOHGimpqa1N3dbXIXAIAEoa4AABCcUpMb\nmzNnjkpKSvTTn/5Ue/bs0Zw5c1RZWakbbrjB5G4AAAlBXQEAIDjGwmFXV5eGDBmiWbNmafjw4ZKk\niy++WJs3bza1CwBAglBXAAAIlrFppWVlZbr77rt7Cvgbb7yhDRs2aPz48aZ2AQBIEOoKAADB8uWC\nNNOmTdPll1+uyspKXXvttX7sAgCQINQVAAD8V5JKpVJ2n9zZ2am2tjbLn9XU1Kh///6SpNdff137\n9u3TokWLdPzxx+uBBx6wf0AlzbafCwDoK5U6PexDsM3vukJNAQDv4lRXMjWXlIS279PtR6xIcRQO\nX3nlFU2fPl0lFn/o5cuX68ILL+z12Pbt23XllVdqw4YNGjZsmL0DopADgCdxKuJ+1xVqCgB4F6e6\nkolw6JyjC9KcffbZamlpsfzZgQMHtG7dOl166aU9j5100kmSpD179tgOhwCA5KCuAAAQHcbWHB48\neFB1dXXaunVrz2Pbt29XaWmpRowYYWo3AICEoK4AABAsY+FwyJAhmjJlihYtWqSdO3dq06ZNWrhw\noaZNm6YBAwaY2g0AICGoKwAABMvRmsNCDhw4oLvuuksbNmyQJE2dOlW33HKLSkvtz15lfQgAeBPX\ntSFWvNYVagoAeBfXusKaQ+eMhkMTKOQA4E1ci7gfqCkA4F1c6wrh0Dlf7nMIAAAAAIgXwiEAAAAA\ngHAIAAAAACAcAgAAAABEOAQAAAAAiHAIAAAAABDhEAAAAAAgwiEAAAAAQIRDAAAAAIAIhwAAAAAA\nEQ4BAAAAACIcAgAAAABEOAQAAAAAiHAIAAAAABDhEAAAAAAgwiEAAAAAQIRDAAAAAIAIhwAAAAAA\nEQ4BAAAAAJJKwz4AFJlrTu/9/SPN4RwHnOF1AwAASDzCIZzLDhJunkv4CIfd1y7f83jtAAAAihLh\nEIU5CYNutknY8Jfp14/XDgAAoCgRDpGbH6Ew334IGmYF8foRFAEAAIoG4RC9BRUIc+2bgGFGGK8j\nIR8AACDWjIbD/fv3a8mSJfr3f/93dXd363Of+5zmz5+vgQMHmtwN/BBmKMxEQPQmCq8jIREGUVcA\nAAiO0VtZ3H777frDH/6gFStW6Ac/+IFaW1t12223mdwF/BCFQJEpascTF1H7u11zevSOCbFDXQEA\nIDglqVQqZWJDHR0dGjdunB555BGddtppkqQtW7bouuuu06uvvqqysjJ7B1TCaENgot5xZ+TJGV5P\n/J9UKuJtwSYTdYWaAgDexbWuNJeUhLbv081ErMAZGzns16+fHnzwQdXW1vY8lkqldPjwYX388cem\ndgMT4jKiE4djjIo4/K3icIyIFOoKAADBMrbmsLy8XBMnTuz12I9+9COdcsopqqqqMrUbeOWxg162\nfJ/t53bdVOlpX5JYg2iHodAVyGvLekQ4QF0BACBYjsJhZ2en2traLH9WU1Oj/v3793y/Zs0aPfvs\ns1q5cqW3I4Q5LkKEk8CQ73eNBEUY5/b1zf49x68voR//h7oCAEB0OAqHW7du1fTp01ViMX93+fLl\nuvDCCyVJP/7xj7V48WItWLBAEyZMMHOkcC/gUJhrewREH7gcNfTj9ZUchkQCIkRdAQAgSoxdkCZt\n5cqVWrZsmb7xjW/oS1/6kvMD4uIBZjkMD6ZDgxVXIZEQYS0Cwd+K49eY19eouF44IBcvdYWaAgDe\nxbWucEEa54zeyuIXv/iFmpqatGDBAlfBEIY5CA5ly/cFEhrS+0I4gnyNHe2Li9UgB+oKAADBMXZB\nmg8//FCNjY2aOnWqLrnkErW3t/f8rLq6Wv36Gc2hKMRmZ5uglhxhvNaOppsyzRRZqCsAAATLWDh8\n6aWX1NHRoSeeeEJPPPGEpCOXHC8pKdHzzz+vYcOGmdoVColBMGQNYrLYfr0JiMhAXQEAIFjG1xx6\nxfoQj2wEQyehcG71UtvPXfrBXNvPlRyuSyMwWPPpRICT111y9trbet15vT2J69oQP1BTAMC7uNYV\n1hw6RzgsJoaCodNgkM14UJAIC7n4EA69vP52X3sCor/iWsT9QE0BAO/iWlcIh84Zm1aKkBkIhl5D\nYeZ2nI4iInwmXv/0Ngq9/rammTLFFIATdk6W8ZmCMNFGEQOEw2JQ4MMmiNFCq+0REJPLTkgkIALw\nxM1VjjN/h88WBMFpO81+Pu0UAWNaadx5DIamQ2G2QgGRaaUeGZpWGmY7YIqpeXGd/uMHakoR8uPW\nN3zGwKQibKNxrStMK3WOkcM48xAM/Q4DQKZ8I8lcuRaALX7eDzW9bUIivKCNoggQDouUqWA453BT\nn8eajprj6pgQTU5PFLhtE/mmmhYMiEwvBZLLzw631b74rIEbQbVTQiJ8xrTSuMrzIeQ1GFp1/q3Y\nDYlMKfSRgWmlJtuEZK9d5GoTBdsDbcGWuE7/8QM1JeaCDIbZ+LyBHQlpo3GtK3GZVtrV1aWGhgb9\n+te/VkVFhb7yla/oy1/+suVzf/vb32rZsmV6++23NWbMGN122236i7/4i56f/+pXv9L999+v9vZ2\nffazn1VjY6MGDRpk+1gIh3HkcjppvhDgpPOfrVAYIBz6zOOVaguFQ7dtw227ICB6F9ci7gdqSkw5\n6HA7vY+rxHp3GOLTvYal6N0LOq51JS7hsLGxUZs3b9aSJUu0a9cuzZ07V3fddZemTJnS63lvvPGG\nvvCFL+gf//Ef9Td/8zf62c9+pl/+8pd69tln1b9/fzU3N2v69OlatGiRamtr1djYqAEDBujBBx+0\nfSz9bD8T0RCxYIji5qVtzDnclPf3c7VJN0UUQBGxecIr/eWG7d+/5vRwR4YQTTbbRSBtNH08iK2O\njg499thjWrhwoWprazV58mTNnDlTa9as6fPcRx99VGPGjNFNN92kESNG6NZbb1VlZaV++ctfSpJ+\n/OMf65JLLtHnP/95/eVf/qWWLVumF198Uf/zP/9j+3gIh0XEaTAs1Hm3y9dwyVnb2MvXzlwFRIog\nULxsnAA1fQLJdkgEJNtt1GQ7pY0Wt5aWFh0+fFijR4/ueWzs2LFqbu7bB37nnXd0xhln9Hrs5JNP\n1muvvSZJ2rJli8aNG9fzs2OPPVaf+cxntHXrVtvHQziMExfrDPMFw7BxhUpDYhKgCYgA8ipQ4/ye\nVVBwH3zuwMB9pb2gjRan3bt3q6qqSqWln1wndPDgwers7NSePXt6PXfw4MFqa2vr9di7777b87zd\nu3dr6NChvX4+ZMgQvffee7aPh6uVxkWRBUMEq+umSsdFy492Mudwk+VaxHy3ugCQAC4vsiY5v+Jy\noc+avFdP5mqmyeWhjUpm2ylttLh0dHSorKys12Pp77u6uno9fumll+rrX/+6LrvsMk2aNElPPfWU\ntm/frvHjx0uSDh48aLmt7O3kQzgsUkURDPlws++R5ticMXQSECmAQALk+Ozy6169mb+b7/6rUo4Z\nLnz2JI+PV4i387u5bgEl0UYLOf2WsI+gsPLy8j7hLf19//79ez0+adIk3XTTTZo1a5a6u7s1fvx4\nTZ06Vfv378+7rYqKCtvHQziMA4cfSm6DYeXc3GcV9i0ty/mzfBgNKm5u20yugGiFgAgUMYfB0Etn\nu9D2HN2Dlc+e5Aj45EW+7dFGi9MxxxyjvXv3qru7W/36HVnx197eroqKClVW9n1tb7zxRn3lK1/R\n/v37VV1drX/+53/WcccdJ0kaOnSo2tvbez2/vb29z1TTfAiHMeZkmmC+YJivg5/5HLcB0Qq3Kwhe\nrqmlSz+Y66qQFWo36Z/najdWAZHppUCCOOh02/2MylfrCp2QyvX5Q+c7wXw6eZGrndppo1LfkEgb\njbeRI0eqtLRUW7Zs0ZlnnilJ2rRpk0aNGtXnuU8//bS2bt2q+fPnq7q6WgcPHtTLL7+spUuPtI3R\no0dr8+bNmjp1qqQj6xHfe++9PhexyYdwGHUOpwpafTDl+hCyEwoRIwFOLXXSdvKdWLAbEBk9BIqM\nxWeV37diyn5uruntEp1vyNjJC9NtNL0/qzYqWZx8p41GXkVFha644grV19frzjvvVFtbm1atWqUl\nS5ZIOjLyN3DgQJWXl2vEiBGaP3++xo0bp5NPPlnLli3TsGHDdO6550qSrrnmGk2fPl1nnHGGRo0a\npTvvvFPnn39+z8iiHSWplIM7NAaAGxZn8fjhZDoY5urk5/oAc3Wjcz7E3DN8H8ww249V28nZbmgz\nvcT1ZsV+oKZEkM1gGNTaeSf1i8+ghDAQDCPXTj220djWlTkl4e27yX7EOnjwoO644w49++yzGjhw\noGbOnKlp06ZJkmpra7VkyZKe0cBf/OIXWr58uT788EOdc845uv322zVkyJCebT3xxBO6//779eGH\nH2rixIlqbGzUn/3Zn9k+FsJhlEUsGEreO/cSU0p95yIgBhUOJfttyFHHTKLdZIhtEfcBNSViPARD\nvy+oFnbnGxFhs+8V1oX/PJ1M9dBGY1tXYhIOo4RppVHlcXpgbKeSUlxD4XbdoRt21686nl4KINp8\nDIZOalu+ae5S38637c8ipu/FX8TbaHo/tFH4qV/YBwBnvCzUj0IwpGMfgAIf/FF4DazaolVhdRRY\nY3IrDyCRXHa65xxuynuyM/3lRKHfs9qn1WeR5TR9PofiqwjaaPax0UbhBuGwCFl9SAUdDF1dcZKz\nWYliNyBmc3KVXgARYKPTbdWxzVXL3HS2cynUAc8+xmx0vouEzdfMThuV8rcrp/K1eTsnVWmjcIpw\nGEUe1hqGEQzt3q8uCiNWieEiaFsFeruvrVt22iajh0CM2QyGmXKNxPhZy+x2vm0HRMSenXaazWQo\ntJKrjRZqpwREOMGaQ0QDo4bm5bm1Ra57HkaB1XqKbKw9BGLAZTDMZrez3fzNws85/Zb8P7e6P2v2\nWsR8NyTvwdqu+HDYTqPYRtPHlVk7s9chsgYRdvk2cnj99dfriSee8GvzxStmo4ZWuIk5nHB7tj4n\nzoYWLepKvDkNhoVGYZq/2fvLDrvPd/q5xMhMTAUQDL200Xy/Y/X+cDWCCGQxHg5TqZQaGxv1u9/9\nzvSmkcFOZ9nJB5bdD61sRqaUctbKP3n+tlavialgn9223HTECqHIJQd1JYYKBCM7wTAXLzXLaju5\ntuW0801AjBmPwTDfyQuvfSurbeXiOSDSRpHFaDhsa2vTjBkz9MILL6iykilfjjkYNcxmorAW+hCz\nc/sBFB9T6w6dFklPo4coGtSVGCrQ6XYbDE11tp1uu1DnOxMBMSYMBEMrUW2jBesnbRQZjIbDHTt2\naNiwYXr88cc1YMAAk5uGIX59aKVx4/II8mH00O2JAifFzY6cJ04odEWDuhIzPgRDPzvcVvuykq/z\nzcU/4s9UMAxCrvcDo9wwxWg4PP/887VkyRJVVVWZ3CyyeDnrapLfV7JEPJlsZ4wegroSIz4Fw6C5\n6XyztitGstqp12AY5MmL7P1my57qSkCEG47CYWdnp95++23Lr46ODr+OMRk8TCmNCkYNI8zh6GE2\n0ycCGD1EGnWleOWrX5mdVqu1W2F1uLOPIZungMhnUvh8CIZhsnMiw/E0aCSeo3C4detWTZkyRRdf\nfHGfLy4UEIywRg2NrzckGEZaEFec9W3tBGKFulIkXHa6o9jhzmTV+XYSEPsgIIbHwUWSCvWtonDy\nIpOTgMhJDBTiKByeffbZamlp0c6dO/t8XXjhhX4dY2KZOqPjxweYnZEk7kMXMR5HD7OZOGEQpeKK\ncFBXikCRBsNMbgMiU/ciwsGUZzvBMIoIiDDFt/scwgGbb0Ivl/72m6ORJkYNYyH7NbU7tbTQDXz9\nwNRSICQuR2PiFAzTjAZEhKrYgmGal4DYB/UzsQiHRc7EB5mbESJGDSMqgsHc61XXAESHnU53HINh\nmrGASMc7OHlGtvPVkrgFwzS3AZFRbqT5Fg5LSkr82nQiOD3TGGSxzR5BYtSwOLi5MA33vkSQqCsR\nY7PTne+CGFJ8Ot1phQJiLgTEEBSY8pwp3wmMuLdRyUNAROL4Fg6ff/55TZ061a/NFw9DU0oB2xwE\n9CAuTCO5Gz3MRlErftSVCLHZ6Y77NL1c8gVEpu5FhKEpz8XSRrMxyo1cmFYK43KOQDFqGHlxmg5s\ne2ophQ0wy0GnO1OxdLrT3AREpu6Fx82U52Juo/kQEJONcFgETE0ptbqQSPa0QU9TShEdHkYP/Zpa\nGvcijIiiU2OWy6s+BtXpPv2W3F9+MBYQYZaLKc+0Udby4wjCYQQVKhxRnlIap5En+MdUkWNqKYwg\nIPom7E63k861Xx1xIwGRNmpOxIJh0bRRJAbhMEwu1xsWEtQIDBeiibkcr0mcAj5TS2EbbcC7CHW6\nTXSeTXbA7Xa+MxEQfWDzb0gbJSAiN8JhzPl5b8NCU0qRHG7veegUU0uBiHJxARo/Ot1+TL8ztU07\n/z4uUOMjB1Oec0laG3V1EgNFj3AYM7GcUsqoYXQZGj3085YWfp4AQcLQ8XbHxQVoTAdDJ53jfUvL\n+nyZ3kcumf9ORmbC5XRk21QwtCMqbVSyrrGsP0w2wmHEZBeKoKaUOv2w4UI0KMSvxfVRPkGCGCAg\neub0foYmgmE+djrZTjrioQdE2qhzHqc8+33yIsptNBMnMSARDsMTsw//QtMIGTUsfmHe8zCf7BMo\nFDFk4tYBHoXY6c7X4XY64pLr9/Pt2xQCos8MTHn2wk4odKNQG/c6iuj0JAaSg3AYY0GuN0QRczm1\nNHZrUOlsJRYnDFzy2On2GgyteOlsO92el863m7VdffCZVViIU57tnLwwpVA7dYuACCulYR8A7CtU\nULiYB8K0b2mZbycsKud2ccICnpQt39f7hMc1pzOzIR+bnW4/RmPydbjzsXPCKlcdTW/b6t9w+i3u\n6mvzN3v/W6w+x+ZWL+2ZldGnjcKxoKY8h9FG09s32UazZbbROYeb4ncSGEYwcgj/1hvS8So6UZla\nyrpDuMHUPfcKXfXR5GhMtnyjJk1Hzen5sqPQ83Pty9QIYhrTSw0IacpzsbdRLlADwmGEeL0YjV9c\nrzdEfPh0z0O/LkoD2MWtA1xw0enOZLrTnc1pZzuXQh1wO8dnBxeo8UFIU57ttlEpf/uyK19bD/ok\nBpKDcIg+mL4Hu6Iy5YSL0iCfglfdo/P9iYiMxuQaHbH7mbP0g7k9X4UE3flm/aFHIU15dnryohAn\nbTTXdq3eJ27XynpuoygarDkMg4EP+cjf+40ppYnk57pDwAvWdtlgszYFEQyz5etsF+pcW/3cakQ5\nvY/Mf5/VWkQT67tcrT9knayrG92bmPLstY1K+dup3Taa3k92YLOqvW7aaeY62ez1hzpqsbONIbYY\nOSwSQV2MJvsDjA5WEbHZ6QjrHpdGQidn3xONqXt5GOh0u1Go051vFMbJqIuT37Uziuh1ZCYT95Zz\nL4gpz16CodPRQavftZJrFDGb16UdnOxNJsJhTPg1rF/ogyMq0wYRHk4AIM7mHG5i6l4hDoJhkJ1u\nP0Kh3W3Z6Xz7NXWPkxg5hDDl2e3JCz/aaK52mu/43ODK9yAcAvDErxMIXLEUfmD9YWFR6XRnc9rh\ndnJiy23nO4i1XQREhXIBmqi10fT2swXZRpEMhMOIiMr0ES5GA9O4YimiwNHUvSR1vmPQ6S7U4e66\nqdLyK9fP8rHalx+d70yubh2Q4DaazY8pz26DYT5O2mi+dhpUGyUgJhfhMKKichsLV5K+YD7hOMGA\nKHE1dU9KRuc7Yp1uqyl6dkKhU3Y74JlMd76NrD9MaBv1e8qzyZMXdk9K5PvdXKzaaOax+blOFsWN\ncAjbuBhNAhDsUYRcTd0rdhHrdDsZiXHb2Xa6rSADIicxLLhso15Gtt0EQytBttF87dT0KDeSgXAY\nQwzvI2xhXbE0H+51iHzsTN1LzNquiHW67Y7EmOxw29129rEQEANSoI1milIwDKONWh2LyYDI6GHy\nEA6RE1cqRRqjxCgmuTreloqt4x2DYJjNzw631b6shB0QMyUmIGbI/jf7dWuVtHzBMN/JiyDkO5GR\niYAItwiHQSvyD3DAT4yawylGZjJEbDTGbjB05JHm3F822el8+3ELgTRXF6iRirqdZvJ7ynOhYJjN\n8ckLA200vd9sTDGFCcbDYVdXl+bPn69x48Zp0qRJWrVqleldIIvbMzp8MMAURpnhJ691hYAoW8Ew\nyNEYY8HQSefaYUfcS0AM5QI1xSDkkW03wdCWkNqolD8gAlaMh8OlS5dqx44dWr16terr67V8+XI9\n99xzpneDqOKCJgAMM1FXEj11z8ZxBzkaUygY2hqJcTHS4mYbTgNiJtYfOhSBKc+ZPAdDlyOCltvI\nw+r9YjcgMkgAK0bDYUdHhx577DEtXLhQtbW1mjx5smbOnKk1a9aY3A0AICH8qiuuLlAjxa/zneN4\nwxyNyeT4KtgmQqHDbRbqfPu1tstTQCyCdhqVtbCOT16E0EbTx5WJgAi3jIbDlpYWHT58WKNHj+55\nbOzYsWpuZjQJiA0fRn8pPnDLZF1xOnVPinlAjEAwzFao052THx1uh/sIKiBmsjOtN+cU0xi3UzvT\nZr1MefYSDPOKcBvNhxqNTEbD4e7du1VVVaXS0tKexwYPHqzOzk7t2bPH5K7gA+aiI2q4QhpM1xWn\nIzNSTAOioWDolC+d7qCXK/gQEJ0wdhJDinY7zTHCmW8trKkpz5mMtdEg22nETmKguBifVlpW1rux\npb/v6uIqgwAAZ/yoK0YDYhQ73zaCoV2hdrqD7nBn7zsHN51vv6eXSjELiDbbaJBTnmNz8iJzvzn2\nHcRJDBQvo+GwvLy8T7FOf9+/f3+TuwIAJEAQdaWoOt/F0umOwsXNDHe+CYj/p5jaaFTaqYUg2iiK\nk9FweMwxx2jv3r3q7u7ueay9vV0VFRWqrOQm2gAAZ/yqK9mdyth3vvOMYsay0x0lNjvfmXJdxdTU\n+sNYttE8x+CmjTrl5m9fbG2UgAg7jIbDkSNHqrS0VFu2bOl5bNOmTRo1apTJ3QBICIoU/KwrdgNQ\n5DvfefYbZqe7KIJhmo3Od66Lf3hZ2+XLSYww2qnhkxeS/1Oei7GNSgREFGY0HFZUVOiKK65QfX29\ntm3bpvXzUP9eAAAgAElEQVTr12vVqlWaMWOGyd3AJyZvdowYi8LZZeD/+F1X7N7/MJKd7wL7CrvT\nnUvsOt1pDgOi3xeokVwGRCnYz3mXbTQfv0e2i7mNSvauYkpATC6j4VCS5s2bp1GjRmnGjBlqbGzU\nzTffrMmTJ5veDYAY4aqj8MLvumI8IEr+h8QC2w4qGCam051mKCCaWn+YLXJt1EMwDHPKs6U4tVGL\nY8313uMCNchmPBxWVFTorrvu0quvvqoXX3xR06ZNM70LAECCBF1XjAVEyXwH3EaH282tACSCoW02\njjesC9RI0W+jVscURDDM5HjKc9zaqFQwIDK9FLkYD4dIOKYkAoghu2u7JBedb8lbBzz9uw473FL+\nYBjkaEzRBMM0GyMzuTrfmfy4QI1k3UZth0Q37dRBG3UbDL2yGhUr2mCYRkCEC4TDoPnwIeP2jctU\nP5iSbzQCiAuvAdFRSMzXmS70cwuFgmG2oEdjLMW50y05DohpJtcfOmmjkoN7XYbURoOe8pyp6IKh\nTX6fxED8EA6REx1+RA3rIeCUl6l7ko+d70wuOtqZ+/Pa6fYikaMxmQys7Sr2Npprn2EHw4Ij2wlq\no6ZPYiDeCIcACnLVmQiYrZEKJJLXzncmu53vIN4zufYRdqc7U1F3utNCmLpnIiAG1UYJhhHABWrg\nAOEwhnizImx2Lzcepnw3qAYKyXd1SDudb8m/Dni+Dnf2GkM63QFxGRAz+R0QgwyJ+bYbVDDMRBuV\nkZMYSAbCYURFcRTE1UUYEC8GLijE/TIRRV7XzNgJiH6HRLcdbolOdyAK/Pv8mLrnJCBK/p/IKNRG\n/QyG2exMee4jgW2UgIhshMOIKKpRDq5YCiAi/Jq6JznrfEufdJyddMIL/U6hDrdEpztQWf9Ov9cf\nSoUDot2RbsmfNmq1T9Oj2pKBKc8JbaOS81FuFDfCIXph1Aem+XVVXIoW3AgqIBaaep3Zoc73lY/V\nPvwIhnS6HcoTEP26dYDTdhpkG3V68sINY7dWSQoXo9xIDsJhwnE7C3jl11VtuXQ2TPFj6l6h0RnJ\nv7W5uTr2YQXDvJ3upAXDHPxefyj5dyLDDbdtVGLKc2BcjHIjGQiHMVHozelXR7pQx591h0Ukx3Tg\nQjcsBuLA9MiM5KzzbeJ9k6/D7XcwzMRojA0upu5ln8QwUdfttFHJXEjMtx0/gmG2QlOeCYb5Mb0U\nEuEQDtkuHqw7hGFGFsTTCUgcP6fuSdadb5Md8ELh0mpffgRDOt0uROACNZK7Nuqkndppo34FQ6dT\nnvugjfZ5KFdARHKUhn0AifRIs+fwtG9pGesDETlhtUkKGPJpOmqOZWf49Fucd0ibv9k3OFXO7erT\nqZ9zuClnB9XECI2dUCiZWTpg1enmAjQ2ZdX7rpsqLWfcZLbRzPrudxtN79uK13aaK4AGGQyZ8mxD\ngT7p0g/mMmMoYRg5RB+ETqQVmjbs13pDr5jehlxMTN2zMzoj5R+hcSvXNk0GQy5AY1hIF6gptFY2\nzXQ7zddG/QiGmQiGLhVYf8gJ2GQhHEZIVN+MrtcdMrU0Pmy+Vk7PHpronFph/QPsSn+Omr5nV66A\nmK/z7aUDnu/3gw6GdLq9C+ICNZL9ExmS95Dopo2GNuUZfdm8QA2KH+EwRvy6KI3TD2emFwCII5Mj\nM1Luzm2+2Rd2g2Lm8/J1uP0Khpm4AI0BBtYfmg6IhUJioXYaRhvNxpRnf/EeTybWHAIwws/pyEYu\nRoPEylwzY3JtV1quNV7pfeTidpQm33vNVDDkRvc+8Lj+0AurNipZr0XMZrqdegmGTHn2mcX6w1zt\nFMWLkcMY87PD7LQYMbU0xmzewiJbVNYb2p5+Tacg0fycuifl7vDmG6FxKt+23E7Rs0Kn20chrD9M\nK9RGg2qnbjHlOSAFrmCK4kc4DIvND6morjtkamnyRGW9YSEUMeTi59Q9KX+bd9sBt/N7XkMhne6A\nhRwQ7bRTpwq1U68nL5jyHDDe04nGtNKI8Tp873ZqVK4pJyhyMRzZ5WI0cMLN1D2vU0zT28jF1AiN\niZFCz8EQRmS2Uzu3DvDSRqXCNd/kMgGTJy8kpjwHxsBt1xBPjBzGTJQ7xkwtLR5Op5Sy3hBR5nRk\nxgST0zxzbd8rOycEudG9T2z+3fK1Ua8ndINoo34GQ6Y8A/4gHMZckOsOmVqaXEG91k47O6w3hF1B\nTt3LZLoDbmp7bjrdffC+8sbj9FLJTDuNextlyrOP+PslEuEwTD6tO4zk9FBGD6PH5YVowlSo08qU\nN+QTVkCUvHWY079rqgNPpztCIhIQJW/tLJJtFGbwXk8cwmEEFfqAi+XUUhQNO1NK/bhqImCanRNv\npk+2ZXaic3WmC/3cC4Jh9IUZENPCbKPZXAdD2qk5/C0Txbdw2NXVpcsvv1wbN270axf4P1YdaDeF\nws7NnD1NLWX0MDocvBZRnVKK5HFVV/KMzGQKsuOdLaiONqMxEVWg4x1WQMwUZBjMdZGkTFyABvCP\nL+Gwq6tLdXV1evPNN/3YfCJF5ZYWdjB6GF9Rfu2yR8xZb5gsnupKhKbuhcVNMLTE+8kfNk9iFFJs\n7TQTF6BBMevq6tL8+fM1btw4TZo0SatWrbJ83rRp01RbW9vna8GCBZKkffv2qba2ViNHjuz52YQJ\nExwdi/Fw2Nraqquvvlq7du0yvenilONDzOnUUlOjh75j9DB8HkYN/brxvVVbdTqllBGO4uVHXUlS\nQLRzKwArdLoDZuAkhlSc7ZQpzyh2S5cu1Y4dO7R69WrV19dr+fLleu655/o87zvf+Y5eeumlnq/v\nfOc7Kisr0z/8wz9Ikt58800NGjSo13OefvppR8diPBy+8sormjBhgtauXatUKmV68wiBm6mlUR6B\ngjU3r5mft7AA0ozUlRhM3fNDoWBIpzvakhAQT7+FKc9Ito6ODj322GNauHChamtrNXnyZM2cOVNr\n1qzp89zKykoNHjxYgwcP1qBBg3Tvvffqq1/9qk499VRJR06mjhgxQtXV1T3Pq66udnQ8xsPhNddc\no7lz56q8vNz0phPPzdTSSBYIRg/DE8G/vZ1RQ6aUJpuxulJg6l6xBURjwRDBsfjMKuaAWOjznynP\nSIKWlhYdPnxYo0eP7nls7Nixam7O36Z//vOfa+/evZo5c2bPY+lw6AVXK40CH6eWBoV7Hsab1aih\n2ymlTi5WYKrzQmcWtjlY21UoIEa58200GNLpDpahUW4pXm1Ush8MGdlGMdm9e7eqqqpUWlra89jg\nwYPV2dmpPXv25Py9lStX6stf/rL69+/f81hra6vee+89XXXVVTr33HNVV1en3bt3OzoewmEC+FEc\n7ASFvNMUIziCVfQM/839mlJqbNQQsCnXyIwUz86322BoiU53OAyNckvxaKMSwRDJ1dHRobKy3u/d\n9PddXdZ9rf/6r/9SW1ubrrrqql6Pv/XWW/roo4+0YMEC3XfffXr//fd14403OlqSQTiMmXwdFyn3\n6KHX4sDasuJlZ9TQD76PGtJhQC4Opu5li3Lnu9DaLYlOd6wYGuWWotNGJWfBMBuzRFCMysvL+4TA\n9PeZo4KZnnvuOU2aNEmVlb3fE+vWrdOaNWs0evRonXnmmfrWt76llpYWbd261fbxlBZ+CgLxSLPl\nyE7XTZXGLu5y+i3+3p9obvXSPp2NsuX7cn+YX3M6nY+geBw1NH2V0lwdFW56j8BYfOZmft4u/WBu\nz0mSpqPm9HoP7Fta1ueEWbpN+30PuFycrt0lGMZTrjYq9W6ncWmjUv5gyJRneBVmv8LuqYxjjjlG\ne/fuVXd3t/r1OzJu197eroqKij7hL+0//uM/NGvWrD6PZ6/Nr66uVlVVldra2mwfNyOHMeR29FAy\ne/bQr9saIN4KtTEnbZAppfCVy7Vdkn+zNJzKtfbRUzBEdDgc5c4eQYzKKGKuNkowBKSRI0eqtLRU\nW7Zs6Xls06ZNGjVqlOXz9+zZo3feeUdjx47t9fiBAwc0fvx4bdy4seextrY27dmzRyeccILt4/E1\nHJaUlPi5+cRwU7iDCojZHN/WgrWH/svzN3Y7pdTtNON8bc/N2T2mlCaP8bricm2XlL/zHUQH3O4I\nvONperx/osVDQJRyTzMNqo36ckVq2iiKSEVFha644grV19dr27ZtWr9+vVatWqUZM2ZIOjKK2NnZ\n2fP8N954QxUVFTruuON6befTn/60xo4dqzvvvFPbtm3T73//e9XV1em8887TySefbPt4fA2HO3fu\n1Lhx4/zcRXFx8GFXqBgUYvVhXahQsO4wmdyMEGe3pUIdEavOC6OGsBJEXSkUEJ2MIvrRAc+3XSed\nboJhjDgY5ZacjXRHuY1KTHlGMsybN0+jRo3SjBkz1NjYqJtvvlmTJ0+WJE2cOFHPPPNMz3Pb29s1\ncOBAy+0sXbpUp556qm688UbNmDFDw4cP17JlyxwdS0kqYneqLylJ+Js+xyiP21sNmA50XjrxeUdA\n+bD3h4FRwzDaldXJDkdT4RLenlIpRuTTbNcUi/dK9nuk0GduofeFl7VehTrwTj+bCYYxldVOnbZR\nyb92aidkeg6GEu00JHGtK/sOh3ff9cqjOgs/KYIIh1EU4YBopwMiWQdEOvMBKzBlNyrh0O0JB6aU\n5hbXIu4HRzXFQECUnL0/rDriTkdy3LyHGI2JqYi0UclZOzXSRiXaaYjiWlcIh85xQZoi43R6aZBY\nexgddq+A6/dFh+yebMjGBTTgiwJruyR7U/qdrJ9NT7/L/LLLas2j1bRXLkBTRAy2Ubvt1KqN2m2n\nufZDMASii3AYRTk+AK0+LO2sw/L7Mr5WASKI++QhDxdhO+jXzG67dLTWkM4DvAqh8+2Ukw43ne4i\n5LKNBtlO87VRgiEQbYTDmLFzxtfrmezQMHoYCFP3zfSDp+mkgCk+dL5NfAbn247r9w6d7nhy0Ual\n3DMzwmijEsEQiCLWHEaZh7WHkj9rxbx86KcV7NxTCLxxsdZQst+GJH/akee1q7SbHnFdG+IH1zXF\n5uevk/dNtkLvIzuddU/vG94z8WdjDaLkvp3SRpEW17rCmkPnCIdR5vFKk3507J2EQ8nlyA/FwBtD\nF6KRgmtDXNTIrLgWcT94qik+nKAzxfNnLu+X4uEhIErBt1PaaDzFta4QDp1jWmmUefxgzDd9JEwF\npzUyvdQ9l6OGVkx3GJxMW3J8T0M6EfCDx/XfuaaaepFrm1brCyU63YlgY4qplPtz1XQ7TW+PYAjE\nU2nYBwB3um6q7NPRX/rB3D5nBpuOmmPZyU930p2MAIUdKhEd+5aWGWs7djslrDVEKB5ptjzpkusz\nWOo7QpPZxt2cdCn0HmH6NazaqZM2Kn3SztyeGMzXTl0vMwEQOKaVxoFPNzJPs9vJd9PB91QQ6MQ4\n42HU0M2UODvtptAJBaaT+iOu03/8YKymOJhiKgVz9V/Hn6+8V4pfTNopn+nxE9e6wrRS55hWWoSc\nXKFMKnw1PT8udR3lK2YmjdvOgdc24+QiRjnRiUBQ8kwxzTWFz3F7tinftgmGCedgKrTk4jPXAUdT\nndNop0DoGDmMCwOjh5J/C899mU5CkbDPxjpNp2eOo3CRAomOhBtxPcPrB+M1xeFncZqJEZpCHXmC\nIXo4HEFM89pOXbdRiXYacXGtK4wcOkc4jJMIB0Tf1hpQLOzxIRxKwbYVzjCbE9ci7gdfaorLgJjJ\nbifczsgO7xNY8thOaaPIFNe6Qjh0jgvSFDGrC9RIuS9SE7Sy5ftYjB6QKEzjdXoCgbaByMpxkRrp\nk3Zb6D1nYjofsy+QV4F2GkQbTe8rJ9ooEDmsOYyTPB+iTtcT+HGJdV9wW4vCYvA3cjOynBcdCoTt\nkeaCn8l+neCwtW3eI5BCa6O2tk8bBSKJcBg3BgOiZP82AvnEImTCFb/bB+sMEXsF2qPJDrjtbfEe\nQSabbdREO7W9LdooEFlMKy0yuaaK5JpiKnm7txHBELm4uTebRDBEDOWZvpeW3a7tTvV21GHn/YFc\nbLRRqXd786WNpo8FQGRxQZq4cnlPOzsLzO2GRLvBMN/IJWfBPbI5pdTEVRTttAs7bcJTe6At2BLX\nCwf4IfCaEtY0b94bsIs2ChfiWle4II1zhMM4M3zTcz94uqx1JoqKtQDDoQkEw2DEtYj7IZSaEmTn\nm/cF3Ag6INJOYy+udYVw6BxrDuPMxjqCXPy88S3iJai2QDBEYhS4WI3R/QBuBNlGaadArLDmsMjl\nu1x1vnWIgCmeR4/pWCCu0m3X9CgN7wmYQhsFkIVppcXAww3Q0/wIiZ5vjJuJQmPNwLTStDDaAMHQ\nH3Gd/uOHSNUULx1w3gsIitt2ShstanGtK0wrdY5wWCwMBETJXECwO1WRcOiRwXAoBfv6czEi/8S1\niPsh8jXF6j1Mu0eU5KoztNNEiWtdIRw6RzgsJoYCouQtJDhZw0Y49MhwOJSCee0Jhv6KaxH3AzUF\nALyLa10hHDpnPBy2tbVp8eLFevnll1VRUaFLLrlEdXV1Kisrs3dAFHJvQgwLTi9swv27DHAw/cfJ\nay7Ze919e815vT2JaxG3Qk0BgPDFta4QDp0zfkGa2bNnq6qqSj/5yU+0d+9ezZ8/X0cddZRuvfVW\n07uCFQc3urUbFriyaTKZft0JhnCDmgIAQHCM3srirbfeUnNzs+666y6deOKJGjt2rGbPnq1f/epX\nJneDQmx2rh2N3BkW5r6TKqy/eddNlQRDuEJNAQAgWEbDYU1NjVasWKHq6uqex1KplPbv329yN7DD\nQUAMOjQQDJODqcPwgpoCAECwjIbDgQMHauLEiT3fp1IprVmzRuecc47J3cAuB53toEIiwTBcQf39\nHY8WEgxhgZoCAECwjK85zHT33XerpaVFP//5z/3cDfJxeIPbdIfe6cVLnGzbMYJDbjbXmGby6zV2\n9fry2sIBagoAAP7yLRwuW7ZMq1ev1n333acTTzzRr93ALochIrOj7zVEMFoYTaZCIqEfQaCmAADg\nP1/CYWNjo9auXatly5Zp8uTJfuwCbrgYZZL6dv7thAljgZAA4TunJwI8v7a8pnCImgIAQDCM3+dw\n+fLlevDBB3Xvvffqoosucn5A3JMqGC5CYigIEvbweiJDXO9HZYWaAgDhi2td4T6HzhkNh62trfr8\n5z+vG2+8Uddee22vnw0ZMsTeAVHIgxP1QEGQcCbKryevZaDiWsSzUVMAIBriWlcIh84ZDYff//73\nde+99/Z6LJVKqaSkRDt37rR3QBTy4EUxVBAm3OG1hOJbxLNRUwAgGuJaVwiHzhmfVuoVhTxEUQkW\nhAlveB0TL65F3A/UFADwLq51hXDoHOEQfYUZLggUZvAaJlpci7gfqCkA4F1c6wrh0DnCIXILMmAQ\nKPwR1GvI6xcpcS3ifqCmAIB3ca0rhEPnCIewx6+QQagIBq9fosS1iPuBmgIA3sW1rhAOnSMcwh0v\nYYNAES5eu6IX1yLuB2oKAHgX17pCOHSOcAgARSauRdwP1BQA8C6udYVw6Fy/sA8AAAAAABA+wiEA\nAAAAgHAIAAAAACAcAgAAAABEOAQAAAAAiHAIAAAAABDhEAAAAAAgwiEAAAAAQIRDAAAAAIAIhwAA\nAAAAEQ4BAAAAACIcAgAAAABEOAQAAAAAiHAIAAAAABDhEAAAAAAgwiEAAAAAQIRDAAAAAIAIhwAA\nAAAA+RAO3377bV1//fUaM2aMLrjgAq1cudL0LgAACUJdAQAgGKUmN5ZKpXTDDTfojDPO0JNPPqk/\n/vGPqqur07HHHqvLLrvM5K4AAAlAXQEAIDhGRw7b29t16qmnqr6+Xn/+53+uc889VxMmTNDmzZtN\n7gYAkBDUFQAAgmM0HNbU1Oiee+7R0UcfLUnavHmzNm7cqPHjx5vcDQAgIagrAAAEx+i00kwXXHCB\n3n33XX3uc5/TlClT/NoNACAhqCsAAPjLt6uVfvvb39aDDz6onTt3avHixX7tBgCQENQVAAD8VZJK\npVJ+7uDZZ5/VrbfeqldffVWlpYUHKktKmv08HAAoeqnU6WEfgq+c1BVqCgB4F9e6su9weWj7rjyq\nM7R9e2F05PBPf/qT1q9f3+uxk046SYcOHdKBAwdM7goAkADUFQAAgmM0HO7atUuzZs3S7t27ex7b\ntm2bqqurVVVVZXJXAIAEoK4AABAco+HwtNNO06hRozRv3jy1trbqxRdfVFNTk772ta+Z3A0AICGo\nKwAABMf4msPdu3ersbFR//mf/6n+/fvruuuu0w033GD/gFgfAgCexHVtSC5e6go1BQC8i2tdYc2h\nc75fkMYpCjkAeBPXIu4HagoAeBfXukI4dM63W1kAAAAAAOKDcAgAAAAAIBwCAAAAAAiHAAAAAAAR\nDgEAAAAAIhwCAAAAAEQ4BAAAAACIcAgAAAAAoenq6tL8+fM1btw4TZo0SatWrcr53Ndff13XXnut\nzjjjDH3+85/Xyy+/3OvnP/zhD3Xuuedq7NixWrBggTo7nd1vkXAIAAAAACFZunSpduzYodWrV6u+\nvl7Lly/Xc8891+d5Bw4c0PXXX6+TTz5Zv/rVr3TRRRfppptu0gcffCBJevbZZ/Xd735XjY2Nevjh\nh7V161YtW7bM0bEQDgEAAAAgBB0dHXrssce0cOFC1dbWavLkyZo5c6bWrFnT57mPP/64BgwYoDvu\nuEPDhw/XrFmzNGLECG3fvl2StHr1as2YMUPnnXeeRo0apTvuuEOPPfaYo9FDwiEAAAAAhKClpUWH\nDx/W6NGjex4bO3asmpub+zx348aNuuCCC3o99rOf/Uznnnuuuru7tW3bNp111lk9Pxs9erQOHTqk\nlpYW28dDOAQAAACAEOzevVtVVVUqLS3teWzw4MHq7OzUnj17ej33nXfe0aBBg3T77bdr4sSJ+uIX\nv6hXX31VkrRv3z51dnZq6NChPc8/6qijVFVVpffee8/28RAOAQAAACAEHR0dKisr6/VY+vuurq5e\nj3/88cd66KGHNHToUD300EM666yzdP3116utrU0HDx5USUmJ5bayt5NPaeGnAAAAAEC8NB01J7R9\nL7L5vPLy8j7hLf19//79ez1+1FFHaeTIkbrpppskSbW1tXrppZf05JNP6sorr1QqlbLcVvZ28mHk\nEAAAAABCcMwxx2jv3r3q7u7ueay9vV0VFRWqrKzs9dyamhqdcMIJvR4bMWKE3n33XQ0aNEjl5eVq\nb2/v+dnhw4e1d+9e1dTU2D4ewiEAAAAAhGDkyJEqLS3Vli1beh7btGmTRo0a1ee5o0eP7nNxmbfe\nekvHH3+8SkpKdNppp2nz5s09P3vttdf0qU99SrW1tbaPh3AIAAAAACGoqKjQFVdcofr6em3btk3r\n16/XqlWrNGPGDElHRhHTt6L44he/qNdff13Lly/X22+/rfvvv1+7du3S5ZdfLkm69tprtXLlSq1f\nv17Nzc264447dPXVV6u8vNz28ZSkUqmU+X+meyUlfS/bCgCwL5U6PexDiAxqCgB4F9e6crsWhLbv\nRVps+7kHDx7UHXfcoWeffVYDBw7UzJkzNW3aNElH1hUuWbJEU6dOlXRkNLCxsVGtra068cQTtXDh\nQp155pk921qxYoV++MMf6tChQ7r44ot122239blITT6EQwAoMnEt4n6gpgCAd3GtK3EJh1HCtFIA\nAAAAAOEQAAAAAEA4BAAAAACIcAgAAAAAEOEQAAAAACDCIQAAAABAhEMAAAAAgAiHAAAAAAARDgEA\nAAAAIhwCAAAAACSVhn0A2VKp08M+BABAkaCmAABgHyOHAAAAAADCIQAAAACAcAgAAAAAEOEQAAAA\nACDCIQAAAABAhEMAAAAAgAiHAAAAAAARDgEAAAAAIhwCAAAAAEQ4BAAAAAAo4uGwoaFB06ZNC/sw\ncvrggw80e/ZsnXXWWZo4caKamprU3d0d9mH1sX//fi1YsECf/exnNWHCBM2bN0/79+8P+7Dyuv76\n6/XEE0+EfRi9dHV1af78+Ro3bpwmTZqkVatWhX1IeXV1denyyy/Xxo0bwz4US21tbZo9e7bGjx+v\n8847T0uWLFFXV1fYh2Xp7bff1vXXX68xY8boggsu0MqVK8M+JLhEXTGDuuJd3GqKRF0xhZqCKIts\nOHz11Ve1du1alZSUhH0oOc2ZM0cfffSRfvrTn+r+++/X008/rYceeijsw+rj9ttv1x/+8AetWLFC\nP/jBD9Ta2qrbbrst7MOylEql1NjYqN/97ndhH0ofS5cu1Y4dO7R69WrV19dr+fLleu6558I+LEtd\nXV2qq6vTm2++Gfah5DR79mx1dnbqJz/5ie655x698MILuv/++8M+rD5SqZRuuOEGDRkyRE8++aQa\nGhr0wAMP6Omnnw770OAQdcUc6op3caopEnXFFGoKoq407AOwcujQIdXX12vMmDFhH0pOXV1dGjJk\niGbNmqXhw4dLki6++GJt3rw55CPrraOjQ7/+9a/1yCOP6NRTT5UkzZ8/X9ddd526urpUVlYW8hF+\noq2tTbfeeqt27dqlysrKsA+nl46ODj322GNauXKlamtrVVtbq5kzZ2rNmjWaMmVK2IfXS2trq265\n5ZawDyOvt956S83NzXrppZdUXV0t6UhRv/vuu3XrrbeGfHS9tbe369RTT1V9fb2OPvpo/fmf/7km\nTJigzZs367LLLgv78GATdcUc6op3caopEnXFJGoKoi6SI4ff+973dMopp+icc84J+1ByKisr0913\n391TwN944w1t2LBB48ePD/nIeuvXr58efPBB1dbW9jyWSqV0+PBhffzxxyEeWV87duzQsGHD9Pjj\nj2vAgAFhH04vLS0tOnz4sEaPHt3z2NixY9Xc3BziUVl75ZVXNGHCBK1du1apVCrsw7FUU1OjFStW\n9BRw6Ui7jOK0tJqaGt1zzz06+uijJUmbN2/Wxo0bI/deR37UFXOoK97FqaZI1BWTqCmIusiNHLa2\ntqj9F64AACAASURBVOrRRx/VU089pZ/85CdhH44t06ZN08aNGzVq1Chde+21YR9OL+Xl5Zo4cWKv\nx370ox/plFNOUVVVVUhHZe3888/X+eefH/ZhWNq9e7eqqqpUWvrJW2bw4MHq7OzUnj17NGjQoBCP\nrrdrrrkm7EMoaODAgb3aZSqV0po1ayLdcZekCy64QO+++64+97nPRfLsPqxRV8yirngXp5oiUVf8\nQk1BFAUeDjs7O9XW1mb5s5qaGtXX1+vmm2/udeYnLIWOtX///pKkhQsXat++fVq0aJH+5V/+RQ88\n8ECQh2n7OCVpzZo1evbZZ0NZ/OzkOKOmo6Ojz1Sp9PdRXOweN3fffbdaWlr085//POxDyevb3/62\n2tvbVV9fr8WLF2vhwoVhHxJEXfEDdcVf1BT/xaGuUFMQRYGHw61bt2r69OmWFwSoq6tTd3e3rrrq\nqqAPy1K+Y12+fLkuvPBCSdIpp5wiSbrrrrt05ZVX6n//9381bNiwyB3nj3/8Yy1evFgLFizQhAkT\nAju+NLvHGUXl5eV9Cnb6+6h2PuJi2bJlWr16te677z6deOKJYR9OXn/1V38lSZo3b55uvfVWfeMb\n3+h15h/hoK6Ed5zUFXeoKf6KS12hpiCKAm+BZ599tlpaWix/Nn36dG3fvr3nggGHDh1Sd3e3zjzz\nTK1bt07HHntskIea91gPHDigdevW6dJLL+157KSTTpIk7dmzJ9Ainu8401auXKlly5bpG9/4hq67\n7rqAjqw3O8cZVcccc4z27t2r7u5u9et3ZKlue3u7KioqInWRg7hpbGzU2rVrtWzZMk2ePDnsw7H0\npz/9Sa+99lqv4zvppJN06NAhHThwIHLT6JKIumIedcVf1BT/RL2uUFMQdZE6PdHU1KTOzs6e7x9+\n+GFt27ZNTU1NGjp0aIhH1tfBgwdVV1en4447TmeccYYkafv27SotLdWIESPCPbgsv/jFL9TU1KQF\nCxZE+v5eUTZy5EiVlpZqy5YtOvPMMyVJmzZt0qhRo0I+svhavny51q5dq3vvvVcXXXRR2IeT065d\nuzRr1iz95je/UU1NjSRp27Ztqq6upojHAHXFH9QVb6gp/ohDXaGmIOoidbXSoUOHavjw4T1fVVVV\nKi8v1/Dhw3vOrEXFkCFDNGXKFC1atEg7d+7Upk2btHDhQk2bNi1SV0T78MMP1djYqKlTp+qSSy5R\ne3t7z1cUb6wcVRUVFbriiitUX1+vbdu2af369Vq1apVmzJgR9qHFUmtrqx544AHdcMMNGjNmTK92\nGTWnnXaaRo0apXnz5qm1tVUvvviimpqa9LWvfS3sQ4MN1BXzqCveUVPMi0tdoaYg6iI1chg3d955\np+666y595StfkSRNnTo1cvcBeumll9TR0aEnnnhCTzzxhKQjV/AqKSnR888/H+g0JSeieJPqefPm\n6Y477tCMGTM0cOBA3XzzzZGcspIpin9HSXr++efV3d2tBx54oOdCG+l2uXPnzpCPrrd+/frpu9/9\nrhobG/XFL35R/fv31/Tp00ObRofiRl3xT9Q+D+NYU6To/R3T4lJXqCmIupJUVG9YAwAAAAAu3a4F\noe17kRaHtm8vojWnBgAAAAAQCsIhAAAAAIBwCAAAAAAgHAIAAAAARDgEAAAAAIhwCAAAAAAQ4RAA\nAAAAIMIhAAAAAECEQwAAAACACIcAAAAAABEOAQAAAAAiHAIAAAAARDgEAAAAAIhwCAAAAAAQ4RAA\nAAAAIMIhAAAAAECEQwAAAACACIcAAAAAABEOAQAAAAAiHAIAAAAARDgEAAAAAIhwCAAAAAAQ4RAA\nAAAAIMIhAAAAAECEQwAAAACACIcAAAAAABEOAQAAAADyMRw2NDRo2rRpfm0eAJAw1BUAAPzlSzh8\n9dVXtXbtWpWUlPixeQBAwlBXAADwn/FweOjQIdXX12vMmDGmNw0ASCDqCgAAwTAeDr/3ve/plFNO\n0TnnnGN60wCABKKuAAAQDKPhsLW1VY8++qjmz59vcrMAgISirgAAEJxSJ0/u7OxUW1ub5c9qampU\nX1+vm2++WdXV1UYODgBQ3KgrAABEh6NwuHXrVk2fPt3yggB1dXXq7u7WVVdd5emASkpe9vT7AJB0\nqdT4sA/BNr/rCjUFQEHnW3xmNvR96JJzH/f9UEx45jd/2/fBBosnvmD/8zFOdQXelKRSqZSJDU2f\nPl1btmzRUUcdJenIBQS6u7tVUVGhdevW6dhjj7V3QBRyAPCkWIq4ibpCTQGQV5EFwzTTATGudeV2\nLQht34u0OLR9e2EsHL7//vvq7Ozs+f7hhx/Wtm3b1NTUpOOOO079+tlb3kghBwBv4lrEs5moK9QU\nAHllh8OGvk+JWzBMMxkQ41pXCIfOOZpWms/QoUN7fV9VVaXy8nINHz7c1C4AAAlCXQHgK6tRwyxx\nDYbSkWPvExAb1Dcgnj/e0RRTFDfjt7IAAAAAIs3GdNI4B8M0y39Dg8UTbQRlJIOxaaWmMAUIALyJ\n6/QfP1BTAPSRkGCYydYU0zyjh3GtK0wrdY6RQwAAACRXQ+9viy0YSjb/TYweQoRDAAAAJIWNC9AU\nqz4BscHiSQTExCMcAgAAoPgV+QVo7CAgohDCIQAAAJKnofe3xR4M0wiIyIdwCAAAgOJWYDppGMHw\nmd/8rfWFYsLQEPYBJFtXV5fmz5+vcePGadKkSVq1alXB39m1a5fGjBmjjRs39jy2b98+1dbWauTI\nkaqtrVVtba0mTJjg6FiM3ecQAAAAiJwIBMN8IdDqZ34fk+U9ELNx/8PALF26VDt27NDq1au1a9cu\nzZ07V8cdd5ymTJmS83caGhp08ODBXo+9+eabGjRokJ5++mmlb0hRUlLi6FgIhwAAAIBhXkYFM3/X\nr6DYJyA2qO8IIgHRdx0dHXrssce0cuXKntG+mTNnas2aNTnD4VNPPaWPP/64z+Otra0aMWKEqqur\nXR8P00oBAABQnEIYNTQ9XdTP6aesPwxfS0uLDh8+rNGjR/c8NnbsWDU3N1s+f8+ePfrmN7+pRYsW\nKft29elw6AXhEAAAAMUnpGDol1ADInyze/duVVVVqbT0kwmdgwcPVmdnp/bs2dPn+UuWLNEXvvAF\nnXTSSX1+1traqvfee09XXXWVzj33XNXV1Wn37t2OjodwCAAAgEQxHQyDurhMYBexafB/Fziio6ND\nZWVlvR5Lf9/V1dXr8d/97nd67bXX9PWvf91yW2+99ZY++ugjLViwQPfdd5/ef/993XjjjX1GGPNh\nzSEAAACKS4A3u7cV1pzs38Zzn/nN3xoNuLYuUANflJeX9wmB6e/79+/f81hnZ6caGhpUX1/fJ0ym\nrVu3TiUlJT0//9a3vqWJEydq69atvaat5kM4BAAAQPEIaDqp8VBo9Tt5fj+9f1P/HlsXqImZpR/M\nDW3fi2xeE+aYY47R3r171d3drX79jkzqbG9vV0VFhSorK3ue19zcrHfeeUezZs3qNRL41a9+VVOn\nTlVDQ4PKy8t7bbu6ulpVVVVqa2uzfdxMKwUAAEAiBBYMG2QmWNnYjskRP9YfBm/kyJEqLS3Vli1b\neh7btGmTRo0a1et5Z5xxhp577jk9+eSTeuqpp/TUU09JkhYvXqzZs2frwIEDGj9+fK/7Hra1tWnP\nnj064YQTbB8P4RAAAADFIc+oYSDBsEH+BKoC2yUgxldFRYWuuOIK1dfXa9u2bVq/fr1WrVqlGTNm\nSDoyitjZ2amysjINHz6815ckDR06VNXV1fr0pz+tsWPH6s4779S2bdv0+9//XnV1dTrvvPN08skn\n2z4ewiEAAACKT4P5TeYMYQ3+7M/JflgzGF/z5s3TqFGjNGPGDDU2Nurmm2/W5MmTJUkTJ07UM888\nY/l72Te4X7p0qU499VTdeOONmjFjhoYPH65ly5Y5OpaSlJPL1wSgpIQbbQKAF6kU96RKo6YACeLz\nqGHeYBiGHPv1Y4Q0da6RTQau/IN9oe27s7qy8JMiiHAIAEWGcPgJagqQEHEIhi/Y/DxyctP5HPs3\nHRAJh84RDg2hkAOAN4TDT1BTgITIDFQNvX/kNShZBsOGvg9ZshsIc7EbFBv6PmQyIBIOnYtrOGTN\nIQAAAOIrT4AKLRi+8LL3YOhkOw19H2INItxg5BAAigwjh5+gpgBFzsfppK6CoYlAmE+hkcSGvg+Z\nGEFcp3gGTUYOnWPkEAAAAMjgOBiaGikspNB+Gvo+xAginCAcAgAAIH58GjV0FQyDRkCETwiHAAAA\niLeGkLYdRjC0s++Gvg8REGGH0XD4wQcfaPbs2TrrrLM0ceJENTU1qbu72+QuAAAJQl0BYMmni9D0\nCVANOZ4Y1DTSQvIdR0PfhwiIKMRoOJwzZ44++ugj/fSnP9X999+vp59+Wg899JDJXQAAEoS6AqCg\nhk/+N7BgGDUERBhSampDXV1dGjJkiGbNmqXhw4dLki6++GJt3rzZ1C4AAAlCXQFgKc89Dd2KdTBM\ne+Fl6xHVBvk77RZFxdjIYVlZme6+++6eAv7GG29ow4YNGj+eS6oDAJyjrgDow4fppLZH0qIcDNNs\njiAyeohcfLkgzbRp03T55ZersrJS1157rR+7AAAkCHUFQB8Nn/yviXv5WW23RxyCYRoBER6UpFKp\nlN0nd3Z2qq2tzfJnNTU16t+/vyTp9ddf1759+7Ro0SIdf/zxeuCBB+wfEDcsBgBPUqn4jKz5XVeo\nKUAR8eHWFbamk8YpGGbKNcra0PtbO3+7dYpnkCz/YF9o++6srgxt3144CoevvPKKpk+frpKSkj4/\nW758uS688MJej23fvl1XXnmlNmzYoGHDhtk7IAo5AHgSp3Dod12hpgBFJMdaQ4JhHrnWIGYp9Dck\nHDoX13Do6II0Z599tlpaWix/duDAAa1bt06XXnppz2MnnXSSJGnPnj22wyEAIDmoKwBsybPW0I1E\nBEPJ+iI1DeICNcjJ2JrDgwcPqq6uTlu3bu15bPv27SotLdWIESNM7QYAkBDUFQCWGj75XyNrDRss\nHiuGYJhm9W9p6P0t6w+RZiwcDhkyRFOmTNGiRYu0c+dObdq0SQsXLtS0adM0YMAAU7sBACQEdQWA\nJP+nkyYBARE2OVpzWMiBAwd01113acOGDZKkqVOn6pZbblFpqf3Zq6wPAQBv4rTmsBCvdYWaAsSc\n4YvQJGY6aS421iBa/V1Zc+hcXNccGg2HJlDIAcCbYgqHXlFTgJjzc9SwweIJxRwM01wERMKhc3EN\nh77c5xAAAADwxO+L0GRLQjAECiAcAgAAINoaPvlfX6aTJikYsv4QeRAOAQAAEC05Rg19uzpp0hAQ\nkQPhEAAAANHV4O3XWWeYAwERFgiHAAAAiA6DF6FhnWEBNv79BMRkIRwCAACg+DVkfZ/0YJiW/Xdo\nCOUoEBGEQwAAAESDX6OGDTmfBisNYR8AwkI4BAAAQFFhOqlDNtYfIhkIhwAAAAif4RveW21LEsEw\nFwIiJJWGfQAADMp3w2CKIQAgAfJOJ6UW5vfCy337Eg2SNoRxMAgD4RAoBvlCYfZzKIwAgKgxNGrI\nlTUBb5hWCsTZ+ePtBcPs3wEAIOKYThoS/k6JRjgE4spLyHMTKgEA8EOOUUOnmE5qEH+vxCIcAnFk\nKtgREAEAEeR51BDeERATiXAIxI3pQEdABACEhVHDaONvlzhckAaIk0JBrsHlz84fTwEAAESC0YvQ\nUNu8e+FlSZxITgrCIRAX+YJhg43fb8j6r9X2KaIAgKAYGjXsxdR2gIRiWikQB7mCYYOcF0KnzwcA\nICCeRg0bsn7ICU/AMcIhEHX5gqFbDTl+n/WHAIAg+DFqmIlgCLhCOATiqMHH7RAQAQAhMDpqCMAV\n1hwCUWYV1BpyPz1XYc27WL/BYpusPwQA+MXAqCEXoQH8QTgEospBMCx0tjXz55YFtSH3tgEA8Jun\n+xo2GDsMIPGYVgpEkc1geMm5jzsuqDmfn719ppcCAEwzPWqYvQ1GDQFPCIdAHDT0fcjLWVbbv0tA\nBAD4zNOoYSaCIeCZ0XC4f/9+LViwQJ/97Gc1YcIEzZs3T/v37ze5C6D4ZQeyhr5PMVFILUcdLfYF\nhIm6AhQRv0cNAXhmNBzefvvt+sMf/qAVK1boBz/4gVpbW3XbbbeZ3AVQ3GyM1Bk7w5prew1ZT2D0\nECGirgDFjVFDIFqMhcOOjg79+v+3d/dBVlR3+sCfISMzaOGOvPqyWqloLQOLAgJBIr7gC0ZdA/Wr\nTXaxBBK1cFMbsdYshYCbaZ0oKKzGBINZl7AriGFDEC01QSGu7hpKBeUlIrubcdcUuzLLJGIwDjMu\n3N8f0Hf69u2Xc7rP6T6n+/lUUczcube75053f+/T56Vfegnf+ta3MGrUKIwcORKLFi3Cli1b0Nvb\nq2o1ROXi1H6rOhiGLte3XgZEygPrClGBsNWQyArKwmG/fv3w2GOPobW1tfpYpVLB0aNH8cknn6ha\nDVFxxXQn1RUMiUzFukJUbInrmuP7nq2GRMooC4dNTU2YMmUKTjrppOpjTzzxBEaMGIGWlhZVqyEq\nJgOCIVsPyTSsK0QF5CR7Weh9DRkMiZSSus9hT08POjs7A382dOhQDBgwoPr92rVrsXnzZqxatSrd\nFhKVXJYthtdeurG+247jecLUSSzEpBTrClEJBFxclKlt7E5KlB2pcLhr1y7Mnj0bDQ0NdT9bsWIF\nrrzySgDAk08+ifvuuw+LFy/G5MmT1WwpUVEJzE6apbqASKQR6wpRiTiKl8eLlUTKNVQqlYrKBa5a\ntQrLli3DXXfdha9+9avyG9TAA51KxIDupGF4k2F7VSrF6gKcpq6wphDlLGAiGmWthqxFmbG1rjT9\n9ne5rbtn0Km5rTsNpbeyePrpp7F8+XIsXrw4UTAkoj5GTUDj+L7n+EPKCOsKUUE4ipfHYEikhbJw\n+NFHH6G9vR0zZszAtddei66uruq/Y8eOqVoNUXFEtBqaEAxN2AYqN9YVIsvpGmvIYEikjdSYwyiv\nvfYauru7sWnTJmzatAnA8SnHGxoasHXrVpx55pmqVkVkP8PGGYapGX/ogJPTUKZYV4iIiLKlfMxh\nWhwfQqVgeKuhV93kNI7vCQyIxrF1bIgOrClEOdE11pA1Jxe21hWOOZSndMwhEQmwKBgCZm4TERGV\nhJP3BhCVC8MhUZYs6U7qVxMQHd8POTkNERF5BbQayuAN74nyo2zMIRGlY1ULnQOOPyQiMyS5QMXz\nVeYS1ThH+WYQUQyGQ8pXWFEvYuG2rDupX83kNEGKGBDLtH8S2SRtjwXv63k86+PIv4SthkT5Yjik\nbIkW9KDn2VwYMupOGlZUVYXPyNlLi6Cs+yeRLXR0Y3eXyWNYjZS3r6hy0m8KEcljOCT9VBVzW6/0\nxvz+KoJbZIue5+fKWygd2N+9VPX+advvT2SDLMY28xhWy5F/SSlaDaP25SL9nmQt3sqC9MmimNtw\nItXcnTQuGAZJu97QKcZdNv5dVMvxPbB1ynEdWFMKII9Jr2w4h5lI1e0rHM+Dtv8t0uy/Bv3uttYV\n3spCHsMhqcdC3iemO6nSkCaptAEx6/0zh/fB1iKuA2uKxUSPVSfBskVfY+p5zFTu38zpe0i01hTu\nvoaqa03O74OtdYXhUB67lZJaqou56PNM7A6kuTtpmmDovl5ZN1MH5o8PkSnUjsLnmbhvEpku7nh1\nUi7fCfk6aDt47IrRMdbQxvde1wVI1hLKCFsOSQ2Rk6GjYD0iyzDhxBn0fjh9X2oNho7gYwq2pW47\n/Osx9W/h5yhaV9xyMno/bL3CqwNrioWijllH43qjlm3Cucx0KbqUFqLVsOC9UmytK2w5lMdwSOmZ\nWMiB/IpKTDAEFAeykHXIPEfZ9gQt36S/g5+jad1xy9X8nthaxHVgTbGIgloSdS4T7m0RtS6bwkrW\nfF1KSzPWMI+hNK4M3yNb6wrDoTyGQ0on7KToxL80rnAoKeRAtgUmj2Do1D8UKeT5SbcrtvUQyL7I\nK+qSlvqDZtR6NL4nthZxHVhTLJEwGGq50BazTmtCS5bK2mqYZzB0sUdKJIZDeQyHlEyCQq59jF3I\negFkc/IUCMq5B8OI12rtXgrk+zdwOeE/yryrL6DtPbG1iOvAmmIByYuMym/JgwQ9MmwILllSMRGN\n57VWvL9JgqGj6DlB2CMlEMOhPIZDkmdjIXfpOnkKvifKWucClp2IbxlWB8QcWh7CSP29NLwnthZx\nHVhTDCdRT+KO1b/AD2JX9xhui/x53seulcrWaigbCp2U65N5PXuk1GE4lMdwSHIUFXKRIg4kLOQh\n21Ol+uRpazAMWZbW7qUulX+DBK2FKj5kAgo/aCreJ20t4jqwphhMQT0RPVaDRB2/wuczk0NMVsrU\naqjz9iqqlskeKTUYDuUxHJI4gfF0gJ4i7gor5okCVNoTqERLlfYWOa+g30syQFkREBWHQl37Zx4B\n0dYirgNriqFS1BMVx6qfKceudcrUapjnJGey62CPlCqGQ3kMhyQm4UQrOoo4YEBIlOzCaEzY0tzK\nCSgKs1ESjivMe//MstutrUVcB9YUAyUMhlHH65d2vSi8+mfHTAv9mf/YZUCMoSIcOp4HTX0fTQmG\nMutjjxQADIdJMBxSvATBMKyIyxRwr9TFHEh/tS1hcdAeDFWFK9/yMw2IQPzvofD9z2r/jN03nYAX\nKSjothZxHVhTDKPwQmPS49XLtGPXKmVpNVQ0+7UWcetmjxSGwwQYDimaomCoooi7gop5qi5BqgQs\nW+v9A4F0J36Nt90w9b3XvW8CYvun7ntD2lrEdWBNMYiB9cQlfdwCDIhlaDVUHAxF3h/h23iJbkfJ\ne6QwHMpjOKRwkoXcX8RjC/gDMetfEP3jRMXc5cSsW1bA8qyY+bMIATFgmUo+YCreP6U/aKb4+9pa\nxHVgTTGI/3zj1H4bd9yGHrNxx2qQkOM3rq4wIHokvOm9Na2GCWfA9tN+Gy+XE/GzEvdIYTiUx3BI\n4SIKeaIinqSAeyUs5oDmoBKwHCuCoSsmIGq5ybQT/LCUgGWItBYq/YDpJbh/Sn3QTPh3trWI68Ca\nYgiJYJhJPfEKOHZTHbeAeUFHtaK3GqYMhpnewsvLifhZSXukMBzKYzikYILBMPMi7pIs5oCGoBLy\nOqVXCYPWoaOISl7Rl5HX+57bvgnU7Z9ZB0Rbi7gOrCkGUBkMdR2zgNrjFjAv8KiU8PYVVrQapgiG\nOkKhX2xIdCJ+VsIeKQyH8hgOqZ6qYBhTxF97KvrnF8+M/jmA/EJiyHNF7p8nfG+toHXoKqAau5cC\nMcXMCf+R6POkg6HAB8y4/RMQ2EcjPmzqDIi2FnEdWFNyljAYqq4nXqYet9Yocquh4cHQK3FdLVmP\nFFvCYW9vLxzHwUsvvYTm5mbcfPPN+NrXvhb43GeffRaPPvooDhw4gFGjRmHhwoW44IILqj9/7rnn\n8Mgjj6CrqwsXX3wx2tvbcdpppwlvi7ZweMstt+CGG27AjBkzpF7HQp6zBMFQtIjLFO8gkQVd4oov\nkPDKW9BjHsrHFfjXp7uAauxeCmTznqcJhVntn0IfNJ2AZUj8/W0t4nGS1BXWlBxpDoZpj1kg4rhl\nQAxX1FbDhMEw61DolWVAtLWu2BIO29vbsWPHDixduhT79+/HggULsGTJEkybVvt5dvv27bj55ptx\n//33Y+zYsXjyySexceNG/PM//zMGDBiA3bt3Y/bs2bj33nvR2tqK9vZ2nHLKKXjssceEt0V5OKxU\nKvj2t7+NdevWYcmSJQyHNkkbDDUWcL8kBR1IEBIFiRSHVK2GWRXPFN1L434/QO/7nSQY6tg3gZD9\nM2TfFA6IJQ6HaeoKa0pOJC422VJTGBBRzlZDJ/wleQZDr0Q9okrSI8WGcNjd3Y2LLroIq1atwoQJ\nEwAAK1euxLZt2/DEE0/UPPdnP/sZ3n//fdx22/Fz0Mcff4wJEybgxz/+Mc4//3wsWLAA/fr1w5Il\nSwAABw4cwNSpU7FlyxacddZZQtvTT/QXFNHZ2Yk5c+bg5Zdfxqmn2tnPtrQ0BMPXntL34Tt02b7t\n8AcEf4C49tKNqU/uoq9P1Z00K/5i4Yi/NC4YAmoKaVBroX+/rNs3ffuFzn0zdPkh+6Z3+2t+N8f3\nepH7PBYQ60pBOH1f2lpTvOcW73bXndcc3zJKeuxaRTIYqvjsoFLotjgRL+J+aYx9+/bh6NGjGDt2\nbPWx8ePHY/fu3XXP/eIXv1gNhj09PfiHf/gHDBkyBOeddx4AYOfOnZg4cWL1+aeffjrOOOMM7Nq1\nS3h7lIbDvXv34swzz8TGjRtxyimnqFw06aQ4GOr+4B27Ll8Y8IeFoNsaJD3Ji7YYht10HRBoTcv6\nympEQIzb1qjf05W0qAa9zqTWwiB1+2fAvuliQAzGumKhItYUj9gLO0Axj92ithomCIYmShQQyQgH\nDx5ES0sLGhsbq48NHjwYPT09+PDDDwNfs23bNowbNw7f//73sWjRIgwYMKC6rGHDhtU8d8iQIThw\n4IDw9igNh1OnTsXSpUvR0tKicrGUJafvy9gi7vugm2UB95Mp6EBwYHPDh8iJX/R57jpEWtUAWNEd\nKSogPobbhAIiIP4ehj0vMhjm0FoYJWrfZECMxrpiGclgGNjS72FMTZG9sBOkZMeuFQoSDF3SAZH7\npBG6u7vRv3//msfc73t7ewNfM2LECGzcuBHz5s3DggULqq2MR44cCVxW2HKCNMY/pU9PTw86OzsD\nfzZ06NBqaiWLhJwYwop4lSEF3O+1p3xjR9ztPDFu5Eu7XqwZNxI2Tk5FARAJhkaMMwzy8uv1V4kd\n8ZeLjD90yb7Xsq2FRu+bAful+95de+nG2qvtjue1UycZeeEgCdaVAkkQDKssP25dkcctYO+xq6LV\n0MuE96BgwdBVsw96OQj+3WzdJwukqampLry534fVwEGDBmHQoEFobW3Fzp078dRTT+GCCy4Ii9Hp\nvgAAIABJREFUXVZzc7Pw9kiFw127dmH27NloaGio+9mKFStw5ZVXyiyO8hYz+QhgVxF3udsTVdAB\n1HwYByRa9wSItJ6pmpxFm4iA+NNX/19ooXRbD2UCoihbg6Grbt9M+0GzAFhXCqKAwdAlEhC957sy\nHLeJOHlvgEdBg6HL3d7AyZKcgBcUOCD2fiPHserrxJ42fPhwHDp0CMeOHUO/fsc7dXZ1daG5ublu\nrP2ePXvwmc98BqNGjao+du6556KjowMAMGzYMHR1ddW8pqurq66raRSpbqWf//znsW/fPrz77rt1\n/1jALSNQyOOKeJ7d9ETIdDMF4scGivIuo0jdSf1Ewq2K99NdTpJupKaq6652QuxkF47ndQXpDsS6\nUmxR98YFYM1xK9I1XOi4Bew7dgNaDUUZ22oYxAl+2LZg6BW47U7Ik23bLwtk5MiRaGxsxM6dO6uP\nbd++HaNHj6577oYNG/C3f/u3NY+988471Qlpxo4dix07dlR/9sEHH+DAgQMYM2aM8PYoHXNIlpAI\nhjV8wdAGsgERSBdoRIOhsd1J/RLOXur93dOE7rBQaMrkFWkkDoheLOZkAoleKIBv3LqH6cdt4DjE\nEwofEH1ExoHXcPRti7SY26x42RwMXQyI5mtubsb06dPR1taGPXv2YMuWLVi9ejXmzJkD4HjLX09P\nDwDgz/7sz/D6669jzZo1eP/99/Hd734Xe/bswaxZswAAM2fOxDPPPIMNGzZg3759WLBgAaZOnSp8\nGwuA4ZCcvi+DgmFQETe9gPvJzmYKyAca//OFg6ENEs5e6n8P0ryfLtu6kcaR+aBZ5fi+ZzGnPEle\nbLQ1GHqVKiBKtBoK3Uc474ufEu95EYJhJCfvDSCvhQsXYvTo0ZgzZw7a29txxx134KqrrgIATJky\nBT/96U8BAKNGjcKjjz6KH//4x5g+fTr+5V/+BT/84Q8xfPhwAMdbDu+99148+uijuPHGG9HS0oL7\n779falsaKpVKRe2vd9yVV16J22+/XepmxQBvWKxdyIm+qMHQr+5GxyE3Jg8jGngK0WroFdMyEFVE\nRd+juPBYtGDoVbNfBtx0232vRPcdW29WHCdJXWFN0UhBMCzDcQtYet4HinX7ColxhkUMhqEXc52A\nxwL+TrbWlYYb81t3RXDMoWm0tRxu3bpVOhiSZgIzk7qKVsRdSbqZerktWlEti4ULhkBs99IkraEi\n7yUQ3LJbpGAIiLdERLZClADrikEEJ6BxFbGmhB23LuHeEqa2IAq2GqoaW54Lp/6hIgZDQPL3MnWf\npEywW2lZJJmZtEBF3CttQIxSqO6kfgrGH8oKDIUFC4au1AGRxZwMEDkzaQFrStBxK33vUosEBYyg\nc7wVrYZO/UNFDYYujj8kEQyHZeT0fRk6M2kGRfzimfH/dBEJiGlCol/gdNJeJrcaCpIZfygirrUQ\n0PsBM4/9U7QlghPUUK6S3gYpYzpriJeSgGjasVukVkMGwxpSAZFKSduYw6Q4PkQDwXGGuoNh2kKt\nIwjEjUEE4schugrZnTRIwvGHMh8i8giGJu2fQWOZIscxObWvr/xc3bbYruEK2HeMmUzRbZBUkz1+\ntW+DyHELmFkPQv7GYTOURk5Ec+K1uf1eguMMyxIMvQIv6DoBTzzxt+OYQ3kcc0hmihlnGHV1V1Xx\nVNXKoqPFJq4FERC78l3o7qR+CccfirYeirzfqoOYyv1ThaiWCKEZTKmWaa0ytjLsNkhpaoKO1n/R\n49aqFkQn/EdBwVD0tZmQvNF92Qi3IJq2T5J25oVD7oTqSFz9qzpR0FQU8Njiu0DgX9JlpxEQEJMq\nQ3dSGXEBMTAYahpjqGv/VLVvinzQLMo4pkywtqjl9H2Z9QQ0Ks//2i84wrKAKDFxXRAjb1/h5dQ/\nVMZWQ5fw787zZ6mYFw5JD6f+obDupGkLeGixFQx+oa+RWZcEkd83qjUrLPAUOhhqmL00kMKg7spq\n/1S+b4oERIrGDzjJCYQG3UMUIo8pkYs5GVzUATy/q+33QHTCfxQ7TCDitZmwZJyhkT2LnLw3gPJm\n3pjDK058YfMHZxPkNM4w9EO3ahrGoCW9B2LkTG1eju/7IuzjCcYfhn2o0N1qGPmhUiWd+6ZvHBMQ\ncA9Eh2MOvao1xVWE4y5rScYZKgyGWo/dkAtQSrc56b1LgWz316KMNTQ0GMoGwSy3T+Qzi611hWMO\n5ZkbDgEW8aRiTvCZBcOQwi06wYsrcgyawg/iWsOh43tCUfZtgSIsGhB13ctQ9oOlsv1TcUgMC4hB\nHzIrlyZfT9E0vIriHn9ZSDkBjZZgKBgKnx0zTXy2VN0XHEUu7AD5BUTJ+1ZGhkPPazM/1gybgEZF\ny2Cu2+r0fclwKI/hUBFe5U0ph2CocsbPODo/iIt8CPH/HqUOhi5FrYc6wqHoB0ut+6eO1s+YgMhw\n2Kfh1RNfOL4fFO041EWwFwqgZ4hCDUUXdADxWlKKgKggGAIGh0On9tsswpaO7qK5bbdz/D+GQ3kM\nh4rUXeVlARcn0CVE9TjDpK1tQPzkJHFjGlTf7kBFOIwNhkAx92kFATFs2nuX6r8lEP+hMmofjdo/\nM2kFjQiIDId9quEQYECUlaKmAIpb+zVd1BG5oAMo7FWT9BYXgL79VSIcun9va1oNnfqHdIasLMYQ\n6g6JYZ9jGA7lMRwqEniVlwVcTMJxhjqCoUwXTFFCrU1A4sKeSTgs6r6c4GqtrnCY5O/o0r5/qv7g\nHPJB8wUYOMlBTmrCIcDaIiqncYYqeqIEHcfSFxsBfWOeTQqIRWk1LEkw9NL1u4S1HjIcymM4VIRd\ngBLKMxgm6HqZRpIuia6o31c2VLDV0Eey9VB1OEwaCjPfP1V2l2U4jHUdju93gR9egWIfk0kZGgxV\nXtQRnhhLY7dwIOeAGBOopFsNva/POxgCmXUnTRwKnZCvJWT5O9naI4XhUJ5x4fA6bGQRl5XxOMO4\nWdhcKm4MH3XiUx0StYfDou+/kq2HqsJh0i5oWe+fWQZEhsM+deEQKN+xKSOnCWhke6KovKiTxQUd\nl7KACKTbbyWDIWBwl9IcxxkKB0MnwcIFX5PV78ZwKI/hUJGacAiwiMcRHBNiQjBM2+Ui7AQY+SEc\nSHevPMFwWPpg6JJoPZQNh8IStmRnsX9mFRC/hM3yCysoNxwCDIhC8h67nrInStRxHPchWqaW6OgW\nDmQQEAVa2RIFQ+8yDAuGgJ4AJVQzArYlEYHl6P4dGQ7l2RoO++W9AUEibwxLfWI+iAPhXWd0BcPH\ncFtg8VDRF99djn9Z/nU+O2Za7QcKmZuaUzr+DwVOxutPOLus7v3Tuz3VbfLtl6luwH3iQ6vwtP0l\n8hf4QfU8aPxNx/MmUVOCLjgmoSIYhh17ss8LqiU1FByvNbU34LgNHRvphCxw6iTxfVew+2UiKpYh\nK6dgKFQznOBtSUxgebbOkErmMa7l8FlcE3hT5xq8wntchuMMRYOhV+qralE/O0FqHBsg/0EmScuh\nE7CcMu2zghMcKG05VHHLkSBOwp8hfobWsN838S1YPK0QbDns8yyuARByvDq+J5fpOPXLYZxhnj1R\nvNLMpqy7BVFoLLufdz8WCY6eZVrTapjTOEOhUKhbzDp0/c5sOZRna8uhkeEQCDgxOr4nlrmIA4nH\nGWYRDCNPno78+kVeqy0kqgiHZdtXE3YtTRwOJcaFAtnvn1H7pq6AiDFGndbztash9H6QAHi8ArmM\nM0waDHXOCJl00iwjA6Io37KsGGtoYjB0wn+kTcQ6dfzuDIfyGA4V4VVeARlOQCMzPgKIvoGqMiHL\nkw6IQHwIYThMJkHroXQ4THubkYBtUyJkeTK/d+rxhwyHfXY1AAi+H2SV43tNmY5ZyWPV2J4ojsQG\nxDw3iws6rsQBEUh/7vK9Pq4F1ehw6NQ/pDIgGRcMBdatOiDaOtEZw6E848YcuifewH73ju/7Mo4R\nERgTEibzYOhAz0kzZLn+cQCx40eA6LGIHKeoVeR9x+L+LiqCoYPM909X6H554vdKMp6penynHANW\nVP7awvGHiPw9rQiGDpIdxzGvixs3XKV6DOIJQRcyr710Y/2HfSfZOoN+dwbDcFqC4cuv1/9Lwgnf\nhqzvu0jFYVw4BCSLeJmkGBOiMxgGDs52oJ8TvB7hCUFcC0L+Ub4E/y5REyKF7puO6o0NELAe3QGR\nAkSEZQZEH6f+IZUT0CgLhg7UzgIZsKygi40u1QGxyvPehl0oDwyIjsQ6Ap5rzaQjNgdDkSCYJiyG\nbAsDIiVhXLdS3NgQemPn0ncvzWgCmsRTbfu2K5LM30z0Q1rAuoUnQBHEbqUCVExIIyBxa6EI1fun\nb72xXUzTdgVfZ9ZpPVc3Hu9WGlZXgIjaAhT3+M1zApqkY9ed+oeUC1iHbFd4HS2qqbvJhzwucgse\nIyaiyWGcoZKhCKrelxSfgwA17wu7lcpjt9IMVHdux/eDMlzhDfkdRYp4YqqDYdIrYqKvC1h/XDfT\nwK6mZLyoYJi4tVDn/ulbv3cbA1skUrRGpLlBd6H5bhsgNHQBKGZ9kQiGQQobDEPWk0ULYtAtLoDa\n/dX7Nwn9sO+E/JMQ2eXfv64s2BgM03QVTbM8J/hhtiCSDDPDYZIiXsQC7hK4KXGY1LMeeiQOhipP\nkpIfwoHobqZAyFjENHzrL/S+mTF/oA/qRlrHiVhg2vEeYcsL49sWXQGR+kTdV0546EKRjmHBln2X\n8nGGJ0hddHQQL2gMV9i/OAHrNCEgAqgLiEkDUdhr/Z+3spwtNpDOezPKElmn6lCYZPlO8MMMiCRK\neTjs7e3FokWLMHHiRFxyySVYvXq11Ov9kyqUfhKBFDclVhEMI7tgudsTsE0A9J4kE3wIDyvugFgr\nov/nQldxy0bjMSjdjdRBfDDUJWr/dGq/ZUCMp6yuBChVQEw5M2kSWocpZNwbxcSACMjXHyX1yvF8\nretcKhEMtbcaBqyzhu5QGLS+KE7wwwyIJEJ5OHzggQewd+9erFmzBm1tbVixYgVefFF+bNfxhR3/\nr7QBMa8JaE4QCoZBsjxJSnwIB6K7mQIau5oWab+U4fR9mbR4B/1NhIJhmKz3zyBO7beiAbGsVNSV\nsAuPXoUOiILbntetkIzujeIhGxCTSBoQ486xcc8xrtUwiFP/kBHBMA8MiKSJ0glpuru7cdFFF2HV\nqlWYMGECAGDlypXYtm0bnnjiCaFlvNZwfPKAqKJSivtUZThZgMh7DUgEw7xEffhxar+NuyeiKKkP\nN7bvk3FiZpKLmoxGRuJgmPf7LzDTXtB4ryQXfi42bJ6xNNLWFbemAArPdXnvS7IEj02VN7rXEgzz\nvME6IHQ+U3XfUiDZ7K5JBZ2Tc719hSnjDJ36h2qYci6Q+PwDJHvfOCGNPE5IA2Dfvn04evQoxo4d\nW31s/Pjx2L17t/SyRE6mhW1BTBIMPXIJhll3qQiSoCufS0WBjT3Z2rxPxhGcYjyN2NtUOBHrzHvf\nDNsGp/bboBbEqpLe4kJbXYkY216oFkTJYFgjZbgBIDd+3UH+x7BgK6JQC+IJSo7ZiBbENBfbRF+f\naWtTDuMME49XN4VE6zfA1kOKpjQcHjx4EC0tLWhsbKw+NnjwYPT09ODDDz+UXl7c+EMg5gO5TQXc\nJTlZQFXaIl6kq+gZBsTIouoEPGbjPpmEU/tt2qu7qVoLTdo/EwREbR82LaGtrnhETn4G2HssS7Tm\nu5SOM5Qdvx4kj2NY8EJjbEBUOf4QCA2IQLKQGPV8VS2S0nIYZ5goGJqIAZEUURoOu7u70b9//5rH\n3O97e3sTLVMkIFY5AQuwoYC7Uk4WkHYCmkIEQ1dYcXdqvw0KiGmKYuANiv1s2idFaP59rO1GGiZo\n33Rqvw0NiCWcoCaLuuIVOLYdsO9YTtDNW/k4wxOsHKYQtX6n70uTAiIgFhJTtTY6nq9V/30YDNNj\nQCQFlIbDpqamumLtfj9gwID0KyjyNOQpJwtQecsKl7XB0EswIKrsZipUtGzYJ0VIjKNzyXwoKVww\n9IoJiK6ogFgGWdcVwPKAOHWSGcHQ5vHrXhYExKiQGPQvTi4T0eQ0M6kUU/bJOAyIlJLScDh8+HAc\nOnQIx44dqz7W1dWF5uZmnHrqqYmXK9INyOqAmHSygITdfkSu7BYiGLrCAqJT+1DSgBhbbJ2Qx03e\nJ0UkCIaiglpwhYKhad1I40QExKAPnP4Pm2VoPcykrojOYAqYXV8EPmBHBkMPBkMPgZ4oUbdKAqA1\nIALBf0PlHM/XKv9GNtyywrR9Mg4DIqWgNByOHDkSjY2N2LlzZ/Wx7du3Y/To0amXLd0NyAlYiCkF\n3CVxhVdlEa9KMgssYN9JEsh8ohqhD5SAefukKI0T0AS958LB0EaCAdFVtoCYSV0B5Ma2OwELCzuf\nZyVhMKyhcAKaoLGyVkxsFiVBQNR2iwsgMCCqDonaQ0NON7kvdDB0SQZEIpfScNjc3Izp06ejra0N\ne/bswZYtW7B69WrMmTNHyfKj7lMlFRBN+EAuUchVFvG4cYaFDIauFAExSUgsbEAU3N6gK7xxrayl\nC4YugYAYtQ8WOSBmVlc8EgdEIPvjOaqmOX1fxt56QfEENK7C1RZFATHpMRsXEIF0ITGziWgSfBbL\nZZyhLftlGIntZ+shuZSGQwBYuHAhRo8ejTlz5qC9vR133HEHrrrqKmXLV3IjY8CaK7xp73fmJTIB\nTdC2VNl+kgQSB0QgumgKj6NzQh63JSBqvPdUaYOhSyIglm38YdZ1BVAQELM4plXdk0/zBDSFCYau\ntAHxBKUBMSIkJg2KQl2Ak4g7NgLWxWCYguDnHoABkY5THg6bm5uxZMkSvPXWW3jllVcwa9Ys1avo\nk+Y+VYA5V3gdaJ9eXKQIRZ4UinKSBLQFxCCBBc2pfwiA+QExZTAMC9BhLbOlCoauFAGx6K2HWdcV\nIGVABPQd03Hh0+n7MrNgKHLRMYitx7BEQHSpnHU48G8V8flAaZfTNH+zHIOh6PoA2LtfhmFAJAnK\nw2EWEt/I2AlYWN5XeJ3ab3XMIidSwCOvEBbtJAloCYhh4acQAVFTi2HYe1nKYOgSmMW0jAFRt6C6\n4pUqIKqqMyKh0LMtscHQQ+U4Q+EJaGw/hgUDoo4ZTAH5gAhET1wTGugd4U2KlnMwFA49tu+XYSQC\nIpWbleEQ0DRORMcHc8HxIIC+6cWrGAxrsQUxnsT+GyUoODMYRggJiCIfbhgQk4ubwTS2tjgxK0gS\nFEVf41u3UDBMMQFN2DAFr1LUlpjfw7aA6FLaepRg/wVympm0KPtlmBSfe6g8rA2HgIZxIkC2V3g9\ntN93isEwWIrfMag7ZNTYQ+sCosT+C8gVcgZDASG/Z9nHH+qWKiAC4hdNvKEv6l8cp36dMsEwidQT\n0BTtGJa8Z2kNnQExxd9YCZl92Ce3W1aUgeDxx4BYXsaFQ9mTY6qA6EQsWNcV3oD1ar/vVJIJaIpW\nvKPEdA0Cok+SMq2I0gExr5CoMBj6AzODYUJO35cMiOJS31NOICAmakVUIWAdssEwlwloiijpPUsB\nJQExU3F1KcWFDSCDm9wHrBNAueqLwOcegAGxrIwLh6nIBkRArHCpvMLr4f9QERgM085MGqF60Pu2\nq1QnSJfCgCg8c2nEumpkGRAlW7wBBkOtOIOpEroDIpCyFVGWE7xs3cGwRplnvQ6TY0AM/VsKth4K\njxONInNBU2a5KQjVFqDY+6UMp/4hBsTyMTIcJm49BIQCYuZXeQOW7y/iKoNhDZHupF5lPkFmFBBD\nw5QT/DAA/a2IGYwJYTBMKMkENVQn14Do/ksrZDlhFxoBtcFQpDdKaWa9FuX0fSl6iwsgn4AoJckF\ncpcT/iNOQJMhzmBKIcwLhwln3RMJiEKtiI7ceiOFLC/r8SDs8iMhoy6miQIioD4kahwT4t23U91Y\nuayF20tighoGRB8NLan+gBh7AdLlQP6c6yDydf51ZRUMg5RqnGEYRbe4ACzpYirLCf8RJ6DJAWcw\npQANlUqlkvdG1NjVoPy+S0DtByb/B9XQD/uO3LrjXpdFEecENIqE3Y/SQ6Q7ZVwoSr3vJf2bpez6\noyoYstVQkvfv5vR96f49qj0QsDnDjTLcrobj/6uYmRMIrSuARG1RIOgY1B4MAdYWUTE1xH/MAuH3\nM0586yovX/CM2ne1XECOWY7qcYasLZIEPvNULs1kS5RruDG/dVfW5bfuNMxrOQRS9b2Pa0EEBLsC\nAbVXbJ2IlQo8L9NgKMLxfc+TZK2U74dbaOPGHyZuQXSJdutJ0v0nZBtUdCUFWLwT8b4fTt+XQdPl\nk0+Ke0HW1RVPF9O42qL6Q2/QMv2tlwyGBhD8veNucQEUoAXRCf+RjmNEeJwh9RGcoIbKwbiWw2dx\nTd3VM11XeoM+SKm+0hsVCoFsx4NwApoE/EHKqf1WtAUtcQtiwDozEbJOkSIu8jszGKYQsU+6f58X\nwPEhrqCaAiioK4BUK6IraY0JO/Z01hRAcpyh43sxj2Gh4xWInpDOJfI3VN5yCCjvReXSMSspu5Om\nFNGCyJZDeba2HBoZDoH67hVZdgUC0oVEkSJed6uKrIIhUHui5AkyWsrupcDxv7toi47yLs6yItbD\nYGiQkA+cDIf1wmoKkF9A9Ao75mW6bgPqawoQPM6QrYYJKAyIQPTfM0k4jO0O7YSvL5DA8xkMDRXW\ns8hhOEyC4VCR67Ax9CbwKgMiIFfI48KizHT+aa8M+jEYamZKQAxYrzIxy1UVDAGBDx7cL8VEBESG\nwz6qawoQHRCBZCFRVFBXda3BEGB3UhUUB0Sg/m8b2f00TTgExGqPyHPAYGi8kIBY+XnG26EIw6E8\nq8IhoP9KL6CukMcWcZXBEGB3Up1MCogB605MYDmihVzk92MwVCxkghpbr/Dq4K0pQH4B0ZW0vsiG\nQoDB0CiCE9QAcgFRWNpwqIiuG9yzR4piAfsrw6E8hkNFroNv1r0MAiKgNiRmVcQZDDOWcvwhoDgg\nBmyDMIHXcfY4SwQERIbDPrpqiitpSHSFnQ/iJrPKMxgCrC2JpJnB1JU0JOYcDnWFQoA3utfGt78y\nHMpjOFSk4dWAadkzutILRBfyJEW87sQOqP9Qwu6k2VEQEGVJFWpH8vEQDIYWCdgnGQ77iNQUQG9A\nBNTcfzKwngDKgiEQXlcA1hYlBGuI0oAYsz/qnpwv02AIsL6ownCYGsOhIg2vHv9fV0AEsinkcaEQ\nYDC0koLupUno7ObjldtYEO6X6RSkiOvgDYdAhgERCL1hvGx9EQ2FgNq6ArA7qRaS4w+BlAExp3Co\nMxQCDIaZ8OyrttYVhkN59oVDILOACMgV8bwKOLv8ZCyngAgUdCwI90k1ClDEdQirKYDai46AXED0\nc8/joXXES3FNATjOMHMqAiIQHxIFP9uoDIi6Q6GLPVIycmJftbWuMBzKMy8cXoG6adlNuNKbWJ5X\ndh3fi3mSVCfHgAhY3OXHCXgS90t1LC/iOrjhENB/0dGlrbYIzlopi+MMc6IqIALB+4bERe+09+XM\nKhC6OM4wY1MnWVtXGA7lGR0OAf0BERC/L5CwHAs4u5NmKIfxh34mX9llMMwBw2GdPGoKIHdbgUgR\nLUPKt5PBMFsJZjB1CbUwh0gzAZ+7H2QdBoO2oYYT8ETum0pVKiH3QDQcw6E8M8MhYE5AdMUVcwMK\nOINhxnJuPfRLcy9O1RgMc2TxFV4d/DUli/GHXrG1JQEt25ZkAhqAx3FaOQREnbfw0o3BMD8Mh/IY\nDhWpFnJAqpibWshzD4YAT5I6GRYQTcAJaPJnaxHXIWlNAdTVFUBNbVG5PQAnoDGGRC+UtAExyYzs\npmAwzJetdYXhUF6/vDcgknP8v6ATQvUEd6KoqQp1rz3V9y/PZbhEu/xUOb7veZLUK+j9dWq/zWqm\nURMIF2+iHInUFEDtxcI0dUFVPfFiMDSI/710ar/1/h38df/ZMdOU3CbFdKwtRNkxr+Ww4fXAmzoD\n2U0mECTuQ4KO9Up3+XF8C2Dxzg5bEAGwO6kpbL3Cq0NNyyEgXVMAfXXF6+KZ2azHXVcVxxmaQ3Ic\ne9A9lqNaEuNCpMkth5yAJn+21hW2HMozMxwCRgbErKW6sssTZLYEwiFQ7IDIYGgOW4u4DnUXHAFj\nA2IWpIMhwNqSlQQXGYMCYhomBkR2JzWDrXWF4VCetm6lvb29uOGGG/Dmm2+mX5jT96V7knBPYEFX\nwnRMAJA1BkPLCHQvBYrbxZTBkLKQuK7EdNsDQmqKpi6meYnrjeKK7E5K+iQYpmBimFOJwZAoe1rC\nYW9vL+6880786le/Sr6QJMW8AIX84pkKxoJQPkoaEBkMKQup60rIPhg0nquIAZEzk1oiYUBUFRJV\nt0SmwXGGRPlQHg47Ojrwla98Bfv370+/sJCAGFnMLS7kYdvLCWgsUrKAWJTfg8ymtK64nL4vix4Q\ng4KhF4OhYRJOdFakgCgVDLlvEimlPBy+8cYbmDx5MtavXw8lwxkFrva6wgKiDcW8bhs5SYC9ShIQ\n2d2HsqKsrkT0SImsKYC1ATEsGPKio+FSBEQVIfEv8IPcQiKDIVG+lIfDmTNnYsGCBWhqalK96OOc\nvi/94w+B4IAImF3M64q36CQBXjxBms+pf8jWgMhgSFlSWlcEAmJgTQGsCoh1F0ZjgqGt56JCS3Gr\nJFtDIoMhUf7Mvs+hK2kxtyAgxhVvL44ztEhY0XLqH7LtQxmDIVmv4AExrCcKIDkzKcDjOG+CATEu\nJKYNim5I9IZF1aFRqhZyvyTSxo5wCKQLiIYWc5FgyLEgFitgQGQwpDKxMSAyGJaEU/9QXB3xBsWg\nfyK8z1U5U2rotjvKVkFEgrTe57C1tRVr1qzBxIkTxTeoIaYYSdyvCgi/ZxWQ332rRIo3wGBYGIL3\nQATMvQ+iVOHmPpk7W+9HJUK2rkTWFIF6AphfU1wMhgUWVEeAQtxPVzoYct/Mha11hffBxgXmAAAZ\nKklEQVQ5lGdPy6FL4hYXQPjVXiCfK74MhiUkOEENYGYLIq/oUmEJTlBjck1x1xs2dh1gMCwEyZ4o\nJtaSIAyGROaxLxxGSFrMsyjogethMCwPyYBoSmFn4abC0xAQswyJUXUFSBAMyVwSAREw82KjK7LO\nOSEvYn0hyoTWcNjQ0KBnwSqKeUYFPTQUygRDKoawgOgEPz3vfYCFm0ykpa5YVFMil++rK4lmJeVx\nbLYEATHvWuIXuT1OyOPcL4kyo3XMYRKxYw690o4XAerGjLjSjB2J/EAQEwqBgBOn41sGT5J2kxg7\nAmQ/foSF2362jg3RQbimxIwNzrOmuEJrS0hrIcDaUliSdcSV93hE1hd72VpXOOZQnt3hEEgUEAHx\ngu4VVtyFrg6zeJMrrKgDkYVdd1GPvbrshDzOfdI4thZxHVJdcAS01BSZoBhbX1hbysvQWhKE9cV+\nttYVhkN59odDIDIgAhJXfAGhkChlQf1DLN6UtKgD6gt74qINcJ80lK1FXIfUFxwBrRcdE2NtIcCo\nWhKGNaYYbK0rDIfyihEOgcQBEdAQEgOKNlB/Y3sW75JLUdRdaYq70DiUqO3gPmksW4u4DkouOAKh\nAREQuOgIqAuKMaEQkKwtAI9l26WsJbpCImtMsdhaVxgO5RUnHAKpAiIQUdRdYcU9JAy6ogo3wGBY\nalFFHZCeSTCsyCeakCBq3dwnjWZrEdchiwuOgOBFR5dsUBS84AgwGJaWgouNQPqgKFxrnJifc780\njq11heFQXrHCIZC6oAMCIVFQXOEGGAzpBEWFXYm49XGfNJ6tRVwHpRccAeEWRJeqeuKX+qIjwGO5\naDK62OgnffExbju4XxrJ1rpiSzjs7e2F4zh46aWX0NzcjJtvvhlf+9rXIl+zfft23HXXXdiyZUvN\n4xMmTMDvf/97uBGvoaEBb731FgYMGCC0LcULh4B0QASCizogX9iDAiFQX7gBBkPyUVzYE4lbB/dJ\nK9haxHVQfsERyO2io2h9YTAsMRPqSJr1c780lq11xZZw2N7ejh07dmDp0qXYv38/FixYgCVLlmDa\ntODz/r/927/hlltuQVNTE7Zu3Vp9vLOzE5dffjm2bNmC5ubm6uODBw8W3pZihkMgNiACciExqUSh\nEOAJsqziCjugp7iLLJP7pDVsLeI6aLngCCQKiC6ZoBgWCAHWFwqRVx1Juz7ul0azta7YEA67u7tx\n0UUXYdWqVZgwYQIAYOXKldi2bRueeOKJuuf/6Ec/woMPPohzzjkHhw8frgmH27Ztw4IFC/Dqq68m\n3u7ihkMgUUF3pQ2JQUUbYOEmASKFHUhf3EVfz/3ROrYWcR20hUMgVa+UNIRCIcD6UmZZ1RFV6+B+\naTxb64oN4fDtt9/GrFmzsHPnTjQ2NgIA3njjDcydOxc7d+6se/43vvENzJgxA4cPH8aKFStqwuGa\nNWvw0ksvBYZKUcUOh4BQQATi+9bHFfiwMOhi4SYpooXdy1H0HC/uj1aytYjroK03isupfyj3i44A\n6wvlHxBFl8v90gq21hUbwuGLL76Ie++9F//6r/9afayjowN/8id/gl/84hc47bTTAl/39NNP14VD\nx3HwzjvvoLm5Gf/5n/+JUaNGYdGiRfjsZz8rvN2Nws+01cuvB3cxdWofcotrWFGPC39hhIs2wBMk\n9XH3BZmQ6GjaBiIKriVAaD0JqiX+OpLZRUeAx3MZidYRJ+TrJGRez32SCMDxbqX9+/evecz9vre3\nV2pZ7733Hn73u9/hm9/8Jk455RQ8/vjj+OpXv4oXXngBJ598stAyih8OgfCA6P3/BG+xTTqlc+TM\nXU7I4zxJUpCwD6S610lE9cI+bDu+/xF/wRFQfNHRt/4aPKbLTaaOOCFfi75GFPdJoqqmpqa6EOh+\nLzrDqGvVqlX4v//7v+rrli9fjssuuwwvv/wyrr/+eqFllCMcAtFXfb3/e/gLcFCRV3JPH54kKUqS\nVsS06yKicJKtiC5t95Bzgh8GwGOajktyodHRsiXcJylbT+3Ob93rLhB62vDhw3Ho0CEcO3YM/fr1\nAwB0dXWhubkZp556qtQqTzrpJJx00knV7/v3748//MM/RGdnp/AyyhMOgeiTo4PYE6HyG4kDPEmS\nOJ0hkfshkRxNFxyjnh8oYD1VPK7JK8sLjXHbQERVI0eORGNjI3bu3IkLL7wQwPF7GI4ePVp6WVdf\nfTX+8i//EjNmzAAAfPLJJ3j//ffxuc99TngZ5QqHQHxADPo6ibjX8wRJSakq8NwHidLJ44Kjd/lR\neHxTGA5XIDJKc3Mzpk+fjra2Ntx///3o7OzE6tWrsXTpUgDHWxEHDhyIpqam2GVddtll+O53v4sz\nzzwTp512Gh555BGcccYZuOyyy4S3p/izlUZROdlH3M+9eJIkHUT2Z+57pWDrrHI6ZFJTsryvXNxy\neIyTDN0hkftjYdhaVxoa8utWWqmIdSsFgCNHjuCee+7B5s2bMXDgQNx6662YNWsWAKC1tRVLly6t\ntga6gmYr7e3txcMPP4znn38ehw8fxuTJk9HW1obhw4cLb0u5wyGQ7dUzniSJKAO2FnEdMqspOm8b\nIPoa1hhKSvVnIe6LhWNrXbElHJqE4dDFq2dEVBC2FnEdCn3B0Ys1hlTgcAUKYWtdYTiUx3Dox6tn\nRGQ5W4u4DoW94OhijSGd4vZj7n+lYWtdYTiUx3AYJWlx58mSiHJkaxHXIdeaojMgss4QUYZsrSsM\nh/LKN1upDBZfIiJKSsetA1iXiIhII4ZDIiIinbyBLklQZCAkIqKMMBwSERFlhUGPiIgM1i/vDSAi\nIiIiIqL8KQ+HnZ2dmDdvHiZNmoTLLrsMS5cuRW9vr+rVEBFRCbCmEBERZUd5t9J58+ahpaUF69at\nw6FDh7Bo0SJ85jOfwfz581WvioiICo41hYiIKDtKWw7fe+897N69G0uWLMG5556L8ePHY968eXju\nuedUroaIiEqANYWIiChbSsPh0KFD8fjjj2PQoEHVxyqVCg4fPqxyNUREVAKsKURERNlSGg4HDhyI\nKVOmVL+vVCpYu3YtvvCFL6hcDRERlQBrChERUba03sriwQcfxL59+/CTn/xE52qIiKgEWFOIiIj0\n0hYOly1bhjVr1uA73/kOzj33XF2rISKiEmBNISIi0k9LOGxvb8f69euxbNkyXHXVVTpWQUREJcGa\nQkRElA3l4XDFihVYv349Hn74YVx99dWqF09ERCXCmkJERJQdpeGwo6MDK1euxG233YZx48ahq6ur\n+rMhQ4aoXBURERUcawoREVG2GiqVSkXVwv7u7/4ODz/8cM1jlUoFDQ0NePfdd8U2qOF1VZtDRFRK\nlcqkvDdBCdYUIiIz2FpXGhp257buSuWC3NadhtJwqAILORFROrYWcR1YU4iI0rO1rjAcylN6n0Mi\nIiIiIiKyE8MhERERERERMRwSERERERERwyERERERERGB4ZCIiIiIiIjAcEhERERERERgOCQiIiIi\nIiIwHBIREREREREYDomIiIiIiAgMh0RERERERASGQyIiIiIiIgLDIREREREREYHhkIiIiIiIiMBw\nSERERERERGA4JCIiIiIiIjAcEhERERERERgOiYiIiIiICAyHREREREREBIZDIiIiIiIiAsMhERER\nERERgeGQiIiIiIiIwHBIREREREREYDgkIiIiIiIiaAiHv/71r3HLLbdg3LhxuOKKK7Bq1SrVqyAi\nohJhXSEiIspGo8qFVSoVzJ07F2PGjMEzzzyD//qv/8Kdd96J008/Hddff73KVRERUQmwrhAREWVH\nacthV1cXRo0ahba2Npxzzjm49NJLMXnyZOzYsUPlaoiIqCRYV4iIiLKjNBwOHToUDz30EE4++WQA\nwI4dO/Dmm29i0qRJKldDREQlwbpCRESUHaXdSr2uuOIKfPDBB7j88ssxbdo0XashIqKSYF0hIiLS\nS9tspd/73vfw2GOP4d1338V9992nazVERFQSrCtERER6NVQqlYrOFWzevBnz58/HW2+9hcbG+IbK\nhobXdW4OEVHhVSrF7nIpU1dYU4iI0rO1rjQ07M5t3ZXKBbmtOw2lLYe/+c1vsGXLlprHzjvvPHz6\n6af4+OOPVa6KiIhKgHWFiIgoO0rD4f79+3H77bfj4MGD1cf27NmDQYMGoaWlReWqiIioBFhXiIiI\nsqM0HJ5//vkYPXo0Fi5ciI6ODrzyyitYvnw5vv71r6tcDRERlQTrChERUXaUjzk8ePAg2tvbsW3b\nNgwYMAA33XQT5s6dK75BHB9CRJSKrWNDwqSpK6wpRETp2VpXOOZQnvYJaWSxkBMRpWNrEdeBNYWI\nKD1b6wrDoTxtt7IgIiIiIiIiezAcEhEREREREcMhERERERERMRwSERERERERGA6JiIiIiIgIDIdE\nREREREQEhkMiIiIiIiICwyERERERERGB4ZCIiIiIiIjAcEhERERERERgOCQiIiIiIiIwHBIRERER\nEREYDomIiIiIiAgMh0RERERERASGQyIiIiIiIgLDIREREREREYHhkIiIiIiIiMBwSERERERERGA4\nJCIiIiIiIjAcEhERERERERgOiYiIiIiICAyHREREREREBIZDIiIiIiIiAsMhERERERFRbnp7e7Fo\n0SJMnDgRl1xyCVavXh363L179+IrX/kKxo4diy9/+ct45513an7+3HPP4eqrr8a4cePwjW98Ax9+\n+KHUtjAcEhERERER5eSBBx7A3r17sWbNGrS1tWHFihV48cUX657X3d2NuXPnYuLEidi4cSPGjh2L\n2267DUeOHAEA7N69G3fffTduv/12rF+/Hh999BEWLlwotS0Mh0RERERERDno7u7Ghg0bcPfdd6O1\ntRVXXXUVbr31Vqxdu7buuc8//zwGDBiA+fPn43Of+xwWL16MU045BT/72c8AAE8++SSuvfZafOlL\nX8If/dEfYdmyZXjllVfw3//938Lbw3BIRERERESUg3379uHo0aMYO3Zs9bHx48dj9+7ddc/dvXs3\nxo8fX/PYhRdeiLfffhsAsHPnTkycOLH6s9NPPx1nnHEGdu3aJbw9DIdEREREREQ5OHjwIFpaWtDY\n2Fh9bPDgwejp6akbL/i///u/GDZsWM1jgwcPRmdnZ3VZ/p8PGTIEBw4cEN4ehkMiIiIiIqIcdHd3\no3///jWPud/39vbWPH7kyJHA57rPi/u5iMb4p2SrUpmU9yYQEVFBsKYQEZVXpXJB3psQq6mpqS68\nud8PGDBA6LnNzc1CPxfBlkMiIiIiIqIcDB8+HIcOHcKxY8eqj3V1daG5uRmnnnpq3XMPHjxY81hX\nVxeGDh0KABg2bBi6urrqfu7vahqF4ZCIiIiIiCgHI0eORGNjI3bu3Fl9bPv27Rg9enTdc8eMGVOd\nfMb19ttvY9y4cQCAsWPHYseOHdWfffDBBzhw4ADGjBkjvD0Mh0RERERERDlobm7G9OnT0dbWhj17\n9mDLli1YvXo15syZA+B4y19PTw8A4JprrsHhw4dx//33o6OjA9/+9rfxySef4Itf/CIAYObMmXjm\nmWewYcMG7Nu3DwsWLMDUqVNx1llnCW9PQ6VSqaj/NYmIiIiIiCjOkSNHcM8992Dz5s0YOHAgbr31\nVsyaNQsA0NraiqVLl2LGjBkAgD179qCtrQ3vvfceRowYgXvuuQetra3VZW3atAmPPPIIPvroI0yZ\nMgXt7e34gz/4A+FtYTgkIiIiIiIidislIiIiIiIihkMiIiIiIiICwyERERERERGB4ZCIiIiIiIjA\ncEhEREREREQwPBw6jlOdxtVEv/3tbzFv3jxMmDABU6ZMwfLly3Hs2LG8N6vO4cOHsXjxYlx88cWY\nPHkyFi5ciMOHD+e9WZFuueUWbNq0Ke/NqNHb24tFixZh4sSJuOSSS7B69eq8NylSb28vbrjhBrz5\n5pt5b0qgzs5OzJs3D5MmTcJll12GpUuXore3N+/NCvTrX/8at9xyC8aNG4crrrgCq1atynuTKCHW\nFTVYV9KzraYArCuqsKaQyYwNh2+99RbWr1+PhoaGvDcl1F//9V/j97//Pf7pn/4JjzzyCJ5//nn8\n/d//fd6bVedb3/oW/v3f/x2PP/44fvjDH6KjowN/8zd/k/dmBapUKmhvb8cvfvGLvDelzgMPPIC9\ne/dizZo1aGtrw4oVK/Diiy/mvVmBent7ceedd+JXv/pV3psSat68eejp6cG6devw0EMP4eWXX8Yj\njzyS92bVqVQqmDt3LoYMGYJnnnkGjuNg5cqVeP755/PeNJLEuqIO60p6NtUUgHVFFdYUMl1j3hsQ\n5NNPP0VbWxvGjRuX96aE6u3txZAhQ3D77bfj7LPPBgBcc8012LFjR85bVqu7uxsvvfQSnnrqKYwa\nNQoAsGjRItx0003o7e1F//79c97CPp2dnZg/fz7279+PU089Ne/NqdHd3Y0NGzZg1apVaG1tRWtr\nK2699VasXbsW06ZNy3vzanR0dOCb3/xm3psR6b333sPu3bvx2muvYdCgQQCOF/UHH3wQ8+fPz3nr\nanV1dWHUqFFoa2vDySefjHPOOQeTJ0/Gjh07cP311+e9eSSIdUUd1pX0bKopAOuKSqwpZDojWw5/\n8IMfYMSIEfjCF76Q96aE6t+/Px588MFqAf+P//gP/PznP8ekSZNy3rJa/fr1w2OPPYbW1tbqY5VK\nBUePHsUnn3yS45bV27t3L84880xs3LgRp5xySt6bU2Pfvn04evQoxo4dW31s/Pjx2L17d45bFeyN\nN97A5MmTsX79elQqlbw3J9DQoUPx+OOPVws4cHy/NLFb2tChQ/HQQw/h5JNPBgDs2LEDb775pnHH\nOkVjXVGHdSU9m2oKwLqiEmsKmc64lsOOjg786Ec/wrPPPot169blvTlCZs2ahTfffBOjR4/GjTfe\nmPfm1GhqasKUKVNqHnviiScwYsQItLS05LRVwaZOnYqpU6fmvRmBDh48iJaWFjQ29h0ygwcPRk9P\nDz788EOcdtppOW5drZkzZ+a9CbEGDhxYs19WKhWsXbvW6A/uAHDFFVfggw8+wOWXX27k1X0Kxrqi\nFutKejbVFIB1RRfWFDJR5uGwp6cHnZ2dgT8bOnQo2tracMcdd9Rc+clL3LYOGDAAAHD33Xfjd7/7\nHe6991781V/9FVauXJnlZgpvJwCsXbsWmzdvzmXws8x2mqa7u7uuq5T7vYmD3W3z4IMPYt++ffjJ\nT36S96ZE+t73voeuri60tbXhvvvuw9133533JhFYV3RgXdGLNUU/G+oKawqZKPNwuGvXLsyePTtw\nQoA777wTx44dw5e//OWsNytQ1LauWLECV155JQBgxIgRAIAlS5bgT//0T/E///M/OPPMM43bzief\nfBL33XcfFi9ejMmTJ2e2fS7R7TRRU1NTXcF2vzf1w4ctli1bhjVr1uA73/kOzj333Lw3J9If//Ef\nAwAWLlyI+fPn46677qq58k/5YF3JbztZV5JhTdHLlrrCmkImynwP/PznP499+/YF/mz27Nn45S9/\nWZ0w4NNPP8WxY8dw4YUX4oUXXsDpp5+e5aZGbuvHH3+MF154Adddd131sfPOOw8A8OGHH2ZaxKO2\n07Vq1SosW7YMd911F2666aaMtqyWyHaaavjw4Th06BCOHTuGfv2OD9Xt6upCc3OzUZMc2Ka9vR3r\n16/HsmXLcNVVV+W9OYF+85vf4O23367ZvvPOOw+ffvopPv74Y+O60ZUR64p6rCt6saboY3pdYU0h\n0xl1eWL58uXo6empfv+P//iP2LNnD5YvX45hw4bluGX1jhw5gjvvvBNnnXUWxowZAwD45S9/icbG\nRnz2s5/Nd+N8nn76aSxfvhyLFy82+v5eJhs5ciQaGxuxc+dOXHjhhQCA7du3Y/To0Tlvmb1WrFiB\n9evX4+GHH8bVV1+d9+aE2r9/P26//Xa8+uqrGDp0KABgz549GDRoEIu4BVhX9GBdSYc1RQ8b6gpr\nCpnOqNlKhw0bhrPPPrv6r6WlBU1NTTj77LOrV9ZMMWTIEEybNg333nsv3n33XWzfvh133303Zs2a\nZdSMaB999BHa29sxY8YMXHvttejq6qr+M/HGyqZqbm7G9OnT0dbWhj179mDLli1YvXo15syZk/em\nWamjowMrV67E3LlzMW7cuJr90jTnn38+Ro8ejYULF6KjowOvvPIKli9fjq9//et5bxoJYF1Rj3Ul\nPdYU9WypK6wpZDqjWg5tc//992PJkiW4+eabAQAzZsww7j5Ar732Grq7u7Fp0yZs2rQJwPEZvBoa\nGrB169ZMuynJMPEm1QsXLsQ999yDOXPmYODAgbjjjjuM7LLiZeL7CABbt27FsWPHsHLlyupEG+5+\n+e677+a8dbX69euH73//+2hvb8ef//mfY8CAAZg9e3Zu3eio2FhX9DHtfGhjTQHMex9dttQV1hQy\nXUPF1BvWEBERERERUWbM6lNDREREREREuWA4JCIiIiIiIoZDIiIiIiIiYjgkIiIiIiIiMBwSERER\nERERGA6JiIiIiIgIDIdEREREREQEhkMiIiIiIiICwyERERERERGB4ZCIiIiIiIjAcEhEREREREQA\n/j9oYvooeAiQ9AAAAABJRU5ErkJggg==\n", 109 | "text/plain": [ 110 | "" 111 | ] 112 | }, 113 | "metadata": {}, 114 | "output_type": "display_data" 115 | } 116 | ], 117 | "source": [ 118 | "fig, axs = plt.subplots(2, 2, figsize=(12,8))\n", 119 | "\n", 120 | "Z1 = p1(XY)\n", 121 | "Z2 = p2(X, Y)\n", 122 | "Z3 = p3(XY)\n", 123 | "Z4 = p4(XY)\n", 124 | "\n", 125 | "cf = axs[0,0].contourf(X, Y, Z1)\n", 126 | "axs[0,1].contourf(X, Y, Z2)\n", 127 | "axs[1,0].contourf(X, Y, Z3)\n", 128 | "axs[1,1].contourf(X, Y, Z4)\n", 129 | "\n", 130 | "axs[0,0].invert_yaxis()\n", 131 | "axs[0,1].invert_yaxis()\n", 132 | "axs[1,0].invert_yaxis()\n", 133 | "axs[1,1].invert_yaxis()\n", 134 | "\n", 135 | "fig.colorbar(cf, ax=axs.ravel().tolist());" 136 | ] 137 | }, 138 | { 139 | "cell_type": "markdown", 140 | "metadata": {}, 141 | "source": [ 142 | "#### Distribution 4.1 from [A family of non-parametric density estimation algorithms](http://math.nyu.edu/faculty/tabak/publications/Tabak-Turner.pdf)" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": 47, 148 | "metadata": { 149 | "collapsed": false 150 | }, 151 | "outputs": [], 152 | "source": [ 153 | "def cart2pol(x, y):\n", 154 | " rho = np.sqrt(x**2 + y**2)\n", 155 | " theta = np.arctan2(y, x)\n", 156 | " return(theta, rho)\n", 157 | "\n", 158 | "def p(x, y):\n", 159 | " t, r = cart2pol(x,y)\n", 160 | " return np.exp(-0.5*t**2 - 0.5*((r-1.)/0.1)**2)" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 48, 166 | "metadata": { 167 | "collapsed": false 168 | }, 169 | "outputs": [ 170 | { 171 | "data": { 172 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoIAAAHcCAYAAABCnMsPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xt0VOWh9/HfQEgmlaSBECJgz0KommCEQABFEygatbZF\nqAItXcVIpVBr0FZRhKCRiwiCtnBSUWlMj0ApFRE9WkRprVcOEG6JxLQl0SqnEDJvuJZkpk3m/YOT\nMZPJZU+y55b9/ayVZbPnyeSZTqVfnr2fPTa32+0WAAAALKdbqCcAAACA0CAEAQAALIoQBAAAsChC\nEAAAwKIIQQAAAIsiBAEAACyKEAQAALAoQhAAAMCiCEEAAACLMj0Eq6qqdO+99+rqq6/WuHHjtHz5\ncrlcrhbHlpWVaerUqUpPT9eUKVN0+PBhs6cDAAAQllwulyZMmKC9e/e2Oqa9Vho5cqRSU1OVkpKi\nlJQUpaamqra21vAcTA/Be++9V06nU7/97W/19NNP65133tHq1at9xtXW1mrWrFkaNWqUtm7dqvT0\ndM2ePVt1dXVmTwkAACCsuFwu3X///Tpy5EirY9prpaqqKv3zn//Uzp079eGHH+rDDz/UBx98oNjY\nWMPzMDUEKysrVVJSoieeeEKDBw9WRkaG7r33Xr3++us+Y9944w3FxsbqwQcf1KBBg5SXl6eLLrpI\nb775pplTAgAACCsVFRWaOnWqjh492ua49lqpsrJSSUlJGjBggBITEz1f/jA1BJOSkrRu3Tr17t3b\nc8ztduvs2bM+Y0tKSpSRkeF1bMSIETpw4ICZUwIAAAgre/bs0ZgxY7R582a53e5Wx7XXSkeOHNHA\ngQM7NZeoTv10M3FxccrMzPR873a7tWHDBl177bU+Y0+cOKHLL7/c61hiYmKbS6QAAACRbtq0aYbG\ntddKFRUVqq2t1fTp0/Xpp59qyJAhWrBggV9xGNBdw08++aTKy8v185//3Oexuro6RUdHex2Ljo5u\ndWMJAACAlbTXSpWVlTpz5ozuuecerV27Vna7XXfeeafOnz9v+HeYuiLY1MqVK7V+/Xr98pe/1ODB\ng30ej4mJ8Yk+l8slu91u+He43W7ZbLZOzxUAAFhDSYC6YWgbp3g7qr1WKiws1L///W/P5pBVq1Zp\n3Lhxeuedd/Ttb3/b0O8ISAguWbJEmzdv1sqVK5Wdnd3imOTkZFVXV3sdczgcSkpKMvx7unUr7dQ8\nAQBAcJ0+fZni443varWy9lqpR48e6tGjh+ex6OhoXXLJJaqqqjL8O0w/NVxQUKDNmzfrF7/4hW65\n5ZZWxw0bNsxnY8iBAweUnp5u9pQAAECYqK9vCPUUIkZrrTR8+HBJ0o033qht27Z5Hjt//rz+/ve/\na9CgQYZ/h6khWFFRobVr12rWrFkaPny4HA6H50u6ULFOp1OSdPPNN+vs2bNatmyZKioqtHTpUp0/\nf77NeAQAAOjKjLTSN7/5TUnSuHHjtGbNGu3Zs0d/+9vf9NBDD6lfv34aN26c4d9nagj+8Y9/VEND\ng9auXausrCxlZWUpMzNTWVlZkqTMzExt375dktSzZ089++yzKi4u1u23367S0lKtW7fOr2sEAQAA\nIlnzvQ7+tNJDDz2km2++WXPnztXUqVPV0NCg559/3q/9EzZ3WzewCXM2W0mopwAAAPxQUzNYvXpd\nFLLfH0mbRYIhoLePAQAAQPgiBAEAACyKEAQAALAoQhAAAMCiCEEAAACLIgQBAAAsihAEAACwKEIQ\nAADAoghBAAAAiyIEAQAALIoQBAAAsChCEAAAwKIIQQAAAIsiBAEAACyKEAQAALAoQhAAAMCiCEEA\nAACLIgQBAAAsihAEAACwKEIQAADAoghBAAAAiyIEAQAALIoQBAAAsChCEAAAwKIIQQAAAIsiBAEA\nACyKEAQAALAoQhAAAMCiCEEAAACLIgQBAAAsihAEAACwKEIQAADAoghBAAAAiyIEAQAALIoQBAAA\nsChCEAAAwKIIQQAAAIsiBAEAACyKEAQAALAoQhAAAMCiCEEAAACLClgIulwuTZgwQXv37m11zN13\n362UlBSlpqZ6/vnuu+8GakoAAABoIioQT+pyuXT//ffryJEjbY6rrKzUU089pWuuucZzLD4+PhBT\nAgAAQDOmh2BFRYUeeOCBdse5XC4dPXpUaWlpSkxMNHsaAAAAaIfpp4b37NmjMWPGaPPmzXK73a2O\n+/TTT2Wz2XTJJZeYPQUAAAAYYPqK4LRp0wyNq6ioUM+ePfXQQw9p9+7d6tevn+bMmaOxY8eaPSUA\nAAC0IGS7hisrK+V0OpWVlaXCwkKNGzdOd999tw4fPhyqKQEAgADr3p0bloSTgGwWMSI3N1c5OTmK\ni4uTJF1xxRX6+OOPtXnzZi1evDhU0wIAAAEUHx8b6imgiZBmeWMENho8eLBOnDgRotkAAIBAO3Om\nNtRTQBMhC8H58+crLy/P61h5ebkuvfTSEM0IAAAEWn19Q6inEHIul0sLFizQqFGjlJWVpaKiolbH\nfvDBB5o4caKGDx+uH/3oR/r000+9Hn/99dd14403avjw4crNzdXJkyf9mktQQ9DhcMjpdEqSbrjh\nBr322mvatm2bPv/8cxUUFGj//v2aPn16MKcEAAAQVCtWrFBZWZnWr1+v/Px8FRQU6K233vIZ97e/\n/U0/+clPdOONN+qVV15RamqqcnJyVFt7YVW1pKRECxcu1Jw5c7R582adPn1a8+fP92suAQ1Bm83m\n9X1mZqa2b98uScrOzlZ+fr7Wrl2rCRMm6J133tGvf/1r9e/fP5BTAgAACJna2lpt2bJFCxcuVEpK\nirKzszVz5kxt2LDBZ+zvfvc7z0rfwIED9eCDDyo+Pl7//d//LUnauHGjbrnlFt166626/PLLtXLl\nSr377rv63//9X8PzCehmkU8++cTr+/Lycq/vJ0+erMmTJwdyCgAAAGGjvLxc9fX1Sk9P9xzLyMjQ\nc8895zP2iy++0LBhw7yOXXbZZTpw4ICmTp2qgwcPavbs2Z7HLr74YvXr10+HDh3SgAEDDM2HPdwA\nAABBUl1drYSEBEVFfbkWl5iYKKfT6XN9X2JioqqqqryOHTt2zDOuurpaffv29Xq8T58+On78uOH5\nEIIAAABBUltbq+joaK9jjd+7XC6v49/61rf05ptv6s9//rPq6+v1yiuv6OOPP9a//vUvSVJdXV2L\nz9X8edoSsvsIAgAABNvQB0L7+2NiYnxCrfH72FjveyxmZWUpNzdXc+bMUUNDg66++mpNmjRJZ8+e\nbfO57Ha74fmwIggAABAkycnJOnXqlBoavryNjsPhkN1uV3x8vM/42bNna//+/Xr//ff1wgsv6Ny5\nc57r//r27SuHw+E13uFw+JwubgshCAAAECSpqamKiorSwYMHPceKi4uVlpbmM/aNN97QsmXL1KNH\nD/Xu3Vt1dXXavXu3rrnmGklSenq69u3b5xl/7NgxHT9+3GeDSVsIQQAAgCCx2+2aOHGi8vPzVVpa\nqp07d6qoqEg5OTmSvO+5PHDgQG3evFlvv/22PvvsMz3wwAPq37+/xo4dK0maNm2aXn31VW3ZskXl\n5eWaN2+exo8fb3jHsCTZ3G632/yXGRw2W0mopwAAAPxQUzNYvXpdFLoJzLW1P6YjVhnPqbq6Oi1a\ntEg7duxQXFycZs6c6flAjZSUFC1fvlyTJk2SJL3yyisqKCjQ6dOnde211+rRRx9Vnz59PM+1bds2\nrV69WqdPn1ZmZqaWLFmir371q4bnQggCAICgIQTDC6eGAQAALIoQBAAAsChCEAAAwKIIQQAAAIsi\nBAEAACyKEAQAALAoQhAAAMCiCEEAAACLIgQBAAAsihAEAACwKEIQAADAoghBAAAAiyIEAQAALIoQ\nBAAAsChCEAAAwKIIQQAAAIsiBAEAACyKEAQAALAoQhAAAMCiCEEAAACLIgQBAAAsihAEAACwKEIQ\nAADAoghBAAAAiyIEAQAALIoQBAAAsKioUE8AANo1bWjgnntTSeCeGwDCHCEIIPQCGXpm/G5iEUAX\nRQgCCJ5QBl9ntDZvAhFAhCMEAQRGpEafP1p6jcQhgAhCCAIwhxXCz4jm/z0QhgDCGCEIoOOIv/YR\nhgDCGCEIwD/EX+c0/e+PKAQQYoQggPYRf4FBFAIIsYCFoMvl0u23365HH31Uo0aNanFMWVmZHnvs\nMf31r3/VZZddpscee0xXXnlloKYEwF8RFIDRBWeC8ntcufGBeWKiEEAIBCQEXS6X7r//fh05cqTV\nMbW1tZo1a5YmTpyo5cuXa9OmTZo9e7Z27twpu90eiGkBMCoCAjBY4efP7zUtEolCAEFieghWVFTo\ngQceaHfcG2+8odjYWD344IOSpLy8PL333nt68803NWnSJLOnBcCIMA/AUMWfUS3Nr9NxSBQCCCDT\nQ3DPnj0aM2aMfvazn2nYsGGtjispKVFGRobXsREjRujAgQOEIBBsBGDANJ97p8Kw8X0iCAGYxPQQ\nnDZtmqFxJ06c0OWXX+51LDExsc3TyQBMFuYBKEV2BLak6evpcBQShABMErJdw3V1dYqOjvY6Fh0d\nLZfLFaIZARYSAQFoBZ1eLeS0MSJQ9+7dQj0FNBGyEIyJifGJPpfLxUYRINCIwLDVqdVCVgkRIeLj\nY0M9BTQRsixPTk5WdXW11zGHw6GkpKQQzQiwgAiMwIDdriXMRRec6dhp8WlDI/J9hnWcOVMb6img\niZCtCA4bNkzr1q3zOnbgwAH95Cc/CdGMgC4uguPAlRsfltcKzuu9otXHVtTMM+V3dHiVkBVChKn6\n+oZQTwFNBDUEHQ6H4uLiFBMTo5tvvllPP/20li1bpu9973vatGmTzp8/r1tuuSWYUwKsIYIjsFFj\nBIU6CNuKP6PjOhqJja+dIARgloCeGrbZbF7fZ2Zmavv27ZKknj176tlnn1VxcbFuv/12lZaWat26\ndVwjCJitC0RgU67c+JCdLjYagUaep+mXvxpPG/sVxV3sfwcAzGFzu93uUE+io2w2/oYLtMsCARCs\nVUKzQrA9HVkx9CuOWR1ECNXUDFavXheFbgJzbe2P6YhVkZlThCDQ1VkgBBsFIwiDFYON/I1CghDh\njhAMLyHbLAIAZmsaQaG+ltAsTcPTSBT6dR0h1w8ClseKIGAFFloVbInZURjsVcHm/FklNLxCSAwi\nSFgRDC+EIGAlFg9CydwoDHUQSsajkCBEuCAEL3yAxmOPPaa3335bdrtdP/rRjzRjxgyfcdOnT9fe\nvXt9jt9+++16/PHHdebMGY0ePVo2m02NOderVy/t2rXL8FwIQQBt68LxaEYUhkMMSiYHITGIACIE\npSVLlmjfvn1avny5jh49qnnz5umJJ57QTTfd5DXuzJkz+te//uX5/uDBg/r5z3+u3/3udxoyZIj2\n79+ve+65R2+88YYnBG02m3r37m14LoQggNAIo8DsbBCGSwxKxoKQ1UGEktVDsLa2Vtdcc40KCws1\ncuRISdLatWu1a9cuvfjii63+XENDg2699VbdfPPNmjNnjiTppZde0tatW7Vp06YOT5vNIgBCw0hk\nBCkWO7vJpDG+wiEIG+fQVhAa3lAybSgxCJisvLxc9fX1Sk9P9xzLyMjQc8891+bPvfzyyzp16pRm\nzpzpOVZRUaGBAwd2aj6EIIDw1VqEBDAQO/MJJitq5oVFDErGg9BQDEoEIWCS6upqJSQkKCrqywRL\nTEyU0+nUyZMn1atXrxZ/rrCwUDNmzFBsbKznWEVFhf79739rypQpqqqq0siRIzV//nwlJSUZng8h\nCCDytBQlJsdhR1cJw2l1UGo/CFkdBIKrtrZW0dHRXscav3e5XC3+zP/8z/+oqqpKU6ZM8TpeWVmp\nxMRE5eXlqaGhQU8//bRmz56tl19+2efT3VpDCALoGgIYhx1ZJTRrdXBu/apWH1vVfa7h5zElCIlB\ndAFnVkS3P6gDjN7KPSYmxif4Gr9vutrX1FtvvaWsrCzFx3v/lj/84Q+y2WyekFyzZo0yMzN16NAh\nr1PPbQnoZw0DQEhtKvnyywT+fs7xipp5Hfq4uEZtRWDj402/jGgvTtuN3TDa5ANEouTkZJ06dUoN\nDQ2eYw6HQ3a73Sf0Gr3//vvKzs72OR4TE+O1uti7d28lJCSoqqrK8HwIQQDW0DQKOxmGHQnCjvBn\nxU+S4SCc13tFm0FoKAYJQqBDUlNTFRUVpYMHD3qOFRcXKy0trcXxJ0+e1BdffKGMjAyv4+fOndPV\nV1/tdZ/BqqoqnTx5UoMGDTI8H24fAwCdjBp/Thn7e7rY6Epfa4zEZFuh2m7wcqoYfgr17WPO1McE\n5HnjuzsNj83Pz9f+/fu1bNkyVVVV6eGHH9by5cuVnZ0th8OhuLg4xcRcmOeePXs0a9Ysr3Bs9NOf\n/lTHjh3T4sWL1a1bNy1btkxxcXF69tlnDc+FEASApjoRhUaDMNgxKLUfhMQggoUQlOrq6rRo0SLt\n2LFDcXFxmjlzpqZPny5JSklJ0fLlyzVp0iRJF64DfOKJJ/T+++/7PM/Zs2e1fPlyvfPOO3K5XMrO\nzlZeXp7i4uIMz4UQBIDWdDAKzQ5CM0KwUUeDkBiEWQjB8EIIAkB7AhiEkRSDUjtBSAzCAEIwvBCC\nAOCPDkShWUFoZgxKrA4iNAjB8EIIAkBH+BmE4RqDUttBSAzCbIRgeCEEAaAzTA5CYhBdHSEYXghB\nADBDkIMwEDEotR6ExCDMQgiGF0IQAMzkRxBGWgxKHQxCYhBNEILhhRAEgEAwKQhDFYOSyauDxCD+\nDyEYXghBAAgkg0HYmdVBYhCRhBAML4QgAASDCUFIDKIrIATDCyEIAMESwTEotRyExCD8RQiGl26h\nngAAWMamEkMR1FZEtfWpH+3dILqzWgrN1sLU6MfsAQgtVgQBIBQ6uTpo1spg/DyX5z+fWRFt6GdM\nWRlkVdCyWBEML4QgAISSgSAMVAw2jcDm2otCYhAdRQiGF04NA0AodeJUcVuniY1oK/baikTJpNPE\nHfjcZgDmYkUQAMJFO2Hk7yaSzq4KNmorGI2uDLJ5BI1YEQwvrAgCQLhoJ4j83URi1uaRtmLR6Mpg\nm5tHWBkEQoYVQQAINyZfN9jeyqCRVcFGra0OsjIIo1gRDC+sCAJAuDFwmxl/rhs087YyrUWjKSuD\nAIKOEASAcBWkGDR625hGAYtBThEDQUcIAkA468R1g81FxMogMQgEFSEIAOGuAzHY2q1lWotBf1cF\npc7HYKuIQSBoCEEAiAQmxqCZ4ue5WgxCI7eu4XpBIPQIQQCIFO1sIjEag2auCjYyEoOcIgbCDyEI\nAJGGGARgEkIQACJRAGPQbEZiEEBoEIIAEKlMiMGWmL0q2JLmMciqIBAapoegy+XSggULNGrUKGVl\nZamoqKjVsXfffbdSUlKUmprq+ee7775r9pQAoOvy89M4msdgIFYFTd88QgwCAWN6CK5YsUJlZWVa\nv3698vPzVVBQoLfeeqvFsZWVlXrqqaf0wQcf6MMPP9QHH3yga6+91uwpAUDX1koMGr3HYEsx2JlV\nQanj1wsCCC5TQ7C2tlZbtmzRwoULlZKSouzsbM2cOVMbNmzwGetyuXT06FGlpaUpMTHR89WjRw8z\npwQA1uBHDBq9XjAUMciqIBBcpoZgeXm56uvrlZ6e7jmWkZGhkhLfP6A+/fRT2Ww2XXLJJWZOAQDQ\nTKjuMSgZv2awKe4vCASPqSFYXV2thIQERUVFeY4lJibK6XTq5MmTXmMrKirUs2dPPfTQQ8rMzNSU\nKVP03nvvmTkdALCWTm4eCcSqoOQbgx0+RcyqIGA6008NR0d7/6HR+L3L5f0HQWVlpZxOp7KyslRY\nWKhx48bp7rvv1uHDh82cEgBYi58x2Fy43FKGU8RdV/fu3LAknJj6bsTExPgEX+P3sbGxXsdzc3P1\n3nvvadKkSbriiiuUm5urrKwsbd682cwpAYD1+LGT2Mgp4kCsChrBKeKuKT4+tv1BCBpTQzA5OVmn\nTp1SQ0OD55jD4ZDdbld8vO/fROPi4ry+Hzx4sE6cOGHmlADAmjqxeSRQq4KcIoYknTlTG+opoAlT\nQzA1NVVRUVE6ePCg51hxcbHS0tJ8xs6fP195eXlex8rLy3XppZeaOSUAsK5O3lamKTNWBVvS3v0F\nWRXseurrG9ofhKAxNQTtdrsmTpyo/Px8lZaWaufOnSoqKlJOTo6kC6uDTqdTknTDDTfotdde07Zt\n2/T555+roKBA+/fv1/Tp082cEgDAgFCtCjbHqiAQXDa32+028wnr6uq0aNEi7dixQ3FxcZo5c6Yn\n7lJSUrR8+XJNmjRJkrRlyxatW7dOx48f19e//nUtWLBAGRkZxidv8++O+gBgSa1EU0urbc1DrPmK\nXUeu9WtJ8xXG5uHZPExbXMX081NVEB5qagarV6+LQvb7z9THBOR547s7A/K8gWZ6CAYTIQgABhmM\nQTNCsOQpaegDbY8xJQQlYjACEYLhhT3cAACP9k4RG71WsOSpC1+t8XfjCNcKAoFBCAKAFZi4caQt\nzeOvMzFoCNcKAp1CCAKAVRiMwc6sCrZ0SritGGwLq4JA4BGCAAAfZn8WcWsxyKogEFqEIABYSQdP\nEXf0WsGmOrIyyKogEFiEIABYjcGdtmavCraGVUEgdAhBAIAkVgUBKyIEAcCKArgq2N49BFuKQVNW\nBQH4jRAEAHj4uyoYLIY+eo7Tw4DfCEEAsKoO3k6mqY6cHpaMrQq2hdPDgDkIQQCAX4K1KsjpYSDw\nCEEAsDITPqu3pVXB9q4TlPxfFTS0aYTTw4gALpdLCxYs0KhRo5SVlaWioqJWx/7lL3/RD37wAw0b\nNky33nqrdu/e7fX4b37zG40dO1YZGRnKy8uT0+nfZx4TggAAH/6cHg4kVgXRFa1YsUJlZWVav369\n8vPzVVBQoLfeestn3Llz53TXXXfpsssu0+uvv64bb7xRubm5qqmpkSTt2LFDzzzzjJYsWaL/+q//\n0qFDh7Ry5Uq/5kIIAoDVdWBV0KzTw529VhCINLW1tdqyZYsWLlyolJQUZWdna+bMmdqwYYPP2K1b\nt+qiiy7SokWL9LWvfU1z5szRwIED9fHHH0uS1q9fr5ycHI0bN05paWlatGiRtmzZ4teqICEIAGhR\nMDaN+IvTw4h05eXlqq+vV3p6uudYRkaGSkp8/0K2d+9eXX/99V7HXnrpJY0dO1YNDQ0qLS3VyJEj\nPY+lp6frX//6l8rLyw3PhxAEAARkVdDIdYJS+zeZ5vQwupLq6molJCQoKirKcywxMVFOp1MnT570\nGvvFF1+oV69eevTRR5WZmanvf//72r9/vyTpzJkzcjqd6tu3r2d89+7dlZCQoOPHjxueDyEIAGhV\ne/cVDBR/No0AkaS2tlbR0d4r6I3fu1ze/7s/f/68fv3rX6tv37769a9/rZEjR+quu+5SVVWV6urq\nZLPZWnyu5s/Tlqj2hwAALGFTSbunVVfUzGs1xM6siO7w9X0lT7W9gji3flWrK5DRBWd8g3XaUFN2\nRKPrCdTtjxYbHBcTE+MTao3fx8bGeh3v3r27UlNTlZubK0lKSUnRhx9+qFdffVWTJ0+W2+1u8bma\nP09bWBEEAHRYqD5pBIhUycnJOnXqlBoaGjzHHA6H7Ha74uO9/0KTlJSkQYMGeR0bOHCgjh07pl69\neikmJkYOh8PzWH19vU6dOqWkpCTD8yEEAQBt6sytZIxeJ9gSdg+jK0pNTVVUVJQOHjzoOVZcXKy0\ntDSfsenp6T4bPyorK3XJJZfIZrPpqquu0r59+zyPHThwQD169FBKSorh+RCCAIAvdfJ0amd2D7e3\naaQpPnsYkcput2vixInKz89XaWmpdu7cqaKiIuXk5Ei6sDrYePuX73//+/rLX/6igoICff7551q9\nerWOHj2qCRMmSJJ+8IMfqLCwUDt37lRJSYkWLVqkqVOnKiYmxvB8CEEAQERoa/cwnz2MSDJ//nyl\npaUpJydHS5Ys0X333afs7GxJUmZmprZv3y5J6t+/vwoLC/WnP/1JEyZM0Lvvvqt169Z5dgp/61vf\n0qxZs5Sfn6+ZM2cqPT1dc+f6d7mGze12u819ecFjs3EhMAAERAuraU1jq/mKXNNIa35K15+VPsn3\ndHLTVcam1yQ2P0Xd4g5nNoyEnZqawerV66KQ/f5HlReQ512sxwPyvIHGiiAAwG/Buk6wLdxGBug8\nQhAAYKpAfsoIN5cGzEUIAgAMaevm0uFwGxk+bg7wHyEIAPBl4No6f04P+6P5NYXcRgYIHEIQAADA\noghBAEDEYsMI0DmEIADAsLauE2yq+YYRM3cOs2EEMA8hCABomZ/34DNzw0hHrxNkwwjgH0IQANBh\ngdowAiA4CEEAAACLIgQBAAAsihAEAEQ0dg4DHUcIAgBa18KGkY7uHPZX8w0jTbFzGDAHIQgAME1b\nO4c7ewsZPmEEMB8hCADoUlq8hQyAFhGCAIBOiYhbyHAvQaBFhCAAAIBFEYIAAAAWZXoIulwuLViw\nQKNGjVJWVpaKiopaHVtWVqapU6cqPT1dU6ZM0eHDh82eDgAAAFphegiuWLFCZWVlWr9+vfLz81VQ\nUKC33nrLZ1xtba1mzZqlUaNGaevWrUpPT9fs2bNVV1dn9pQAAADQAlNDsLa2Vlu2bNHChQuVkpKi\n7OxszZw5Uxs2bPAZ+8Ybbyg2NlYPPvigBg0apLy8PF100UV68803zZwSAAAAWmFqCJaXl6u+vl7p\n6emeYxkZGSop8b0haUlJiTIyMryOjRgxQgcOHDBzSgAAAGiFqSFYXV2thIQERUVFeY4lJibK6XTq\n5MmTXmNPnDihvn37eh1LTExUVVWVmVMCAHRWC58uAqBrMP3UcHS090cKNX7vcnnfEb6urq7Fsc3H\nAQCArqN7d25YEk5MfTdiYmJ8Qq7x+9jYWENj7Xa7mVMCAABhJD4+tv1BCBpTQzA5OVmnTp1SQ0OD\n55jD4ZDdbld8fLzP2Orqaq9jDodDSUlJZk4JAACEkTNnakM9BTRhagimpqYqKipKBw8e9BwrLi5W\nWlqaz9jSsxNaAAAgAElEQVRhw4b5bAw5cOCA10YTAADQtdTXN7Q/CEFjagja7XZNnDhR+fn5Ki0t\n1c6dO1VUVKScnBxJF1b8nE6nJOnmm2/W2bNntWzZMlVUVGjp0qU6f/68brnlFjOnBADoLD6nF+iy\nTL9ic/78+UpLS1NOTo6WLFmi++67T9nZ2ZKkzMxMbd++XZLUs2dPPfvssyouLtbtt9+u0tJSrVu3\njmsEAQAAgiSq/SH+sdvteuKJJ/TEE0/4PFZeXu71/VVXXaWtW7eaPQUAAAAYwB5uAAAAiyIEAQAA\nLIoQBAAAsChCEADQ9fExeUCLCEEAQKfM670i1FPw4sqNb38QAEmEIADARHPrV7X6WMlTnXvuMyui\n2x8EwC+EIAAgLA19oPXHVnWfG7yJAF0YIQgAaF0LnyoSXXDG0I/Gz3OZPZsWraiZF5TfA3RFhCAA\nAIBFEYIAAAAWRQgCADos3HYMA/APIQgAMEVbO4Y7ix3DQGAQggCAlnVio0hndXTHcIv3EORm0kCr\nCEEAgOma7xju7D0EW8OOYaBzCEEAAACLIgQBAB3CRhEg8hGCAABffl4faOZGkebXB7JRBAgcQhAA\nYKpAfqIIG0UAcxGCAICAYqMIEL4IQQCANwOnhQN1fSCnhWEFLpdLCxYs0KhRo5SVlaWioqJ2f+bo\n0aMaPny49u7d6zl25swZpaSkKDU1VSkpKUpJSdGYMWP8mkuU37MHAKCJQN5IGuiKVqxYobKyMq1f\nv15Hjx7VvHnzNGDAAN10002t/sxjjz2muro6r2NHjhxRr1699MYbb8jtdkuSbDabX3MhBAEApuH6\nQKBttbW12rJliwoLCz2reDNnztSGDRtaDcHXXntN58+f9zleUVGhgQMHqnfv3h2eD6eGAQBfCqPT\nwm3h+kBEqvLyctXX1ys9Pd1zLCMjQyUlLf/F5eTJk3rqqae0ePFiz6pfo8YQ7AxCEADQYe2dFu7M\nRhGuD0RXVF1drYSEBEVFfXlSNjExUU6nUydPnvQZv3z5cn33u9/V17/+dZ/HKioqdPz4cU2ZMkVj\nx47V/fffr+rqar/mQwgCAC5oYTUwWNpbDfT7tDAQpmpraxUd7f2XnMbvXS7vSys++ugjHThwQD/9\n6U9bfK7Kykr985//VF5enn75y1/qxIkTmj17ts/KYVu4RhAA0Cp/TgubeX1gW6uBhk4Lc30gWhGo\nywoWG7xMLyYmxif4Gr+PjY31HHM6nXrssceUn5/vE46N/vCHP8hms3keX7NmjTIzM3Xo0CGvU89t\nIQQBAB3CbmHAf8nJyTp16pQaGhrUrduFE7MOh0N2u13x8V+ubpeUlOiLL77QnDlzvFb4fvzjH2vS\npEl67LHHFBMT4/XcvXv3VkJCgqqqqgzPhxAEAARkk4jR6wM5LQwrSU1NVVRUlA4ePKgRI0ZIkoqL\ni5WWluY1btiwYXrrrbe8jt144416/PHHNWbMGJ07d0433HCDCgoKNGrUKElSVVWVTp48qUGDBhme\nD9cIAgA6jdPCgDF2u10TJ05Ufn6+SktLtXPnThUVFSknJ0fShdVBp9Op6Ohofe1rX/P6kqS+ffuq\nd+/e6tmzpzIyMrRs2TKVlpbq8OHDuv/++zVu3DhddtllhudDCAKA1XVgk4hZp4U7sxoIRKr58+cr\nLS1NOTk5WrJkie677z5lZ2dLkjIzM7V9+/YWf675zaJXrFihIUOGaPbs2crJydHXvvY1rVy50q+5\n2Nz+bC0JMzYbf+sDgE7rwGnhpiHY0mpgR08LN18NbBqCzVcDWz0tzIpgWKupGaxevS4K2e+PqTnT\n/qAOcPaOzMsUWBEEACsL49XADiECAb8QggAAL8H4JBEjEchpYSDwCEEAsCoTVgNDtUmEzxYGzEEI\nAoAVtRKBZqwGtnd9IKuBQPggBAEAhoT1aiCADiEEAcBqArga2J6ArQZyWhjoEEIQAOATgc2ZsRrY\nWgSyGgiEDiEIAFZicIOI2auBRiOQ1UAguAhBALAKg6eEm/PnvoEtbRTp6P0CWQ0EAo8QBAB4aW81\nsK3Tws2jr60IZDUQCD3TQ3DVqlUaM2aMrr766nY/727p0qVKSUlRamqq558bN240e0oAgA5uEOnM\np4h05pNDWA0EgiPKzCd74YUX9MYbb+iZZ57Rv/71L82dO1d9+vTRjBkzWhxfWVmpuXPn6rvf/a7n\nWM+ePc2cEgCgg6eEW2Jkk4iRAGQ1EAgPpq4Irl+/Xvfdd5+GDx+u0aNHa+7cudqwYUOr4ysqKjRk\nyBAlJiZ6vmJiYsycEgDAIDNXA9vSXgSyGggEj2kheOLECR07dkwjR470HMvIyNA//vEPORwOn/Hn\nzp1TVVWVBg4caNYUAADNBXk1MGhYDQRMYVoIVldXy2azqW/fvp5jffr0kdvt1vHjx33GV1ZWymaz\nae3atRo3bpwmTpyobdu2mTUdAIAfEdjeaqBZEchqIBBe/LpG0Ol0qqqqqsXHzp8/L0mKjv7yX/LG\n/+xy+f4BUllZqW7dumnw4MGaPn269uzZo0ceeUQ9e/ZUdna2P9MCADRn8H6BUuhOCTfXPAJbxWpg\nROvenRuWhBO/QvDQoUO64447ZLPZfB6bO/fC3+pcLpdPAMbGxvqMnzRpkq6//nrFx1/4297ll1+u\nzz77TJs2bSIEAaAz2ojAjnyMXKBOCbe3QYTVwK4pPt63CRA6foXg6NGjVV5e3uJjJ06c0KpVq+Rw\nONS/f39JX54uTkpKavFnGiOw0aBBg7R7925/pgQAMMjIdYHhskGkVawGRrwzZ2qJwTBi2vps3759\n1a9fP+3bt89zrLi4WP369VOfPn18xq9Zs8bntjKffPKJLr30UrOmBADWY+J1gZI5q4HtnRJuCauB\nXVd9fUOop4AmTL2P4Pe//32tWrVKycnJcrvdevrpp3XXXXd5Hq+pqZHdbtdXvvIVjR8/Xs8//7yK\nioqUnZ2t999/X6+99prWr19v5pQAwDo6EYGB0lIEdniDCKuBgOlMDcGZM2fq5MmTmjNnjrp166ap\nU6cqJyfH8/jkyZN12223KTc3V1dddZXWrFmj1atXa/Xq1RowYICeeuopDR1q/AJnAMD/8WNzSEsC\nsRrYkQhsFREIBITN7Xa7Qz2JjrLZ+IMBAPzZHCIZ3yVsdgi2tDmE1UDrqakZrF69LgrZ74+p8f8e\nmkY4e0fm5Qzs4QaASNbJCGxNIFYDmyMCgdAjBAEgUpkQgWF/ShhAQBGCABCJAhSBndXRCGQ1EAgN\nQhAAIk0AI7Azq4FGIrAlRCAQOoQgAEQSPyMwWIxGINcFAuGFEASASNGBCAzGamBHIxBA6BGCABAJ\numAEshoIhJ6pN5QGAJisnRtFmxGBHWX0o+OIQCB8EYIAEK6CFIH+rga2FYCd2iEMIOg4NQwA4ShM\nVwIDGoGsBgJBRwgCQLgxMQLb489qoD8R6DciEAgJTg0DQLhoJwAl/yPQjNXA9q4FZHMIELkIQQAI\nBx1cBZSIQAAdRwgCQCh1YhVQ6lwEtnVauCMBKLE5BIg0XCMIAKESwgiUWo69MyuigxeBrAYCIceK\nIAAEm4EAlDq2KcTf08FG7wXYiAgEuhZCEACCKUCrgJL5N4xujggEuh5CEACCoZOrgFL4RWBrnx1M\nBAKRgxAEgEAyGIBSeEagP6uAEhEIRBpCEAACxYRVQIkIBBA4hCAAmC0Iq4ASEQig8whBADCLSQEo\nhS4C2/qoOCIQ6HoIQQDoLD8CUCICAYQPQhAAOsrkAJRCE4GmB6BEBAIRghAEAH/4GX+SOQEoEYEA\nzEcIAoARAQpAKTQR2FYASkQgYBWEIAC0pQMBKJm3Cmi2jgagRAQCZnG5XHrsscf09ttvy26360c/\n+pFmzJjR4tjXXntNv/rVr3T8+HENGTJE8+fP19ChX/659Prrr2v16tVyOBy67rrrtGTJEvXq1cvw\nXAhBAGiug/EnmbsK2MiM1cD2AlDqRAQSgIBfVqxYobKyMq1fv15Hjx7VvHnzNGDAAN10001e44qL\ni7Vw4UItW7ZM6enp2rhxo3784x/rz3/+s2JjY1VSUqKFCxdq8eLFSklJ0ZIlSzR//nw9++yzhudi\nc7vdbrNfYLDYbPzhA8AknYg/KTABKHU+AgMagBIRCL/V1AxWr14Xhez3x9QY+3fVX87e7fy78n9q\na2t1zTXXqLCwUCNHjpQkrV27Vrt27dKLL77oNfbNN9/U3//+d82ePVuSdO7cOY0cOVIvvfSSrrrq\nKs2bN0/dunXTE088IUk6fvy4xo8fr507d2rAgAGG5sOKIADr6mT8ScYDUAruqWAjASgRgUCwlZeX\nq76+Xunp6Z5jGRkZeu6553zGfvOb3/T8Z6fTqd/85jfq06ePvv71r0uSDh486IlESbr44ovVr18/\nHTp0iBAEgBaZEH9ScALQ39VAo/EndTIAJSIQ6KDq6molJCQoKurLBEtMTJTT6dTJkydbvL5v165d\nuuuuuyRJq1atUmxsrOe5+vbt6zW2T58+On78uOH5EIIAujaTwq+RPwEodW4VcFX3uYZi0KwAlIhA\nINBqa2sVHR3tdazxe5fL1eLPXHHFFdq6dav+/Oc/a968ebrkkks0dOhQ1dXVtfhcrT1PSwhBAF2L\nyeEn+R9/knmngZvHoD/R1xyrgEDoxcTE+IRa4/eNK33N9e7dW71791ZKSooOHjyoTZs2aejQoa0+\nl91uNzwfQhBA5ApA9DXVkQCUzL8WsDPxJ7EKCDRl6H/vHfFbY8OSk5N16tQpNTQ0qFu3bpIkh8Mh\nu92u+HjvuZWWlqp79+4aMmSI59jgwYNVUVEhSerbt68cDofXzzgcDp/TxW0hBAGEvwAHX3PhEoCd\nZUoASkQgYKLU1FRFRUXp4MGDGjFihKQLt4lJS0vzGbtlyxYdPXpUhYWFnmOHDx/2jE1PT9e+ffs0\nadIkSdKxY8d0/PhxDRs2zPB8CEEAoRfk0GtJR+OvUThFYHsBKLEKCISK3W7XxIkTlZ+fr2XLlqmq\nqkpFRUVavny5pAsrenFxcYqJidH3vvc9TZ06VevXr9fYsWP16quvqrS0VCtWXPjzZtq0abrjjjs0\nbNgwpaWladmyZRo/frzhHcMS9xEEYKYwCDp/dDb+pC4agBIRiIAJ9X0EbT8IzPO6DZ4alqS6ujot\nWrRIO3bsUFxcnGbOnKnp06dLklJSUrR8+XLPKt+7776rp556Sp9//rkuu+wyLVy40GvFb9u2bVq9\nerVOnz6tzMxMLVmyRF/96lcNz4UQBKwowoLNbGYEoBQ+EUgAIpIQguGFU8OAVRB/pj5fqCPQSPxJ\nfl4YTwQClkMIAl2dhQPQ7PhrFMoINBqAEquAANpHCALoUgIVf41CEYH+xJ9EAAIwjhAE0CUEOgCl\n4Eagv/EncRoYgP8CFoJ33XWXJkyY4Nn10pKjR4/qkUce0cGDBzVgwADNnz9f1113XaCmBKCLCUb8\nBUtHwk/qwM1xCUAATZgegm63W0uXLtVHH32kCRMmtDn2nnvuUUpKil5++WXt3LlTubm52r59uy6+\n+GKzpwVY16aSLnedYKgCcEXNPNNWBTsafo0IQABmMDUEq6qq9OCDD+ro0aM+H5PS3K5du/TFF1/o\n97//vWJiYjRr1izt2rVLW7ZsUW5urpnTAtBFhMMKYEdisLPR1xQBCMBMpoZgWVmZ+vfvrzVr1ui2\n225rc2xJSYmuvPJKxcTEeI5lZGTo4MGDZk4JgBTxq4LhEIBNmRl2RnTos1EJQAAGmBqC48eP1/jx\n4w2Nra6u9vlQ5MTERFVVVZk5JQCNGsMgwoIw3CIwWDoUfxIBCMAvfoWg0+lsNdSSkpIUGxtr+Llq\na2sVHR3tdSw6Oloul8ufKQHwV4SvDnZ1BCC6uu7du4V6CmjCrxA8dOiQ7rjjDtlsNp/HCgoKdMMN\nNxh+rpiYGJ0+fdrrmMvlkt1u92dKADqCGAwbHQ6/RgQgIkx8vPFFIwSeXyE4evRolZeXm/KLk5OT\ndeTIEa9jDodDSUlJpjw/gHZEyKliV258lzs93On4kwhARKwzZ2qJwTASshtKDxs2TOvWrZPL5fKc\nIt63b59GjhwZqikB1hQBQRjpMWhK+EnEH7qE+vqGUE8BTQQ1BGtqamS32/WVr3xFo0ePVr9+/fTw\nww/rpz/9qf70pz+ptLRUy5cvD+aUADQK8yBsGlPhHIWmRV9TBCCAAAlYCLZ0HeHkyZN12223KTc3\nV926ddMzzzyjBQsW6Pbbb9d//Md/6Fe/+hU3kwZCLcyDUPKNrVCFYUCirxHxByAIbG632x3qSXSU\nzcYflEDAhXEQGtGRSAxo4LWF+IMF1NQMVq9eF4Xs99t+EJjndf82MM8baCG7RhBAhGgaJxEYhSGL\nOqOIPwAhRAgCMC7CozBsEH8AwgQhCKBjiELjCD8AYYoQBNB5zUPH6mFI+AGIEIQgAPNZLQwJPwAR\nihAEEHithVKkBSLBB6CLIQQBhE5bYRWKSCT0AFgMIQggPBFlABBw3UI9AQAAAIQGIQgAAGBRhCAA\nAIBFEYIAAAAWRQgCAABYFCEIAABgUYQgAACARRGCAAAAFkUIAgAAWBQhCAAAYFGEIAAAgEURggAA\nABZFCAIAAFgUIQgAAGBRhCAAAIBFEYIAAAAWRQgCAABYFCEIAABgUYQgAACARRGCAAAAFkUIAgAA\nWBQhCAAAYFGEIAAAgEURggAAABZFCAIAAFgUIQgAABBELpdLCxYs0KhRo5SVlaWioqJ2f6a4uFjZ\n2dk+x0eOHKnU1FSlpKQoJSVFqampqq2tNTyXKL9mDgAAgE5ZsWKFysrKtH79eh09elTz5s3TgAED\ndNNNN7U4/i9/+Yt+9rOfKSYmxut4VVWV/vnPf2rnzp2y2+2e47GxsYbnwoogAABAkNTW1mrLli1a\nuHChUlJSlJ2drZkzZ2rDhg0tjv/d736nadOmqU+fPj6PVVZWKikpSQMGDFBiYqLnyx+EIAAAQJCU\nl5ervr5e6enpnmMZGRkqKSlpcfwHH3ygJ598Ujk5OT6PHTlyRAMHDuzUfAhBAACAIKmurlZCQoKi\nor68Oi8xMVFOp1MnT570GV9QUNDitYGSVFFRodraWk2fPl2ZmZmaNWuWPvvsM7/mQwgCAAAESW1t\nraKjo72ONX7vcrn8eq7KykqdOXNG99xzj9auXSu73a4777xT58+fN/wcbBYBAAAIkpiYGJ/ga/ze\nn00eklRYWKh///vfnp9btWqVxo0bp3feeUff/va3DT0HIQgAAKxjU8vX4nXab4caGpacnKxTp06p\noaFB3bpdODHrcDhkt9sVHx/v16/s0aOHevTo4fk+Ojpal1xyiaqqqgw/R8BODd91113atm1bm2OW\nLl3quedN4z83btwYqCkBAACEVGpqqqKionTw4EHPseLiYqWlpfn9XDfeeKNXa50/f15///vfNWjQ\nIMPPYfqKoNvt1tKlS/XRRx9pwoQJbY6trKzU3Llz9d3vftdzrGfPnmZPCQAAICzY7XZNnDhR+fn5\nWrZsmaqqqlRUVKTly5dLurA6GBcX53PPwJaMGzdOa9asUf/+/dWrVy+tXr1a/fr107hx4wzPx9QV\nwaqqKuXk5Oidd94xtLxZUVGhIUOGeN37xsgLBwAAiFTz589XWlqacnJytGTJEt13332encGZmZna\nvn27oed56KGHdPPNN2vu3LmaOnWqGhoa9Pzzz8tmsxmei83tdrs79Cpa8M4772jHjh16+OGHddtt\nt+nee+/VpEmTWhx77tw5jRw5Un/605/Uv3//Dv0+my1A5/kBAEBA1NQMVq9eF4Xs9weqHdxuY9cI\nhhtTTw2PHz9e48ePNzS2srJSNptNa9eu1XvvvaeEhATNmDGj1XAEAACAufwKQafT2epOlKSkJL+2\nPVdWVqpbt24aPHiwpk+frj179uiRRx5Rz549W71xIgAAiGzdu3ML43DiVwgeOnRId9xxR4vnngsK\nCnTDDTcYfq5Jkybp+uuv91xLePnll+uzzz7Tpk2bCEEAALqo+Hj/7pWHwPIrBEePHq3y8nLTfnnz\nDSWDBg3S7t27TXt+AAAQXs6cqSUGw0jI1mfXrFmjGTNmeB375JNPdOmll4ZoRgAAINDq6xtCPQU0\nEdQQrKmp8Xz+3fjx47V3714VFRXpiy++0G9/+1u99tprmjlzZjCnBAAAYFkBC8GWriOcPHmyXnjh\nBUnSVVddpTVr1mjbtm2aMGGCNm7cqKeeekpDh0bm9msAAIBIY+p9BION+wgCABBZuI9geGEPNwAA\ngEURggAAABZFCAIAAFgUIQgAAGBRhCAAAIBFEYIAAAAWRQgCAABYFCEIAABgUYQgAACARRGCAAAA\nFkUIAgAAWBQhCAAAYFGEIAAAgEURggAAABZFCAIAAFgUIQgAAGBRhCAAAIBFEYIAAAAWRQgCAABY\nFCEIAABgUYQgAACARRGCAAAAFkUIAgAAWBQhCAAAYFGEIAAAgEURggAAABZFCAIAAFgUIQgAAGBR\nhCAAAIBFEYIAAAAWRQgCAABYFCEIAABgUYQgAACARRGCAAAAFkUIAgAAWBQhCAAAYFGEIAAAgEUR\nggAAABZFCAIAAFgUIQgAAGBRhCAAAIBFmRqCZ8+eVV5enq677jqNGTNG8+fP19mzZ1sdf/ToUc2Y\nMUPDhw/Xd77zHX344YdmTgcAACDsuFwuLViwQKNGjVJWVpaKiopaHVtWVqapU6cqPT1dU6ZM0eHD\nh70ef/3113XjjTdq+PDhys3N1cmTJ/2ai6kh+Oijj+qvf/2r1q1bpxdeeEEVFRV65JFHWh1/zz33\nqG/fvnr55Zd16623Kjc3V8ePHzdzSgAAAGFlxYoVKisr0/r165Wfn6+CggK99dZbPuNqa2s1a9Ys\njRo1Slu3blV6erpmz56turo6SVJJSYkWLlyoOXPmaPPmzTp9+rTmz5/v11xsbrfbbcaLqq2t1ahR\no7Rp0yZdddVVkqSDBw/qhz/8ofbv36/o6Giv8bt27dI999yjXbt2KSYmRpI0Y8YMZWRkKDc319jk\nbSVmTB0AAARJTc1g9ep1Uch+f6Dawe0eamhcbW2trrnmGhUWFmrkyJGSpLVr12rXrl168cUXvcZu\n2bJFzz33nN5++23PsZtvvll33323Jk2apHnz5qlbt2564oknJEnHjx/X+PHjtXPnTg0YMMDQfExb\nEezWrZueffZZpaSkeI653W7V19fr/PnzPuNLSkp05ZVXeiJQkjIyMnTw4EGzpgQAABBWysvLVV9f\nr/T0dM+xjIwMlZT4BmpJSYkyMjK8jo0YMUIHDhyQdGHBbdSoUZ7HLr74YvXr10+HDh0yPB/TQjAm\nJkaZmZnq0aOH59iLL76oK664QgkJCT7jq6ur1bdvX69jiYmJqqqqMmtKAAAAYaW6uloJCQmKiory\nHEtMTJTT6fS5vu/EiRNttlJLLdWnTx+/LrOLan/Il5xOZ6uhlpSUpNjYWM/3GzZs0I4dO1RYWNji\n+NraWp/TxdHR0XK5XP5MCQAARJDu3a19w5LW+keSTwPV1dW12UrtPW6EXyF46NAh3XHHHbLZbD6P\nFRQU6IYbbpAkbdy4UY8//rjy8vI0ZsyYFp8rJiZGp0+f9jrmcrlkt9sNz8fo+XgAAAAp9O0QExPj\nE2qN3zddUGtrbGMrtfe4EX6F4OjRo1VeXt7mmMLCQq1cuVIPP/ywfvjDH7Y6Ljk5WUeOHPE65nA4\nlJSU5M+UAAAAIkZycrJOnTqlhoYGdet2YXXU4XDIbrcrPj7eZ2x1dbXXsaat1LdvXzkcDp/Hm58u\nboup67OvvPKKVq1apby8PN15551tjh02bJjKysq8Snbfvn1eF08CAAB0JampqYqKivLaHFtcXKy0\ntDSfscOGDfNsDGl04MABDR8+XJKUnp6uffv2eR47duyYjh8/rmHDhhmej2khePr0aS1ZskSTJk3S\nLbfcIofD4flqaGiQJNXU1Hh2EI8ePVr9+vXTww8/rCNHjuj5559XaWmpJk+ebNaUAAAAwordbtfE\niROVn5+v0tJS7dy5U0VFRcrJyZF0YUXP6XRKunCrmLNnz2rZsmWqqKjQ0qVLdf78eX3zm9+UJE2b\nNk2vvvqqtmzZovLycs2bN0/jx483fOsYycT7CP7hD3/QAw884HXM7XbLZrPpj3/8o/r376/rr79e\nt912m+c+gV988YUWLFigkpIS/cd//Ify8vJ0zTXXmDEdAACAsFRXV6dFixZpx44diouL08yZMzV9\n+nRJUkpKipYvX65JkyZJkkpLS5Wfn6/KykpdccUVWrRokdet+rZt26bVq1fr9OnTyszM1JIlS/TV\nr37V8FxMC0EAAABEFmvv4QYAALAwQhAAAMCiCEEAAACLIgQBAAAsihAEAACwqIgJwbNnzyovL0/X\nXXedxowZo/nz5+vs2bOtjj969KhmzJih4cOH6zvf+Y4+/PDDIM42MO666y5t27atzTFLly5VSkqK\nUlNTPf/cuHFjkGYYGEZed1d6v1etWqUxY8bo6quv1sqVK9scG8nvt8vl0oIFCzRq1ChlZWWpqKio\n1bFlZWWaOnWq0tPTNWXKFB0+fDiIMzWXP6/77rvv9nl/33333SDO1nwul0sTJkzQ3r17Wx3Tld7v\nRkZed1d6v6uqqnTvvffq6quv1rhx47R8+fJWP/+2K77fEcUdIX72s5+5J0+e7D58+LC7rKzMPWXK\nFPd9993X6vhbb73V/dBDD7krKirczz33nDs9Pd197NixIM7YPA0NDe7Fixe7U1JS3K+88kqbY2fM\nmOFet26d2+FweL7q6uqCNFNz+fO6u8r7XVhY6P7GN77h3r9/v3v37t3urKws9wsvvNDq+Eh+vxcv\nXuyeOHGi+5NPPnG//fbb7hEjRrh37NjhM+78+fPu6667zv3kk0+6Kyoq3EuXLnVfd9117tra2hDM\nukhac8IAAAgrSURBVPOMvm632+2+6aab3K+//rrX++tyuYI8Y/M4nU73Pffc405JSXHv2bOnxTFd\n7f12u429bre7a73fU6dOdc+aNct95MgRd3Fxsfumm25yP/nkkz7juuL7HWkiIgTPnz/vvvLKK90l\nJSWeYwcOHHBfeeWVbqfT6TP+o48+cg8fPtzr/xDvvPNO93/+538GZb5mOn78uHv69Onu8ePHu0eP\nHt1uEI0dO9b94YcfBml2gePP6+5K7/c3vvENr9f66quvuq+//vpWx0fq+33+/Hn30KFD3Xv37vUc\ne+aZZ9zTp0/3GfvSSy+5s7OzvY7ddNNN7f67EI78ed1Op9M9ZMgQ92effRbMKQbMkSNH3BMnTnRP\nnDixzSDqSu+32238dXel97uiosKdkpLi/n//7/95jr3++uvusWPH+oztau93JIqIU8PdunXTs88+\n63Unbbfbrfr6es9H1jVVUlKiK6+8UjExMZ5jGRkZXp/rFynKysrUv39/bd26VRdddFGbY8+dO6eq\nqioNHDgwOJMLIH9ed1d5v0+cOKFjx45p5MiRnmMZGRn6xz/+4fOh4lJkv9/l5eWqr6/3+mzxjIwM\nlZSU+IwtKSlRRkaG17ERI0b4fP5mJPDndX/66aey2Wy65JJLgjnFgNmzZ4/GjBmjzZs3y93G5xh0\npfdbMv66u9L7nZSUpHXr1ql3796eY263u8XLubra+x2JokI9ASNiYmKUmZnpdezFF1/UFVdcoYSE\nBJ/x1dXV6tu3r9exxMREVVVVBXSegTB+/HiNHz/e0NjKykrZbDatXbtW7733nhISEjRjxgzPx9RE\nEn9ed1d5v6urq2Wz2bxeS58+feR2u3X8+HH16dPHa3wkv9/V1dVKSEhQVNSXfwQlJibK6XTq5MmT\n6tWrl+f4iRMndPnll3v9fGJioo4cORK0+ZrFn9ddUVGhnj176qGHHtLu3bvVr18/zZkzR2PHjg3F\n1Dtt2rRphsZ1pfdbMv66u9L7HRcX5/X/2W63Wxs2bNC1117rM7arvd+RKGxC0Ol0tvp/3ElJSYqN\njfV8v2HDBu3YsUOFhYUtjq+trVV0dLTXsejo6FYvVA0lf153eyorK9WtWzcNHjxY06dP1549e/TI\nI4+oZ8+eys7ONmvKpjDzdXeV97txdbvpa2n8zy29lkh6v5tr7T2TfF9rXV1dxLy/7fHndVdWVsrp\ndCorK0uzZs3S22+/rbvvvlu///3vdeWVVwZtzsHWld5vf3Tl9/vJJ59UeXm5Xn75ZZ/HrPp+h5Ow\nCcFDhw7pjjvukM1m83msoKBAN9xwgyRp48aNevzxx5WXl6cxY8a0+FwxMTE6ffq01zGXyyW73W7+\nxDvJ6Os2YtKkSbr++usVHx8vSbr88sv12WefadOmTWEXBma+7q7yfs+dO1fShbk3j4OWwjiS3u/m\nYmJifP6gb+21tjY2HN/f9vjzunNzc5WTk6O4uDhJ0hVXXKGPP/5Ymzdv1uLFi4Mz4RDoSu+3P7rq\n+71y5UqtX79ev/zlLzV48GCfx636foeTsAnB0aNHq7y8vM0xhYWFWrlypR5++GH98Ic/bHVccnKy\nz7Kyw+FQUlKSKXM1k5HX7Y/GKGg0aNAg7d6927TnN4uZr7urvN8nTpzQqlWr5HA41L9/f0lfni5u\n7bVEyvvdXHJysk6dOqWGhgZ163bhUmWHwyG73e7zmpKTk1VdXe11LFzf3/b487oleaKg0eDBg1VR\nURGUuYZKV3q//dXV3u8lS5Zo8+bNWrlyZat/ObXy+x0uImKziCS98sorWrVqlfLy8nTnnXe2OXbY\nsGEqKyvz+lvGvn37vC7Q7orWrFmjGTNmeB375JNPdOmll4ZoRsHRVd7vvn37ql+/ftq3b5/nWHFx\nsfr16+dzfaAU2e93amqqoqKivDb0FBcXKy0tzWfssGHDfC4cP3DgQMS9v5J/r3v+/PnKy8vzOlZe\nXh4R729ndKX32x9d7f0uKCjQ5s2b9Ytf/EK33HJLq+Os+n6Hk4gIwdOnT2vJkiWaNGmSbrnlFjkc\nDs9XQ0ODJKmmpsZzjdX/b+f+XdKJ4ziOv79b/0J/wTmduBxNgQYJEkYi0aC4iH/DbS0iDuEkiC4H\nDS0tceBUzZGKgze4eIvS1hYUDvJui+zLtx9wfa/ung+46bO837yGzwsOPpZlyebmpti2LbPZTHq9\nnnieJ8ViMcw1vsXrvdPptAyHQ3EcRxaLhZydnYnrulKtVkOeMnhRzfvo6EhOTk5kMBjI7e2ttFot\nqVQqL+dRyXtjY0P29/fl+PhYPM+Tq6srcRznZdf7+3tZLpciIpLNZuXh4UEajYb4vi/1el0eHx/f\nvVx+qq/svbOzI67rysXFhcznc2m32zIej6VcLoe5wreIat4fiWrevu9Lp9ORWq0mqVRq7c4WiW/e\nP1aYb9d8Vr/f10QisfYZhqGJRELv7u5UVTWdTq+9Gzefz7VUKqlpmrq3t6c3NzdhjR+YTCbz19tK\nb/e+vr7WfD6vyWRSc7mcXl5e/u8xA/eZvaOS92q10mazqZZl6dbWlrZarbXzKOX99PSktm1rKpXS\n7e1tPT09fTkzDGMt88lkogcHB5pMJvXw8FCn02kYIwfiK3ufn5/r7u6umqaphUJBR6NRGCMH7u17\nelHO+7WP9o5K3t1u9593tmp88v4t/qi+87ARAAAAIutX/BoGAABA8CiCAAAAMUURBAAAiCmKIAAA\nQExRBAEAAGKKIggAABBTFEEAAICYoggCAADEFEUQAAAgpiiCAAAAMUURBAAAiKlnjIkzY083YrUA\nAAAASUVORK5CYII=\n", 173 | "text/plain": [ 174 | "" 175 | ] 176 | }, 177 | "metadata": {}, 178 | "output_type": "display_data" 179 | } 180 | ], 181 | "source": [ 182 | "x = np.arange(-2., 2., 0.01)\n", 183 | "y = np.arange(-2., 2., 0.01)\n", 184 | "X, Y = np.meshgrid(x, y)\n", 185 | "\n", 186 | "Z = p(X, Y)\n", 187 | "plt.figure()\n", 188 | "plt.contourf(X, Y, Z)\n", 189 | "plt.colorbar();\n", 190 | "plt.ylim([-2, 2])\n", 191 | "plt.xlim([-2, 2]);" 192 | ] 193 | }, 194 | { 195 | "cell_type": "markdown", 196 | "metadata": {}, 197 | "source": [ 198 | "Sample from `p` using Rejection Sampling \n", 199 | "\n", 200 | "[Reference](http://python-for-signal-processing.blogspot.com/2014/02/methods-of-random-sampling-using.html)" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "metadata": { 207 | "collapsed": false 208 | }, 209 | "outputs": [], 210 | "source": [ 211 | "M = 1.1 # scale factor\n", 212 | "m = 10000\n", 213 | "ux = np.random.uniform(-0.5, 1.3, m)\n", 214 | "uy = np.random.uniform(-1.2, 1.3, m)\n", 215 | "uz = np.random.uniform(size=m)\n", 216 | "idx= where(uz <= p(ux,uy)/M)[0] # rejection criterion\n", 217 | "ux_, uy_ = ux[idx], uy[idx]\n", 218 | "\n", 219 | "ax = sns.kdeplot(ux_, uy_)\n", 220 | "ax.set_title('Estimated Efficency=%3.1f%%'%(100*len(ux_)/len(ux)))\n", 221 | "plt.ylim([-2, 2])\n", 222 | "plt.xlim([-2, 2]);" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": null, 228 | "metadata": { 229 | "collapsed": true 230 | }, 231 | "outputs": [], 232 | "source": [] 233 | } 234 | ], 235 | "metadata": { 236 | "kernelspec": { 237 | "display_name": "Python 2", 238 | "language": "python", 239 | "name": "python2" 240 | }, 241 | "language_info": { 242 | "codemirror_mode": { 243 | "name": "ipython", 244 | "version": 2 245 | }, 246 | "file_extension": ".py", 247 | "mimetype": "text/x-python", 248 | "name": "python", 249 | "nbconvert_exporter": "python", 250 | "pygments_lexer": "ipython2", 251 | "version": "2.7.10" 252 | } 253 | }, 254 | "nbformat": 4, 255 | "nbformat_minor": 1 256 | } 257 | --------------------------------------------------------------------------------