├── .gitignore ├── _config.yml ├── dis_loss.png ├── gen_loss.png ├── experiment.pkl ├── loss.py ├── discriminator.py ├── README.md ├── rollout.py ├── target_lstm.py ├── data_iter.py ├── generator.py ├── main.py └── experiment_notebook.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-cayman -------------------------------------------------------------------------------- /dis_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-czh/SeqGAN-PyTorch/HEAD/dis_loss.png -------------------------------------------------------------------------------- /gen_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-czh/SeqGAN-PyTorch/HEAD/gen_loss.png -------------------------------------------------------------------------------- /experiment.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-czh/SeqGAN-PyTorch/HEAD/experiment.pkl -------------------------------------------------------------------------------- /loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | class PGLoss(nn.Module): 6 | """ 7 | Pseudo-loss that gives corresponding policy gradients (on calling .backward()) 8 | for adversial training of Generator 9 | """ 10 | 11 | def __init__(self): 12 | super(PGLoss, self).__init__() 13 | 14 | def forward(self, pred, target, reward): 15 | """ 16 | Inputs: pred, target, reward 17 | - pred: (batch_size, seq_len), 18 | - target : (batch_size, seq_len), 19 | - reward : (batch_size, ), reward of each whole sentence 20 | """ 21 | one_hot = torch.zeros(pred.size(), dtype=torch.uint8) 22 | if pred.is_cuda: 23 | one_hot = one_hot.cuda() 24 | one_hot.scatter_(1, target.data.view(-1, 1), 1) 25 | loss = torch.masked_select(pred, one_hot) 26 | loss = loss * reward.contiguous().view(-1) 27 | loss = -torch.sum(loss) 28 | return loss 29 | -------------------------------------------------------------------------------- /discriminator.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | 6 | class Discriminator(nn.Module): 7 | """ 8 | A CNN for text classification. 9 | Uses an embedding layer, followed by a convolutional, max-pooling and softmax layer. 10 | Highway architecture based on the pooled feature maps is added. Dropout is adopted. 11 | """ 12 | 13 | def __init__(self, num_classes, vocab_size, embedding_dim, filter_sizes, num_filters, dropout_prob): 14 | super(Discriminator, self).__init__() 15 | self.embed = nn.Embedding(vocab_size, embedding_dim) 16 | self.convs = nn.ModuleList([ 17 | nn.Conv2d(1, num_f, (f_size, embedding_dim)) for f_size, num_f in zip(filter_sizes, num_filters) 18 | ]) 19 | self.highway = nn.Linear(sum(num_filters), sum(num_filters)) 20 | self.dropout = nn.Dropout(p = dropout_prob) 21 | self.fc = nn.Linear(sum(num_filters), num_classes) 22 | 23 | def forward(self, x): 24 | """ 25 | Inputs: x 26 | - x: (batch_size, seq_len) 27 | Outputs: out 28 | - out: (batch_size, num_classes) 29 | """ 30 | emb = self.embed(x).unsqueeze(1) # batch_size, 1 * seq_len * emb_dim 31 | convs = [F.relu(conv(emb)).squeeze(3) for conv in self.convs] # [batch_size * num_filter * seq_len] 32 | pools = [F.max_pool1d(conv, conv.size(2)).squeeze(2) for conv in convs] # [batch_size * num_filter] 33 | out = torch.cat(pools, 1) # batch_size * sum(num_filters) 34 | highway = self.highway(out) 35 | transform = F.sigmoid(highway) 36 | out = transform * F.relu(highway) + (1. - transform) * out # sets C = 1 - T 37 | out = F.log_softmax(self.fc(self.dropout(out)), dim=1) # batch * num_classes 38 | return out 39 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SeqGAN-PyTorch 2 | An implementation of SeqGAN (Paper: [SeqGAN: Sequence Generative Adversarial Nets with Policy Gradient](https://arxiv.org/pdf/1609.05473.pdf)) in PyTorch. The code performs the experiment on synthetic data as described in the paper. 3 | 4 | ## Usage 5 | ``` 6 | $ python main.py 7 | ``` 8 | Please refer to ```main.py``` for supported arguments. You can also change model parameters there. 9 | 10 | ## Dependency 11 | * PyTorch 0.4.0+ (1.0 ready) 12 | * Python 3.5+ 13 | * CUDA 8.0+ & cuDNN (For GPU) 14 | * numpy 15 | 16 | ## Hacks and Observations 17 | - Using Adam for Generator and SGD for Discriminator 18 | - Discriminator should neither be trained too powerful (fail to provide useful feedback) nor too ill-performed (randomly guessing, unable to guide generation) 19 | - The GAN phase may not always lead to massive drops in NLL (sometimes very minimal or even increases NLL) 20 | 21 | ## Sample Learning Curve 22 | Learning curve of generator obtained after MLE training for 120 steps (1 epoch per round) followed by adversarial training for 150 rounds (1 epoch per round): 23 | 24 | ![alt tag](https://raw.githubusercontent.com/X-czh/SeqGAN-PyTorch/master/gen_loss.png) 25 | 26 | Learning curve of discriminator obtained after MLE training for 50 steps (3 epochs per step) followed by adversarial training for 150 rounds (9 epoch per round): 27 | 28 | ![alt tag](https://raw.githubusercontent.com/X-czh/SeqGAN-PyTorch/master/dis_loss.png) 29 | 30 | 31 | ## Acknowledgement 32 | This code is based on Zhao Zijian's [SeqGAN-PyTorch](https://github.com/ZiJianZhao/SeqGAN-PyTorch), Surag Nair's [SeqGAN](https://github.com/suragnair/seqGAN) and Lantao Yu's original [implementation](https://github.com/LantaoYu/SeqGAN) in Tensorflow. Many thanks to [Zhao Zijian](https://github.com/ZiJianZhao), [Surag Nair](https://github.com/suragnair) and [Lantao Yu](https://github.com/LantaoYu)! 33 | -------------------------------------------------------------------------------- /rollout.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.optim as optim 7 | 8 | 9 | class Rollout(object): 10 | """ Rollout Policy """ 11 | 12 | def __init__(self, model, update_rate): 13 | self.ori_model = model 14 | self.own_model = copy.deepcopy(model) 15 | self.update_rate = update_rate 16 | 17 | def get_reward(self, x, num, discriminator): 18 | """ 19 | Inputs: x, num, discriminator 20 | - x: (batch_size, seq_len) input data 21 | - num: rollout number 22 | - discriminator: discrimanator model 23 | """ 24 | rewards = [] 25 | batch_size = x.size(0) 26 | seq_len = x.size(1) 27 | for i in range(num): 28 | for l in range(1, seq_len): 29 | data = x[:, 0:l] 30 | samples = self.own_model.sample(batch_size, seq_len, data) 31 | pred = discriminator(samples) 32 | pred = pred.cpu().data[:,1].numpy() 33 | if i == 0: 34 | rewards.append(pred) 35 | else: 36 | rewards[l-1] += pred 37 | 38 | # for the last token 39 | pred = discriminator(x) 40 | pred = pred.cpu().data[:, 1].numpy() 41 | if i == 0: 42 | rewards.append(pred) 43 | else: 44 | rewards[seq_len-1] += pred 45 | rewards = np.transpose(np.array(rewards)) / (1.0 * num) # batch_size * seq_len 46 | return rewards 47 | 48 | def update_params(self): 49 | dic = {} 50 | for name, param in self.ori_model.named_parameters(): 51 | dic[name] = param.data 52 | for name, param in self.own_model.named_parameters(): 53 | if name.startswith('emb'): 54 | param.data = dic[name] 55 | else: 56 | param.data = self.update_rate * param.data + (1 - self.update_rate) * dic[name] 57 | -------------------------------------------------------------------------------- /target_lstm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | 6 | class TargetLSTM(nn.Module): 7 | """ Target LSTM """ 8 | 9 | def __init__(self, vocab_size, embedding_dim, hidden_dim, use_cuda): 10 | super(TargetLSTM, self).__init__() 11 | self.hidden_dim = hidden_dim 12 | self.use_cuda = use_cuda 13 | self.embed = nn.Embedding(vocab_size, embedding_dim) 14 | self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True) 15 | self.fc = nn.Linear(hidden_dim, vocab_size) 16 | self.log_softmax = nn.LogSoftmax(dim=1) 17 | self.init_params() 18 | 19 | def forward(self, x): 20 | """ 21 | Embeds input and applies LSTM on the input sequence. 22 | 23 | Inputs: x 24 | - x: (batch_size, seq_len), sequence of tokens generated by generator 25 | Outputs: out 26 | - out: (batch_size, vocab_size), lstm output prediction 27 | """ 28 | self.lstm.flatten_parameters() 29 | h0, c0 = self.init_hidden(x.size(0)) 30 | emb = self.embed(x) # batch_size * seq_len * emb_dim 31 | out, _ = self.lstm(emb, (h0, c0)) # out: seq_len * batch_size * hidden_dim 32 | out = self.log_softmax(self.fc(out.contiguous().view(-1, self.hidden_dim))) # seq_len * batch_size * vocab_size 33 | return out 34 | 35 | def step(self, x, h, c): 36 | """ 37 | Embeds input and applies LSTM one token at a time (seq_len = 1). 38 | 39 | Inputs: x, h, c 40 | - x: (batch_size, 1), sequence of tokens generated by generator 41 | - h: (1, batch_size, hidden_dim), lstm hidden state 42 | - c: (1, batch_size, hidden_dim), lstm cell state 43 | Outputs: out, h, c 44 | - out: (batch_size, 1, vocab_size), lstm output prediction 45 | - h: (1, batch_size, hidden_dim), lstm hidden state 46 | - c: (1, batch_size, hidden_dim), lstm cell state 47 | """ 48 | self.lstm.flatten_parameters() 49 | emb = self.embed(x) # batch_size * 1 * emb_dim 50 | out, (h, c) = self.lstm(emb, (h, c)) # out: batch_size * 1 * hidden_dim 51 | out = self.log_softmax(self.fc(out.contiguous().view(-1, self.hidden_dim))) # batch_size * vocab_size 52 | return out, h, c 53 | 54 | def init_hidden(self, batch_size): 55 | h = torch.zeros((1, batch_size, self.hidden_dim)) 56 | c = torch.zeros((1, batch_size, self.hidden_dim)) 57 | if self.use_cuda: 58 | h, c = h.cuda(), c.cuda() 59 | return h, c 60 | 61 | def init_params(self): 62 | for param in self.parameters(): 63 | param.data.normal_(0, 1) 64 | 65 | def sample(self, batch_size, seq_len): 66 | """ 67 | Samples the network and returns a batch of samples of length seq_len. 68 | 69 | Outputs: out 70 | - out: (batch_size * seq_len) 71 | """ 72 | samples = [] 73 | h, c = self.init_hidden(batch_size) 74 | x = torch.zeros(batch_size, 1, dtype=torch.int64) 75 | if self.use_cuda: 76 | x = x.cuda() 77 | for _ in range(seq_len): 78 | out, h, c = self.step(x, h, c) 79 | prob = torch.exp(out) 80 | x = torch.multinomial(prob, 1) 81 | samples.append(x) 82 | out = torch.cat(samples, dim=1) # along the batch_size dimension 83 | return out 84 | -------------------------------------------------------------------------------- /data_iter.py: -------------------------------------------------------------------------------- 1 | import math 2 | import random 3 | import torch 4 | 5 | 6 | class GenDataIter: 7 | """ Toy data iter to load digits """ 8 | 9 | def __init__(self, data_file, batch_size): 10 | super(GenDataIter, self).__init__() 11 | self.batch_size = batch_size 12 | self.data_lis = self.read_file(data_file) 13 | self.data_num = len(self.data_lis) 14 | self.indices = range(self.data_num) 15 | self.num_batches = math.ceil(self.data_num / self.batch_size) 16 | self.idx = 0 17 | self.reset() 18 | 19 | def __len__(self): 20 | return self.num_batches 21 | 22 | def __iter__(self): 23 | return self 24 | 25 | def __next__(self): 26 | return self.next() 27 | 28 | def reset(self): 29 | self.idx = 0 30 | random.shuffle(self.data_lis) 31 | 32 | def next(self): 33 | if self.idx >= self.data_num: 34 | raise StopIteration 35 | index = self.indices[self.idx : self.idx + self.batch_size] 36 | d = [self.data_lis[i] for i in index] 37 | d = torch.tensor(d) 38 | 39 | # 0 is prepended to d as start symbol 40 | data = torch.cat([torch.zeros(len(index), 1, dtype=torch.int64), d], dim=1) 41 | target = torch.cat([d, torch.zeros(len(index), 1, dtype=torch.int64)], dim=1) 42 | 43 | self.idx += self.batch_size 44 | return data, target 45 | 46 | def read_file(self, data_file): 47 | with open(data_file, 'r') as f: 48 | lines = f.readlines() 49 | lis = [] 50 | for line in lines: 51 | l = [int(s) for s in list(line.strip().split())] 52 | lis.append(l) 53 | return lis 54 | 55 | 56 | class DisDataIter: 57 | """ Toy data iter to load digits """ 58 | 59 | def __init__(self, real_data_file, fake_data_file, batch_size): 60 | super(DisDataIter, self).__init__() 61 | self.batch_size = batch_size 62 | real_data_lis = self.read_file(real_data_file) 63 | fake_data_lis = self.read_file(fake_data_file) 64 | self.data = real_data_lis + fake_data_lis 65 | self.labels = [1 for _ in range(len(real_data_lis))] +\ 66 | [0 for _ in range(len(fake_data_lis))] 67 | self.pairs = list(zip(self.data, self.labels)) 68 | self.data_num = len(self.pairs) 69 | self.indices = range(self.data_num) 70 | self.num_batches = math.ceil(self.data_num / self.batch_size) 71 | self.idx = 0 72 | self.reset() 73 | 74 | def __len__(self): 75 | return self.num_batches 76 | 77 | def __iter__(self): 78 | return self 79 | 80 | def __next__(self): 81 | return self.next() 82 | 83 | def reset(self): 84 | self.idx = 0 85 | random.shuffle(self.pairs) 86 | 87 | def next(self): 88 | if self.idx >= self.data_num: 89 | raise StopIteration 90 | index = self.indices[self.idx : self.idx + self.batch_size] 91 | pairs = [self.pairs[i] for i in index] 92 | data = [p[0] for p in pairs] 93 | label = [p[1] for p in pairs] 94 | data = torch.tensor(data) 95 | label = torch.tensor(label) 96 | self.idx += self.batch_size 97 | return data, label 98 | 99 | def read_file(self, data_file): 100 | with open(data_file, 'r') as f: 101 | lines = f.readlines() 102 | lis = [] 103 | for line in lines: 104 | l = [int(s) for s in list(line.strip().split())] 105 | lis.append(l) 106 | return lis 107 | -------------------------------------------------------------------------------- /generator.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | 6 | class Generator(nn.Module): 7 | """ Generator """ 8 | 9 | def __init__(self, vocab_size, embedding_dim, hidden_dim, use_cuda): 10 | super(Generator, self).__init__() 11 | self.hidden_dim = hidden_dim 12 | self.use_cuda = use_cuda 13 | self.embed = nn.Embedding(vocab_size, embedding_dim) 14 | self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True) 15 | self.fc = nn.Linear(hidden_dim, vocab_size) 16 | self.log_softmax = nn.LogSoftmax(dim=1) 17 | self.init_params() 18 | 19 | def forward(self, x): 20 | """ 21 | Embeds input and applies LSTM on the input sequence. 22 | 23 | Inputs: x 24 | - x: (batch_size, seq_len), sequence of tokens generated by generator 25 | Outputs: out 26 | - out: (batch_size * seq_len, vocab_size), lstm output prediction 27 | """ 28 | self.lstm.flatten_parameters() 29 | h0, c0 = self.init_hidden(x.size(0)) 30 | emb = self.embed(x) # batch_size * seq_len * emb_dim 31 | out, _ = self.lstm(emb, (h0, c0)) # out: batch_size * seq_len * hidden_dim 32 | out = self.log_softmax(self.fc(out.contiguous().view(-1, self.hidden_dim))) # (batch_size*seq_len) * vocab_size 33 | return out 34 | 35 | def step(self, x, h, c): 36 | """ 37 | Embeds input and applies LSTM one token at a time (seq_len = 1). 38 | 39 | Inputs: x, h, c 40 | - x: (batch_size, 1), sequence of tokens generated by generator 41 | - h: (1, batch_size, hidden_dim), lstm hidden state 42 | - c: (1, batch_size, hidden_dim), lstm cell state 43 | Outputs: out, h, c 44 | - out: (batch_size, vocab_size), lstm output prediction 45 | - h: (1, batch_size, hidden_dim), lstm hidden state 46 | - c: (1, batch_size, hidden_dim), lstm cell state 47 | """ 48 | self.lstm.flatten_parameters() 49 | emb = self.embed(x) # batch_size * 1 * emb_dim 50 | out, (h, c) = self.lstm(emb, (h, c)) # out: batch_size * 1 * hidden_dim 51 | out = self.log_softmax(self.fc(out.contiguous().view(-1, self.hidden_dim))) # batch_size * vocab_size 52 | return out, h, c 53 | 54 | def init_hidden(self, batch_size): 55 | h = torch.zeros(1, batch_size, self.hidden_dim) 56 | c = torch.zeros(1, batch_size, self.hidden_dim) 57 | if self.use_cuda: 58 | h, c = h.cuda(), c.cuda() 59 | return h, c 60 | 61 | def init_params(self): 62 | for param in self.parameters(): 63 | param.data.uniform_(-0.05, 0.05) 64 | 65 | def sample(self, batch_size, seq_len, x=None): 66 | """ 67 | Samples the network and returns a batch of samples of length seq_len. 68 | 69 | Outputs: out 70 | - out: (batch_size * seq_len) 71 | """ 72 | samples = [] 73 | if x is None: 74 | h, c = self.init_hidden(batch_size) 75 | x = torch.zeros(batch_size, 1, dtype=torch.int64) 76 | if self.use_cuda: 77 | x = x.cuda() 78 | for _ in range(seq_len): 79 | out, h, c = self.step(x, h, c) 80 | prob = torch.exp(out) 81 | x = torch.multinomial(prob, 1) 82 | samples.append(x) 83 | else: 84 | h, c = self.init_hidden(x.size(0)) 85 | given_len = x.size(1) 86 | lis = x.chunk(x.size(1), dim=1) 87 | for i in range(given_len): 88 | out, h, c = self.step(lis[i], h, c) 89 | samples.append(lis[i]) 90 | prob = torch.exp(out) 91 | x = torch.multinomial(prob, 1) 92 | for _ in range(given_len, seq_len): 93 | samples.append(x) 94 | out, h, c = self.step(x, h, c) 95 | prob = torch.exp(out) 96 | x = torch.multinomial(prob, 1) 97 | out = torch.cat(samples, dim=1) # along the batch_size dimension 98 | return out 99 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import pickle as pkl 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.optim as optim 7 | import torch.backends.cudnn as cudnn 8 | 9 | from data_iter import DisDataIter, GenDataIter 10 | from generator import Generator 11 | from discriminator import Discriminator 12 | from target_lstm import TargetLSTM 13 | from rollout import Rollout 14 | from loss import PGLoss 15 | 16 | 17 | # Arguemnts 18 | parser = argparse.ArgumentParser(description='SeqGAN') 19 | parser.add_argument('--hpc', action='store_true', default=False, 20 | help='set to hpc mode') 21 | parser.add_argument('--data_path', type=str, default='/scratch/zc807/seq_gan/', metavar='PATH', 22 | help='data path to save files (default: /scratch/zc807/seq_gan/)') 23 | parser.add_argument('--rounds', type=int, default=150, metavar='N', 24 | help='rounds of adversarial training (default: 150)') 25 | parser.add_argument('--g_pretrain_steps', type=int, default=120, metavar='N', 26 | help='steps of pre-training of generators (default: 120)') 27 | parser.add_argument('--d_pretrain_steps', type=int, default=50, metavar='N', 28 | help='steps of pre-training of discriminators (default: 50)') 29 | parser.add_argument('--g_steps', type=int, default=1, metavar='N', 30 | help='steps of generator updates in one round of adverarial training (default: 1)') 31 | parser.add_argument('--d_steps', type=int, default=3, metavar='N', 32 | help='steps of discriminator updates in one round of adverarial training (default: 3)') 33 | parser.add_argument('--gk_epochs', type=int, default=1, metavar='N', 34 | help='epochs of generator updates in one step of generate update (default: 1)') 35 | parser.add_argument('--dk_epochs', type=int, default=3, metavar='N', 36 | help='epochs of discriminator updates in one step of discriminator update (default: 3)') 37 | parser.add_argument('--update_rate', type=float, default=0.8, metavar='UR', 38 | help='update rate of roll-out model (default: 0.8)') 39 | parser.add_argument('--n_rollout', type=int, default=16, metavar='N', 40 | help='number of roll-out (default: 16)') 41 | parser.add_argument('--vocab_size', type=int, default=10, metavar='N', 42 | help='vocabulary size (default: 10)') 43 | parser.add_argument('--batch_size', type=int, default=64, metavar='N', 44 | help='batch size (default: 64)') 45 | parser.add_argument('--n_samples', type=int, default=6400, metavar='N', 46 | help='number of samples gerenated per time (default: 6400)') 47 | parser.add_argument('--gen_lr', type=float, default=1e-3, metavar='LR', 48 | help='learning rate of generator optimizer (default: 1e-3)') 49 | parser.add_argument('--dis_lr', type=float, default=1e-3, metavar='LR', 50 | help='learning rate of discriminator optimizer (default: 1e-3)') 51 | parser.add_argument('--no_cuda', action='store_true', default=False, 52 | help='disables CUDA training') 53 | parser.add_argument('--seed', type=int, default=1, metavar='S', 54 | help='random seed (default: 1)') 55 | 56 | 57 | # Files 58 | POSITIVE_FILE = 'real.data' 59 | NEGATIVE_FILE = 'gene.data' 60 | 61 | 62 | # Genrator Parameters 63 | g_embed_dim = 32 64 | g_hidden_dim = 32 65 | g_seq_len = 20 66 | 67 | 68 | # Discriminator Parameters 69 | d_num_class = 2 70 | d_embed_dim = 64 71 | d_filter_sizes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20] 72 | d_num_filters = [100, 200, 200, 200, 200, 100, 100, 100, 100, 100, 160, 160] 73 | d_dropout_prob = 0.2 74 | 75 | 76 | def generate_samples(model, batch_size, generated_num, output_file): 77 | samples = [] 78 | for _ in range(int(generated_num / batch_size)): 79 | sample = model.sample(batch_size, g_seq_len).cpu().data.numpy().tolist() 80 | samples.extend(sample) 81 | with open(output_file, 'w') as fout: 82 | for sample in samples: 83 | string = ' '.join([str(s) for s in sample]) 84 | fout.write('{}\n'.format(string)) 85 | 86 | 87 | def train_generator_MLE(gen, data_iter, criterion, optimizer, epochs, 88 | gen_pretrain_train_loss, args): 89 | """ 90 | Train generator with MLE 91 | """ 92 | for epoch in range(epochs): 93 | total_loss = 0. 94 | for data, target in data_iter: 95 | if args.cuda: 96 | data, target = data.cuda(), target.cuda() 97 | target = target.contiguous().view(-1) 98 | output = gen(data) 99 | loss = criterion(output, target) 100 | total_loss += loss.item() 101 | optimizer.zero_grad() 102 | loss.backward() 103 | optimizer.step() 104 | data_iter.reset() 105 | avg_loss = total_loss / len(data_iter) 106 | print("Epoch {}, train loss: {:.5f}".format(epoch, avg_loss)) 107 | gen_pretrain_train_loss.append(avg_loss) 108 | 109 | 110 | def train_generator_PG(gen, dis, rollout, pg_loss, optimizer, epochs, args): 111 | """ 112 | Train generator with the guidance of policy gradient 113 | """ 114 | for epoch in range(epochs): 115 | # construct the input to the genrator, add zeros before samples and delete the last column 116 | samples = generator.sample(args.batch_size, g_seq_len) 117 | zeros = torch.zeros(args.batch_size, 1, dtype=torch.int64) 118 | if samples.is_cuda: 119 | zeros = zeros.cuda() 120 | inputs = torch.cat([zeros, samples.data], dim = 1)[:, :-1].contiguous() 121 | targets = samples.data.contiguous().view((-1,)) 122 | 123 | # calculate the reward 124 | rewards = torch.tensor(rollout.get_reward(samples, args.n_rollout, dis)) 125 | if args.cuda: 126 | rewards = rewards.cuda() 127 | 128 | # update generator 129 | output = gen(inputs) 130 | loss = pg_loss(output, targets, rewards) 131 | optimizer.zero_grad() 132 | loss.backward() 133 | optimizer.step() 134 | 135 | 136 | def eval_generator(model, data_iter, criterion, args): 137 | """ 138 | Evaluate generator with NLL 139 | """ 140 | total_loss = 0. 141 | with torch.no_grad(): 142 | for data, target in data_iter: 143 | if args.cuda: 144 | data, target = data.cuda(), target.cuda() 145 | target = target.contiguous().view(-1) 146 | pred = model(data) 147 | loss = criterion(pred, target) 148 | total_loss += loss.item() 149 | avg_loss = total_loss / len(data_iter) 150 | return avg_loss 151 | 152 | 153 | def train_discriminator(dis, gen, criterion, optimizer, epochs, 154 | dis_adversarial_train_loss, dis_adversarial_train_acc, args): 155 | """ 156 | Train discriminator 157 | """ 158 | generate_samples(gen, args.batch_size, args.n_samples, NEGATIVE_FILE) 159 | data_iter = DisDataIter(POSITIVE_FILE, NEGATIVE_FILE, args.batch_size) 160 | for epoch in range(epochs): 161 | correct = 0 162 | total_loss = 0. 163 | for data, target in data_iter: 164 | if args.cuda: 165 | data, target = data.cuda(), target.cuda() 166 | target = target.contiguous().view(-1) 167 | output = dis(data) 168 | pred = output.data.max(1)[1] 169 | correct += pred.eq(target.data).cpu().sum() 170 | loss = criterion(output, target) 171 | total_loss += loss.item() 172 | optimizer.zero_grad() 173 | loss.backward() 174 | optimizer.step() 175 | data_iter.reset() 176 | avg_loss = total_loss / len(data_iter) 177 | acc = correct.item() / data_iter.data_num 178 | print("Epoch {}, train loss: {:.5f}, train acc: {:.3f}".format(epoch, avg_loss, acc)) 179 | dis_adversarial_train_loss.append(avg_loss) 180 | dis_adversarial_train_acc.append(acc) 181 | 182 | 183 | def eval_discriminator(model, data_iter, criterion, args): 184 | """ 185 | Evaluate discriminator, dropout is enabled 186 | """ 187 | correct = 0 188 | total_loss = 0. 189 | with torch.no_grad(): 190 | for data, target in data_iter: 191 | if args.cuda: 192 | data, target = data.cuda(), target.cuda() 193 | target = target.contiguous().view(-1) 194 | output = model(data) 195 | pred = output.data.max(1)[1] 196 | correct += pred.eq(target.data).cpu().sum() 197 | loss = criterion(output, target) 198 | total_loss += loss.item() 199 | avg_loss = total_loss / len(data_iter) 200 | acc = correct.item() / data_iter.data_num 201 | return avg_loss, acc 202 | 203 | 204 | def adversarial_train(gen, dis, rollout, pg_loss, nll_loss, gen_optimizer, dis_optimizer, 205 | dis_adversarial_train_loss, dis_adversarial_train_acc, args): 206 | """ 207 | Adversarially train generator and discriminator 208 | """ 209 | # train generator for g_steps 210 | print("#Train generator") 211 | for i in range(args.g_steps): 212 | print("##G-Step {}".format(i)) 213 | train_generator_PG(gen, dis, rollout, pg_loss, gen_optimizer, args.gk_epochs, args) 214 | 215 | # train discriminator for d_steps 216 | print("#Train discriminator") 217 | for i in range(args.d_steps): 218 | print("##D-Step {}".format(i)) 219 | train_discriminator(dis, gen, nll_loss, dis_optimizer, args.dk_epochs, 220 | dis_adversarial_train_loss, dis_adversarial_train_acc, args) 221 | 222 | # update roll-out model 223 | rollout.update_params() 224 | 225 | 226 | if __name__ == '__main__': 227 | # Parse arguments 228 | args = parser.parse_args() 229 | args.cuda = not args.no_cuda and torch.cuda.is_available() 230 | torch.manual_seed(args.seed) 231 | if args.cuda: 232 | torch.cuda.manual_seed(args.seed) 233 | if not args.hpc: 234 | args.data_path = '' 235 | POSITIVE_FILE = args.data_path + POSITIVE_FILE 236 | NEGATIVE_FILE = args.data_path + NEGATIVE_FILE 237 | 238 | # Set models, criteria, optimizers 239 | generator = Generator(args.vocab_size, g_embed_dim, g_hidden_dim, args.cuda) 240 | discriminator = Discriminator(d_num_class, args.vocab_size, d_embed_dim, d_filter_sizes, d_num_filters, d_dropout_prob) 241 | target_lstm = TargetLSTM(args.vocab_size, g_embed_dim, g_hidden_dim, args.cuda) 242 | nll_loss = nn.NLLLoss() 243 | pg_loss = PGLoss() 244 | if args.cuda: 245 | generator = generator.cuda() 246 | discriminator = discriminator.cuda() 247 | target_lstm = target_lstm.cuda() 248 | nll_loss = nll_loss.cuda() 249 | pg_loss = pg_loss.cuda() 250 | cudnn.benchmark = True 251 | gen_optimizer = optim.Adam(params=generator.parameters(), lr=args.gen_lr) 252 | dis_optimizer = optim.SGD(params=discriminator.parameters(), lr=args.dis_lr) 253 | 254 | # Container of experiment data 255 | gen_pretrain_train_loss = [] 256 | gen_pretrain_eval_loss = [] 257 | dis_pretrain_train_loss = [] 258 | dis_pretrain_train_acc = [] 259 | dis_pretrain_eval_loss = [] 260 | dis_pretrain_eval_acc = [] 261 | gen_adversarial_eval_loss = [] 262 | dis_adversarial_train_loss = [] 263 | dis_adversarial_train_acc = [] 264 | dis_adversarial_eval_loss = [] 265 | dis_adversarial_eval_acc = [] 266 | 267 | # Generate toy data using target LSTM 268 | print('#####################################################') 269 | print('Generating data ...') 270 | print('#####################################################\n\n') 271 | generate_samples(target_lstm, args.batch_size, args.n_samples, POSITIVE_FILE) 272 | 273 | # Pre-train generator using MLE 274 | print('#####################################################') 275 | print('Start pre-training generator with MLE...') 276 | print('#####################################################\n') 277 | gen_data_iter = GenDataIter(POSITIVE_FILE, args.batch_size) 278 | for i in range(args.g_pretrain_steps): 279 | print("G-Step {}".format(i)) 280 | train_generator_MLE(generator, gen_data_iter, nll_loss, 281 | gen_optimizer, args.gk_epochs, gen_pretrain_train_loss, args) 282 | generate_samples(generator, args.batch_size, args.n_samples, NEGATIVE_FILE) 283 | eval_iter = GenDataIter(NEGATIVE_FILE, args.batch_size) 284 | gen_loss = eval_generator(target_lstm, eval_iter, nll_loss, args) 285 | gen_pretrain_eval_loss.append(gen_loss) 286 | print("eval loss: {:.5f}\n".format(gen_loss)) 287 | print('#####################################################\n\n') 288 | 289 | # Pre-train discriminator 290 | print('#####################################################') 291 | print('Start pre-training discriminator...') 292 | print('#####################################################\n') 293 | for i in range(args.d_pretrain_steps): 294 | print("D-Step {}".format(i)) 295 | train_discriminator(discriminator, generator, nll_loss, 296 | dis_optimizer, args.dk_epochs, dis_adversarial_train_loss, dis_adversarial_train_acc, args) 297 | generate_samples(generator, args.batch_size, args.n_samples, NEGATIVE_FILE) 298 | eval_iter = DisDataIter(POSITIVE_FILE, NEGATIVE_FILE, args.batch_size) 299 | dis_loss, dis_acc = eval_discriminator(discriminator, eval_iter, nll_loss, args) 300 | dis_pretrain_eval_loss.append(dis_loss) 301 | dis_pretrain_eval_acc.append(dis_acc) 302 | print("eval loss: {:.5f}, eval acc: {:.3f}\n".format(dis_loss, dis_acc)) 303 | print('#####################################################\n\n') 304 | 305 | # Adversarial training 306 | print('#####################################################') 307 | print('Start adversarial training...') 308 | print('#####################################################\n') 309 | rollout = Rollout(generator, args.update_rate) 310 | for i in range(args.rounds): 311 | print("Round {}".format(i)) 312 | adversarial_train(generator, discriminator, rollout, 313 | pg_loss, nll_loss, gen_optimizer, dis_optimizer, 314 | dis_adversarial_train_loss, dis_adversarial_train_acc, args) 315 | generate_samples(generator, args.batch_size, args.n_samples, NEGATIVE_FILE) 316 | gen_eval_iter = GenDataIter(NEGATIVE_FILE, args.batch_size) 317 | dis_eval_iter = DisDataIter(POSITIVE_FILE, NEGATIVE_FILE, args.batch_size) 318 | gen_loss = eval_generator(target_lstm, gen_eval_iter, nll_loss, args) 319 | gen_adversarial_eval_loss.append(gen_loss) 320 | dis_loss, dis_acc = eval_discriminator(discriminator, dis_eval_iter, nll_loss, args) 321 | dis_adversarial_eval_loss.append(dis_loss) 322 | dis_adversarial_eval_acc.append(dis_acc) 323 | print("gen eval loss: {:.5f}, dis eval loss: {:.5f}, dis eval acc: {:.3f}\n" 324 | .format(gen_loss, dis_loss, dis_acc)) 325 | 326 | # Save experiment data 327 | with open(args.data_path + 'experiment.pkl', 'wb') as f: 328 | pkl.dump( 329 | (gen_pretrain_train_loss, 330 | gen_pretrain_eval_loss, 331 | dis_pretrain_train_loss, 332 | dis_pretrain_train_acc, 333 | dis_pretrain_eval_loss, 334 | dis_pretrain_eval_acc, 335 | gen_adversarial_eval_loss, 336 | dis_adversarial_train_loss, 337 | dis_adversarial_train_acc, 338 | dis_adversarial_eval_loss, 339 | dis_adversarial_eval_acc), 340 | f, 341 | protocol=pkl.HIGHEST_PROTOCOL 342 | ) 343 | -------------------------------------------------------------------------------- /experiment_notebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import pickle as pkl\n", 10 | "import matplotlib.pyplot as plt\n", 11 | "%matplotlib inline" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 2, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "with open('experiment.pkl', 'rb') as f:\n", 21 | " log = pkl.load(f)" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 3, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "(gen_pretrain_train_loss,\n", 31 | " gen_pretrain_eval_loss,\n", 32 | " dis_pretrain_train_loss,\n", 33 | " dis_pretrain_train_acc,\n", 34 | " dis_pretrain_eval_loss,\n", 35 | " dis_pretrain_eval_acc,\n", 36 | " gen_adversarial_eval_loss,\n", 37 | " dis_adversarial_train_loss,\n", 38 | " dis_adversarial_train_acc,\n", 39 | " dis_adversarial_eval_loss,\n", 40 | " dis_adversarial_eval_acc) = log" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "# Training Settings" 48 | ] 49 | }, 50 | { 51 | "cell_type": "markdown", 52 | "metadata": {}, 53 | "source": [ 54 | "same as the default value in code" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": {}, 60 | "source": [ 61 | "# Generator" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 4, 67 | "metadata": {}, 68 | "outputs": [ 69 | { 70 | "data": { 71 | "text/plain": [ 72 | "Text(0,0.5,'Oracle NLL')" 73 | ] 74 | }, 75 | "execution_count": 4, 76 | "metadata": {}, 77 | "output_type": "execute_result" 78 | }, 79 | { 80 | "data": { 81 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XecVdW5//HPM71SZxjK0EGqUhwBBXuJnWvU2GIwajBGTTTe/NTkJrkpJjcx0cQee4k1URSNvQuo9N6lD21oA8PA1Of3x9mDB5hyZpjDmYHv+/WaF+esvfY+z2bDeWavtfZa5u6IiIjUJS7WAYiISPOghCEiIhFRwhARkYgoYYiISESUMEREJCJKGCIiEhElDBERiYgShoiIREQJQ0REIpIQrQObWQrwGZAcfM6/3f3X+9T5KXAtUA4UAFe7+8pgWwUwJ6i6yt3Pr+szs7KyvFu3bo12DiIih7pp06ZtcvfsSOpGLWEAJcAp7l5kZonABDN7292/DKszA8hz92Izux74M3BJsG2Xuw+uzwd269aNqVOnNkrwIiKHAzNbGWndqDVJeUhR8DYx+PF96nzs7sXB2y+B3GjFIyIiByaqfRhmFm9mM4GNwPvu/lUt1a8B3g57n2JmU83sSzP7r2jGKSIidYtmkxTuXgEMNrNWwDgzG+juc/etZ2bfBfKAE8OKu7p7vpn1AD4ysznu/nU1+44FxgJ06dIlKuchIiIHaZSUu28DPgbO3HebmZ0G/AI4391LwvbJD/5cBnwCDKnh2I+4e56752VnR9RvIyIiDRC1hGFm2cGdBWaWCpwOLNynzhDgH4SSxcaw8tZmlhy8zgJGAvOjFauIiNQtmk1SHYCnzSyeUGJ62d3fNLPfAlPdfTxwF5AB/MvM4Jvhs/2Af5hZZbDv/7m7EoaISAxFLWG4+2yqaUZy91+FvT6thn0nAUdGKzYREak/PekN3PfhEj5dXBDrMEREmjQlDODhT7/mMyUMEZFaKWEAqUnxFJdWxDoMEZEmTQmDUMLYXaaEISJSGyUMIC0xgeLS8liHISLSpClhAClqkhIRqZMSBpCWqCYpEZG6KGEAabrDEBGpkxIGoSapXUoYIiK1UsIg1CS1S01SIiK1UsJATVIiIpFQwkBNUiIikVDCIPQcRmlFJeUVlbEORUSkyVLCINQkBagfQ0SkFkoYhJqkADVLiYjUQgmD0Cgp0B2GiEhtorlEa4qZTTazWWY2z8x+U02dZDN7ycyWmtlXZtYtbNsdQfkiM/tWtOKEb5qkNFJKRKRm0bzDKAFOcfdBwGDgTDMbsU+da4Ct7t4LuAf4E4CZ9QcuBQYAZwIPBku9RkWKEoaISJ2iljA8pCh4mxj8+D7VRgNPB6//DZxqocW9RwMvunuJuy8HlgLDohVrVZOU5pMSEalZVPswzCzezGYCG4H33f2rfap0AlYDuHs5UAi0DS8PrAnKoiItKbS0ue4wRERqFtWE4e4V7j4YyAWGmdnAxv4MMxtrZlPNbGpBQcOWWU3d0ySlNTFERGpyUEZJufs24GNC/RHh8oHOAGaWALQENoeXB3KDsuqO/Yi757l7XnZ2doPiq0oYapISEalZNEdJZZtZq+B1KnA6sHCfauOBMcHri4CP3N2D8kuDUVTdgd7A5GjFWtWHoSYpEZGaJUTx2B2Ap4PRTXHAy+7+ppn9Fpjq7uOBx4FnzWwpsIXQyCjcfZ6ZvQzMB8qBG9w9at/mqRolJSJSp6glDHefDQyppvxXYa93AxfXsP+dwJ3Rii9cckIcZmqSEhGpjZ70BsyMtERNcS4iUhsljEBqUoIShohILZQwAqlJcWqSEhGphRJGIC0xQc9hiIjUQgkjkKplWkVEaqWEEUhNjFeTlIhILZQwAmm6wxARqZUSRiA1KV4r7omI1EIJI5CaGK8V90REaqGEEVCTlIhI7ZQwAqlJCWqSEhGphRJGIDUxntKKSsorKmMdiohIk6SEEUgLZqxVP4aISPWUMAJVU5yrWUpEpHpKGIHURN1hiIjURgkjkKZFlEREaqWEEdCqeyIitYvaintm1hl4BsgBHHjE3f++T52fAVeExdIPyHb3LWa2AtgBVADl7p4XrVgB0pJCfxWaT0pEpHrRXNO7HLjV3aebWSYwzczed/f5VRXc/S7gLgAzOw+4xd23hB3jZHffFMUY96jqw9AdhohI9aLWJOXu69x9evB6B7AA6FTLLpcBL0Qrnrp80ySlNTFERKpzUPowzKwbMAT4qobtacCZwCthxQ68Z2bTzGxsLccea2ZTzWxqQUFBg2Os6vRWk5SISPWinjDMLINQIrjZ3bfXUO08YOI+zVGj3H0ocBZwg5mdUN2O7v6Iu+e5e152dnaD41STlIhI7aKaMMwskVCyeM7dX62l6qXs0xzl7vnBnxuBccCwaMUJGiUlIlKXqCUMMzPgcWCBu99dS72WwInA62Fl6UFHOWaWDpwBzI1WrADJCXHEmZqkRERqEs1RUiOBK4E5ZjYzKPs50AXA3R8Oyi4A3nP3nWH75gDjQjmHBOB5d38nirFiZqQmaopzEZGaRC1huPsEwCKo9xTw1D5ly4BBUQmsFqlJCUoYIiI10JPeYdKS4tUkJSJSAyWMMKEmKT2HISJSHSWMMKlaplVEpEZKGGHSkuK1HoaISA2UMMKkJsZrPQwRkRooYYRJ1R2GiEiNlDDCpKkPQ0SkRkoYYdQkJSJSMyWMMKlJCWqSEhGpgRJGmLSkeEorKimvqIx1KCIiTY4SRpiqNTHULCUisj8ljDBVU5zvLFHCEBHZlxJGmNZpSQBsLS6NcSQiIk2PEkaYNumhhLFlpxKGiMi+lDDCtFXCEBGpkRJGmNZKGCIiNYrmEq2dzexjM5tvZvPM7CfV1DnJzArNbGbw86uwbWea2SIzW2pmt0crznCt05Iwg81KGCIi+4nmEq3lwK3uPj1Yn3uamb3v7vP3qfe5u58bXmBm8cADwOnAGmCKmY2vZt9GFR9ntEpNZMvOkmh+jIhIsxS1Owx3X+fu04PXO4AFQKcIdx8GLHX3Ze5eCrwIjI5OpHtrk57E1p1lB+OjRESalYPSh2Fm3YAhwFfVbD7WzGaZ2dtmNiAo6wSsDquzhhqSjZmNNbOpZja1oKDggGNtk57EZt1hiIjsJ+oJw8wygFeAm919+z6bpwNd3X0QcB/wWn2P7+6PuHueu+dlZ2cfcLxt0pPU6S0iUo2oJgwzSySULJ5z91f33e7u2929KHj9FpBoZllAPtA5rGpuUBZ1bdKTlTBERKrRoIRhZh0jqGPA48ACd7+7hjrtg3qY2bAgns3AFKC3mXU3syTgUmB8Q2Ktr7bpSWwtLqOy0g/Gx4mINBsNHSX1JdCljjojgSuBOWY2Myj7edV+7v4wcBFwvZmVA7uAS93dgXIzuxF4F4gHnnD3eQ2MtV5apydRUels311Gq2CqEBERaXjCsLoquPuEuuq5+/3A/TVsewt4q0HRHYCqp7037yxVwhARCdPQPoxDtr1G80mJiFSvxjsMM7uP6hODAa2iFlGMVSWMzUVKGCIi4WprkprawG3NWtsMTXEuIlKdGhOGuz9d0zYz+0t0wom9qjUx1CQlIrK3hvZhfKdRo2hCUhLjSU+KV5OUiMg+Gpow6hwl1Zy1yUjSBIQiIvuordO7TU2bONQTRnqypjgXEdlHbZ3e0wiNkqouORzS07m2SUukoEh3GCIi4Wrr9O5+MANpStqkJ7No/Y5YhyEi0qTU1iRV69Qf7r6q8cNpGtpmJLF5ZynuTjDVlYjIYa+2Jqn/sH+TlAPZQDtCczwdktqkJ1FSXklxaQXpydFclFBEpPmorUnqyPD3wSJItwGnAX+IalQx1ibsWQwlDBGRkDqH1ZpZbzN7CnibUEd4f3e/L9qBxZLmkxIR2V9tfRgDgV8AA4A/A9e4e8XBCiyW2mQoYYiI7Ku29pZZhNbV/g8wDBgW3gHs7j+ObmixEz7FuYiIhNSWMK4+aFE0Md80SelZDBGRKg2afDASZtYZeAbIITS66hF3//s+da4g1JFuwA7genefFWxbEZRVAOXunncg8dRHRnICifHGlp2H9POJIiL1Es0hQOXAre4+3cwygWlm9r67zw+rsxw40d23mtlZwCPA8LDtJ7v7pijGWC0zo0265pMSEQkXtYTh7uuAdcHrHWa2AOgEzA+rMylsly+B3GjFU18dW6WycnNxrMMQEWkyGjpbbb0Ez3AMAb6qpdo1hIbuVnHgPTObZmZjazn2WDObamZTCwoKGiNcAPp1aMGCddtxP2RXoxURqZdInsM4wsw+NLO5wfujzOx/Iv0AM8sAXgFudvftNdQ5mVDCuC2seJS7DwXOAm4wsxOq29fdH3H3PHfPy87OjjSsOvXr0ILtu8tZW7i70Y4pItKcRXKH8ShwB8EMte4+G7g0koObWSKhZPGcu79aQ52jgMeA0e6+uarc3fODPzcC4wgN7T1o+nfIBGDB2mpznIjIYSeShJHm7pP3KSuvaycLPbTxOLDA3e+uoU4X4FXgSndfHFaeHnSUY2bpwBnA3AhibTR92rcAYME6JQwREYis03uTmfUk1KeAmV1E0Jldh5HAlcAcM5sZlP0c6ALg7g8DvwLaAg8GDwVWDZ/NAcYFZQnA8+7+TqQn1RgykhPo2jaN+UoYIiJAZAnjBkLDXfuaWT6hobDfrWsnd59AHSvzufu1wLXVlC8DBkUQW1T1a99CdxgiIoE6E0bw5X1a0DQU5+6HzcpC/Tq04N3569lZUq5Za0XksFfb5IM/raEcgJr6JQ4l/Tpk4g4L1+/g6K6tYx2OiEhM1dbpnVnHzyGvf0d1fIuIVKltLqnfHMxAmqJOrVJpkZKghCEiQmQP7j1tZq3C3rc2syeiG1bTYGb07dBCI6VERIjsOYyj3H1b1Rt330pomo/DQv8OLVi0fgeVlZoiREQOb5EkjDgz29Pja2ZtiO4st01Kvw6ZFJdWsHKLJiIUkcNbJF/8fwW+MLN/EXqu4iLgzqhG1YQM7NQSgFmrt9E9Kz3G0YiIxE6ddxju/gxwIbABWA98292fjXZgTUWfnEzSkuKZvmprrEMREYmpiJqW3H2emRUAKRCaA8rdV0U1siYiIT6OwZ1bKWGIyGEvklFS55vZEkJTgnwKrGDvdSsOeUO7tGbBuh0Ul9Y556KIyCErkk7v3wEjgMXu3h04ldDqeIeNo7u2pqLSmbW6MNahiIjETCQJoyxYpyLOzOLc/WMgL8pxNSlDuoQeQ1GzlIgcziLpw9gWrJr3GfCcmW0EdkY3rKalVVoSPbPTmb5SCUNEDl+R3GGMBoqBW4B3gK+B86IZVFM0tEtrZqzepjW+ReSwVWvCMLN44E13r3T3cnd/2t3vDV9K9XAxtGtrtuwsZcVmPcAnIoenWhOGu1cAlWbWsr4HNrPOZvaxmc03s3lm9pNq6piZ3WtmS81stpkNDds2xsyWBD9j6vv5ja1qenM1S4nI4SqSPowiQsusvk9Y34W7/7iO/cqBW919erA+9zQze9/d54fVOQvoHfwMBx4ChgfTj/yaUOe6B/uOD+axiole2RlkpiQwbdVWLjw6N1ZhiIjETCQJ49Xgp17cfR3B2t/uvsPMFgCdgPCEMRp4xkMdA1+aWSsz6wCcBLzv7lsAgmR1JvBCfeNoLHFxxuDOrZi1elvdlUVEDkGRJIyXgF7B66Xuvru+H2Jm3QjNcPvVPps6AavD3q8Jymoqr+7YY4GxAF26dKlvaPXSr0MLnpq0gopKJz6u1uXKRUQOOTX2YZhZgpn9mdCX9dPAM8BqM/uzmSVG+gHBkNxXgJvdvdEXlnD3R9w9z93zsrOzG/vwezkiJ5PS8kpWbD6sRhWLiAC1d3rfBbQBurv70e4+FOgJtAL+EsnBg8TyCvCcu1fXrJUPdA57nxuU1VQeU33bh1amXbR+R4wjERE5+GpLGOcCP3D3Pd+OwR3C9cDZdR3YzAx4HFjg7nfXUG088L1gtNQIoDDo+3gXOCNY3a81cEZQFlO92mUQZ0oYInJ4qq0Pw72ap9TcvcLMInl6bSRwJaERVjODsp8DXYLjPAy8RSj5LCX0cOD3g21bzOx3wJRgv99WdYDHUkpiPN3apithiMhhqbaEMd/Mvhesh7GHmX0XWFjXgd19AqEFl2qr48ANNWx7Amhya4f3aZ+phCEih6XaEsYNwKtmdjUwLSjLA1KBC6IdWFN1RE4m785bz+6yClIS42MdjojIQVNjwnD3fEIP0Z0CDAiK33L3Dw9KZE1U3/aZVDos2VDEkbn1fgBeRKTZqvM5DHf/CPjoIMTSLPSpGim1YYcShogcViKZrVbCdG2bTnJCHIvWN/ojJSIiTZoSRj3Fxxm9czJYqI5vETnMKGE0wBE5mSzeoIQhIocXJYwG6Ns+kw3bS9hWXBrrUEREDholjAbo074FAAvW6S5DRA4fShgNMCi3JUkJcbw1Z12sQxEROWiUMBqgVVoS5x3VkVemr2H77rJYhyMiclAoYTTQmOO6UlxawavT1sQ6FBGRg0IJo4GOym3F4M6teOaLlVRWRjIXo4hI86aEcQDGHNeVZZt2MvHrTbEORUQk6pQwDsDZR3agbXoST09aGetQRESiTgnjACQnxPPtoZ34dPFGdpaUxzocEZGoUsI4QCce0Y6yCmfy8piv7yQiElVRSxhm9oSZbTSzuTVs/5mZzQx+5ppZhZm1CbatMLM5wbap0YqxMeR1a01yQhyfL1E/hogc2qJ5h/EUcGZNG939Lncf7O6DgTuAT/dZhvXkYHteFGM8YCmJ8Qzr3obPlxTEOhQRkaiKWsJw98+ASNtpLgNeiFYs0TaqVxZLNhaxvnB3rEMREYmamPdhmFkaoTuRV8KKHXjPzKaZ2djYRBa543tnAzBhqZqlROTQFfOEAZwHTNynOWqUuw8FzgJuMLMTatrZzMaa2VQzm1pQEJtmob7tM8nKSGKCmqVE5BDWFBLGpezTHBWsJ467bwTGAcNq2tndH3H3PHfPy87OjmqgNYmLM0b2ymLC0k166ltEDlkxTRhm1hI4EXg9rCzdzDKrXgNnANWOtGpKRvXKYlNRqVbiE5FDVkK0DmxmLwAnAVlmtgb4NZAI4O4PB9UuAN5z951hu+YA48ysKr7n3f2daMXZWKr6MT5etJH+HVvEOBoRkcYXtYTh7pdFUOcpQsNvw8uWAYOiE1X0tG+ZwtAurXhj1lpuOLlXrMMREWl0TaEP45DxX0M6sXD9Dhau3x7rUEREGp0SRiM658gOxMcZr81YG+tQREQanRJGI2qbkcwJvbMYPzNfo6VE5JCjhNHIRg/uxNrC3UxZockIReTQooTRyE7vn0NqYjyvzVSzlIgcWpQwGll6cgJnDMjhP7PXUrCjJNbhiIg0GiWMKPjRSb0oKa/klpdmUqG+DBE5RChhREGf9pn85vwBTFi6iQc/XhrrcEREGoUSRpRcckxnRg/uyD0fLOarZZtjHY6IyAFTwogSM+POC46kQ8tU/vzuoliHIyJywJQwoigjOYFrj+/OtJVbmbFqa6zDERE5IEoYUXZxXmcykxN4fMLyWIciInJAlDCiLCM5gcuGd+HtuevJ37Yr1uGIiDSYEsZBMOa4bgA8PWlFTOMQETkQShgHQadWqZw1sD3Pf7WK12bkU1ZRGeuQRETqTQnjIPnp6UfQvmUKN780k5Pu+oQP5m+IdUgiIvUStYRhZk+Y2UYzq3Z5VTM7ycwKzWxm8POrsG1nmtkiM1tqZrdHK8aDqUd2Bu/dfAKPj8kjPTmeW/81i8LisliHJSISsWjeYTwFnFlHnc/dfXDw81sAM4sHHgDOAvoDl5lZ/yjGedDExRmn9svhnksGs313GQ9+qqfARaT5iFrCcPfPgIbM8T0MWOruy9y9FHgRGN2owcXYgI4tuWBIJ56cuII1W4tjHY6ISERi3YdxrJnNMrO3zWxAUNYJWB1WZ01Qdkj57zP6APDHtxfy7JcrGX3/BH72r1kxjkpEpGaxTBjTga7uPgi4D3itIQcxs7FmNtXMphYUFDRqgNHUsVUqV4/szn9mr+OXr81l+aadvDJ9jaZEF5EmK2YJw923u3tR8PotINHMsoB8oHNY1dygrKbjPOLuee6el52dHdWYG9tNp/Ti9rP6Mv7Gkfzrh8dR6fD23HXV1v18SQEbd+w+yBGKiHwjZgnDzNqbmQWvhwWxbAamAL3NrLuZJQGXAuNjFWc0pScn8MMTe3JUbiv6tM/kiJwM3pi1/0p901Zu5crHJzP6/oksXL89BpGKiER3WO0LwBdAHzNbY2bXmNkPzeyHQZWLgLlmNgu4F7jUQ8qBG4F3gQXAy+4+L1pxNiXnHtWRKSu2sq5w7ylE/vbBYtqkJ1HpzsUPfcHEpZtiFKGIHM6iOUrqMnfv4O6J7p7r7o+7+8Pu/nCw/X53H+Dug9x9hLtPCtv3LXc/wt17uvud0YqxqTn3qA4A/Gf2N81Sk5dv4fMlm7j+xJ6M+9FIOrZKZcwTk3nuq5WxClNEDlOxHiUlYXpkZzCwU4u9mqXueX8xWRnJfHdEVzq2SuVf1x/L8b2z+MW4ufzq9bmaZkREDholjCbm3KM6MmtNIfd/tIS731/MF8s286OTepKaFA9Ai5REHhtzDNed0INnvljJhQ9NYm5+YYyjFpHDgRJGE3P+oI5kJCfwl/cWc++HS+jUKpXLh3fZq058nHHH2f148IqhrN22m9EPTOT3b85nw3aNohKR6DF3j3UMjSYvL8+nTp0a6zAOWEWls6usgt1lFaQnJey5u6hOYXEZ//fOQl6YvIqEOOOMATlcOaIbI3q0IRiEhrvjHpqaREQknJlNc/e8iOoqYRwaVmzayfOTV/Hy1NVsKy6jT04m5w/uyNKNRXy+pICiknL6dWhB3/YtqKx0tu0qJSUxntP65XBy33ZkJCfE+hREJAaUMA5ju8sqGD9zLU9NWsH8ddtpm57EqN5ZtElPYt7a7SzesIOk+DhapyWxqaiEzTtLSUqI4+qR3bnl9N4kJ9R8N1Ob0vJK/vnlSs4d1IF2mSl7ym56YTrfyevMqf1yDui8pq7YQs/sDFqnJx3QcURkb/VJGPq18hCTkhjPd47pzMV5uWzYXkK7zOQam6IqKp2pK7bw0pTVPPzp13yyaCN3XTSIAR1bVLvP9t1lLN1YRFl5JQnxxpDOrYmLM9ydX4+fxwuTVzHp6008NuYYAP755UrenbeBgh0lB5QwPl9SwJWPTyYtKZ4rR3Tl2uN7kJ2Z3ODjiUjDKGEcosyM9i1Taq0TH2cM79GW4T3acs5RHbjtldmcd/8EUhLj6NomncuHd+F7x3bFzFi1uZjv/OML1od1rB/Xsy13f2cw789fzwuTV9G3fSYfLNjIx4s2MrRLa+79aAkpiXFMX7WNZQVF9MjOqDGW374xn8QE446z+u1VXlHp/P7NBeS2TuXorq159PNlvDI9n4//+0QyUxJrPN60lVv45WvzeOr7x9CuRe1/DyISGY2SEgBO7ZfDe7ecyO//ayDfHd6VjJQEfj1+Hv87fh6rtxRz2aNfsru8ggcuH8pz1w7nt6MHMGPVNr71t8/43zfmc2rfdrx2w0h6ZKXzuzfm87cPFlO4q4yHvns0cQavTF9T42e/PWcdT0xczj8+Xcbbc/aeS+ulKatZtGEHPz+7H3+/dAgvXXcsm4pKeGriij111m7bxRMTllNaHnomZWdJOTe/NJP567bznlY2FGk0usOQPdqkJ/HdEV0BqKx0/vj2Ah79fDkvTllNUnwcz/9gBEfmtgRgZK8sRvXK4paXZ1FaXsnfLh1MSmI8vzqvP1c9OYVlm3Zy0dG5nNynHScckc246fncenof4uKMsopK4s2IizO2FZfyy9fnhZrBzPjFa3PJ69aG7Mxkduwu4+73FzGsWxvOGtgegGO6teG0fu149PNljBnZjZSEeK57dhpz8gv5bEkBD11xNHe+tYA1W3eRmZLAZ4sL9pzTgXp56mrapicdcH+MSHOlhCHVioszfnFOf3Jbp/H4hOXcc8mgPcmiSo/sDF6/YSSVlb6nz+OkPu04vX8OE5Zs4tYzjgDgwqG53PTCDL5YtpmM5ASue3YayYlxjDm2GzNXb2NbcSlPX30MSfFxnHPfBG57ZTbH987izdnr2FRUyhNX9dszRBjgJ6cewXn3T+CpiSvYWVLOnPxCLsnrzMvTVjP6gQks3lDE2BN6sGN3OW/MWktZRSWJ8Qd2M71mazF3vDoHA56+ehgje2VRUek8+PFSKtz58Sm9q+33qah04jWcWQ4RGiUlja6kvILNRaV0bJUKhEZuHXPnB+S2TuPrgiKyM5Lp0DKFqSu3AqFp3m8NFpR69LNl3PnWAgC6Z6Vz5YiuXD2q+36fce3TU5m4dBO7yiq4YngX7rzgSN6YtZZbXppJt6x03rxpFJ8s2sgP/zmdl687lmHd2+y1f1FJOVNXbGHJhiLWFu7iiuFd6dWu5j6W37wxj2e/WEnXtmls3FHCk1cdw70fLeWzxaE1WEYP7shdFw0iKeGbxDR/7XaufPwrrj+pJ9ce32PP380tL82kT04LbjqlV9SejZm9Zhv3friEP114FG0zIh8gEJ785fCgUVISU8kJ8XuSBYRGbp03qCPPf7WKYd3b8NAVQ2mbkcys1duYvHwL3zvumyaja0Z1p2e7dHq3y6Rzm7QaP+Pm03rzwYIN9G6Xwf+cE1ry/bxBHTkiJ5M26UmkJMZzbM8s4uOMz5cU7EkYlZXOuBn5/PHthWwqCi1WFR9nvDdvA2/cNIo21Qzb3bqzlBcnr+b8wR356elH8F8PTOKih78gMd74wwVHUrirjD+9s5BtxWX8/dLBtEpLYuP23Vzz9BQ27yzlT+8sZFTvLPq2b8Ff31vMW3PW89ac9czJL+TuSwbx9cYi3py9jiUbi9i0owQH/nThkRyV26pBf/9Vo9ZmrAoljd+MHhjRfvd9uIRxM/L51w+P3ZNkCneVsWj9jv0S7sHw1MTlJMTH7dWkuHHHbnbsLqdnLQMoJHp0hyEHxaaiEt6dt56Lj+6812/hB+KjhRvo277FXslpXxc+NInyikpev3HLk7GQAAATOUlEQVQUm4tK+OE/pzFlxVYGd27FT08/giM7tWT11mIufvgLhnRpxbPXDGf2mkKe+WIFAzq24OqR3Xnok6/56/uLeefm4+nbvgWz12zjz+8s4ubTepPXLfRF+tKUVdzx6hzSkxK49vgefLhwA0s3FvHIlXn85MUZdGiVwn+f0YernpzC5cO70Ccnk9++OZ/4OKO0vJKkhDj6tc8kKyOZ+eu2U1bhvHbDceS2rjlp1uSD+Ru49pmpdGubxpqtu3jvlhNqHaEGUFxazvA/fMiO3eUc3zuLp74/jKKSci575Evmr9vOA5cP5ZxgNuVwBTtKWLO1mNZpSbRMTaTSnfJKZ/mmnUxauok5+YWcObA9Fx/duV53LvnbdnHSXR/jzp74S8srOe++CWzfXcak20/Zq5myKXD3vWJyd16bmc/2XeUA9G2fyfAebet93Ln5hVz/3DR+c/4ATunb+P1nenBPJPD3D5bwtw8X88Xtp3Lj89OZk1/I70YP5KKjc/f6Ahs3Yw23vDSLbm3TWLG5mLSkeIpLKxiU25I1W3dxZG5Lnvr+sFo/a/GGHfzl3UW8N38DZvDolXmc1j+Hd+au54f/nEZCnNGlbRr/uel4UpPi+eLrzbwyfQ0jerTljAE5tAiGCS/duIMLHpxEh5YpPHjFUKas2MrsNYVcMbwLAzu13O9z5+YXsmj9Ds45qgNJ8XGcfe/nlJRX8uLYEZz6108Z2ast/7iy9u+DFyev4vZX53BJXmdemrqaH57Yk6+Wb2ZufiFd2qSxYXsJb940im5Z6Xv2qax0Tr/nU74u2FntMeMMOrRMJX/bLgbltuTGU3rTOi2RpIQ4clqk0C4zmfJK58tlm/lg/gZG9srijAGhwQ2/fG0uL05ZRXJCPMO7t+Hxq47hgY+Xcte7i4BQEjkiJ7PG85mbX8hzX63k1jP6kFWPJrn6mre2kFen5zNx6SaWb9rJ+BtH0ad9KK5PFxcw5onJe+qmJsYz+Ren1jocvDrX/3Mab89dT3JCHE9fPYwRDUg6tVHCEAnMWLWVCx6ctCcR1PSbMsBf3l3Ei1NWce3xPfjesV35cMFGfvX6XLYWl/H8D4ZzXM+siD5z9ppt7Nhdzshe39S/+cUZvDVnPa/+6Lhqv/T3NWnpJr73xGTKK0P/PxPijMT4OO69bAin9w/9lrl4ww7+9kGoiQugc5tUTu2bw1OTVnDvZUM4f1BH7v9oCX95b3G1/ThV3J1z7p1ApTtv/+R4fvryLMbNyCc+znjg8qEM7NSCc+6dQG7rVF65/jhSEkOzAXyyaCNXPTmFH5/Si65t09m+u4w4M+LjjJwWKQzr3oYWKQm8NjOfP7y1cL/16lMS40iMj2PH7tBv4EkJcfzrumNp3zKF4//8Md8e0onuWen88e2F/Ob8Adz51gIG57Zi8oot/PLc/lxTTd8WhPrMzr73c5YV7KRzm1SevGpYrf1TDbV22y5Ov/tTyiqdY7q1ZsryrVw+vAv/e/4AAG55aSYfLtjABz89kSUbi7jisa/4wwVH7jeZaG1WbS7mpL98zCXHdGbKiq2sL9zN8z8Y3uDmyuo0iYRhZk8A5wIb3X2/RlQzuwK4DTBgB3C9u88Ktq0IyiqA8khPRglD9lVR6Rz9+/fZVlzGz77VhxtO7lWv/TcVlTA3v5ATj8g+oCaQikpnU1EJOfV4iPCjhRtYvKGIk/u0o3V6Ij94eiqz8ws5a2B75uZvZ9WWYtKT4rn2+B4M6tySP761kCUbi+jfoQVv3jSKuDhjV2kFJ//lE9pmJPH6DSNJqGa02LSVW7nwoUmhZ3BGdKW4tJzbXpnDmQPa70muVc1cVx3Xbc8X4lVPTmbe2u1MvO2UOpsZd5aUMze/kNKKSkrKKllXuIuVm4vZWVrByX2yOTK3JRc99AUVlc5xPdvy+qy1fHzrSeS0TOaMez5j5eZiMpMT+ODWE7n0kS/p1jaNJ4M7vsJdZXxdUMTQLq0B+PM7C3nwk6/5+dl9eeSzZZRVOI9+L2+vhPnSlFVMX7mNrllpdG+bTsdWqXRolUJWes0zI4Rzd37wzDQmLC3gvZtPpEvbNG54bjqTvt7EVz8/jfLKSvJ+/wHnD+rI/114FO7OmX/7nJSkeF6/YWRk/wD4ZrDFhNtOAeCihyexfVcZT37/GI7u+s35HMhovKbS6f0UcD/wTA3blwMnuvtWMzsLeAQYHrb9ZHfXWqRyQOLjjLEn9KCwuIwfndSz3vtnZSRzUp92jRJHfZIFwCl9c/Zqs35x7LH87N+zmLh0E3nd2nD1yG6cN6jjng7q43tnM37mWgZ1brXnSy81KZ5fn9ef65+bzuMTlnPdifv/Hfzzy5VkJCdwwZBOAKQlJXDfZUP2qnNa/xy+P7IbT05cwSl925HbOpVPFhVwy2lHRNQnlZ6cUGf7/SPfO5oLH5rEqzPy+fbQTnRpG+q/+eU5/fnBs1O54+x+5LRIYVSvLF6ZvmZP38/Px83hP7PXcWrfdlyc15l/fLaM7+TlMvaEnpw1sANjnpzMmCcm88RVx3Bsz7Y8MWE5v31zPpkpCXvubqoMym3JS9cdu+cuqsrusgpemrKanBYpnNE/h3fnreeDBRv4+dl998R50dG5/GfOOj5auJGS8gqKSysYPTj0d2pmXHJMZ3775nwWrNtOvw4t2F1Wwby1heS0SKFDy9T9vvC37y7j5SmrOW9Qxz2zNrw4dgRXPj6Z7z42mQevGIpZ6Pmg/G2765WIGiqqTVJm1g14s7o7jH3qtQbmunun4P0KIK++CUN3GCL7c3fGPjuNz5d889twlffmrefG52dw2bDOdY6m2l1WsafT+bieWbw5ey2Tbj+1Uef1+s/sdfz53YU89f1hdA/rL9lUVLKnL+Ldeeu57tlpvDh2BO1bpHDKXz/h6K6tmbd2O8WlFWRnJvPBLSfSMi3UV1Cwo4TLH/2S1VuL+e7wrjw2YTlnDmjP/ZcPYVdZBSs3F7OucDcL123nr+8v5roTe+w1Rc2s1du49V+zWLqxCIA+OZls3llKu8xkxt/4zV1beUUlx/7fRwzKbUWlOwvWhe6+qpL31p2lDP/Dh1w+vAs/PrU3Y56YzJxg8bPEeOO2M/vuGX4N3wwxf/OmUXs1YxbsKGHME5OZv247AK3TErlgSC63n9W3QQNKmsodRn1cA7wd9t6B98zMgX+4+yOxCUuk+TMzfjd6IKfd/Sl3jJvNbWf2xR2emLic12eupX+HFlx/Ut1NdSmJ8dxzyWAueHAi42bk8+0hnRp9EshzjupQbR9TeMf1sT3bEh9nTFy6iS07S0mIi+OBy4dS4c4DHy/lnCM77kkWANmZybwwdgSXP/olj01YzvG9s/j7ZYNJiI8jMz6OgZ1aMrBTS07vn8Pawl08+tkyvjWgPf07tODeD5fwj8+WkZ2RzJPfP4btu8q476OlFO4q5Ymr8vZq4kuIj+PbQzrx2ITlGKEh4uHNW63TkzhjQA7jZoQ6yVduKebOCwYSZ8Ybs9byp3cWcuIR2fTOyWR94W4e+vRrRvRos1+fV9X5PPjJUgbntuLUfjmNNvKwLjG/wzCzk4EHgVHuvjko6+Tu+WbWDngfuMndP6th/7HAWIAuXbocvXLlysY9CZFDxLNfrOCXr8/b8z4hzrjxlF7ccHKvej0J/9AnX/PX9xYx7kcj93v6/2D59oMT2barjDVbd3Hh0E788dtH1bnP5qISXpu5lsuGdSYtqfrflXfsLuPMv31OQrwRb8ayTTu5+Ohc/ufc/rRMDSWhikpnW3FptQ9ELt6wgzPuCX1VvfXj4+nfscVe2ycs2cR3H/+K9KR4Hh2Tt2cgxaaiEk7966cckZPBc9eGktv8ddsZf+OoqHTYh2sSnd5BIN2oJWGY2VHAOOAsd19cQ53/BYrc/S91fZ6apERq5u5MX7WVbcVluEPPdhl7NfvUx9adpTFdm+Tu90NLGJvBhz89sc7nTOqj6ku9c5tU/njBUYzqHdnouCrffnAiu8oqeevHo/YbKFFZ6Tz06dec0Dt7v2T78tTV/L9/z+bITi2Zk1+4Z6RbtDWLJikz6wK8ClwZnizMLB2Ic/cdweszgN/GKEyRQ4aZ7TWy5kDEeiGr43tnce+HS/hW//aNmiwARvXO4p2bj6dLm7Qa70Rq89iYY6jc5yG+KnFxVuNIvYuPzuXV6Wv4ctkWvnds14OSLOoragnDzF4ATgKyzGwN8GsgEcDdHwZ+BbQFHgz+YquGz+YA44KyBOB5d38nWnGKSPMztEtrrjuhB5cNi/yZhvro275F3ZVqUN30MpEwM+65ZDCvzVjL1aO6Nfjzo0kP7omIHMbq0ySlBZRERCQiShgiIhIRJQwREYmIEoaIiERECUNERCKihCEiIhFRwhARkYgoYYiISEQOqQf3zKwAaOjsg1nAobL+hs6ladK5NF2H0vnU91y6unt2JBUPqYRxIMxsaqRPOzZ1OpemSefSdB1K5xPNc1GTlIiIREQJQ0REIqKE8Y1DaVU/nUvTpHNpug6l84nauagPQ0REIqI7DBERichhnzDM7EwzW2RmS83s9ljHUx9m1tnMPjaz+WY2z8x+EpS3MbP3zWxJ8GfrWMcaKTOLN7MZZvZm8L67mX0VXJ+XzCy2S73Vg5m1MrN/m9lCM1tgZsc212tjZrcE/8bmmtkLZpbSXK6NmT1hZhvNbG5YWbXXwULuDc5ptpkNjV3k+6vhXO4K/o3NNrNxZtYqbNsdwbksMrNvHejnH9YJw8zigQeAs4D+wGVm1j+2UdVLOXCru/cHRgA3BPHfDnzo7r2BD4P3zcVPgAVh7/8E3OPuvYCtwDUxiaph/g684+59gUGEzqvZXRsz6wT8GMhz94FAPHApzefaPAWcuU9ZTdfhLKB38DMWeOggxRipp9j/XN4HBrr7UcBi4A6A4LvgUmBAsM+DwXdegx3WCQMYBix192XuXgq8CIyOcUwRc/d17j49eL2D0BdSJ0Ln8HRQ7Wngv2ITYf2YWS5wDvBY8N6AU4B/B1Wa07m0BE4AHgdw91J330YzvTaElktONbMEIA1YRzO5Nu7+GbBln+KarsNo4BkP+RJoZWYdDk6kdavuXNz9PXcvD95+CeQGr0cDL7p7ibsvB5YS+s5rsMM9YXQCVoe9XxOUNTtm1g0YAnwF5Lj7umDTekLrpDcHfwP+H1AZvG8LbAv7z9Ccrk93oAB4Mmhie8zM0mmG18bd84G/AKsIJYpCYBrN99pAzdehuX8nXA28Hbxu9HM53BPGIcHMMoBXgJvdfXv4Ng8Ng2vyQ+HM7Fxgo7tPi3UsjSQBGAo85O5DgJ3s0/zUjK5Na0K/rXYHOgLp7N8s0mw1l+tQFzP7BaFm6uei9RmHe8LIBzqHvc8NypoNM0sklCyec/dXg+INVbfRwZ8bYxVfPYwEzjezFYSaBk8h1AfQKmgGgeZ1fdYAa9z9q+D9vwklkOZ4bU4Dlrt7gbuXAa8Sul7N9dpAzdehWX4nmNlVwLnAFf7NsxKNfi6He8KYAvQORnskEeogGh/jmCIWtPE/Dixw97vDNo0HxgSvxwCvH+zY6svd73D3XHfvRug6fOTuVwAfAxcF1ZrFuQC4+3pgtZn1CYpOBebTDK8NoaaoEWaWFvybqzqXZnltAjVdh/HA94LRUiOAwrCmqybJzM4k1JR7vrsXh20aD1xqZslm1p1QR/7kA/owdz+sf4CzCY0s+Br4RazjqWfsowjdSs8GZgY/ZxNq+/8QWAJ8ALSJdaz1PK+TgDeD1z2Cf+RLgX8BybGOrx7nMRiYGlyf14DWzfXaAL8BFgJzgWeB5OZybYAXCPW9lBG687umpusAGKGRk18DcwiNDIv5OdRxLksJ9VVUfQc8HFb/F8G5LALOOtDP15PeIiISkcO9SUpERCKkhCEiIhFRwhARkYgoYYiISESUMEREJCJKGCJ1MLMKM5sZ9tNoEwaaWbfwmUdFmrKEuquIHPZ2ufvgWAchEmu6wxBpIDNbYWZ/NrM5ZjbZzHoF5d3M7KNgfYIPzaxLUJ4TrFcwK/g5LjhUvJk9Gqw38Z6ZpQb1f2yhtU5mm9mLMTpNkT2UMETqlrpPk9QlYdsK3f1I4H5Cs+0C3Ac87aH1CZ4D7g3K7wU+dfdBhOaVmheU9wYecPcBwDbgwqD8dmBIcJwfRuvkRCKlJ71F6mBmRe6eUU35CuAUd18WTAK53t3bmtkmoIO7lwXl69w9y8wKgFx3Lwk7RjfgfQ8t5IOZ3QYkuvvvzewdoIjQtCKvuXtRlE9VpFa6wxA5MF7D6/ooCXtdwTd9i+cQmtdoKDAlbGZYkZhQwhA5MJeE/flF8HoSoRl3Aa4APg9efwhcD3vWLm9Z00HNLA7o7O4fA7cBLYH97nJEDib9xiJSt1Qzmxn2/h13rxpa29rMZhO6S7gsKLuJ0Ep7PyO06t73g/KfAI+Y2TWE7iSuJzTzaHXigX8GScWAez20xKtIzKgPQ6SBgj6MPHffFOtYRA4GNUmJiEhEdIchIiIR0R2GiIhERAlDREQiooQhIiIRUcIQEZGIKGGIiEhElDBERCQi/x/tC2ztIM2JDwAAAABJRU5ErkJggg==\n", 82 | "text/plain": [ 83 | "" 84 | ] 85 | }, 86 | "metadata": {}, 87 | "output_type": "display_data" 88 | } 89 | ], 90 | "source": [ 91 | "x1 = range(len(gen_pretrain_eval_loss))\n", 92 | "plt.plot(x1, gen_pretrain_eval_loss)\n", 93 | "plt.xlabel('Epochs')\n", 94 | "plt.ylabel('Oracle NLL')" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": 5, 100 | "metadata": {}, 101 | "outputs": [ 102 | { 103 | "data": { 104 | "text/plain": [ 105 | "Text(0,0.5,'Oracle NLL')" 106 | ] 107 | }, 108 | "execution_count": 5, 109 | "metadata": {}, 110 | "output_type": "execute_result" 111 | }, 112 | { 113 | "data": { 114 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd8nXX5//HXld1mN0lHVtNJaekuHWwEAaGCCigVAZmKiuBC/Sluvw5wgQNlyBAZIiAie1NpKS3dpbtNm3SlI11pm3Gu3x/nNKalSU6Tc3KSnPfz8ciDnPvc5z5X7pLzzmfcn9vcHREREYCEWBcgIiKdh0JBREQaKRRERKSRQkFERBopFEREpJFCQUREGikURESkkUJBREQaKRRERKRRUqwLOFr5+fleVlYW6zJERLqUOXPmbHX3gtb263KhUFZWxuzZs2NdhohIl2Jm5eHsp+4jERFppFAQEZFGCgUREWmkUBARkUYKBRERaaRQEBGRRgoFERFpFLehcKC+gQdnlrOvtiHWpYiIdBpxGwpPvFfJLU8t4ufPvR/rUkREOo24DYV/zqkA4P4Z5cxYtS3G1YiIdA5xGQprt+5ldvkOvnzGEMryevKNx+ez90B9rMsSEYm5uAyFJ96rIMHg0xNLufXi0VRW7+Pnzy2NdVkiIjEXd6EQCDj/fK+SEwfn0zc7jePLenHViQN4cGY5b6/cGuvyRERiKu5CYe76HVRW7+PCccWN275+1jEMyE/nG48vYI+6kUQkjsVdKKzcsgeA8f1zG7f1SEnkZ58YSWX1Pp6ZvyFWpYmIxFzchULljn0kGPTNTjtk+6QBvcjtmcyc8h0xqkxEJPbiLxSq99MnK43kxEN/dDNjfP9chYKIxLU4DIUaCnN6HPG5cf1zWb11L9v31nZwVSIinUPchcKG6v0UNRMKE/r3AuA9tRZEJE7FVSgEAs7GnfuabSmMKs4mKcGYs06hICLxKa5CoWrPAeoanKLcI4dCWnIiI4qymbNWoSAi8SmuQqFixz4AinLSmt1nQv9c5ldUU1sf6KiyREQ6jbgKhQ3VB0OhZ7P7jO+fy4H6AEs27uqoskREOo24DIXCFloKY0pyAFhYUd0hNYmIdCZxFQqV1fvISksiMy252X36ZafRIzmRtdtqOrAyEZHOIa5CYUN18zOPDjIz+uf1ZO3WvR1UlYhI5xFXoVCxYx/Fzcw8aqosL5212xQKIhJ/4ioUwmkpAJTlp7N++z4aAt4BVYmIdB5xEwq799exa399s1czN1WW15PahkDjwLSISLyIWiiY2b1mtsXMFrWy3/FmVm9mF0WrFggubwGE3VIA1IUkInEnmi2F+4BzWtrBzBKBXwAvRrEOILgQHtDs1cxNleUdDAXNQBKR+BK1UHD3N4Htrex2A/BPYEu06jgou0cK543sR2mv5i9cO6hPVippyQmagSQicScpVm9sZkXAx4HTgeNb2fc64DqA0tLSNr3f+P65h9xtrZX3oywvnXJ1H4lInInlQPNvgW+6e6uLDLn7X9x9grtPKCgo6IDSgl1Ia9RSEJE4E7OWAjABeMTMAPKBc82s3t2fimFNjfrn9+TVpVtoCDiJCRbrckREOkTMQsHdBxz83szuA57pLIEAMCAvvXFaakkY4xAiIt1B1ELBzB4GTgPyzawC+D6QDODud0brfSOlf2gGUvm2GoWCiMSNqIWCu087in0/G6062mpgQTAUlm7axUlD8mNcjYhIx4ibK5qPVp+sNIb1zeSZBRtjXYqISIdRKLTgE+OKmLe+mlVVe2JdiohIh1AotOCCMUUkGDw1tzLWpYiIdAiFQgv6ZKVx4uB8nnivkoBWTBWROKBQaMWF44qprN7HrLWtrdghItL1KRRacdaIPmSmJXHff9fGuhQRkahTKLSiZ0oSV55QxvOLN7F88+5YlyMiElUKhTBceeIA0lMS+f2rK2NdiohIVCkUwpCbnsJnpvTnmQUbWK3pqSLSjSkUwnTNSQNJS07k03e9w39Xbo11OSIiUaFQCFNBZiqPXjeF9NRELr37HZ6cWxHrkkREIk6hcBRGFmfzzA0nU5bXk6fnbYh1OSIiEadQOEo9UhKZNCCPueurcdcFbSLSvSgU2mBsaQ7VNXWs3VYT61JERCJKodAGY0uD93qeu25HjCsREYkshUIbDO6dQUZqEnPXVce6FBGRiFIotEFigjG6JJu569VSEJHuRaHQRmNLcnl/42721TbEuhQRkYhRKLTR2NIcGgLOwsqdsS5FRCRiFAptNKYkB9Bgs4h0LwqFNsrLSKV/Xk8NNotIt6JQaIexJTm8t26HLmITkW5DodAOY0tz2bL7ABt37o91KSIiEaFQaIexpQfHFdSFJCLdg0KhHYb1zSI1KUGDzSLSbSgU2iElKYGRRdm8p1AQkW5CodBOY0tzWLRhFwfqdRGbiHR9UQsFM7vXzLaY2aJmnr/UzBaY2UIze9vMRkerlmgaW5pLbX2A9zfujnUpIiLtFs2Wwn3AOS08vwY41d1HAj8G/hLFWqLmf4PN6kISka4vaqHg7m8C21t4/m13P/hJOhMojlYt0dQvuwd9s9I0A0lEuoXOMqZwNfBcrItoq5HF2SzSGkgi0g3EPBTM7HSCofDNFva5zsxmm9nsqqqqjisuTKOKslm9dS+799fFuhQRkXaJaSiY2SjgbuACd9/W3H7u/hd3n+DuEwoKCjquwDAdV5wNwOINuyJyvLdXbmXqHW9x6d0z+ckzSzSzSUQ6TMxCwcxKgSeAy9x9eazqiISRRcFQWFjR/i4kd+dnzy1lQ/V+9hxo4O7pa7hn+pp2H1dEJBzRnJL6MDADOMbMKszsajP7vJl9PrTL94A84I9mNs/MZkerlmjLz0ilMDstIvdWeGvFVhZW7uTms4/hX188kbOG9+GOV1ayoXpfBCoVEWlZNGcfTXP3fu6e7O7F7n6Pu9/p7neGnr/G3XPdfUzoa0K0aukIxxVFZrD5j6+vpG9WGh8fVwTALVOHE3Dnp/95v93HFhFpTcwHmruLkaHB5l3tGGyevXY7M1dv59pTBpKalAhASa+eXH/aIP6zcCNLN0VmzEJEpDkKhQgZeXCwubJtH9zVNbV89bH59MlKZdrEkkOeu3RSf8zg+UWb2l2niEhLFAoRcnCwuS1dSPUNAW54eC6bdu7nT58ZT8+UpEOeL8hMZUL/XIWCiESdQiFC8jJSKcrpwZzyo1/u4u7pa3hrxVZ+8rHjGFeae8R9zh7Rl6WbdlO+bW97SxURaZZCIYI+PLwPry7dwo69tWG/piHgPDijnBMH5/HJ40ua3e/sEX0BeGGxWgsiEj0KhQi6ZGIJtQ0BnphbGfZr3lxeRWX1Pi6d1L/F/Up69WREYRYvLN7c3jJFRJrVplAws8JIF9IdDOubxZiSHB6ZtQ53b9z+8pLN3PHKiiMug/H3WevIz0jhzGP7tHr8s0f0ZU75Drbs0j2hRSQ6klrf5YhmAqWRLKS7uOT4Er71xELeW7eD8f17AfB/z73P6qq93D9jLR8dXci+2gYy05I4YVA+ry7dwrUnDyQlqfV8PmVoAb9+aTlzynfwkZH9ovyTiEg8amv3kUW0im7ko6MLSU9J5OFZ6wEo37aX1VV7+czkUgb3zuDv76zjlaVbuP/tcq68710aAv6BKajNGdY3k6QEY4FWZBWRKGlrS8Fb3yU+pacmcf6YQp6au4HvfXQ4ry7dAsC1Jw+kf1564347a+p4btFGHA7Z3pK05ESG9ctkQYXu3SAi0dFsKJjZHRz5w9+AnKhV1A1ccnwpD89az9PzNvDasioGFqR/4IM/u2cyl0w8+h64kUU5PLNgA+6OmRpsIhJZLbUUWlqgrssuXtcRRhVnM6xvJg/OKGfNtr1cPrnlmUVHe+yHZ62jfFsNZfnhtTBERMLVbCi4+/3NPWdmt0WnnO7BzJg2sZTvP70YgNOH9Y7YsUeFltNYULlToSAiEdfWgeZPRrSKbuhjY4pITUogIzWJ48t6Rey4Q/tkkpqUwIL1GlcQkchr60CzOrNbkd0zmS+fMYSGgIc13TRcyYkJDC/M0gwkEYmKlgaam/vz1lAohOWLpw+OynFHFWXz+JwKGgJOYoL+KUQkclpqKcwhOPvoSJ86ukN9DI0szuH+GeWsrtrDkD6ZsS5HRLqRlgaaB3RkIRK+YX2DQbB8s0JBRCKrpe6jFifRu/u6yJcj4RhYEJx1tLpqT4wrEZHupqXuo//wwe4jBwqA3kBiFOuSFvRMSaJfdhqrt+reCiISWS11H41s+tjMyoBvAmcC/xfVqqRVAwvS1VIQkYhrda6kmQ0xs/uA5wgOPg939zuiXZi0bGB+Bqur9h6yRLeISHs1GwpmdpyZPQz8E3gZOM7d73Z3zTzqBAYWpLP7QD1Vew7EuhQR6UZaGlOYD6wnOLYwEZjYdAE2d/9ydEuTlgwsyABgddVeememxbgaEekuWgqFqzqsCjlqA/MPzkDay+SBeTGuRkS6izYtiCexV5TTg9SkBA02i0hERW5RHulQCQnGgPx0TUsVkYhSKHRhmpYqIpEWtVAws3vNbIuZLWrmeTOz281spZktMLNx0aqluxqYn8H6HfuorQ/EuhQR6SbCuU5hqJm9cvDD3cxGmdl3wzj2fcA5LTz/EWBI6Os64E9hHFOaGFiQTkPAWbddXUgiEhnhtBTuAr5NaGVUd18AXNLai9z9TWB7C7tcADzgQTOBHDPrF0Y9EjK4d3Ba6vLN6kISkcgIJxR6uvusw7bVR+C9iwheB3FQRWibhGlon0wSE4z3N+6KdSki0k2EEwpbzWwQwcXwMLOLgI1RreowZnadmc02s9lVVVUd+dadWlpyIoMK0lm8QaEgIpERTih8EfgzMMzMKoGbgOsj8N6VQEmTx8WhbR/g7n9x9wnuPqGgoCACb919jCjMZolCQUQipNVQcPfV7n4mwSWzh7n7Se6+NgLv/TRweWgW0mRgp7t3aAukOxjeL4tNu/azTWsgiUgEtHSTna82sx0Ad/91SwcOLaZ3GpBvZhXA94Hk0GvvBJ4FzgVWAjXAlUddvTCiMAuAJRt3cfIQtaJEpH1aWvuoXfd5dPdprTzvBLumpB2Gh0Jh8QaFgoi0X0trH/2wIwuRtsnpmUJRTg+NK4hIRIRz8dr9ZpbT5HGumd0b3bLkaBzbL4vFG3bGugwR6QbCmX00yt2rDz5w9x3A2OiVJEdrRGEWq7fupaY2EpePiEg8CycUEsws9+ADM+tFy2MR0sGGF2bhDks37Y51KSLSxYXz4f4rYIaZ/QMw4CLgp1GtSo5K4wykDbsYV5rbyt4iIs1rNRTc/QEzmwOcHtr0CXdfEt2y5GgU5fQgu0eyrmwWkXYLqxvI3RebWRWQBmBmpe6+LqqVSdjMjOH9sliiNZBEpJ3CmX10vpmtANYAbwBrgeeiXJccpeGFWSzduIv6Bt1bQUTaLpyB5h8Dk4Hl7j4AOAOYGdWq5KiNKMziQH2ANbo9p4i0QzihUOfu2wjOQkpw99eACVGuS45S0yubRUTaKpxQqDazDOBN4CEz+x2gP0c7mUEFGaQkJWhcQUTaJZxQuIDggnVfAZ4HVgEfjWZRcvSSExM4pk+mrmwWkXZpcfaRmSUCz7j76UAAuL9DqpI2GVGYxQuLN+HujavZiogcjRZbCu7eAATMLLuD6pF2GFGYxY6aOjbu3B/rUkSkiwrnOoU9wEIze4kmYwnu/uWoVSVt0nSwuTCnR4yrEZGuKJxQeCL0JZ3c8H7ZJCYYCyqq+fDwPrEuR0S6oHBC4VFgcOj7le6uvolOqkdKIsf0yWTe+urWdxYROYJmxxTMLMnMfglUEBxgfgBYb2a/NLPkjipQjs6Y0hzmra8mEPBYlyIiXVBLA823Ar2AAe4+3t3HAYOAHOC2jihOjt6Ykhx2769n9dY9sS5FRLqglkJhKnCtuzcu0u/uu4DrgXOjXZi0zbjS4E3y5q5TF5KIHL2WQsHd/QN9EKFpquqb6KQG5meQmZakcQURaZOWQmGJmV1++EYz+wywNHolSXskJBiji3MUCiLSJi3NPvoi8ISZXQXMCW2bAPQAPh7twqTtxpTk8Kc3VrGvtoEeKYmxLkdEupBmQ8HdK4FJZvYhYERo87Pu/kqHVCZtNrY0h4aAs7ByJxMH9Ip1OSLShYRzO85XgVc7oBaJkDElwcHmd9duVyiIyFEJZ5VU6WLyMlIZ3i+LN5ZXxboUEeliFArd1KnHFPBe+Q5276+LdSki0oUoFLqpU4YUUB9wZqzaFutSRKQLiWoomNk5ZrbMzFaa2beO8Hypmb1mZnPNbIGZ6aK4CBnfP5f0lETeXKEuJBEJX9RCIXSDnj8AHwGGA9PMbPhhu30XeMzdxwKXAH+MVj3xJiUpgSmD8nhjeRVHuAZRROSIotlSmEhwVdXV7l4LPELw1p5NOZAV+j4b2BDFeuLOqUMLWL99H2u31cS6FBHpIqIZCkXA+iaPK0LbmvoB8BkzqwCeBW6IYj1x55ShBQC8qVlIIhKmWA80TwPuc/digovsPWhmH6jJzK4zs9lmNruqSh9w4eqfl07/vJ6amioiYYtmKFQCJU0eF4e2NXU18BiAu88A0oD8ww/k7n9x9wnuPqGgoCBK5XZPpwwpYMaqbRyob4h1KSLSBUQzFN4FhpjZADNLITiQ/PRh+6wDzgAws2MJhoL+rI2gU4cWsK+ugTlrd8S0joaA8/KSzTwwYy33TF9DbX0gpvWIyJGFczvONnH3ejP7EvACkAjc6+6LzexHwGx3fxr4GnCXmX2F4KDzZ4+0XLe03ZRBeSQnGm+sqOKEwR9ohHWY/3v2fe6ZvqbxcYLBlScOiFk9InJkUR1TcPdn3X2ouw9y95+Gtn0vFAi4+xJ3P9HdR7v7GHd/MZr1xKP01CTG98/ljWUd2wBbvnk3Nz8+n+Wbd/P6si3cM30Nn55UyrvfOZNJA3rxx9dXsb9OXVoinU2sB5qlA5w6tDdLN+1my679Hfaev3x+KY/NruDc373FDX+fyzF9Mvne1OEUZKbylQ8PpWr3AR56Z12H1SMi4VEoxIFThga7jV7voFlIq6r28PL7W/jsCWVcNL6YtJREbp82lrTk4L0dJg/MY8rAPP70+iq27TnQITWJSHiiNqYgncfwflkMzE/nr/9dy0XjiklIsKi+3z3T15CSlMCXPjSY/IzUI+7z9bOH8sk/z+TkX77GFSeUcd3JA8lNT4lqXSLSOrUU4oCZccMZg3l/4y5eXLIpqu+1bc8B/jmngk+MLWo2EADG9+/F8zeezBnH9uHON1Zx0i9e5dYXlrKvVuMMIrGkUIgT548uYmBBOr95aQWBQHQmeO2sqeOmR+dxoD7ANSe3PrNoSJ9M7pg2lhduOoXTh/XmD6+t4mv/mKe1mkRiSKEQJxITjBvPGMKyzbt5blHkWwvl2/by8T/+l5mrt/HLC0cxuHdm2K8d2ieT3396HP/v3GE8u3ATf3htZcTrE5HwKBTiyNRRhQzuncHvXlke0dZCTW091z4wm+01tfz92sl88viS1l90BNeePJCPjy3itheXM33F1ojVJyLhUyjEkcQE48tnDGH55j38Z+HGiBzT3fnuk4tYsWUPv582juPL2n5PaDPjZ58YSWmvnvzkP0ui1s0lIs1TKMSZ80b2Y0jvDH73ygoaIvCh+8R7lTwxt5KbzhjKSUPaf8V0WnIiXztrKEs37eZf8w9fKktEok2hEGcSE4wbzxzCyi17eHJu+z5099c1cOsLyxhbmsMNHxocoQrho6MKGVGYxa9eXK6F/EQ6mEIhDp17XD/GlOTwnScXMmvN9jYf5+FZ69i0az/fOOuYiF77kJBg3HzOMCp27OOx2RURO66ItE6hEIcSEox7rphAcW4Prr7vXZ5buJG9B+qP6hj7ahv44+urmDywV1QW2jtlSD6jirO5/+21mqIq0oEUCnEqLyOVB6+eRE56Mtc/9B5jfvQiv35pedivf3DmWqp2H+CrHz4mKvWZGZdN7s/KLXuYsWpbVN5DRD5IoRDHCnN68NJXTuVvV0/ijGF9uP2VFby+bEurr9u65wB3vLqSU4cWMHFA22cbteajowvJ7ZnMAzPKo/YeInIohUKcS0tO5KQh+fz2kjEM7ZPBNx5fwPa9tS2+5lcvLmdfbQO3TD026rV96vhSXlyyicrqfVF9LxEJUigIEPwA/t0lY9lZU8dH75jOj59ZwouLN7Fs025qaoPjDe7OjFXbeOTddVw+peyorlpuq89MLsWBR2dpmW2RjqBVUqXRsf2y+PNl47nv7bU8OKP8kDul5WekUh8IUF1TR156CjeeMaRDairO7clJg/OD10KcOTTqK7yKxDuFghzi9GG9OX1Yb/YeqGfllj2Ub69h/fYayrftJcGMsaU5nDq0N9k9kzuspgvHFXPTo/OYtXY7kwfmddj7isQjhYIcUXpqEqNLchhdkhPrUjhrRB/SUxJ54r0KhYJIlGlMQTq9nilJnDuyH88u3KT7LYhEmUJBuoRPjCtmz4F67p+hi9lEokmhIF3CpAG9mDywFz9/bimX3zuLjTs1RVUkGhQK0iUkJBgPXTOZH54/gjnlO/juk4tiXZJIt6SBZukyEhOMK04oo7qmjt+8vJyVW3Z3yLUSIvFELQXpci6b0p+05ATuenNN6zuLyFFRKEiX0ys9hYvHl/Dk3Eq27Nof63JEuhWFgnRJ15w8gLpAgK8/voD122sat9fU1nPP9DU8F6HbjYrEG40pSJfUPy+dW84bzq0vLOOMX73BlEF5FGSm8vqyKrbuOUBSgvFoVhrj++fy1NxKdu6r4/Ip/TELf5mMt1ZUcf/b5aSnJlLaqyefO3UQGan6lZHuzaI559vMzgF+ByQCd7v7z4+wzyeBHwAOzHf3T7d0zAkTJvjs2bOjUK10RZt27uf2V1ewoKKazbsOMLRPBteePJBb/rWI+gbnzGP78ODM4NLbV0zpz/c/OqLV9ZMCAefON1dx6wvL6J2ZSlpyIuu31zC0TyZ/vfJ4+mX3aNy3riFAcqIa3NL5mdkcd5/Q6n7RCgUzSwSWAx8GKoB3gWnuvqTJPkOAx4APufsOM+vt7i0u6K9QkHAsrNjJhX96m9qGAFeeWEZSgnHXW2uY0D+XYf0ymTggj/NHFx7ympmrt/HgzHLeXrmVHTV1fHR0Ib+4cCQ9U5J4c3kVX3joPdJTE3ngqkkc0zeTp+dv4Bv/mM9NZw7l+tMGxegnFQlPZwiFKcAP3P3s0ONvA7j7z5rs80tgubvfHe5xFQoSrteXbWHPgXqmjirE3bln+hoen1PBxp372bmvjm+cfQxfPH0wyzfv5tYXlvHSks3kZ6Ry6tACzji2Nx85ru8h3U1LN+3iintncaA+wBdOG8Qvn19GRloS1TV1fO6Ugdx8zjASO9EqroGAM6+imjVVe/nIyL70TFHXVzzrDKFwEXCOu18TenwZMMndv9Rkn6cItiZOJNjF9AN3f/4Ix7oOuA6gtLR0fHm57sQlbdcQcL722DyemreB0SU5zF9fTUZqEl84fRBXnTiAtOTEZl+7blsNl94zk/Xb9zGiMIu/XzOZ215cxoMzyynITOX80YV87pSB9M5K68Cf6FCBgPPgzHLueHUlW/ccAKBvVhrfPncYF4wpilldEltdJRSeAeqATwLFwJvASHevbu64ailIJDQEnJsfX8CrSzdzxQllXDGljNz0lLBeu3nXfh6YsZYrTxxAfkYq7s6LSzbzxHsVvLp0C+mpSfz0YyM5b1S/6P4QR7Bm615ufnw+767dwYmD8/jkhBLy0lP5+fPvs6hyF3d+ZjznHNe3w+uS2OsMoRBO99GdwDvu/tfQ41eAb7n7u80dV6EgkeTuRzUjqTWrqvbw1UfnMb9iJ188fRBfP+uYiB6/OQ0B597pa7jtxWWkJiVwy9ThXDS+uPG96xsCTL1jOrv31/PyV0+lR0rzrSHpnsINhWhOm3gXGGJmA8wsBbgEePqwfZ4CTgMws3xgKLA6ijWJHCLSH9iDCjL45/UnMG1iCX94bRW3v7Iyosc/XEPAeWpuJR/+zRv89Nn3OXlIPi999VQunlByyM+WlJjAD88fQWX1Pv70enRrkq4taiNP7l5vZl8CXiA4XnCvuy82sx8Bs9396dBzZ5nZEqAB+Ia7b4tWTSIdISkxgZ9+bCT1Dc5vXl5OUqLxxdMHR/x9ausDfOGh93j5/c0M65vJnZ8Zx9kj+jYbdJMG5nHBmELufHM1540q5Ji+WjdKPiiq1ylEg7qPpKtoOqD9nXOP5dpTBkbs2HUNAW74+1yeX7yJW6YO58oTysK6f/WWXfs5747ppCUn8NQXTiQvIzViNUnnFvMxhWhRKEhXUt8Q4MZH5vGfhRvpnZnKgfoA/bLTGFWczVUnDWBY36w2HffHzyzhnulruGXqcK4+acBRvXbe+mo+9ecZjCrO5m/XTCI1SeML8aAzjCmIxL2kxAR+e8kYbjxjCB8a1psLxhTSJyuNZxdu4pr7Z1NTW3/Ux1y2aTf3vb2WSyeVHnUgAIwpyeHWi0fz7trgfSm62h+GEl26mkUkypITE/jKh4cesm3Wmu188s8z+NWLy7ll6vCwj+Xu/PDfi8lITeLrZx3T5prOH13Iqi17+N0rKxjSJ4PrTtEV2RKkloJIDEwc0IvLJvfn3v+uYU759sbt9Q0B6hsCzb7uuUWbeHvVNr5+1tCwr6tozo1nDOG8Uf342XNLeX7RpnYdS7oPhYJIjNx8zjEUZvfgsntm8dTcSv49fwMn/PxVLv7zjCN2K23etZ/vPrWIEYVZTJtY2u73T0gwbrtoNKOLc7jxkbnMKd/R7mNK16dQEImRzLRkHr9+CiMKs7jp0Xnc8PBcsnokM399NV9+eC71DQHWbN3Lyi27qW8IcNMj89hX28DvLhlLUoRWZu2Rksg9V0ygX3Ya19z/Luu21bT+IunWNPtIJMbqGwLc9dYaMtOSmDaxlIfeKed7/wqOG+w5EGwx9ExJpKa2gVsvGsXFE0oiXkP5tr1MvWM6x/bL4pFrJ4c1vVW6lnBnH2mgWSTGkhITDll6+/IpZeyva2Dpxt1MKOt2FwjpAAANfElEQVRFUqLx7prt9MtO46LxxVGpoX9eOrdMHc7Njy/goXfKuWxKWVTeJ15s2b2fl5ds4bxR/cjukcy2PQeYU76DEwbnk5GaxLMLN/LnN1Zx04eHcvoxvWNd7iHUUhARIDiz6fJ7ZzGnfAePXDeZUcU5UXmf/67cypqte/n42CLSu+Gd7Cqr9/Hpu2ZSvq2GzNQkThqSz6tLt3CgPkBWWhKjinOYvnIrqUkJ1AecH10wgpMG51PX4AwqSI/aWlm6eE1Ejlpl9T4u+P10tu6pZeqofnxv6vCILQO+fW8tP35mCU/OrQQgPyOFL54+mM9M7t9497rNu/Yzd90ONu3cz7j+uYwozO5U96hozaqqPVxx7yx21tTxo4+N4IVFm5m+citTR/XjzGP78MTcCt5asZWrThzAVScO4MuPzOWN5VWNrz9vVD9+dfHoFpdvbyuFgoi0ya79ddz15mruems1Jbk9eexzU9o9/fXtlVu56dF57Kip5frTBnPS4Hx++/Jy3l61jWF9M/nY2CKeXbiRBRU7D3ldXnoKXz/7GD41oaRTj3O4O4+8u54f/XsJackJPHDVJEYWZ7f6urqGAM8u3Ehdg1O+bS93vLqSiWW9+Mvl48np2b5zfjiFgoi0y4xV27jir7MY1jeTv10ziay05KN6fSDgzC7fweNz1vOPORUMzE/n9mljGVEY/LA8eB+KH/17CZXV+xjWN5NPjCtiQlkv+mSlMXvtdh56Zx2z1mxnTEkOP/nYcRxX1PoHbSz87Nn3+fObqzlpcD63XTyavtlta139e/4GvvbYfEp69eC+KyfSNzuNRZU7GdQ746jP/+EUCiLSbi8v2czn/jaHvPQUbj5nGJ8YWxTWX+ybd+3n83+bw9x11fRMSeTi8cV88yPDjnhL0P11DWzetZ/SXj0/0J/u7jw5t5L/e/Z9tu+t5fIpZXz1rKHt/oCMpH/Nq+TGR+bx6Uml/OSC49rdonln9TaufWB247Tj7Xtr6ZuVxm0Xj+akIfltPq5CQUQiYu66Hfzg30uYv76aM4/tze3TxtIzJYm60JXXyYkJ7D1Qz/yKarbuqWVfbT2/enE5ew7Uc8vU4VwwprDd94feWVPHbS8u42/vlJOfkcp3zzuWqaMKYzbe8MLiTfxjdgW90pP5V+i2rg9dM6lxbKS9Vm7ZzXeeXERBZionD8nnz2+uZnXV3sb7ireFQkFEIiYQcO6fsZYfP7OE44qyGdY3k/8s2Mje2gZyeyazc18dgSYfJSW9enDX5RPavApscxZUVPPdpxaxoGInhdlpXDi+mHOO68vwflkdcoc7gLdWVHHlX98lLyOFhgD0zkzlgasnkh/FZcj31Tbwi+eXctaIPpwwqG2tBYWCiETcy0s2c8PDc0kwOHdkP4pze7Jl937y0lMY2z+X4pweJCYYRbk9orYkd0PAeX7RJh6bvZ43V1ThDoXZaVw2pYzLpvQnI0rTXA/UNzB9xVZufGQexbk9eOzzUzpVN1ZrFAoiEhV7DtSTYLS7SygSqnYf4LWlW/j3gg28tWIrOT2T+fypg/jsCWURm9a5qHIn905fw/OLN1FT20BRTg8ev34K/bJ7ROT4HUWhICJxZd76an778nJeX1ZF36w0bjpzCBeNL251nahAwPnaP+aTlpzID84fTmpSIg0B55X3N3PP9DW8s2Y76SmJnD+miDOP7c0Jg/LpkdL1bkykUBCRuDRz9TZ+/txS5q2vZmBBOgPz09m5r46GgJOUmMC0iSV8fOz/lgt5cGY5tzy1CIBJA3pxznF9ue/ttZRvq6EopwefPaGMT00s6VJdRUeiUBCRuOXuvLB4M3e+sYra+gBZPZJITkxg0879rNiyh++edyzXnDyQ9dtrOPu3bzK+fy4XjivmG4/Pp67BGVuawzUnDeTsEX0itiJtrCkUREQOU1sf4MZH5vLcok2NLYgD9QFe+MopFOX0YPGGndQ1OGNKorPuUyxplVQRkcOkJCVwx7Sx/P61lazYvAcMPjWhhKKc4KDxwaut45lCQUTiSlJiAjedObT1HeNU9+gsExGRiFAoiIhII4WCiIg0UiiIiEijqIaCmZ1jZsvMbKWZfauF/S40MzezVqdLiYhI9EQtFMwsEfgD8BFgODDNzIYfYb9M4EbgnWjVIiIi4YlmS2EisNLdV7t7LfAIcMER9vsx8AtgfxRrERGRMEQzFIqA9U0eV4S2NTKzcUCJu/8ninWIiEiYYnbxmpklAL8GPhvGvtcB14Ue7jGzZW1823xgaxtf21FUY2SoxshQje3XWerrH85O0QyFSqCkyePi0LaDMoHjgNdDd0zqCzxtZue7+yGLG7n7X4C/tLcgM5sdztofsaQaI0M1RoZqbL/OXt/hotl99C4wxMwGmFkKcAnw9MEn3X2nu+e7e5m7lwEzgQ8EgoiIdJyohYK71wNfAl4A3gcec/fFZvYjMzs/Wu8rIiJtF9UxBXd/Fnj2sG3fa2bf06JZS0i7u6A6gGqMDNUYGaqx/Tp7fYfocvdTEBGR6NEyFyIi0ihuQiHcJTc6kpmVmNlrZrbEzBab2Y2h7b3M7CUzWxH6b26M60w0s7lm9kzo8QAzeyd0Lh8NTSSIZX05Zva4mS01s/fNbEonPIdfCf0bLzKzh80sLdbn0czuNbMtZraoybYjnjcLuj1U64LQNUaxqvHW0L/1AjN70sxymjz37VCNy8zs7FjV2OS5r4WW8MkPPY7JeTwacREK4S65EQP1wNfcfTgwGfhiqK5vAa+4+xDgldDjWLqR4GSBg34B/MbdBwM7gKtjUtX//A543t2HAaMJ1tppzqGZFQFfBia4+3FAIsHZeLE+j/cB5xy2rbnz9hFgSOjrOuBPMazxJeA4dx8FLAe+DRD63bkEGBF6zR9Dv/uxqBEzKwHOAtY12Ryr8xi2uAgFwl9yo0O5+0Z3fy/0/W6CH2ZFBGu7P7Tb/cDHYlMhmFkxcB5wd+ixAR8CHg/tEuv6soFTgHsA3L3W3avpROcwJAnoYWZJQE9gIzE+j+7+JrD9sM3NnbcLgAc8aCaQY2b9YlGju78Ymt0IwansxU1qfMTdD7j7GmAlwd/9Dq8x5DfAzUDTgduYnMejES+h0OqSG7FmZmXAWIILA/Zx942hpzYBfWJUFsBvCf6PHQg9zgOqm/xSxvpcDgCqgL+GurjuNrN0OtE5dPdK4DaCfzFuBHYCc+hc5/Gg5s5bZ/0dugp4LvR9p6nRzC4AKt19/mFPdZoamxMvodCpmVkG8E/gJnff1fQ5D04Pi8kUMTObCmxx9zmxeP8wJQHjgD+5+1hgL4d1FcXyHAKE+uUvIBhghUA6R+hu6Gxifd5aY2bfIdgF+1Csa2nKzHoC/w844vT7zi5eQqG1JTdixsySCQbCQ+7+RGjz5oNNytB/t8SovBOB881sLcEutw8R7L/PCXWDQOzPZQVQ4e4Hl15/nGBIdJZzCHAmsMbdq9y9DniC4LntTOfxoObOW6f6HTKzzwJTgUv9f/PqO0uNgwj+ATA/9LtTDLxnZn3pPDU2K15CocUlN2Il1D9/D/C+u/+6yVNPA1eEvr8C+FdH1wbg7t929+LQMiSXAK+6+6XAa8BFsa4PwN03AevN7JjQpjOAJXSScxiyDphsZj1D/+YHa+w057GJ5s7b08Dlodkzk4GdTbqZOpSZnUOwS/N8d69p8tTTwCVmlmpmAwgO5s7q6PrcfaG7926yhE8FMC70/2qnOY/Ncve4+ALOJThTYRXwnVjXE6rpJILN8wXAvNDXuQT77V8BVgAvA706Qa2nAc+Evh9I8JdtJfAPIDXGtY0BZofO41NAbmc7h8APgaXAIuBBIDXW5xF4mOAYRx3BD66rmztvgBGcwbcKWEhwJlWsalxJsF/+4O/MnU32/06oxmXAR2JV42HPrwXyY3kej+ZLVzSLiEijeOk+EhGRMCgURESkkUJBREQaKRRERKSRQkFERBopFERCzKzBzOY1+YrYInpmVnakVTRFOpuo3nlNpIvZ5+5jYl2ESCyppSDSCjNba2a/NLOFZjbLzAaHtpeZ2auhdfFfMbPS0PY+oXX+54e+TggdKtHM7rLgfRVeNLMeof2/bMF7aiwws0di9GOKAAoFkaZ6HNZ99Kkmz+1095HA7wmuHAtwB3C/B9f1fwi4PbT9duANdx9NcB2mxaHtQ4A/uPsIoBq4MLT9W8DY0HE+H60fTiQcuqJZJMTM9rh7xhG2rwU+5O6rQwsYbnL3PDPbCvRz97rQ9o3unm9mVUCxux9ocowy4CUP3rwGM/smkOzuPzGz54E9BJfoeMrd90T5RxVplloKIuHxZr4/GgeafN/A/8b0ziO4Hs444N0mK6eKdDiFgkh4PtXkvzNC379NcPVYgEuBt0LfvwJcD433t85u7qBmlgCUuPtrwDeBbOADrRWRjqK/SET+p4eZzWvy+Hl3PzgtNdfMFhD8a39aaNsNBO/49g2Cd3+7MrT9RuAvZnY1wRbB9QRX0TySROBvoeAw4HYP3k5UJCY0piDSitCYwgR33xrrWkSiTd1HIiLSSC0FERFppJaCiIg0UiiIiEgjhYKIiDRSKIiISCOFgoiINFIoiIhIo/8P4l2jQ2j17U8AAAAASUVORK5CYII=\n", 115 | "text/plain": [ 116 | "" 117 | ] 118 | }, 119 | "metadata": {}, 120 | "output_type": "display_data" 121 | } 122 | ], 123 | "source": [ 124 | "x2 = range(len(gen_adversarial_eval_loss))\n", 125 | "plt.plot(x2, gen_adversarial_eval_loss)\n", 126 | "plt.xlabel('Epochs')\n", 127 | "plt.ylabel('Oracle NLL')" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": 6, 133 | "metadata": {}, 134 | "outputs": [ 135 | { 136 | "data": { 137 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd0XNW5/vHvq1EvVrEkS7Yld9kYd8tgCJgWCCWBEHpCICTgBBIICb97IfWSfiFwc0NvIZRwKQkldDBgTDPGsnGVm9yLmi2rWV2zf3/MeJBtNZfRSJrns5YWM2eOzrzbw9Iz++x99jHnHCIiIgARoS5ARER6D4WCiIgEKBRERCRAoSAiIgEKBRERCVAoiIhIgEJBREQCFAoiIhKgUBARkYDIUBdwsNLT093w4cNDXYaISJ+yaNGinc65jK7263OhMHz4cAoKCkJdhohIn2Jmm7uzn04fiYhIgEJBREQCFAoiIhKgUBARkQCFgoiIBCgUREQkQKEgIiIBYRMKa0pquPPtNeyqbQx1KSIivVbYhML68lrufq+IcoWCiEiHwiYUojy+pja3uBBXIiLSe4VNKERH+pra1Noa4kpERHqvsAmFKI8B0KSegohIh8ImFGICPQVviCsREem9wiYUoj0eAJpbFAoiIh0Jm1CIivSfPlJPQUSkQ2ETCtF7Zx8pFEREOhQ2obB3SmqjTh+JiHQobEIhMNCsUBAR6VDQQsHMYs3sMzNbamYrzew37ewTY2bPmlmRmS0ws+HBqidKp49ERLoUzJ5CI3Cqc24yMAU408xm7rfP94DdzrnRwF+A24JVTLR6CiIiXQpaKDifWv/TKP/P/leOnQc87n/8L+A0M7Ng1KOegohI14I6pmBmHjNbApQBc5xzC/bbZQiwFcA51wJUAQODUcsXVzQrFEREOhLUUHDOtTrnpgBDgWPMbMKhHMfMZptZgZkVlJeXH1ItZkZ0ZARNrVrmQkSkIz0y+8g5VwnMBc7c76XtQA6AmUUCycCudn7/IedcvnMuPyMj45DriPZEqKcgItKJYM4+yjCzFP/jOOB0YPV+u70MXOl/fCHwnnMuaF/lfT0FrZIqItKRyCAeOxt43Mw8+MLnOefcq2b2W6DAOfcy8DfgSTMrAiqAS4NYD1Ee0/0UREQ6EbRQcM4tA6a2s/3XbR43ABcFq4b9+XoKOn0kItKRsLmiGXzTUhUKIiIdC6tQ0ECziEjnwioUYiIjdPGaiEgnwioUotRTEBHpVFiFQnSkQkFEpDNhFQpRHp0+EhHpTFiFQnRkhG6yIyLSifAKBfUUREQ6FV6hoIvXREQ6FVahoGUuREQ6F1ahoJ6CiEjnwisUPB6aNdAsItKhsAqFqEijUT0FEZEOhVUoxPivaA7iLRtERPq0sAqFKI+vuS1ehYKISHvCKhSiI33N1VIXIiLtC6tQ2NtT0AVsIiLtC6tQUE9BRKRz4RkK6imIiLQrvELBo56CiEhnwisU1FMQEelUWIVCYKBZ6x+JiLQrrELhi55Ca4grERHpncIqFKI8BkCTegoiIu0Kq1CI0ZiCiEinwioUoj0eABqbdfpIRKQ9YRUK8TG+UKhrUiiIiLQnaKFgZjlmNtfMCs1spZn9uJ19TjazKjNb4v/5dbDqAUiMiQSgtrElmG8jItJnRQbx2C3ATc65xWaWBCwysznOucL99vvQOffVINYRkOAPhbomhYKISHuC1lNwzhU75xb7H9cAq4AhwXq/7oiP8p0+qm3U6SMRkfb0yJiCmQ0HpgIL2nn5ODNbamZvmNnRwawjIsKIj/awR6ePRETaFczTRwCYWSLwPHCjc656v5cXA8Occ7VmdjbwEjCmnWPMBmYD5ObmHlY9CTGRCgURkQ4EtadgZlH4AuEp59wL+7/unKt2ztX6H78ORJlZejv7PeScy3fO5WdkZBxWTYkxkezR7CMRkXYFc/aRAX8DVjnn/qeDfbL8+2Fmx/jr2RWsmgASYnT6SESkI8E8ffQl4NvAcjNb4t/2cyAXwDn3AHAhcK2ZtQD1wKXOuaCuQREfHakpqSIiHQhaKDjnPgKsi33uAe4JVg3tSYyJpKymoSffUkSkzwirK5ph70CzxhRERNoTdqGQGOPR6SMRkQ6EXSjER2tKqohIR8IuFBJiIqlrasXr1T0VRET2F3ahkLh3pVQtny0icoCwC4W9i+LpFJKIyIHCLxSitXy2iEhHwi8U9i6frWmpIiIHCMNQ2Lt8tnoKIiL7C79QiNaYgohIR8IvFPYONOvuayIiBwi7UEgMzD7SmIKIyP7CLhSS46IA2F3XFOJKRER6n7ALhbhoDynxUeyorA91KSIivU7YhQJAdnIcJVVaPltEZH9hGQqDk2PZoVAQETlAWIZCdkosxVU6fSQisr9DCgUzG3ykC+lJ2clxVNY1U9+kGUgiIm0dak/h0yNaRQ/LTo4FYId6CyIi+zjUUOj03su9XXZyHADFlRpXEBFp61BDoU/foWZwinoKIiLtiezoBTO7m/b/+BuQErSKekCW//SRegoiIvvqMBSAgkN8rdeLifSQnhjDtt11oS5FRKRX6TAUnHOPd/Samd0RnHJ6zrCB8WyuUCiIiLR1qGMKFx/RKkJg2MB4tuxSKIiItBWWs48AhqUlUFLdQEOzrlUQEdmrs4HmtI5eoj+EwsB4ALZW1DFmUFKIqxER6R06G2hehG/2UXsB0ByccnrO3lDYvEuhICKyV2cDzSMO58BmlgM8AQzCFy4POef+ut8+BvwVOBuoA77jnFt8OO/bXcMGJgBosFlEpI3OTh/ldvaLzrktXRy7BbjJObfYzJKARWY2xzlX2Gafs4Ax/p9jgfv9/w261PgokmIi2bJrT0+8nYhIn9DZ6aPXOPD0kQMygEzA09mBnXPFQLH/cY2ZrQKGAG1D4TzgCeecAz41sxQzy/b/blCZGTlp8WxRT0FEJKCz00cT2z43s+HAzcCXgT8ezJv4f3cqsGC/l4YAW9s83+bftk8omNlsYDZAbm6nHZiDMjgllu26qllEJKDLKalmNsbMHgPewDf4PN45d3d338DMEoHngRudc9WHUqRz7iHnXL5zLj8jI+NQDtGuQQNiKdH6RyIiAR2GgplNMLOn8f1BfweY4Jx7xDnX7ZlHZhbl//2nnHMvtLPLdiCnzfOh/m09ImtALLvrmnWtgoiIX2djCkvxndp5DTgGOMY3WcjHOXdDZwf2zyz6G7DKOfc/Hez2MvAjM3sG3wBzVU+MJ+y1d2G80uqGwGwkEZFw1lkofPcwj/0l4NvAcjNb4t/2cyAXwDn3APA6vumoRfimpF51mO95UPaGQkmVQkFEBA5xQbzucM59RBdXPvtnHf3wcN7ncGQN8IdCtQabRUTg0Nc+6hfa9hRERCTMQyEpNoqEaI96CiIifmEdCgCDkmPVUxAR8evOdQp5Zvauma3wP59kZr8Mfmk9Izs5lmKFgogI0L2ewsPAz/CvjOqcWwZcGsyietKYzCRWl1TT2KJrFUREuhMK8c65z/bb1hKMYkLh+FEDaWj2smRLZahLEREJue6Ewk4zG4VvMTzM7EL2W5uoLzt25EAiDD5ZvyvUpYiIhFx3QuGHwIPAODPbDtwIXBvUqnpQclwUE4Yk88n6naEuRUQk5Dq7ohkA59wG4MtmlgBEOOdqgl9Wz5o5ciCPfbyJllYvkZ6wn5AlImGss5vs/LSD7QB0sp5RnzN2UBJNrV427apjdGZiqMsREQmZznoKYXPj4jz/PZqLymoUCiIS1jpb++g3PVlIKI3K9C2Gt7a0ljMnhLgYEZEQ6s7Fa4+bWUqb56lm9mhwy+pZ8dGR5KTFsa6sNtSliIiEVHdGVSc55wKT+J1zu/HdWrNfGZOZxLrSfjeGLiJyULoTChFmlrr3iZml0Y1ZS33NmEGJbCjfQ0urN9SliIiETHf+uN8JzDezf+K7P8KFwB+CWlUIjMvyzUBaV1bLUdkDQl2OiEhIdNlTcM49AVwAlAIlwDecc08Gu7CeNjXH1xlaslXLXYhI+OrWlVrOuZXAc/juqVxrZrlBrSoEhg2MJzU+SmsgiUhY687so3PNbB2wEZgHbALeCHJdPc7MmJqbyudbd4e6FBGRkOlOT+F3wExgrXNuBHAa8GlQqwqRKTkprCurpbqhOdSliIiERHdCodk5twvfLKQI59xcID/IdYXEjOFpOAfvrioNdSkiIiHRnVCoNLNE4APgKTP7K7AnuGWFxrEj0sgblMiD8zbgnAt1OSIiPa47oXAeUAf8BHgTWA98LZhFhUpEhHHNiSNZXVKj+yuISFjqNBTMzAO86pzzOudanHOPO+fu8p9O6pe+NnkwMZERvKNTSCIShjoNBedcK+A1s+QeqifkYqM8HDMijY/W6aY7IhJ+unNFcy2w3Mzm0GYswTl3Q9CqCrFZYzL4w+urKK6qJzs5LtTliIj0mO6MKbwA/ArfQPOiNj/91ol56QA8u3CrBpxFJKx0JxSe5YsgeNY/rvB4V79kZo+aWZmZrejg9ZPNrMrMlvh/fn1wpQfP2EFJnDouk/99Zx33z1sf6nKkB1XVNXPv3CKq6nWtioSnDkPBzCLN7HZgG/A48ASw1cxuN7Oobhz7MeDMLvb50Dk3xf/z2+4WHWxmxsNX5POl0QP5x/zN6i2Ekb+8s5Y/v7WG7z62kLqmllCXI9LjOusp/BlIA0Y456Y756YBo4AU4I6uDuyc+wCoOCJVhoAnwvjG1KHsqGrg/TXlFFfVh7okCbJtu+t4asFmJuek8PmW3Xz/yUU0trSGuiyRHtVZKHwVuMY5F7jzjHOuGrgWOPsIvf9xZrbUzN4ws6M72snMZptZgZkVlJeXH6G37trpRw8i2hPBVY8t5OIH5/fY+0povLK0mOZWx33fmsbtF07mw3U7eeTDjaEuS6RHdRYKzrVz3sQ/TfVInE9ZDAxzzk0G7gZe6qSQh5xz+c65/IyMjCPw1t0zIDaKy2cOY2hqHFsr6impaqCl1cvzi7bR0KxvkP3N2tIasgbEMiQljgunDyV/WCqvLSsOdVkiPaqzUCg0syv232hmlwOrD/eNnXPVzrla/+PXgSgzSz/c4x5pv/7aeO6+zHf30SVbK3lrZSk3/XMpv3llZYgrkyNtbWkNYwYlBp6fOSGLwuJqtuyqC2FVIj2rs1D4IfBDM3vfzO70/8wDbsB3CumwmFmWmZn/8TH+WnrlldJHZQ8gymMs3VbJ24UlADz92VZeX65vkf1Fq9dRVFZL3qCkwLavHJ0FwJsr9TlL+Ojw4jXn3HbgWDM7Fdh7vv9159y73TmwmT0NnAykm9k24L+AKP+xH8B3W89rzawFqAcube90VW8QG+VhXNYACjZVsLqkhvOnDmFDeS23PL+MyTkpDEnRBW593daKOhpbvOS16SnkpMUzOjORBRsqmD1rVAirE+k5XV7R7Jx7D3jvYA/snLusi9fvAe452OOGyuScZP7x6RYAzp6YTd6gRM656yO+8+hnPD17JumJMSGuUA7H2lLffIoxbXoKAJOGJPNRkZY8kfDRrdtxClx53HAuzh/K5TNzmZWXzrCBCTx8RT5bd9dx3j0f80nRTrxep4ue+qh1ZbUAjMlM3Gf7hCHJlNU0UlrdEIqyRHpcd9Y+EnzfIG+/cPI+244bNZBnZh/HT59dwjcfWcCQlDi2V9YzMj2BFq/ja5Ozue7k0STEtP/P3Op1eCKMkqoG4mM8DIjtzjWBEgybd+0hIymGpP0+g4lDfWtBLt9WxaDxsaEoTaRHqadwmKbkpPDaDSdy9QkjGDYwnutPHc2ozERGZiRw79z1XP14AU0t3sD+ZTUNVNU1s3HnHibd+hZvLC/m/Ps+5r/+3f5sJuccVz++kN++UrjPceTIKq5qYHDygX/0x2cPIMJg+faqEFQl0vPUUzgC4qI9/PKr4w/Y/sLibfz0uaUc/9/v8f1ZI6lpbOGud9cR7YngqMED2NPUyn+9vJKymkbmr9+Fcw4zY1VxNbe+vJL7vjWNzRV1vLOqDID65hb+9I1JABSV1XLz88t44PLpZCRpPONw7aisZ0xm0gHbE2IifYPNG3vlxDiRI06hEETfmDaUhJhIHv9kE394fRWeCOPLRw1i2+46lm6tJDU+irKaRgBKqhvYUdXAkJQ4Hpy3ngUbK3j6sy3srG0iOjKC08Zl8urSYqblpvLmihISYyNZtHk3n22s4JxJ2R3WsL68llEZiR2+Lr7eWHFVA7Py2r8w8vypQ7ntzdUs2lzB9GFpPVydSM/S6aMg+8rRWfztyhmMzEggPtrDf18wkb9fNYPZs0byyJX5wBeDm798cTmznyjg9RW+ayH+8ekWXlm6g9PGZXLBtKHUNLbw8xeX8+7qMv69ZAfwxayZ9izYsIvT7pzHi59v44n5m1hdUg2A1+u0pk8b1Q0t1DW1MriDe2dcefww0hOjue3NNVocUfo9hUIPiIv28PwPjufV608gPTGG7OQ4fn72UUwflsZtF0zk/sunEx/tYe6act4uLKWpxctNp+dRUt1AU6uXq08cwQlj0omL8tDc6jgqewAAiTGRgVB4bVkxs58o4IE2S33vvaXoz15Yzq//vZJrnihgT2MLv321kNPunMeizRWc+b8fsHJHFVX1zXy+ZTe1je2vDNrc6uX8+z7mpc+377O9qcXLh+vKD/mPZavXdfiePWXvYofZKe0PJMdHR3LTGWP5bGMFT366uSdLE+lxOn3UQ1IToklNiD5g+yUzcgE4buRANu7cwz3fnMb2ynq+fFQmM0akMWFIMon+2UtnHD2INSU1vHjd8awqruaBeetZU1rD/e+v57Y3VxMf7eG91WVMHppCZV0T768pJyMphvKaRiYNTWb59ique2oxHxXtpNXr+P6Ti9hZ28SVjy6kur6ZplYvZ4wfxENX5B9Q54INFXy+pRLnNvH1qUMC219YvI1bXljOnRdN5qkFm7nxy3nMysvAOUdji5fYKA9bK+qYv34X504ZTGyUZ5/jPvTBBh78YD3Pff84fvnSCv73kikMPgIXAxZX1VNe08iYzCTiovd9T6/Xce/cIs6bMoTcgfEUV/qmm3Z2l71LZ+Tw5ooSbn15JSu2V/HH8ycS6dF3Kul/FAq9xD3fnIbXORJiIhk/2NcTmDly4D77/PnCybR6HbFRHqbmppI3KIm3C0u57c3VfG3yYP7zK2M57c55XPbwp4Hf+cXZRzE5J4UJQwbwj08388fXVxMbFUFmUgzFVQ2cNi6TzzZW8NXJ2cREenj6sy389pVCBqfEcvWJI9m9p4nnF28LzL5ZsrWSrRV15KTFAwQu7PrP55fR6nX85Z21zMrL4Jbnl/NswVbiojzU+xcPfGXZDi6dkcvw9Hjmri4jd2ACryzdQWVdMz94chEbdu7h46KdXJSfA8Dry4uprGvmm8fmdvnvV9PQzB1vreHj9bt44PLpnHfPR+xpauXi/KEHTCVetr2KO+es5ZP1u/i/a45lx96eQjuzj/YyM+7+5lRue2M1Ty3YwlcnDe5wDEKkL1Mo9BL7f5ttT3Tkvt9M8wYl4Rykxkfxx/MnkBQbxbUnj2Le2nKyBsQyZ1Upp4zLZLR/zGL2rFG+K3YdbNq1hzveWsOfvjGRjKQYzIw9jS3MKSzh0Y99y0XXNbXywuJtbPIvCDc5J4WlWyt5dVkx1548Cuccn27YRVJMJDWNLQxNjePzLZUUbKrgrcISJuekcMzwVNISYoiJjOD3rxXy4bovrg6OjYqgodk3zXbDTt/tv4v8F5E1NLfyixeXs7uumcEpsZyUl8G6slpy0+IP6G0A3Pf+ep74dDPOwd3vrWNPUysDYiP5uOjAWUMfrvUtvz5/wy7eXFFCSVUDEQaZXcziGhAbxa++Op4XFm9nTmGpQkH6JetrA2f5+fmuoKAg1GX0Cht37uGUO97nF2cfxTWzRu7zmtfrKK5u6HBdJucce5paA6em9ioqq6GuqZVfvrSCZduqyEyK4XsnjODBDzbwl0um8Nd31lLT0MLbP5nFurJazvjLB/zu6xNobG7la5MHc/r/zCMrOZa1pbX85ZLJnD91aODYO2sbKatuZE1pNVGeCK5/+nOcg9PHD2JOYSmREcaXRqdT09BMZEQEn22qIC0hGk+E8Z3jh/Pnt9bgiTDGZSVx2wWTmDAkmaKyGkqrG7nx2SVMHprMsm1VVOxposXr+NEpo7lnbhHv/PQkPBHGsLR4IiKMix+YT01jC845KvY0MTYriaKyWub/7LRu/bvPfqKA5dur+OSWU/Gv6SjS65nZIufcgeeG96OeQh82Ij2Bt38y64ClGQAiIqzThfrM7IBAABjtn6v/yJX5LN5cySnjMoiJ9DB71kjMjJKqem5+fjmLt1QGBrJPzssInE667NhcHpy3AYDjR+27Enp6YgzpiTGB02PvrSqjsLia2y6YxLEjtrF4y27eXllKi9f3RSU3LZ67L5vK+fd9zJ/fWsO03BSOH5XOUws2c8fba7j5zHFc8uB8qht8A9WXzMglOnIbry8vISctjrMmZnHP3CIufOATKuuaGZ2ZyNPXzGTxlt1cM2sk50zM5rx7P+bDdTu54rhh3f53P338IN4uLGXF9urAFc8i/YVCoY/LG3TgBVdHQmZSLGdOyAo83/uN+JxJg/nNK4X8+JnP2ba7nq9PGRwIBPCtEfXIhxsZmZ7AoAGdLwtx+4WTaPGPkVx94kjuencdry8vIcLglrPGMWFwMpNzUrjy+OG+8ZBvTGRc1gAiIoy731vH8m1VxEV7OHNCFuvL93DK2Ay2VNTx+vISpuakMi5rAEkxkVTWNXPm0Vm8ubKEHz/zOS1ex6njMpkwJJl7LpuKA85q09aunHbUICIM5hSWKBSk31EoyEFJjInkp6fn8eqyYk4ck8Fvzt33LqqDU+L45TlHkZnU9TpBkZ4IItsMD+zt8UzJSdlnqepfnTOea08aRaY/ZC47Jod75xbR6hzPfG/mPiubzhieCsDU3BQ8EcZXJmRRVtPIvd+axql3vs8n63cxeWgy+cN8+501seML/zqSlhBN/rA03i4s5adnjD3o3xfpzRQKctCuPnEkV584ssPXr/rSiEM6bl6W74/7SXmZ+2yPiLBAIIBv6ugjV+aTkxoXON2118Qhyfz10imcPn4QAHdcNDmwfMjF+Tn8+a01XHvyqMMeCzh9/CD+8PqqfWZiifQHmmgtvcaojET+eukUrjpheJf7njI284BAAN9prvOmDCE+OnKfbQDfO2EE935zGmeM7/6poo7sDZ13/eMqIv2FQkF6lfOmDAnaEuKxUR7OmZRNRMThzxganp7AiPQE5vmnt3bG63Xc+Mzn5P9+Djc+8znNrVrtVnovhYLIITopL4P5G3bR0Nz5OlJPLdjMS0t2cFT2AF5asoPfvVrYQxWKHDyFgsghOmlsBg3NXhZuquhwn38v2c7vXl3FrLwMnvjuMVz1peE8MX9zpwsZioSSQkHkEM0cMZDYqAj+tWhbu6+v3FHFjc8uYWpuCndfOhUz4/pTxxAbFcEjH27o4WpFukezj0QOUVy0h+9+aQT3vb+eq08YycJNFTyzcAunjM0kJsrDh+vKSYmL4qEr8kmO842TpCVEc+H0oTy3cBs/P/soUuIPXCRRJJTUUxA5DNeePIqBCdH87tVC7nu/iJKqBh76cAP3vLeOz7dU8pPT8wKBsNcF04bS1OrlgzbrQIn0FuopiByGpNgobvzyGH7lv8f2Y1fN4KS8DBpbvKwvr2W8/94XbU0amkJqfBTvry7j3MmDe7pkkU6ppyBymC49JpeRGQkMGxjPrDEZmBmxUR6OHpzc7kVyngjjpLwM5q0tx+vtWwtSSv+nnoLIYYryRPD0NTNpbvV2+xqIk8dm8tKSHazcoUX1pHdRT0HkCBg0IJahqd1f7uKYEWkALN6yO1gliRwShYJICGQnx5KZFMOSrZWhLkVkH0ELBTN71MzKzGxFB6+bmd1lZkVmtszMpgWrFpHexsyYkpOiUJBeJ5g9hceAMzt5/SxgjP9nNnB/EGsR6XUm56SwceceKuuaQl2KSEDQQsE59wHQ8fX/cB7whPP5FEgxs4Nf3F6kj5qakwKg3oL0KqEcUxgCbG3zfJt/m0hYmJKbQpTHmL9+V6hLEQnoEwPNZjbbzArMrKC8vOulikX6gvjoSPKHpXVr+W2RnhLKUNgO5LR5PtS/7QDOuYecc/nOufyMjIweKU6kJ8zKy2B1SQ2l1Q2hLkUECG0ovAxc4Z+FNBOocs4Vh7AekR53Up7vS84H6i1ILxHMKalPA/OBsWa2zcy+Z2Y/MLMf+Hd5HdgAFAEPA9cFqxaR3uqo7CQykmK0OJ70GkFb5sI5d1kXrzvgh8F6f5G+wMw4cUw6c1eX0ep1eI7ArUJFDkefGGgW6c9Oystgd10zy7dXhboUEYWCSKidMDodM40rSO+gUBAJsYGJMUwckqxQkF5BoSDSC8wak8HnWyupbmgOdSkS5hQKIr3ArLwMWr2OT4o0C0lCS6Eg0gtMzU0hKSZSVzdLyCkURHqBKE8Es8Zm8NbKUhpbWkNdjoQxhYJIL3Fxfg4Ve5qYU1ga6lIkjCkURHqJE0anMyQljmcXbu16Z5EgUSiI9BKeCOPi/Bw+XLeTrRV1h328Vq9j0eYKFm2uwLeAgEjXFAoivchF+UMxg+cKDr238EnRTu6dW8T1Ty/mgvvnc8H983nwgw1HsErpz4K29pGIHLzBKXGclJfBcwVbue7k0cRFew7q93fVNnLd/y2mss53vcNNp+exfHsVd769hgiD86YMYdCA2GCULv2Eegoivcy1J42irKaR379WeFC/t6Oynh8/s4Q9jS38/TszeGb2TK4/bQy3XziJKTkp/PH11cz807v8/MXlNLd6g1S99HXqKYj0MseOHMjsE0fy4AcbOHlsJqePH9Tl7yzavJvvPraQhuZWbj33aE4Zlxl4LSU+mn/+4HjWl9fy5PzNPPbJJsprGnnw8ulEaFVW2Y96CiK90E/PyOPowQO4+fllbCiv7XTfDeW1XPG3BaTGR/H2T2bxrWOHtbvfqIxEbj33aH711fHMKSzlgQ/WB6N06eMUCiK9UEzYY764AAANKElEQVSkh79eOhXnHOfe8zE3PbeUFe0srd3Y0soNz3xOVGQET8+eybCBCV0e+7tfGs45k7L5y5y1rO8icCT8KBREeqnRmYm8dsOJzMpL551VpVz20Kf8e8l2Nu/aQ31TK82tXn790kpWbK/m9gsmkZ0c163jmhm3fu1oYqM83PrySk1XlX1oTEGkFxucEsd935pOcVU933x4AT9+ZkngtQgDr4PrTx3NGUdnHdRxM5JiuPHLefzu1UI+KtrJiWMyjnTp0kdZX/uWkJ+f7woKCkJdhkiPa2xpZcX2KjburKO8ppH6phbGZg3grAlZhzRg3NjSyql3zCM90TcQHR2pEwf9mZktcs7ld7mfQkEkfP1r0Tb+3z+XMmhADI9ddQxHZQ8IdUkSJN0NBX01EAljF04fyuPfPYZWL/zHv5bSousXwp7GFETC3El5Gfz2vKO57qnFfP2+j8kflsaybZXERnn40amjWbK1kstnDmNAbNQBvztvbTlzV5eRkRTD5ccOIzn+wH2kb9HpIxHBOcdzBVt58tPNbNpZx5CUODbu3EOTv+eQnhjN+MHJzD5xJCnxUcwpLGVOYSmFxdXERXloaGklPTGGH548imcLtjE+ewA3nzmWzCO4pMbWijq2V9YzLTdV4x+HQGMKInJYPttYwSfrdzJjeBpPzt/M8u1VbK+sB8AMpuemcu6UwVw6I5e1pTXc/PwyVu6oZmBCNFX1zbR4HdNyU7jxy3nMyju02U1er+Pd1WU8MG89izbvBnwzpx789nSm5aYesbaGA4WCiBxR9U2tPPnpJlLiojn1qEzSE2P2eb2l1cvbhaUcOyKN3XXNvLWyhKc/28K23fVcPjOXX54zntio7i3w19Lq5W8fbeSpBVvYUlHH0NQ4rjxuOENS4/jvN1ZT19TCC9d+idyB8cFoar+kUBCRkGtq8XLH22t46IMN5KbFk5sWT3ZyLN8/aSSjM5Pa/Z2G5lau+vtC5m/YxfGjBnLJjBzOmZhNpMd3yqiorIYL7p9PZITx4Lenkz88rSeb1GcpFESk15i3tpz75hbR1OqlqLSW2GgPz8yeycCEaBpbvIHlvL1ex43PLuGVZTu47YJJXJyf0+7x1pfXcvXjBWzfXc8fvzGRC6cP7cnm9Em9IhTM7Ezgr4AHeMQ599/7vf4d4M/Adv+me5xzj3R2TIWCSN+2rrSGix6cz57GFiLM8EQYPzp1NM7B2tIa/r1kB/955liuO3l0p8eprGviuqcW88n6XfzolNHcdEYeZlr1tSMhDwUz8wBrgdOBbcBC4DLnXGGbfb4D5DvnftTd4yoURPq+4qp6Hnh/PV4HK3ZU8fmWysBr1586mp+e3r0/8M2tXn754gqeLdjKdSeP4rpTRpMY072Z9k0tXt5ZVUpWcixTc1IC7+ec65FweXtlCf9YsIVxWUmsL6tlSGocy7dXERfl4crjh5OdHMukoSlH7P16QygcB9zqnPuK//nPAJxzf2qzz3dQKIiEtVavY0dlPakJ0dQ2tJCVfHDTWL1ex3/8axnPL95GUmwkt10wibMnZne4f3FVPW+uKOGhDzZQXNUAwLisJIamxrFkayVV9c3kDUriwulDuSg/p9sh013ry2v5/auFzF1TTmZSDOW1jeSkxlNS3cDg5FiqG1qo2NMEwDkTs/n118bjdY5BSbGHdf+L3hAKFwJnOueu9j//NnBs2wDwh8KfgHJ8vYqfOOc6vTmtQkFE9uecY/GW3fzu1VUs2VrJpKHJ3HTGWFLioqhraqW2sYWdtY18VLST15YVA5A/LJXrThlFSVUjzy7cQlV9MzOGp5GWEM2nGytYurWSxJhILsofygXThjI+e8Ah/1F2zvHqsmIKi6t56tPNmBnfP2kkV58wEq9zxEZ5aGrxEhlh1Da1sK60lvnrd3LXe0U0t3pxDsZkJvKfZ47r1k2X2tNXQmEgUOucazSz7wOXOOdObedYs4HZALm5udM3b94clJpFpG9ravHyzMIt/O2jjWzeVXfA6zGREVxz4kjOmZTNuKykTk8TLdlayd8/3shry4pp8TrSE6M5cUwGJ4/N4IzxWe3eP9s5h3MEwmNnbSNzV5fx3uoy3lhRAvh6JQ9fkU9OWtfTaTft3MPTC7eQGh/NC4u3ccmMXL53woju/nPsozeEQpenj/bb3wNUOOeSOzuuegoi0pWG5lZeWLydpNhI0hKiiYv2kJ0cy4DYKBIO8nTQztpGPlhb7vtZt5OKPU0kxUTylQlZJMZE4nWOsyZkM2N4Kj95bimfbdzFRdNz+GxjBQWbK/A6SIqN5Krjh3PDaWPwRNghjVl4vY5W54jyHNrV3L0hFCLxnRI6Dd/sooXAN51zK9vsk+2cK/Y/Ph+42Tk3s7PjKhREJFS8XseCjRX8s2Ar764uwzlHi9dR19TKkJQ4tlfWk5sWz5aKOsZlJXHG0VmcMX7QYZ16OlK6GwpBWxDPOddiZj8C3sI3JfVR59xKM/stUOCcexm4wczOBVqACuA7wapHRORwRUQYx40ayHGjBga21Te1cv/7Rawv38MPTxnNJTNyqKpvJi0hOoSVHjpdvCYiEgZ0PwURETloCgUREQlQKIiISIBCQUREAhQKIiISoFAQEZEAhYKIiAQoFEREJKDPXbxmZuXAoa6Ilw7sPILl9EZqY9/X39sH/b+NvbF9w5xzGV3t1OdC4XCYWUF3rujry9TGvq+/tw/6fxv7cvt0+khERAIUCiIiEhBuofBQqAvoAWpj39ff2wf9v419tn1hNaYgIiKdC7eegoiIdCJsQsHMzjSzNWZWZGa3hLqeI8HMNpnZcjNbYmYF/m1pZjbHzNb5/5sa6joPhpk9amZlZraizbZ222Q+d/k/02VmNi10lXdfB2281cy2+z/LJWZ2dpvXfuZv4xoz+0poqu4+M8sxs7lmVmhmK83sx/7t/eZz7KSNff9z9N1oun//4Lvz23pgJBANLAXGh7quI9CuTUD6fttuB27xP74FuC3UdR5km2YB04AVXbUJOBt4AzBgJrAg1PUfRhtvBf5fO/uO9///GgOM8P9/7Al1G7poXzYwzf84Cd9tecf3p8+xkzb2+c8xXHoKxwBFzrkNzrkm4BngvBDXFCznAY/7Hz8OfD2EtRw059wH+G7N2lZHbToPeML5fAqkmFl2z1R66DpoY0fOA55xzjU65zYCRfj+f+61nHPFzrnF/sc1wCpgCP3oc+ykjR3pM59juITCEGBrm+fb6PwD7Csc8LaZLTKz2f5tg5xzxf7HJcCg0JR2RHXUpv72uf7If/rk0Tan/fp0G81sODAVWEA//Rz3ayP08c8xXEKhvzrBOTcNOAv4oZnNavui8/Vb+9X0sv7YJr/7gVHAFKAYuDO05Rw+M0sEngdudM5Vt32tv3yO7bSxz3+O4RIK24GcNs+H+rf1ac657f7/lgEv4uuOlu7tevv/Wxa6Co+YjtrUbz5X51ypc67VOecFHuaLUwt9so1mFoXvj+VTzrkX/Jv71efYXhv7w+cYLqGwEBhjZiPMLBq4FHg5xDUdFjNLMLOkvY+BM4AV+Np1pX+3K4F/h6bCI6qjNr0MXOGfvTITqGpzeqJP2e8c+vn4PkvwtfFSM4sxsxHAGOCznq7vYJiZAX8DVjnn/qfNS/3mc+yojf3icwz1SHdP/eCb4bAW36j/L0JdzxFoz0h8sxmWAiv3tgkYCLwLrAPeAdJCXetBtutpfN3uZnznXb/XUZvwzVa51/+ZLgfyQ13/YbTxSX8bluH7A5LdZv9f+Nu4Bjgr1PV3o30n4Ds1tAxY4v85uz99jp20sc9/jrqiWUREAsLl9JGIiHSDQkFERAIUCiIiEqBQEBGRAIWCiIgEKBRE/Mystc3qlkuO5Gq6Zja87aqoIr1VZKgLEOlF6p1zU0JdhEgoqacg0gX/fStu99+74jMzG+3fPtzM3vMvfvaumeX6tw8ysxfNbKn/53j/oTxm9rB//f23zSzOv/8N/nX5l5nZMyFqpgigUBBpK26/00eXtHmtyjk3EbgH+F//truBx51zk4CngLv82+8C5jnnJuO7b8JK//YxwL3OuaOBSuAC//ZbgKn+4/wgWI0T6Q5d0SziZ2a1zrnEdrZvAk51zm3wL4JW4pwbaGY78S1j0OzfXuycSzezcmCoc66xzTGGA3Occ2P8z28GopxzvzezN4Fa4CXgJedcbZCbKtIh9RREusd18PhgNLZ53MoXY3rn4Fv7Zxqw0Mw01icho1AQ6Z5L2vx3vv/xJ/hW3AX4FvCh//G7wLUAZuYxs+SODmpmEUCOc24ucDOQDBzQWxHpKfpGIvKFODNb0ub5m865vdNSU81sGb5v+5f5t10P/N3M/gMoB67yb/8x8JCZfQ9fj+BafKuitscD/MMfHAbc5ZyrPGItEjlIGlMQ6YJ/TCHfObcz1LWIBJtOH4mISIB6CiIiEqCegoiIBCgUREQkQKEgIiIBCgUREQlQKIiISIBCQUREAv4/UdnForwdZhAAAAAASUVORK5CYII=\n", 138 | "text/plain": [ 139 | "" 140 | ] 141 | }, 142 | "metadata": {}, 143 | "output_type": "display_data" 144 | } 145 | ], 146 | "source": [ 147 | "x3 = range(len(gen_pretrain_eval_loss) + len(gen_adversarial_eval_loss))\n", 148 | "gen_loss = gen_pretrain_eval_loss + gen_adversarial_eval_loss\n", 149 | "plt.plot(x3, gen_loss)\n", 150 | "plt.xlabel('Epochs')\n", 151 | "plt.ylabel('Oracle NLL')\n", 152 | "plt.savefig('gen_loss.png',dpi=600)" 153 | ] 154 | }, 155 | { 156 | "cell_type": "markdown", 157 | "metadata": {}, 158 | "source": [ 159 | "# Discriminator" 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": 7, 165 | "metadata": {}, 166 | "outputs": [ 167 | { 168 | "data": { 169 | "text/plain": [ 170 | "Text(0,0.5,'Oracle NLL')" 171 | ] 172 | }, 173 | "execution_count": 7, 174 | "metadata": {}, 175 | "output_type": "execute_result" 176 | }, 177 | { 178 | "data": { 179 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd8VFX6+PHPMymEhISQAoQUAiQQOkLoqIANu+4qxb5FV1d3193VVbfp4upW13X9+nMtay+gYsEKiKKCtASpCSX0ICUJNdSU5/fHvegY0pPJJJPn/XrNKzNnzj3zXIzz5Jxz7zmiqhhjjDH15fF3AMYYY1o2SyTGGGMaxBKJMcaYBrFEYowxpkEskRhjjGkQSyTGGGMaxBKJMcaYBrFEYowxpkEskRhjjGmQYH8H0BTi4uI0NTXV32EYY0yLkp2dXaiq8TXVaxWJJDU1laysLH+HYYwxLYqIbK1NPRvaMsYY0yCWSIwxxjSIJRJjjDENYonEGGNMg/g0kYjIBBFZJyJ5InJ3Je8/LCLL3cd6Ednv9d71IrLBfVzvVT5ERFa5bf5HRMSX52CMMaZ6PrtqS0SCgMeAc4B8YKmIzFTVnJN1VPWXXvV/BpzmPo8B7gUyAQWy3WP3AY8DNwKLgQ+ACcCHvjoPY4wx1fNlj2QYkKeqm1T1BDANuLSa+lOAV93n5wFzVHWvmzzmABNEJAGIUtVF6mzt+AJwme9OwRhjTE18mUgSge1er/PdslOISFegG/BJDccmus9rbLMxzF6zi1eXbPNV88YYExCay2T7ZOANVS1rrAZF5CYRyRKRrIKCgnq18UZ2PlPfzWHH/qONFZYxxgQcXyaSHUCy1+skt6wyk/l2WKu6Y3e4z2tsU1WfVNVMVc2Mj6/xDv9K/fHiPijK1HfX1Ot4Y4xpDXyZSJYC6SLSTURCcZLFzIqVRCQD6AAs9CqeBZwrIh1EpANwLjBLVXcCB0VkhHu11nXAO746gaQO4fxsfDqz1uzm07V7fPUxxhjTovkskahqKXAbTlLIBV5T1TUiMlVELvGqOhmY5k6enzx2L3A/TjJaCkx1ywB+CjwN5AEb8fEVWzee3p3u8RHcO3MNx0oabeTNGGMChnh9fweszMxMbciijQvyCrn66cX84qx0fnlOz0aMzBhjmi8RyVbVzJrqNZfJ9mZtdFocFw1I4PHPNrK16LC/wzHGmGbFEkkt/eGiPoQGebh35hpaQy/OGGNqyxJJLXWKCuP2s9OZt66AWWt2+zscY4xpNiyR1MENo1LJ6BzJ1HfXcOREqb/DMcaYZsESSR0EB3m4/7J+fH3gGI9+kufvcIwxplmwRFJHQ1Nj+P7gJJ76fBOr8g/4OxxjjPE7SyT18IeLehPXrg23T/+Koyfs3hJjTOtmiaQeosNDeWjiQDYWHObBD3L9HY4xxviVJZJ6Gp0Wx4/HdOPFRVtt+RRjTKtmiaQB7jivFxmdI7nzjRUUFh/3dzjGGOMXlkgaICwkiH9PHsTBY6XcPWOV3ahojGmVLJE0UEbnKO6akMHHubuZtnR7zQcYY0yAsUTSCH4wKpUxaXFMfTeHTQXF/g7HGGOalCWSRuDxCP+8ciChwR5+OX05JWXl/g7JGGOajCWSRtK5fRj3X9aPFfkHeH/lTn+HY4wxTcYSSSO6qH8CqbHhvLhoq79DMcaYJmOJpBF5PMI1I7qSvXUfa7625VOMMa2DJZJGduWQZMJCPLy0aJu/QzHGmCZhiaSRtQ8P4ZKBXXj7qx0cOFri73CMMcbnLJH4wHUjUzlaUsaby/L9HYoxxvicTxOJiEwQkXUikicid1dRZ6KI5IjIGhF5xav8byKy2n1M8ip/TkQ2i8hy9zHIl+dQH/0S2zMoOZoXF221u92NMQHPZ4lERIKAx4DzgT7AFBHpU6FOOnAPMFpV+wK3u+UXAoOBQcBw4A4RifI69E5VHeQ+lvvqHBri2hFd2VRwmC83Fvk7FGOM8Slf9kiGAXmquklVTwDTgEsr1LkReExV9wGo6slldPsAn6tqqaoeBlYCE3wYa6O7cEACHcJDeHGhXQpsjAlsvkwkiYD34lP5bpm3nkBPEVkgIotE5GSyWAFMEJFwEYkDxgHJXsc9ICIrReRhEWnjqxNoiLCQICYOTWZO7m52Hjjq73CMMcZn/D3ZHgykA2OBKcBTIhKtqrOBD4AvgVeBhcDJrQjvATKAoUAMcFdlDYvITSKSJSJZBQUFPj2JqlwzvCvlqry6xBZzNMYELl8mkh18txeR5JZ5ywdmqmqJqm4G1uMkFlT1AXcO5BxA3PdQ1Z3qOA48izOEdgpVfVJVM1U1Mz4+vlFPrLaSY8IZ16sjry7ZxolSW3/LGBOYfJlIlgLpItJNREKBycDMCnXexumN4A5h9QQ2iUiQiMS65QOAAcBs93WC+1OAy4DVPjyHBrt2RFcKDh1nds6uU97bVFDMfz/bSPbWfX6IzBhjGkewrxpW1VIRuQ2YBQQBz6jqGhGZCmSp6kz3vXNFJAdn6OpOVS0SkTDgCydXcBC4RlVL3aZfFpF4nF7KcuBmX51DYzijZzzJMW15ceFWLhrQha1Fh3lv5U7eX7mTnJ0HAeidEMWHvzjdz5EaY0z9SGu4zyEzM1OzsrL89vlPfLaRv3y4lt4JUeS6yWNwSjQXDujCwaMlPDJ3A+//fAx9u7T3W4zGGFORiGSramZN9XzWIzHfujIzmf/N30xosIffXdCb8/t3JqlDOAD7j5zg8XkbeT0rn76XWCIxxrQ8lkiaQExEKEt+d3al70WHh3JOn068s3wHv72gN6HB/r6Qzhhj6sa+tZqBKzKT2HekhLm5u/0dijHG1JklkmbgjPR4OkW14fVsW+TRGNPyWCJpBoI8wvcGJ/HZ+gL2HDzm73CMMaZOLJE0E1cMSaKsXHnrq4r3bBpjTPNmiaSZ6BHfjsEp0byenV/t0vOHj5eyMn9/E0ZmjDHVs0TSjFyZmUzenmKWb688URwvLeOGZ5dwyf8tYP3uQ00cnTHGVM4SSTNy0YAEwkI8vFHJpLuqctcbK1m6ZR9BHmH6UlsI0hjTPFgiaUYiw0I4v18CM1d8zbGSsu+89++PN/D28q+587xenNe3E28uy+d4aVkVLRljTNOxRNLMXDEkiUPHSpm15ttFHmdk5/PI3A1cOSSJn47twaShKew7UsKcHLvvxBjjf5ZImpmR3WNJjG77zfDWok1F3P3mSkb1iOWBy/sjIoxJiyMxuq0NbxljmgVLJM2MxyN8f0gS8/MKmb+hkJ+8mE1KTDiPXz3km+VTgjzClZlJfLGhkO17j/g5YmNMa2eJpBm6YnASqnD9s0sI9gjP3jCM9uEh36lzZWYyIvB6lvVKjDH+ZYmkGUqJDWdk91iCPcJT12eSEht+Sp3E6LackR7Pa1n5lJUH/lYAxpjmyxJJM/XoVafxwS9OZ3BKhyrrTB6azK6Dx/h8vX/2pDfGGLBE0mzFtWtDj/h21dY5q3cnYiNCmbZ0WxNFZYwxp7JE0oKFBnv4/pAk5ubuYc8hW+zRGOMflkhauImZyZSWK28us8UejTH+YYmkhUvr2I6hqR2YvnR7tYs9GmOMr/g0kYjIBBFZJyJ5InJ3FXUmikiOiKwRkVe8yv8mIqvdxySv8m4isthtc7qIhPryHFqCSUNT2Fx4mCWb9/o7FGNMK+SzRCIiQcBjwPlAH2CKiPSpUCcduAcYrap9gdvd8guBwcAgYDhwh4hEuYf9DXhYVdOAfcCPfHUOLcUF/TsT2Sa4Vne6qyqrdxzgX3PWc9GjX3DLS9nsOmDzK8aY+gv2YdvDgDxV3QQgItOAS4Ecrzo3Ao+p6j4AVd3jlvcBPlfVUqBURFYCE0TkdWA8cJVb73ngPuBxH55HsxceGswlg7rwRnY+makxxESEEtsu1PkZEUpEm2Cyt+5j9prdzM7ZRf6+o3gETkvpwKfr9rAgr5B7L+7L9wYnIiL+Ph1jTAvjy0SSCHj/iZyP07vw1hNARBYAQcB9qvoRsAK4V0QeAsKBcTgJKBbY7yaYk20mVvbhInITcBNASkpKY5xPs3bNiK68np3Pb99aVWWd0GAPY9Li+Nn4NM7u3YnYdm3YXHiYO19fwa9fX8GHq3fy4OX96RgV1oSRG2NaOl8mktp+fjowFkgCPheR/qo6W0SGAl8CBcBCoE5rpqvqk8CTAJmZmQE/C907IYpV951LUfEJ9h4+QdHhE+xzf+4/coKMzlGc2Suedm2++5+8W1wE038ykmcXbOYfs9ZxzsOf86dL+nLpoC7WOzHG1IovE8kOINnrdZJb5i0fWKyqJcBmEVmPk1iWquoDwAMA7iT8eqAIiBaRYLdXUlmbrVab4CC6RLelS3TbOh0X5BF+fHp3xmV05M7XV3D79OW8nr2diwZ0YXxGRzpZD8UYUw1fJpKlQLqIdMP5sp/Mt3MbJ70NTAGeFZE4nKGuTe5EfbSqFonIAGAAMFtVVUQ+Ba4ApgHXA+/48BxalR7x7Xj95lE8u2Azzy7Ywj1vOsNk/RPbMz6jI2f17ki/Lu3xeKynYoz5lvjy3gMRuQD4N878xzOq+oCITAWyVHWmOGMnDwETcIauHlDVaSISBixzmzkI3Kyqy902u+MkkRjgK+AaVT1eXRyZmZmalZXlgzMMXKrK+t3FfJy7m0/W7mHZtn2oQqeoNlw0oAuXDupC/8T2NvxlTAATkWxVzayxXmu4ic0SScMVFR9n3roCPlqzi3nr9lBSpnSPi+CSQV24dFAi3eIiGuVzjpWUUVh8nKQOp654bIxpWpZIvFgiaVwHjpTwweqdvLN8B4s370UVBiZHc9u4NM7u3bFBvZQ/vL2a17K2M/uXZ9A1tnGSkzGmfiyReLFE4jtf7z/Keyu/ZtqS7WwqPMyoHrH87sLe9O3Svs5tHT5eyvAH51J8vJQze8bz3A+G2tCZMX5U20Ria22ZBukS3ZabzujBrF+ewdRL+5K78yAXPTqf37yxgj0H63bH/Psrd1J8vJSLB3bhs/UFfLh6l4+iNsY0JkskplGEBHm4bmQq8+4Yx4/HdOOtr3Yw9p/zeHTuBo6V1O4WoFeWbCOtYzsenjiQPglR/OndNRw6VuLjyI0xDWWJxDSq9uEh/O7CPnz8qzM5Iz2eh+as59531tR43NpdB1m+fT+ThyYTHOThgcv7sefQcR6es6EJojbGNIQlEuMTXWMj+O+1Q/jRmG68nr2dvD2Hqq0/bcl2QoM8fG9wEuCsA3bVsBSe+3Izq3ccaIqQjTH1ZInE+NSt49IIDw3mn7PWV1nnWEkZby7L57x+nYmJ+HZXgN+cl0FMRCi/f3s15eWBf1GIMS2VJRLjUzERodx0Rnc+WrOL5dv3V1rnw9U7OXislClDk79T7gyT9Wb59v28avvSG9NsWSIxPvejMd2IjQjlbx+urXQXx1eXbKdrbDgjusee8t5lgxIZ2T2Wv324lsLiahcwMMb4Sb0SiYh0aexATOCKaBPMz8ansXBTEfPzCr/z3saCYpZs3sukocmVruElItx/WT+OlpTx4Pu5TRWyMaYO6tsjWdSoUZiAd9XwriR1aMvfP1r3nfmO6Uu3E+wRrhiSVOWxaR3b8ZMzevDmVzu48D9f8Mz8zRRZ78SYZqO+icRuNzZ1Ehrs4dfn9mTVjgPf3Gh4orScGdn5nNW7Ix0jq1+q/hdnpzP10r4EeYSp7+Uw/MG53PhCFh+t3sWJ0vKmOAVjTBXqu4y8XUJj6uySgYk88dkm/jl7Hef27cScnN0UHT7B5GE172B58obH60amsm7XIWYsy+etr3YwJ2c3MRGhvPijYfValsUY03BVJhIReZTKE4YA0T6LyASsII9w53m9+NHzWbyRnc8Hq3aSGN2WM9Lj69ROr86R/PaC3vzmvF58saGQW17O5vWsfPpeYonEGH+orkdS3SqHtgKiqZfxGR3J7NqBf85aR9HhE9x+djpB9dwoKzjIw7iMjgzrFsuCCpP4xpimU2UiUdXnq3pPRP7pm3BMoBMR7jo/gyv/uxCPwMTM5JoPqsGYtFge/GAtuw8es22BjfGD+k62T2zUKEyrMjQ1hstPS+R7g5PqvL98ZUb1iAOwXokxflLfyXa7ass0yMOTBjVaW30SougQHsL8vMJv1uoyxjSd6ibbY6p6C0skphnxeIRRaXEsyCtEVW0zLGOaWHVDW9k4k+rZFR5ZQK02iRCRCSKyTkTyROTuKupMFJEcEVkjIq94lf/dLcsVkf+I++0gIvPcNpe7j461O1UTyMakxbH74HE2FhT7OxRjWp3qJtu7NaRhEQkCHgPOAfKBpSIyU1VzvOqkA/cAo1V138mkICKjgNHAALfqfOBMYJ77+mpVtSvHzDfGpDnzJPM3FJLWMdLP0RjTulTZIxGRlOoetWh7GJCnqptU9QQwDbi0Qp0bgcdUdR+Aqu5xyxUIA0KBNkAIsLtup2Zak+SYcFJiwpmfV1Rj3ae/2MTdM1bWeudGY0z1qptsfx/nC917wFmBeKAjEFRD24nAdq/X+cDwCnV6AojIAre9+1T1I1VdKCKfAjvdz/8/VfVese9ZESkDZgB/1sqWlDWtzui0ON5d8TWlZeUEB1X+N9KBoyU8NHs9R0vK2L7vCE9em0lEm/pec2KMgWp6JKraX1UHuD/7AxcDC4Bi4PZG+vxgIB0YC0wBnhKRaBFJA3oDSTgJabyInO4ec7Ubz+nu49rKGhaRm0QkS0SyCgoKGilc05yNSYuj+HgpK/Kr3lHx9aztHC0p45axPVi4sYhr/7eYA0dtX3hjGqLG+0hEJF1EngM+xJls76Oqj9ai7R2A991mSW6Zt3xgpqqWqOpmYD1OYrkcWKSqxapa7H72SABV3eH+PAS8gjOEdgpVfVJVM1U1Mz6+bktwmJZpZI9YRODLKu4nKStXnl+4hWGpMdw1IYP/d/VgVu04wJQnF9lqwsY0QHVzJP1E5FWc4aOPgX6q+rSq1vbPt6VAuoh0E5FQYDIws0Kdt3F6I4hIHM5Q1yZgG3CmiASLSAjORHuu+zrOrR8CXASsrmU8JsDFRITSt0vUKXuenPTJ2j1s33uUG0anAjChXwJPXz+UTYXFTHxiIbsOHGvCaI0JHNX1SFbg9AK+wPmr/2H3Mtz/iMh/ampYVUuB24BZQC7wmqquEZGpInKJW20WUCQiOcCnwJ2qWgS8AWwEVrlxrFDVd3Em3meJyEpgOU4P56k6n7UJWKPT4li2bR9HTpSe8t6zCzaT0D6Mc/t0+qbszJ7xPP+DYew+eJwrn/iSbUVHmjJcYwKCVDVPLSLXV3dgdWtxNTeZmZmalWVXC7cGn68v4LpnlvDcD4Yytte3txit23WI8/79Ob+Z0Iufjk075bgV2/dz/bNLaBPs4YUfDqdXZ7uE2BgRyVbVzJrq1WvRRmOaq6GpMYQGeViQV/idRPLcl1toE+xhytDKr1wfmBzN9JtGct0zi7nyv1/y9PVDGdatqsUdjDHe6rtoozHNUtvQIIZ07fCd+0n2HznBW1/lc/lpiXSICK3y2F6dI5lxyyjiIttw7f8WM3vNrqYI2ZgWzxKJCThj0uPI3XmQQvdKrOlLt3OspJzrR6XWeGxSh3DeuHkUGQlR3PxSNtOWbPNxtMa0fJZITMAZ7S6X8uXGIkrLynlh4VZGdI+hd0JUrY6PiQjllR8PZ0x6PHe/uYr/+2QDds+rMVWrzX0kPUVkroisdl8PEJHf+z40Y+qnf2J7IsOCWbChkI9z97Bj/1FuGFW3peMi2gTzv+szufy0RP45ez33zVxDebklE2MqU5u1IZ4C7gSeAFDVle4qvX/2ZWDG1FeQRxjVI5b5eYVsKTpMYnRbzu5d90WiQ4I8PHTlQGIjQnl6/mY6RIRy+9k9fRCxMS1bbYa2wlV1SYWyUy/SN6YZGZMWx479R1m8eS/Xjexa5dpbNfF4hN9d2JvvDU7k3x9vYG6urR1qTEW1+b+rUER64CzYiIhcgbOYojHN1sl5krYhQUyu4pLf2hIRHry8P30Sorh9+nK2FB5ujBCNCRi1SSS34gxrZYjIDpwFG2/xaVTGNFC3uAh6dYrkquEptA8PaXB7YSFBPHHtEII8wk9ezK70znljWqsq72w/paJIBOBxF0tsUezO9taprFwRnOGpxvL5+gKuf3YJFw3own8mD7JtfU1Aa/Cd7SLyqyrKAVDVf9U7OmOaQFAjJpCTzugZzx3n9uIfs9YxMKk9Pz69e6N/hjEtTXVXbdliQ8ZU4qdje7Ayfz9/+XAtfbu0Z2SPWH+HZIxf1XpoqyWzoS3T2A4dK+HSxxZw4EgJPzmzOx4RRASP8M1w2unp8XSLi/B3qMbUW4OHtrwaeh74harud193AB5S1R82PExjWqbIsBCevHYIk55YxIMfrK20zpCuHZhxy6gmjsyYplebGxIHnEwiAKq6T0RO82FMxrQIaR0jWfTbszhWUoYCWg6KUq7w0qKt/GvOejYXHrZeiQl4tbn81+P2QgAQkRhql4CMCXghQR4iw0KICguhfXgI0eGhxESEMmloMh6BGdn5/g7RGJ+rTSJ5CFgoIveLyJ+BL4G/+zYsY1q2TlFhnNEznhnL8imzNbpMgKsxkajqC8D3gd3ALuB7qvqirwMzpqW7YkgSOw8cY+HGoporG9OC1WoBIlVdA7wGzASKRaRha04Y0wqc3bsTUWHBzFhmw1smsNVmGflLRGQDsBn4DNgCfOjjuIxp8cJCgrh4YBc+XL2TQ8dK/B2OMT5Tmx7J/cAIYL2qdgPOAhbVpnERmSAi60QkT0TurqLORBHJEZE17vL0J8v/7pblish/xL2lXkSGiMgqt81vyo1pjq4YksSxknI+WGXrnJrAVZtEUqKqRThXb3lU9VOgxhtURCQIeAw4H+gDTBGRPhXqpAP3AKNVtS/OgpCIyChgNDAA6AcMBc50D3scuBFIdx8TanEOxvjFoORoesRHMCN7h79DqdL2vUf4ywe5HC8t83copoWqTSLZLyLtgM+Bl0XkEaA262gPA/JUdZOqngCmAZdWqHMj8Jiq7gNQ1T1uuQJhQCjQBggBdotIAhClqovUuSX/BeCyWsRijF+ICN8fksSSLXub5fLzqspv31rFE59v4pPcPTUfYEwlapNILgWOAL8EPgI2AhfX4rhEYLvX63y3zFtPoKeILBCRRSIyAUBVFwKf4ux7shOYpaq57vH5NbRpTLPyvdOS8Ai82Qwn3Wet2c0XGwoBmLniaz9HY1qqam8sdIen3lPVcUA58LwPPj8dGAskAZ+LSH8gDujtlgHMEZHTgaO1bVhEbgJuAkhJsYvMjP90bh/GmPR4Zizbwe1n96zTsvZHT5Tx8uKtFB8vJdgjBHk87k8hJNjD+f06E9euTb3iOnqijPvfy6FXp0gyUzvwenY+h46VEBnW8P1bTOtSbSJR1TIRKReR9qp6oI5t7wCSvV4nuWXe8oHFqloCbBaR9XybWBapajGAiHwIjARe5NvkUlWbJ2N/EngSnEUb6xi7MY3qiiFJ/PzVr1i0uYhRPeJqdUxJWTm3vbKMuWurHnL6aus+/jVpUL1i+u9nG9mx/yjTbhpBSJDw8uJtzF6zm+8PSar5YGO81Gapk2JglYjMwWtuRFV/XsNxS4F0EemG82U/GbiqQp23gSnAsyIShzPUtQnoDtwoIn/BWUz1TODfqrpTRA6KyAhgMXAd8GgtzsEYvzq3Tyciw4J5Izu/VolEVbnnzVXMXbuH+y/rx9XDUihTpaxcKS1XysqUv36Uy4xlO/jDRX3oEBFap3i2FR3h8c82cvHALozoHouqktShLe+s+NoSiamz2syRvAn8AWeyPdvrUS1VLQVuA2YBucBrqrpGRKaKyCVutVlAkYjk4MyJ3OleIfYGzlzMKmAFsEJV33WP+SnwNJDn1rF7WkyzFxYSxEUDuvDhql0UH695m96/frSWN7Lzuf3sdK4d0RWPRwgJ8hAWEkS7NsG0Dw/h+lGpnCgt5416rOd1//s5BHuE316QATgXBVw8sAsL8gopKj5e5/ZM61abRDKdb5PHdFV9XlVrNVeiqh+oak9V7aGqD7hlf1TVme5zVdVfqWofVe2vqtPc8jJV/Ymq9nbf+5VXm1mq2s9t8zZtDRuqmIBwxZAkjpaU1XhPyVOfb+KJzzZx7Yiu/OKs9CrrZXSOIrNrB15evJXyOqznNW/dHubk7Oa28WkktG/7TfklA7tQVq4+ueflxYVbuObpxdj/roGpykQiIsEi8neceYzncS613e7eKGizccbU0eCUaLrHRfDYp3m8tnQ7B46cerf7jOx8Hvgglwv7J3DfJX1r3BP+mhFd2VJ0hAUbC2sVw/HSMv70bg7d4iL40Zhu33kvo3Mk6R3bNfrVWwvyCrl35hrm5xWy88CxRm3bNA/V9Uj+AcQA3VR1iKoOBnoA0cA/myI4YwKJiPCHi/qgCr+ZsZLMB+bww+eW8tZXztVSn6zdzW9mrGR0Wiz/mjSwVnvOT+jXmQ7hIby0aGutYnhm/hY2Fx7m3ov70CY46JT4Lh3UhaVb9rFjf60vkKxW/r4j3PbKsm+uBFu361CjtGual+oSyUXAjar6zX95VT0I3AJc4OvAjAlE4zI68tmdY5l522huGJXK2p0H+eX0FQz588fc/NIyeidE8sS1mad8yVclLCSIiZnJfJy7h101/LW/68AxHv1kA+f06cTYXh0rrXPxwC4AvNsIvZJjJWXc/FI2pWXKCz8cBkDuroMNbtc0P9UlEq1s/kFVy3DuPDfG1IOIMCApmt9d2If5d41nxi0juWpYCmPS4njuB8No16Zu+8ZdNTyFsnLl1SXbqqxTXq7c/eZKSsuVP1zYp8p6XWMjGJgczczlDUskqsrv317N6h0HeXjSIAYmR5MY3Za1O61HEoiqSyQ5InJdxUIRuQaofJNqY0ydeDzCkK4x3HdJX565YWi9bi7sGhvBGT3jmbZ0GyVl5ZXWeWTuBuatK+APF/YmJTa82vYuGdiFnJ0HydtTXOdYTnpp8TbeyM7n52elc3afToAzB7PWeiQBqbpEcitwq4jME5GH3MdnwM9xhreMMc3ENcNT2H3wOHMkU/8WAAAZ+UlEQVRzd5/y3idrd/PI3A18b3Ai14zoWmNbFw9IQKT+S6Zkb93L1HfXMD6jI7d7XXWWkRDJpoLDtjhkAKoykajqDlUdDkzF2YNkCzBVVYepavNdytSYVmh8RkcS2ofx8uLvDm9tLTrM7dOW0ychigcv71/jVWAAHaPCGNk9lndXfF3ny3X3HDzGLS8to0t0Wx6eNOg7y8FkdI6itFzZuKf5LV5pGqY2W+1+oqqPuo+5TRGUMaZugoM8TBmWwhcbCtnsrjJ89EQZN7+0DBHhiWuHEBZSuwl8cIa3NhceZtWOmldGKi0rZ/6GQu55cxUTHvmCQ8dKefLaTNq3/e5dAr0TIgFseCsA1WqrXWNM8zd5aDJBHuGVxVu/WR5+7a6DPDJ5EMkx1c+LVHR+vwRCgqTKSfdvk8dKhj04l2v+t5h3lu9gdFocL/14OL06R55yTGpsBKHBHtbaJcABp26Xhxhjmq2OUWGc26cTr2fnEx/Zhre+2sGvzulZ5aW+1WkfHsKZPTvy3sqd/OrcnmzYXUzOzoPk7jxIztfOz8MnyogIDeKs3p24oH8CY3vFV9vrCQ7ykN6xnSWSAGSJxJgAcs2Irny4ehcPfrCWs3t35LZxafVu65JBXfg4dzd9753FyamSdm2C6Z0QyfeHJDE6LY4ze1afPCrK6BzFFxsK6h2TaZ4skRgTQEb1iKVXp0iOl5bx0MRBddr7pKJz+3Ti+pFdad82hD5douidEEVyh/AGtdk7IZIZy/IpKj5ObD33UTHNjyUSYwKIiDD9JyMIDfYQHtqw/73DQoL406X9Gikyx8m5k3W7DjEqzRJJoLDJdmMCTHR4aIOTiK9kdI4CsHmSAGOJxBjTZOIj2xDXLtQuAQ4wlkiMMU2qV+dI65EEGEskxpgmldE5inW7DlFWh824TPNmicQY06QyOkdyvLScrUW2VEqgsERijGlSvRNswj3QWCIxxjSptI7t8Ais3WkT7oHCp4lERCaIyDoRyRORu6uoM1FEckRkjYi84paNE5HlXo9jInKZ+95zIrLZ671BvjwHY0zjCgsJoltcBLnWIwkYPrvYXESCgMeAc4B8YKmIzFTVHK866cA9wGhV3SciHQFU9VNgkFsnBsgDZns1f6eqvuGr2I0xvpWREMWq/JpXFjYtgy97JMOAPFXdpKongGnApRXq3Ag8pqr7AFR1TyXtXAF8qKpHfBirMaYJ9e4cyba9Ryg+XurvUEwj8GUiSQS2e73Od8u89QR6isgCEVkkIhMqaWcy8GqFsgdEZKWIPCwila6zICI3iUiWiGQVFNgiccY0J73cO9zX2fBWQPD3ZHswkA6MBaYAT4lI9Mk3RSQB6A/M8jrmHiADGArEAHdV1rCqPqmqmaqaGR8f75vojTH1ktHZNrkKJL5MJDuAZK/XSW6Zt3xgpqqWqOpmYD1OYjlpIvCWqpacLFDVneo4DjyLM4RmjGlBkjq0pV2b4Gp7JAWHjtd5q1/jH75MJEuBdBHpJiKhOENUMyvUeRunN4KIxOEMdW3yen8KFYa13F4K4mw+fRmw2hfBG2N8R0TI6BzJ2p2VJ5J56/Yw/MGPee7LLU0bmKkXnyUSVS0FbsMZlsoFXlPVNSIyVUQucavNAopEJAf4FOdqrCIAEUnF6dF8VqHpl0VkFbAKiAP+7KtzMMb4Tq/OkeTuOnhKryN/3xFun76ccoXnv9xCuS2l0uz5dK1pVf0A+KBC2R+9nivwK/dR8dgtnDo5j6qOb/RAjTFNLiMhipcXb+PrA8dIjG4LwPHSMm595SvKypRfnt2Thz9ez4KNhZyebvOczZm/J9uNMa1U7282ufp2wv2B93NZsX0//7hyADeP7U5MRCgvL9rmrxBNLVkiMcb4RU83keS68yTvLN/BCwu3cuPp3ZjQL4E2wUFcmZnEnNzd7D54zJ+hmhpYIjHG+EVUWAiJ0W1Zu+sQG3Yf4p43VzE0tQO/mZDxTZ2rhqVQVq5MX7q9mpaMv1kiMcb4Te+ESFbm7+eWl5cRHhrE/101mJCgb7+WusZGcHp6HK8u2UZpWbkfIzXVsURijPGbjM5RbC06wqaCYv4z5TQ6RYWdUufq4V3ZeeAY89bZChXNlSUSY4zf9O3iLJXy63N7MapHXKV1zurdkU5RbXh58damDM3UgSUSY4zfnNOnE6/cOJxbzuxRZZ2QIA+ThqYwb30B2/fa2q3NkSUSY4zfBAd5GNUjDo9Hqq03eWgyAkxbapcCN0eWSIwxzV6X6LaMz+jE9KX5nCi1SffmxhKJMaZFuGZECoXFx5mTs9vfoZgKLJEYY1qEM9LjSerQ1ibdmyFLJMaYFsHjEa4ansKXG4vYWFDs73CMF0skxpgW48ohyYQECS/Y8vLNiiUSY0yLER/Zhu8PTuKVJdvsUuBmxBKJMaZFuf3sngR5hIdmr/N3KMZlicQY06J0bh/GD0d34+3lX7N6xwF/h2OwRGKMaYFuHtuD6PAQ/vbRWn+HYrBEYoxpgaLCQrhtXBpfbChk/oZCf4fT6lkiMca0SNeO7EpidFv++lGu7evuZz5NJCIyQUTWiUieiNxdRZ2JIpIjImtE5BW3bJyILPd6HBORy9z3uonIYrfN6SIS6stzMMY0T22Cg7jjvJ6s3nGQd1d+Xa82io+XNnJUrZPPEomIBAGPAecDfYApItKnQp104B5gtKr2BW4HUNVPVXWQqg4CxgNHgNnuYX8DHlbVNGAf8CNfnYMxpnm7dGAifRKi+OfsdbVeg6u8XJmbu5tJTyyk/32zWLixyMdRBj5f9kiGAXmquklVTwDTgEsr1LkReExV9wGo6p5K2rkC+FBVj4iI4CSWN9z3ngcu80n0xphmz+MR7j4/g+17j9a4dMqxkjKmLdnGOQ9/xo+ez2L73iNEhYXwv/mbmijawBXsw7YTAe+NlvOB4RXq9AQQkQVAEHCfqn5Uoc5k4F/u81hgv6qe7I/mu59jjGmlTk+PY3RaLI9+kscVQ5KIDAsBQFXZe/gEW/ceYf6GQl5YuIXC4hP07RLFI5MHcUH/BB75eAOPzctjW9ERUmLD/XsiLZgvE0ltPz8dGAskAZ+LSH9V3Q8gIglAf2BWXRsWkZuAmwBSUlIaK15jTDMjItw9oTcX/998furu/b5t71G27z3ynTmQcb3iufGM7ozsHoszuAFXj0jh8c828uKiLfzuwj5VfYSpgS8TyQ4g2et1klvmLR9YrKolwGYRWY+TWJa6708E3nLfBygCokUk2O2VVNYmAKr6JPAkQGZmpl3SYUwA65/UnkmZyby9fAfJMeGkxIQzvFsMKTHhdI0Np2enSJJjTu1xJLRvy4S+nZm+dDu/PKcn4aH+/tu6ZfLlv9pSIF1EuuF82U8GrqpQ521gCvCsiMThDHV5D1hOwZmMB0BVVUQ+xZk3mQZcD7zjszMwxrQYf7tiAH/9fv9vehu1df2oVN5ftZO3v/qaq4bb6EV9+Gyy3e0x3IYzLJULvKaqa0Rkqohc4labBRSJSA7wKXCnqhYBiEgqTo/mswpN3wX8SkTycOZM/uerczDGtCx1TSIAQ1M70DshihcWbkHVBi/qQ1rDP1xmZqZmZWX5OwxjTDM1fek27pqximk3jWBE99hK65SVK/e/l0P3+AiuG5natAH6iYhkq2pmTfXsznZjTKt36aBEosNDeL6afU7+8kEuz325hT+/l8vWosNNF1wLYInEGNPqhYUEMSkzmdk5u/l6/9FT3n9p0Vaenr+Z752WSHCQ2GKRFVgiMcYY4JoRXVFVXlr03RsbP19fwL0z1zCuVzx/v2IAN5/Zgw9W7WLJ5r1+irT5sURijDFAckw4Z/XuxLSl2zlWUgbAul2HuPXlZaR3bMejVw0mOMjDjad3J6F9GH9+P8cWi3RZIjHGGNcNo1LZe/gE7674moJDx/nhc0sJCw3imRuG0q6Nc7dE29AgfjOhFyvzD/DOikpvY2t1LJEYY4xrVI9Y0ju249kFW7jxhSyKDh/nf9dn0iW67XfqXTowkQFJ7fn7R+s4eqLMT9E2H5ZIjDHGJSJcNyqVnJ0HWZG/n39POo0BSdGn1PN4hN9f2IedB47x1Be26KOtB2CMMV6+d1oib2Tnc/mgLkzo17nKesO6xXB+v848Pm8jk4Ym0ykqrNafcfBYCfe9s4bIsGC6xUWQGhdB97h2JHZoS5Cn7jdV+pslEmOM8RLRJph3bh1dq7p3n5/B3Nw9PDR7HX+/YmCtP+PZ+Vt486sdRLYJ5pDXwpKhQR66xobz2wt7M65XxzrH7i+WSIwxpp66xkZww+hUnvpiE9eNTKVfYvsajyk+XsozCzZzdu9OPHXdEAqLT7Cl6DCbCw6zqfAwH63eyV1vrOTTO8YS0aZlfEXbHIkxxjTArePS6BAeyv3v5dRqra6XF23lwNESbhufhogQH9mGoakxTByazN3nZ/DwpEHsOXSc/zcvrwmibxyWSIwxpgHatw3h1+f2ZPHmvbz1VfWXAx8rKeOpLzZxenocg5JPncQHOC2lA5eflshTX2xm+94jvgi50VkiMcaYBpoyNIXTUqL58/u57Dt8osp605Zso7D4BLeNS6u2vbsmZBAkwoMf5DZ2qD5hicQYYxrI4xEevLw/B46W8NcPK1+H60RpOU98volhqTEMr2KF4ZM6tw/jp2N78OHqXSzcWOSLkBuVJRJjjGkEvROi+PHp3Zietb3SdbjeXJbPzgPHuHV89b2Rk248ozuJ0W2Z+l4OZc18KRZLJMYY00h+cVY6idFt+e1bqzhRWv5NeWlZOY9/tpEBSe05Iz2uVm2FhQRxzwUZ5O48yPSl230VcqOwRGKMMY0kPDSYP1/Wj7w9xTz5+cZvyt9buZOtRUe4dVxanXZxvLB/AsNSY3ho9joOHC3xRciNwhKJMcY0onEZHbmgf2ce/SSPLYWHKS9XHvs0j16dIjmnd6c6tSUi/PHiPuw9coJH527wUcQNZ4nEGGMa2b0X9yUkyMMf3lnNrDW72LCnmJ+O64GnHsuf9Etsz8QhyTz35RY2FRT7INqGaxm3TRpjTAvSKSqMO8/rxb0z17Ay/wDd4iK4aECXerd3x3m9eH/VTsY/9Fml7wd7hA4RocSEh9IhIoSYiFCiw53X143qSsfI2q8DVh8+TSQiMgF4BAgCnlbVv1ZSZyJwH6DAClW9yi1PAZ4Gkt33LlDVLSLyHHAmcMBt4gZVXe7L8zDGmLq6ZkRX3lyWz4r8A/zuwt4NWowxPrINz/9wGJ+tL/hO+ckWT5SVs//ICfYePsG+wyWs313MvsMn2HfkBFdmJjXgLGrHZ4lERIKAx4BzgHxgqYjMVNUcrzrpwD3AaFXdJyLeq5S9ADygqnNEpB1Q7vXenar6hq9iN8aYhgryCP+efBpvfbWDywYlNri9IV07MKRrhzodU16u1GFuv958OUcyDMhT1U2qegKYBlxaoc6NwGOqug9AVfcAiEgfIFhV57jlxaraMtYKMMYYV7e4CH51Tk9Cg/0zHe3xSJ2uEqv35/iw7UTA++LnfLfMW0+gp4gsEJFF7lDYyfL9IvKmiHwlIv9wezgnPSAiK0XkYRFp47tTMMYYUxN/X7UVDKQDY4EpwFMiEu2Wnw7cAQwFugM3uMfcA2S45THAXZU1LCI3iUiWiGQVFBRUVsUYY0wj8GUi2YEzUX5SklvmLR+YqaolqroZWI+TWPKB5e6wWCnwNjAYQFV3quM48CzOENopVPVJVc1U1cz4+PhGPTFjjDHf8mUiWQqki0g3EQkFJgMzK9R5G6c3gojE4QxpbXKPjRaRkxlgPJDj1ktwfwpwGbDah+dgjDGmBj67aktVS0XkNmAWzuW/z6jqGhGZCmSp6kz3vXNFJAcow7kaqwhARO4A5roJIxt4ym36ZTfBCLAcuNlX52CMMaZmUpsdvVq6zMxMzcrK8ncYxhjToohItqpm1lTP35PtxhhjWjhLJMYYYxqkVQxtiUgBsLWeh8cBhY0YTkth5926tNbzhtZ77rU5766qWuNlr60ikTSEiGTVZoww0Nh5ty6t9byh9Z57Y563DW0ZY4xpEEskxhhjGsQSSc2e9HcAfmLn3bq01vOG1nvujXbeNkdijDGmQaxHYowxpkEskVRDRCaIyDoRyRORu/0dj6+IyDMiskdEVnuVxYjIHBHZ4P6s2446LYCIJIvIpyKSIyJrROQXbnlAn7uIhInIEhFZ4Z73n9zybiKy2P19n+6ukRdwRCTI3Z7iPfd1wJ+3iGwRkVUislxEstyyRvs9t0RSBa8dHs8H+gBT3A23AtFzwIQKZXcDc1U1HZjrvg40pcCvVbUPMAK41f1vHOjnfhwYr6oDgUHABBEZAfwNeFhV04B9wI/8GKMv/QLI9XrdWs57nKoO8rrkt9F+zy2RVK02OzwGBFX9HNhbofhS4Hn3+fM4Ky0HFHdLgmXu80M4Xy6JBPi5u9swFLsvQ9yH4qyyfXIL64A7bwARSQIuBJ52Xwut4Lyr0Gi/55ZIqlabHR4DWSdV3ek+3wV08mcwviYiqcBpwGJawbm7wzvLgT3AHGAjsN/d/wcC9/f938BvgHL3dSyt47wVmC0i2SJyk1vWaL/nPltG3gQOVVURCdjL+0SkHTADuF1VD3rvcR2o566qZcAgd0fSt3B2HQ1oInIRsEdVs0VkrL/jaWJjVHWHiHQE5ojIWu83G/p7bj2SqtVmh8dAtttrE7EEnL9cA46IhOAkkZdV9U23uFWcO4Cq7gc+BUbibCZ38o/LQPx9Hw1cIiJbcIaqxwOPEPjnjarucH/uwfnDYRiN+HtuiaRqtdnhMZDNBK53n18PvOPHWHzCHR//H5Crqv/yeiugz11E4t2eCCLSFjgHZ37oU+AKt1rAnbeq3qOqSaqaivP/8yeqejUBft4iEiEikSefA+fi7CzbaL/ndkNiNUTkApwx1ZM7PD7g55B8QkRexdnyOA7YDdyLsw3ya0AKzsrJE1W14oR8iyYiY4AvgFV8O2b+W5x5koA9dxEZgDO5GoTzx+RrqjpVRLrj/KUeA3wFXKOqx/0Xqe+4Q1t3qOpFgX7e7vm95b4MBl5R1QdEJJZG+j23RGKMMaZBbGjLGGNMg1giMcYY0yCWSIwxxjSIJRJjjDENYonEGGNMg1giMc2KiPzOXZF2pbtS6XC3/HYRCffxZyd4rQg7zP385e4quZf78rOriCfVe0VmfxKRG0SkSwOOv0hEpjZmTKb5sERimg0RGQlcBAxW1QHA2Xy73tntgE8TCfAr4Cn3+WogU1UH4ayM/ITX3c8Bow7ndANQ70QCvA9c7Os/Box/WCIxzUkCUHjyZjBVLVTVr0Xk5zhfYp+KyKcAInKuiCwUkWUi8rq7XtbJfRf+7u69sERE0tzyK0Vktdu7+LyKz/8+8JH72Ue8FvILw1n07hT1iCNVRD5xe1xzRSTFLe8kIm+58a0QkVHuRwSJyFNuL222eyd6xRieE5H/ikiWiKx315Q6uTDjP0Rkqft5P3HLx4rIFyIyE8ip0FaQ295qN/ZfisgVQCbwsttDaysiQ0TkM3EWAZzltdTGPBF5xK23WkSGuf+eCszD+UPBBBpVtYc9msUDaAcsB9YD/w840+u9LUCc+zwO+ByIcF/fBfzRq97v3OfXAe+5z1cBie7z6Eo+uxuQXaFsOLAGKAYur+SY+sTxLnC9+/yHwNvu8+k4i0aCc8d5eyAVZ8+UQW75azh3XVeM4zmcBOgB0nFWsA0DbgJ+79ZpA2S55zkWOAx0q6StIcAcr9fR7s95OD00cJad/xKId19Pwln54WS9p9znZwCrvdq6GnjU379n9mj8h/VITLOhzh4ZQ3C+AAuA6SJyQyVVR+BsNrZAnKXQrwe6er3/qtfPke7zBcBzInIjzhd1RQnuZ3rHs1hV+wJDgXtEJKwR4hgJvOI+fxEY4z4fDzzufm6Zqh5wyzer6nL3eTZOcqnMa6parqobgE04q/meC1znxrYYZ8n0dLf+ElXdXEk7m4DuIvKoiEwADlZSpxfQD2cV2eXA73EWO/zOeauzz03UyXW9cBYFbMjwmGmmAm7M17Rs6ixvPg+YJyKrcL6cn6tQTXD+ap5SVTMVn6vqze7E/YVAtogMUdUir3pHcf6KryymXBEpxvnyzGpIHPXgveZTGXDK0FYV7StOfD9T1Vneb7jrTB2utBHVfSIyEDgPuBmYiNNz+k4TwBpVHVnx+GpiAeff92gVx5gWzHokptkQkV4iku5VNAhnMTmAQ0Ck+3wRMNpr3iFCRHp6HTfJ6+dCt04Pt4fxR5yeh/cWAeAMp6V6xdLt5ES0iHTF+Qt/S4Vj6hwHzpDQZPf51TiLRoKz1ektbjtBItKeurlSRDwi0gPoDqwDZgG3iLNUPiLSU5zVX6skInGAR1Vn4PQ0Brtvef/7rwPi3YsjEJEQEenr1cwkt3wMcMCrd9UT5yIGE2CsR2Kak3bAo+5QSCmQhzPMBfAk8JGIfK2q49whr1dFpI37/u9xkgFABxFZifPX/Mnewj/cJCU4X9orvD9YVQ+LyEYRSVPVPJwhp7tFpARnZeCfqmphhWMK6hHHz4BnReROnIT2A7f8F8CTIvIjnJ7HLcDJ3etqYxuwBIgCblbVYyLyNE5yXCYi4n5eTdupJrrxnfwj8x7353PAf0XkKM7w3BXAf9yEF4yzSvYat+4xEfkKZy7Fuzczzqs9E0Bs9V8TUMTZtCiz4pd+LY+9HBiiqr/3Zxz1+KzncCbz36ipbhPEMg9nefasCuWdcJYvP8svgRmfsh6JMS5VfUucPRpM40sBfu3vIIxvWI/EGGNMg9hkuzHGmAaxRGKMMaZBLJEYY4xpEEskxhhjGsQSiTHGmAaxRGKMMaZB/j9/61uOXjRBqgAAAABJRU5ErkJggg==\n", 180 | "text/plain": [ 181 | "" 182 | ] 183 | }, 184 | "metadata": {}, 185 | "output_type": "display_data" 186 | } 187 | ], 188 | "source": [ 189 | "x4 = range(len(dis_pretrain_eval_loss))\n", 190 | "plt.plot(x4, dis_pretrain_eval_loss)\n", 191 | "plt.xlabel('Steps (3 epoch per step)')\n", 192 | "plt.ylabel('Oracle NLL')" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": 8, 198 | "metadata": {}, 199 | "outputs": [ 200 | { 201 | "data": { 202 | "text/plain": [ 203 | "Text(0,0.5,'Oracle NLL')" 204 | ] 205 | }, 206 | "execution_count": 8, 207 | "metadata": {}, 208 | "output_type": "execute_result" 209 | }, 210 | { 211 | "data": { 212 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd8leX5+PHPlR1CCAkJMwEChL2JDAWxTlQK1lVc1dbR4eiw/VY77E/t1NZq66raoVbFUQe4EBUEZEjYG8JMwgorrIwzrt8fz5NwCNnk5ByS6/16nVfOM891niTnOvd47ltUFWOMMaYmEaEOwBhjTPizZGGMMaZWliyMMcbUypKFMcaYWlmyMMYYUytLFsYYY2plycIYY0ytLFkYY4ypVVCThYhMEJENIpIrIvdVs8+1IrJWRNaIyKsB628WkU3u4+ZgxmmMMaZmEqw7uEUkEtgIXATkA4uB61R1bcA+WcAbwPmqelBE2qvqXhFJAXKAbECBJcAIVT1Y3eulpqZq9+7dg/JejDGmuVqyZMk+VU2rbb+oIMYwEshV1S0AIjIVmAysDdjnduCp8iSgqnvd9ZcAM1X1gHvsTGAC8Fp1L9a9e3dycnIa/U0YY0xzJiLb67JfMKuhugB5Acv57rpAvYHeIvKliCwUkQn1OBYRuUNEckQkp7CwsBFDN8YYEyjUDdxRQBZwHnAd8LyItK3rwar6nKpmq2p2WlqtpShjjDENFMxkUQBkBCynu+sC5QPTVNWjqltx2jiy6nisMcaYJhLMZLEYyBKRTBGJAaYA0yrt8y5OqQIRScWpltoCzAAuFpFkEUkGLnbXGWOMCYGgNXCrqldE7sL5kI8E/qWqa0TkISBHVadxIimsBXzAz1R1P4CIPIyTcAAeKm/sNsYY0/SC1nW2qWVnZ6v1hjLGmPoRkSWqml3bfqFu4DbGGHMGsGQR4IOVu9h3tDTUYRhjTNixZOEqLvNx56tL+de8raEOxRhjwo4lC1eZ1w/A2l2HQxyJMcaEH0sWLo/fSRZrdlqyMMaYyixZuLw+p1dY4ZFS9h4pCXE0xhgTXixZuDw+f8XztVa6MMaYk1iycHn9J+43sXYLY4w5mSULl9dKFsYYUy1LFi6P22YRHSmWLIwxphJLFi6v2xuqX6c2bN1/jGOl3hBHZIwx4cOShau8ZDEkvS2qsH63lS6MMaacJQtXeZvFkAxn7qUVeUWhDMcYY8KKJQuXz+0NlZEcT/9ObXj2i80UFXtCHJUxxoQHSxYuj5ssoiIj+NNVg9l/rIzffbA2xFEZY0x4sGThKq+Gio4UBqUn8d1ze/BGTj5zNxWGODJjjAk9Sxau8gbuqAjnktxzQRYd2sTy6qIdoQzLGGPCgiULV3nX2ahIASAuOpLzerfny9x9J92wZ4wxLZElC5e3omQhFevG9U7lcImXlQXWM8oY07JZsnB5KtosTlySc3qmIgJzN+4LVVjGGBMWgposRGSCiGwQkVwRua+K7beISKGILHcftwVs8wWsnxbMOOHEQILl1VAAyQkxDO6SZI3cxpgWLypYJxaRSOAp4CIgH1gsItNUtXJ/1NdV9a4qTlGsqkODFV9l5e0S5Q3c5cZlpfHMF5s5XOKhTVx0U4VjjDFhJZgli5FArqpuUdUyYCowOYivd1oCBxIMNC4rFZ9fmZ+7PxRhGWNMWAhmsugC5AUs57vrKrtKRFaKyFsikhGwPk5EckRkoYhcEcQ4gcDeUCdfkmFdk2kTF8Vdry5l8lNfMnPtnmCHYowxYSfUDdzTge6qOhiYCbwYsK2bqmYD1wOPi0jPygeLyB1uQskpLDy9dgVPFb2hAGKiInjtjtHcfm4PjhR7+OHUZezYf/y0XssYY840wUwWBUBgSSHdXVdBVferaqm7+AIwImBbgftzCzAbGFb5BVT1OVXNVtXstLS00wrWW1ENdeolGdA5iZ9P6MvLt40iQoSfvbUCf8DMesYY09wFM1ksBrJEJFNEYoApwEm9mkSkU8DiJGCduz5ZRGLd56nAOUBQB2ry+v2IQGSlkkWgLm3jeWBifxZtPcB/F20PZjjGGBNWgpYsVNUL3AXMwEkCb6jqGhF5SEQmubvdIyJrRGQFcA9wi7u+H5Djrp8F/LGKXlSNyutXoiNqvxzXZKfTt2MiM9bsDmY4xhgTVoLWdRZAVT8EPqy07oGA5/cD91dx3HxgUDBjq8zr8590j0V1RISsDomsyDvUBFEZY0x4CHUDd9jw+PSUxu3qZCTHs/NQsY0ZZYxpMSxZuLx+/yndZqvTNaUVXr+yq6gkyFEZY0x4sGTh8tanZJHSCoC8g9aF1hjTMliycHl8WmW32apkJDvJIv9AcTBDMsaYsGHJwuVUQ9WtZNGpbRyREcKOA1ayMMa0DJYsXPWphoqOjKBTUpxVQxljWgxLFi6Pz1/naihwqqLyrGRhjGkhLFm4vH6tczUUQEZKPHkHrc3CGNMyWLJweXz+U+ayqEnXlFYUHimluMwXxKiMMSY8WLJweX16ylwWNSnvPptv7RbGmBbAkoXL669fySI92e61MMa0HJYsXB5f/dosupbfmGf3WhhjWgBLFi6fv+435QGkto4hPjrS7rUwxrQIlixcTgN33UsWIkJ6crwlC2NMi2DJwlXfrrMAvTsmsm7X4SBFZIwx4cOShctbz66zAIO7JJF/sJgDx8qCFJUxxoQHSxau+jZwAwxKTwJgVUFRMEIyxpiwYcnC5fX76zStaqCBXdxkkW+z5hljmjdLFi5vA0oWbeKi6ZGawMp8K1kYY5o3Sxau+g4kWG5QepJVQxljmr2gJgsRmSAiG0QkV0Tuq2L7LSJSKCLL3cdtAdtuFpFN7uPmYMYJbm+oenSdLTeoSxK7ikrYe8SmWDXGNF9RwTqxiEQCTwEXAfnAYhGZpqprK+36uqreVenYFOA3QDagwBL32IPBitephqp/7hyc3haA1QVFnN83rrHDMsaYsBDMksVIIFdVt6hqGTAVmFzHYy8BZqrqATdBzAQmBClOADx+f70GEiw3oHMbRLB2C2NMsxbMZNEFyAtYznfXVXaViKwUkbdEJKOexzYKn19Rpd73WQAkxEbRK601qyxZGGOasVA3cE8HuqvqYJzSw4v1OVhE7hCRHBHJKSwsbHAQHp8foN69ocoN6NzG7uQ2xjRrwUwWBUBGwHK6u66Cqu5X1VJ38QVgRF2PdY9/TlWzVTU7LS2twYF6/QrQoGoogL6d2rCzqISi454Gx2CMMeEsmMliMZAlIpkiEgNMAaYF7iAinQIWJwHr3OczgItFJFlEkoGL3XVB4fM5yaIh1VAAfTsmArB+t5UujDHNU9B6Q6mqV0TuwvmQjwT+paprROQhIEdVpwH3iMgkwAscAG5xjz0gIg/jJByAh1T1QLBi9fhPrxqqX6c2AGzYc4RRPdo1WlzGGBMugpYsAFT1Q+DDSuseCHh+P3B/Ncf+C/hXMOMr5z3NkkX7xFiSW0WzbteRxgzLGGPCRqgbuMPC6TZwiwh9OiZaNZQxptmyZMHpN3AD9O3Yhg27j+B3z2WMMc2JJQucuSyg4dVQAP06JXK8zEfeQZs5zxjT/FiywJnLAk6/ZAFYu4UxplmyZIEzlwWcXsmid4dERKz7rDGmebJkwYmSRUMbuAHiYyLJbJfAeitZGGOaIUsWnGizaMh8FoGyOrQmt/BoY4RkjDFhxZIFJ3pDNWQ+i0BdU1qRf/A4qtYjyhjTvFiyIPA+i9O7HOnJrSjx+Ck8Wlr7zsYYcwaxZMGJO7hPpzcUQEZKPAD5B4tPOyZjjAknliwIrIY6vcuRkdwKgLwDdq+FMaZ5sWRBQNfZ0yxZdEm2koUxpnmyZEHgQIKnlyxaxUSR2jqGfLuL2xjTzFiy4EQD9+l2nQWnkTvvgJUsjDHNiyULAtosTrMaCiA9Od7GhzLGNDuWLGicgQTLZaS0YuehYnw2+qwxphlp0KejiHRu7EBCqTEGEiyXkdwKj0/Zc7jktM9ljDHhoqFfpRc2ahQhdqI3VGO0WTg9oqz7rDGmOWnop+PpfwUPI55G6g0FTjUUQJ51nzXGNCMNTRbNqkL+xB3cp1+y6Nw2DhGs+6wxplmJqm6DiPydqpOCAG2DFlEIeP1+RCCyEUoWsVGRdEiMs+6zxphmpaav0jnAkioeOcDddTm5iEwQkQ0ikisi99Ww31UioiKS7S53F5FiEVnuPp6t6xtqCI9PiW6EnlDlMlKC033W51deX7yDgkOWiIwxTavakoWqvljdNhH5c20nFpFI4CngIiAfWCwi01R1baX9EoEfAosqnWKzqg6t7XUag9fnb5R7LMr1SG3Nx2t24/Nro5RWys1cu4ef/28VcdERfG98T+78Wq9GqTozxpjaNPST5to67DMSyFXVLapaBkwFJlex38PAn4CQ9TX1+rVRGrfLnd2rHUXFHtbsLGq0cwJMX7mTlIQYLuzXgcc/3cSL87c16vmNMaY6wewN1QXIC1jOd9edOInIcCBDVT+o4vhMEVkmIl+IyLgqgxC5Q0RyRCSnsLCwrrGfwuv3N0q32XJn90wFYF7uvkY75/EyL5+v28tlgzry5PXDGdk9hZcWbLeb/4wxTaLaT0gRSanm0Y5G6DorIhHAY8C9VWzeBXRV1WHAT4BXRaRN5Z1U9TlVzVbV7LS0tAbH4vU1bskiLTGWvh0Tmbfp9JPF1n3HUFU+XbeXYo+Prw927oe8+ezu7DhwnNkb9p72axhjTG2qbbPAacxWqk4MnjqcuwDICFhOd9eVSwQGArNFBKAjME1EJqlqDlAKoKpLRGQz0Buncb3ReXza6HX/47JSeXH+dorLfMTHRDboHHM3FXLTP7/iG8O6cPB4GR3axHJW9xQALh7QgY5t4vjP/G1c0K9DY4ZujDGnqPYTUlUzVbWH+/OURx3OvRjIEpFMEYkBpgDTAs5fpKqpqtpdVbvj3BU+SVVzRCTNbSBHRHoAWcCW03ifNXKqoRr3PsNzeqVS5vOzeNuBBp9j9oZCIgTeWVbA7A2FTBzcmQi3BBQdGcENo7oyd9M+cvcebaywjTGmSjVVQ3Wt6VHbiVXVC9wFzADWAW+o6hoReUhEJtVy+LnAShFZDrwFfE9VG/6pW4vGroYCGJXZjpjIiNNqt1iweT8jM1N48vph9ExLYMpZGSdtv25UVxJiIvnpmyso9fpON2RjjKlWTdVQH3BqNZQCaUB7oNa6FVX9EPiw0roHqtn3vIDn/wP+V9v5G4vH52/0aqj4mEhGdEtmzsZCfnFZv3off+h4Get2H+ZHF/Rm4uDOTBx86tiNqa1j+cu1Q/jef5fy/6at4Q9XDm6M0I0x5hQ1VUMNUtXB7s9BwNeBL4GjwI+aKsCm4PVro1dDAVw6qCPrdx9hRd6heh+7aOsBVGFMz3Y17jdhYCd+cF5PXvsqj49X72poqMYYU6Nav06LSJaI/Af4CKfRu7+q/j3YgTUlj8/fKHNZVPaNYV1IiInk5YXb633sgs37iYuOYEhGUq373ntxHzq0iWX6SksWxpjgqKnNYqCIvIZTHfQpMFBVX1DVuvSEOqN4fdooc1lUlhgXzTeGd2H6ip0cPFZ2yvbDJZ5qSx0Lt+wnu1sKsVG196SKjBDG905j7sbCiomcjDGmMdX0dXoFMAaYi3M39l9F5G/ljyaJrol4/cEpWQDcNLo7pV4/by7JO2n9xj1H+Prf5zH5qS9Zv/vwSdv2Hy1l/e4jtVZBBRrfuz2HS7ysyK9/lZcxxtSmpk/I7wC/Ab6i6gEFmw2PLzhtFgB9OiYyMjOl4p4LcKqYvvHUlxwr9REbFcGL80+upprr3sw3ukfdk8XYXqlECHyxoeF3shtjTHVqauB+saZHUwYZbF5/4/eGCvTjC3tTcKiYx2ZuYN/RUu5+bRkdk+KYfvc5TB7amXeXFVB0/ETt3v+W5pOeHM+wjLqPBJ/UKpphXZOZvdGShTGm8dmQpQTnPotAY3q244ZRXXlh3la+/e/FHC7x8NQNw+mUFM/NZ3en2OPjjRynmqrgUDHzcvdx1fD0ihvw6uq83mmszC9i39HSYLwNY0wLZsmC4NxnUdn9l/Wjc1I8qwqK+OVl/ejb0RnqakDnJGdQwIXbKC7z8c7SfFTh6hHp9X6N8X2c8bE+XbunUWM3xhhLFtDo805UpXVsFP+4aQS/vKwf3xrT7aRt3/9aT/IPFnPd8wt5Iyef0T1SKubyro+BnZPo36kND7+/ltUFjTs8ujGmZavLfRa9ReQzEVntLg8WkV8FP7SmE8wG7kADuyRx+7k9cAdOrPC1Pu155oYRrNt1mB0HjnP1iIxqzlCziAjh398+i7atYrjl31+xY7/NA26MaRx1KVk8D9yPO9Ksqq7EGRSw2fD6/Y06rWpDTBjYkal3jOaWs7tz+aBODT5PhzZxvHTrSEo9fh6Zsb4RIzTGtGQ1jQ1VrpWqflXp27A3SPGEhLeJSha1GdY1mWFdk0/7PD3TWnN1djqvLNzB/qOltGsd2wjRGWNasrp8nd4nIj1xBhFERK7GmZyo2WiKBu6mdt3IrpT5/Ly9tKD2nY0xphZ1+YS8E/gH0FdECnAGEfx+UKNqYo09B3c46N0hkRHdknlt8Q5UbepVY8zpqTVZqOoWVb0QZ2jyvqo6VlW3BT2yJuRUQzWvkgXAlLMy2FJ4jK+2Bm0qEGNMC1Ftm4WI/KSa9QCo6mNBiqnJefz+oAwkGGoTB3fmwelreW/FTkbVY+gQY4yprKYG7sQmiyKEfH5FlaANJBhK8TGRjMxMYdGW/aEOxRhzhqs2Wajqg00ZSKh43CG9w6E3VDCMzEzh8/V72Xe0lFTrFWWMaaC63JT3ooi0DVhOFpF/BTespuP1O42/zbEaCpxkAbDY2i2MMaehLnUvg1W1YpIEVT0IDAteSE2rfLKg5lgNBc4QIPHRkSyyZGGMOQ11+YSMEJGKO8VEJIW63cyHiEwQkQ0ikisi99Ww31UioiKSHbDufve4DSJySV1eryE8vuZdsoiJimB4t7bWI8oYc1rq8qH/F2CBiLwJCHA18LvaDhKRSOAp4CIgH1gsItNUdW2l/RKBHwKLAtb1xxlSZADQGfhURHqrqq9O76oe2raKZvpdY+nUNq6xTx02RnZvx+OfbaSo2ENSfHSowzHGnIHqcp/FS8BVwB5gN3Clqr5ch3OPBHLd+zTKgKnA5Cr2exj4E1ASsG4yMFVVS1V1K5Drnq/RRUdGMCg9qVk3/o7MTEEVcrZZ6cIY0zB1qqhX1TXAG8A04KiIdK3DYV2AwImn8911FURkOJChqh/U91hTd8O6tiUmMsLaLYwxDVaX3lCTRGQTsBX4AtgGfHS6LywiEcBjwL2ncY47RCRHRHIKC2060erERUcyvFtb5tiUq8aYBqpLyeJhYDSwUVUzgQuAhXU4rgAInJgh3V1XLhEYCMwWkW3ua0xzG7lrOxYAVX1OVbNVNTstLa0OIbVcX+vTnvW7j7CrqDjUoRhjzkB1SRYeVd2P0ysqQlVnAdm1HQQsBrJEJFNEYnAarKeVb1TVIlVNVdXuqtodJwFNUtUcd78pIhIrIplAFvBV/d6aCXRen/YAfLHBShfGmPqrS7I4JCKtgTnAKyLyBHCstoNU1QvcBcwA1gFvqOoaEXlIRCbVcmx5G8la4GPgzmD0hGpJendoTeekOGZt2BvqUIwxZyCpbfhqEUkAinESyw1AEvCKW9oIG9nZ2ZqTkxPqMMLaL95ZxXvLClj2wMXERDXPmxCNMfUjIktUtdbaoho/Mdx7Jd5XVb+qelX1RVX9W7glClM35/VO41iZj5zt1ivKGFM/NSYLt+rHLyJJTRSPCaJzeqUSHSlMX7Ez1KEYY84wdbmD+yiwSkRmEtBWoar3BC0qExQJsVFcm53BK4t20LFNPD+8MCvUIRljzhB1SRZvuw/TDDw0eSDFHh9//XQjCbGR3DauR6hDMsacAeqSLF4HernPc1W1pKadTXiLjBAevXoIOw8V8+KCbdw6NrNi9kNjjKlOtW0WIhIlIo/gDLXxIvASkCcij4iIjUZ3BouMEC4f1Im8A8Vs23881OEYY84ANTVwPwqkAJmqOkJVhwM9gbbAn5siOBM85/Z27ni3IUCMMXVRU7KYCNyuqkfKV6jqYeD7wGXBDswEV7d2CXRr18qShTGmTmpKFqpV3LHndqet+U4+c0YY3zuN+Zv3U+q1m+ONMTWrKVmsFZFvVV4pIjcC64MXkmkq52alUezxsWTbwVCHYowJczX1hroTeFtEvgMscddlA/HAN4IdmAm+MT3bER0pfLGpkLN7pYY6HGNMGKu2ZKGqBao6CngIZw6LbcBDqjpSVU8ZLtyceRJioxjRLZk5G/eFOhRjTJir9T4LVf0c+LwJYjEhML53e/708Xr2Hi6hfZvmOw+5Meb02NCjLdy5vZ3qpzmbrHRhjKmeJYsWrl/HNqS2jrUutMaYGlmyaOEiIoRze6cyd1MhPr9SdNzD7iIb0cUYczJLFobxvdM4eNzDzLV7uOxvc5n497kUFXsAeG7OZm57MYfaJskyxjRvliwMY3ulIgJ3vrqUQ8fL2H+sjMc/3ciKvEP88aP1fLpuD/NyrU3DmJbMkoWhXetYBndJIjJCeOHms7h+ZFdeWrCdu19bRvvEONolxPDi/G2hDtMYE0KWLAwAT0wZxrS7zmFMz3b89OI+JMZFsePAcX5/5UCuG9mVz9bvJe/AqSPUfrJmN2f/4bOKaitjTPMU1GQhIhNEZIOI5IrIfVVs/56IrBKR5SIyT0T6u+u7i0ixu365iDwbzDgNdE9NoG/HNgAkJ8Tw5HXDeXDSAM7v24EbRnclQoSXF24/5bhpK3ays6iEedb11phmrS6THzWIiEQCTwEX4cyJsVhEpqnq2oDdXlXVZ939JwGPARPcbZtVdWiw4jM1G5uVytgs5x6MTknxTBjYkf98uQ2vT/nB13qS2joWv1/50m3L+GLjXi4f3CmUIRtjgiiYJYuRODPrbVHVMmAqMDlwB3fI83IJ2Gi2YevBSQOYPLQz/5m/lQmPz+VoqZc1Ow9z8LiH1rFRfLGx0HpMGdOMBTNZdAHyApbz3XUnEZE7RWQz8AhwT8CmTBFZJiJfiMi4IMZp6iC1dSyPXjOEV24bzb6jpbyxOI+5uc6NfN8/ryd7DpeyfveRWs5ijDlThbyBW1WfUtWewM+BX7mrdwFdVXUY8BPgVRFpU/lYEblDRHJEJKew0O5AbgpjerYju1sy/56/lS82FNK3YyLXjEgHYPaG6n8HxWU+npuzmYPHypoqVGNMIwpmsigAMgKW09111ZkKXAGgqqWqut99vgTYDPSufICqPqeq2aqanZaW1miBm5rdNi6TvAPFLNp6gLG9UmnfJo5+ndrwxca91R7z/Nwt/P7D9dz/9iqrrjLmDBTMZLEYyBKRTBGJAaYA0wJ3EJGsgMXLgU3u+jS3gRwR6QFkAVuCGKuph4v6dyQjJR6gohH8vD5p5Gw7yP1vr+TNnDxKPCdm3zt4rIzn52whJSGGj9fs5oNVu0IStzGm4YKWLFTVC9wFzADWAW+o6hoRecjt+QRwl4isEZHlONVNN7vrzwVWuuvfAr6nqgeCFaupn8gI4e7zs+jSNp5Rme0A+GZ2Bmf3SuXDVbv52VsrGf/oLF5esI1Dx8t45ovNHC3z8t9bRzEkPYkH3lvD/qOloX0Txph6keZSJZCdna05OTmhDqPFU1UWbjnAXz7ZQM72g0QIiAiTh3bmsWuHsmH3ES7/21yuHpHOH68aHOpwjWnxRGSJqmbXtl/Q7rMwLZOIMKZnO9783hhW5hfx2fq9rCko4qcX9wGgT8dEbjm7O//8cis3jOrGoPSkEEdsjKkLK1mYJne4xMP5f55N93YJvPm9MYhIlfuparXbjDGNo64li5B3nTUtT5u4aH52SR9yth/k5/9bWWV32hKPj0ufmMv9b6/C728eX2iMOZNZNZQJiWtGZLCl8BgvzNvKJ2v38JdrhnBBvw4V219esJ31u4+wfvcRIgR+e8VAK2UYE0JWsjAhEREh3H9ZPz68ZxwZya24/aWcioEKj5R4eHp2LuOyUvne+J68smgHf525McQRG9OyWcnChFSfjom8/t3R3P3qMn797mpmrd9L2/hoDh738NOL+zA4PYn9R0v5+6xcRvdox9m9UkMdsjEtkpUsTMi1ioniHzeN4N6LerMy/xBvLyvg4v4dGJLRFhHhwckD6JGawI9eX84BGy7EmJCw3lAmrJR6fczbtI9hXZNJSYipWL9252GueOpLLhnYkb9fNyyEERrTvFhvKHNGio2K5IJ+HU5KFAD9O7fhu+N7MH3FTlbkHQJgfu4+lrvPjTHBZcnCnDHuOLcHKQkx/PGj9Xy8ehc3/nMR97y2zAYmNKYJWLIwZ4zEuGjuOb8XC7bs585Xl5EUH82OA8dZZqULY4LOkoU5o1w/qhs90xLIat+a6XePJTYqgveW1TTyvTGmMVjXWXNGiYmKYPrdY4mJjCAqMoIL+3Xg/ZW7+NXE/kRH2ncfY4LF/rvMGadVTBRRbmKYPLQz+4+VMS93X4ijMqZ5s2Rhzmjn9WlPUnw071pVlDFBZcnCnNFioiL4+pBOfLx6N0XHPaEOx5hmy5KFOeNdP7IbpV4//1uaX+X2/UdLWZlvPaaMOR2WLMwZr3/nNgzNaMsri7afcs+Fz6/c+mIO1zy7gCMlVvIwpqEsWZhm4YZRXdlceIyPV+/mDx+u4xfvrOJwiYd/ztvC8rxDlHr9fLJmz0nHbCk8yh8+WkeJxxeiqI05c1jXWdMsTBzcmYfeX8v3X1laMe/3FxsKKTxaykX9O7B252Gmr9zJVSPSK47566ebmL5iJ7sOlfDElKE2X4YxNbCShWkW4mMi+fmEvlw1PJ1Pfjzena4VEmIi+d03BjJxSCfmbdpXMWptUbGHGWt206VtPNNW7OTp2ZtD/A6MCW9BTRYiMkFENohIrojcV8X274nIKhFZLiLzRKR/wLb73eM2iMglwYzTNA83ju7GX64dQq/2rRneNZmZPx7Ppz8ZT/vEOCYN6YzXr3y0ehcA76/cSZnXzzM3DmfSkM78+ZNA3eTzAAAbMklEQVQNbN9/rN6vWeb1N/bbMCYsBS1ZiEgk8BRwKdAfuC4wGbheVdVBqjoUeAR4zD22PzAFGABMAJ52z2dMncXHRNKudSwA/Tu1oWdaAu8sLcDr8/PWknz6dEhkUJckfnZJH1Th49W7AThwrIw/z9hAcVnNbRnrdh1m6EOf8N5yu8fDNH/BLFmMBHJVdYuqlgFTgcmBO6jq4YDFBKC8K8tkYKqqlqrqViDXPZ8xDSIiXDeyKznbDzLhibks23GIq0ekIyJkpLRiYJc2fLzGSRb/mLOZJ2fl8tKCbdWer9Tr48evL+d4mY/pK3Y2zZswJoSCmSy6AHkBy/nuupOIyJ0ishmnZHFPPY+9Q0RyRCSnsLCw0QI3zdOtYzN5+obhFJf5iI2KYPKwzhXbJgzoyLIdh9i67xivL3b+9J6fu6XK0oXH5+exTzayfvcRBnZpw9xN+2othRhzpgt5A7eqPqWqPYGfA7+q57HPqWq2qmanpaUFJ0DTbIgIlw3qxGf3jufzn55H+8S4im0TBnYC4EdTl3HouIefXdKHfUfLePWrHRX7zFy7h3MfmUXvX33EP+Zs4ZvZGdx/aT9KvX4bm8o0e8HsOlsAZAQsp7vrqjMVeKaBxxpTZ3HRkXRpG3/Sul7tW9OrfWtW5BfRv1MbfnBeT+Zt2sezX2xGVdm05yiv5+TRr1Mb7jk/i64prbh8cCciI4TEuCg+XbuHi/p3qDifz69ERlhXXNN8BLNksRjIEpFMEYnBabCeFriDiGQFLF4ObHKfTwOmiEisiGQCWcBXQYzVGC4d2BGAW87pjojwk4t7c+BYGb/9YB1vLMnjtrGZvHvn2fz4ot5cNSKduOhIoiMjGN87jc/W78Xvd5rcPD4/5/15Ftc9t5Cdh4pD+ZaMaTRBK1moqldE7gJmAJHAv1R1jYg8BOSo6jTgLhG5EPAAB4Gb3WPXiMgbwFrAC9ypqlYpbILqptHd8KsyeajTlnFW9xTWPHgJpV4/kRFC69iq/10u6u/MqbEi/xDDuiazdPtB8g4Uk3+wmAmPz+GZG0dwTq/UpnwrxjQ6aS7zF2dnZ2tOTk6owzAtUNFxD9m/m8m3xnTn1xP784eP1vGveVt55wfn8OPXl3PwuIdPf3IubVvFhDpUY04hIktUNbu2/ULewG3MmS6pVTQX9uvAO8sKKPP6mbV+L2d1T2FglyQenzKUQ8fLePj9dQBs33+s4i7yYMg/eJyfv7WSY6XeGvcr8fj43Qdr2bH/eNBiMc2LJQtjGsG1Z2Vw4FgZL87fxsY9Rzm/b3sABnRO4nvje/K/pflMeHwO4x+dzbg/fc5zczYH5e7vp2Zt5vWcPD5fv7fG/Z6bs4Xn527l2Tk2zImpG0sWxjSCc7PS6JQUx6OfbACcGfzK3XV+L4akJwHwi8v6MrpHO37/4Xq++3JOxZDqJR4fS7YfZMaa3byRk8fLC7axaMv+esVw6HgZ7yxz5vSYtaH6ZLF9/zGenJVLVIQwfcVOG3XX1ImNOmtMI4iMEK4ekc7fP8+la0oreqYlVGyLi47kvbvGVizfcW5Pnpuzmd9/uJ53lxdwfp8OXPOP+Wzcc/SkcybERLLk1xcRF131SDeqetJIuVMX51Hi8TOgcxvmbCzE71ciKnXfVVX+37Q1REcID185iJ+8sYIZa3Yzeegp97wacxIrWRjTSK7NzkAEzu/bvtbhzm8d24NhXdvy8PvruP2lHLbuO8ajVw/m/bvHMvf/vsYzNwznWJmP2RuqHpng2S82M/L3n3HUbZvw+vy8NH8bo3ukcNu4TPYdLWP1zqKTjjlS4uGuV5cxa0MhP76oN1cM7UKXtvG8taTqGQaNCWTJwphGkpHSiqm3j+ZHF2bVum9khPDHKwdzpMTDV9sO8OjVQ7gmO4OBXZLISGnFRf07kNwqmg9W7Trl2EVb9vPIx+spPFJa0Tbx6bo97Cwq4dvnZHJuVhoiMGu9k2hUlU/X7uHrf5/Hx2t2c/+lfbl1bCYREcJVw7swL3cfuXuP4PX5eXnhdkb9/lOes7YMU4lVQxnTiEb1aFfnfft0TOQv1w7F5/dzxbCTq4GiIiOYMLAT7y0voMTjq6iK2n+0lHumLqNbuwSOlHj5aNUuJg3pzGtf5dEpKY4L+3UgMkIYnN6W2Rv3MjIzhUdnrGfpjkN0b9eKqXeM5qzuKRWvc/WIDJ6evZkLH5tDXHQEJR4/iXFR/HXmJiYN6ULHpDiMAStZGBNSk4Z05hvD0qvcNnFwJ46X+ZjtNlb7/cq9b67g4DEPT14/jEsHdmT2hkJy9x5lzqZCrsnOqBhi5Gt90li24xDXPb+QnYdK+MOVg5j5k/EnJQqAru1aMf3usfx6Yn+uHJ7OszcO54O7x+HzK392G+tN8BUVe7j22QXMXLun9p1DxEoWxoSpUZkptEuI4e2lBVzcvyMvzNvC7A2FPDR5AAM6J3HpQA8vL9zOj19fDsA1AVPGTh7ahc/W7WXSkM7cNKZbtY3kAP06taFfpzYnrfv2Od15bu4Wbjm7OwO7JAXnDZoKD05fw1fbDtC+TexJY4yFEytZGBOmoiIj+MawLnyydg/jHpnFIx9vYMKAjtw0uhsAIzNTSG4VzaqCIsb2SiUjpVXFsZmpCUy/eyy3n9ujxkRRnTvP70VKqxjufWOFDb8eZB+v3sXbSwtIiIlk8bYDhOuoGpYsjAljP7+0L09eP4zuqa3o0zGRP109uKKnVVRkBJcMcAY/nHJW10Z93TZx0Tz2zaFs3HuEX727OigfYKraou/xWLbjIA+8t5qfvrmSQV2SuPfiPuw5XEr+wfAcfNKqoYwJY9GREUwc3JmJgztXuf07YzMRIShVF+N7p3HP+Vk88dkm+nVK5LZxPU7avnjbAaIihGFdkxt0/reW5PPg9LXM/tl5pLrT37YUC7fs57rnFxIbFcH5fdtz/6X9KrpBL9524KRSYriwkoUxZ7DeHRL5w5WDiYkKzr/yPRdkcXH/Dvz2g3X85r3VeH3OECVen5/v/3cpP3trZYPP/f7KXRwt9fJhFd2Dm7NSr49fvLOK9OR4vvrlhTx9wwgyUlrRp0MiiXFRLN52MNQhVsmShTGmWpERwjM3juD2cZm8uGA7D0xbA8DcTfvYd7SU3L1HyTtQ/8EIi8t8LHCHM5m2vGXNYf70rM1sKTzG764YRJu46Ir1ERFCdrdkFm87EMLoqmfJwhhTo8gI4ZeX9+fmMd14fXEeW/cd462l+cRFOx8fszdWfZd5uXW7DvPC3C0UFXsq1i3Yso8yr5/RPVLI2X6QgoBJogqPlPK/Jfl1HmhRVfnnvK08OmM9T8/OZVdReNb5A6wuKOLp2blMHtqZc3ufOhX0WZkp5O49yoq8Q1z77AJ+OHUZWwqPcrjEw+fr9zB/876QtfNYm4Uxpk7uOj+L13Py+O37a5mbu4/rR3bl8/V7+WLD3ooeWuBMKevx+YmLjsTj83P3a8vI3XuUv322iTu/1os7zu3BrPWFxEdH8tsrBnLhY3N4f8VObhzdjefnbuG5OVs4XuZj6Y6D/O4bg2qN6+PVu3n4/bVECPgVNu4+wuNThgXzUtTK71c2Fx6lW7uEiirCouMevv/KElJbx/LAxP5VHld+H8xVz8wnPiaSVQVFTF/hlLzciRiJiYqgZ1pr2ifGMjIzhe+ck0l8TP17vNWXJQtjTJ2kJcZy85ju/GPOFgCuHpGOX5U3c/Ir7jLPO3CcO15ewsFjZbx2x2hmb9hL7t6j/OryfizYvJ8/fLSeCBE+X7+Xc3ql0qt9IkMy2vKf+dt4fu5W9h0t5bJBHWnbKoZXFu2gb8dEbhrTHXDGtlqz8zCrC4rYceA43zwrg17tW/OHj9bTt2MiH9wzjt9MW82bOfk8WOwhKT66hndzehZs3s97ywv4xeX9TqpKUlVmrNnDX2duZMOeIyTGRjE2K5WsDoks2X6AXYdKeP27Y2hXTYP+4PQk2sRF0aFNHM9/K5uE2CheXrANgNE92znVd5v3s3XfMXYVlfDojA28vGA7P7ukD1cO71LrmGSnw2bKM8bU2YFjZYz70+ekJ7fi4x+NY/aGQr79n8W89J2RlHh8/N//VuL3K5ERQlx0JMdKvQztmsyL3z4LVbjrtaV8uGo3AL//xiCuH9WVF+dv4zfT1nBW92Tuv6wfw7sm4/Mrt7+Uw6wNe2mXEENsVORJVVXRkUJUhNOT6INVu/jvraMYm5XK6oIiJv59Hg9PHsBNY7pT5vUTHSmN+iG6eNsBvvXPryj2+BjetS0v3TqK+OhIvti4l8dmbmR1wWF6pCZw05hubNxzhLmb9lXE/uCkAXzLTX7V2XO4hKT46DrdH/PV1gP89oO1JMRE8ertoxr0Pus6U54lC2NMvSzasp/WcVEM6JxEcZmPoQ99QmxUBIdLvPTtmMizN47gWJmX659fxNFSLzN+NI5e7RMBOF7m5ZpnF7Bm52Hm33c+ndvG4/crG/ceoU+HxJM+7I6UePj3l9vYfbiE4jIfPVITGNgliYFdklCU219awoq8Q1zQtz3/vOWsiuMue2IuIvCnqwZzy78Xk9o6hge+3p+ze9Z/HvRXF+3gy9x9PDFlKFGREazdeZhv/mMBaYmx3H5uD3717mq6tI3n0PEyDpd4yUiJ54cX9OaKoZ2JijzRJFzq9XGs1EdKQuNPrev3K0XFHpIbeG5LFsaYJnH3a8tYsHk/P7wwiylnZRDtfkhuKTzKrqISzul18of0/qOlrCooOmmCqIYo8fj478LtTBzc+aQBD19asI0H3ltDfHQkya2iiYgQ8g8W893xPbj/0n6A064yf/M+3l5aQGxUBN8b35PuqQknnX/drsNMenIeHp/y4KQBXDeyK1//+zwOFZfxzg/OoXPbeD5ctYvn5myhb8dEzu6VyoQBHYPWjTlYwiJZiMgE4AkgEnhBVf9YaftPgNsAL1AIfEdVt7vbfMAqd9cdqjqppteyZGFMaPj8isApEy2FStFxD2P++Bmd28bz8q0jSW4Vw2/eW8PrOXk8/61shnVty3f+s5iV+UUkxUdT4vHh9Ss/OK8n917cBwCPz8/kJ79k75ESMlMT2LD7CFcOT+c/87fxz5uzuaBfeI7f1BAhTxYiEglsBC4C8oHFwHWqujZgn68Bi1T1uIh8HzhPVb/pbjuqqq3r+nqWLIwx5fIOHCclIYaEWKcPT6nXx5VPz6fgUDHJrWLYVVTMw5MH8vUhnTlc7OH3H67j3eU7efbGEVwyoAO/+2AdL8zbyrM3jqBnWgKXPjEXr1+ZNKQzf7sutD2tGltdk0Uwy0sjgVxV3aKqZcBUYHLgDqo6S1XL7+hZCFQ9VrMxxtRDRkqrikQBEBsVyZPXD8fj9bP/aCmv3DaKa7IziIuOpH2bOB65eggDu7Th/rdX8uv3VvPCvK18a0w3JgzsSFaHRH5wXk+6tI3nN1+vustrSxDMksXVwARVvc1dvgkYpap3VbP/k8BuVf2tu+wFluNUUf1RVd+t6fWsZGGMqc2mPUeIj4kkPfnUsZdy9x7h8r/No9Tr56bR3Xhw0oCTqtZ8bi+v5qauJYuwuM9CRG4EsoHxAau7qWqBiPQAPheRVaq6udJxdwB3AHTt2rijbhpjmp+sDonVbuvVPpEnrx/Otn3HuG1c5indUJtjoqiPYCaLAiAjYDndXXcSEbkQ+CUwXlVLy9eraoH7c4uIzAaGASclC1V9DngOnJJFI8dvjGlhwnXioXAQzDaLxUCWiGSKSAwwBZgWuIOIDAP+AUxS1b0B65NFJNZ9ngqcA6zFGGNMSAStZKGqXhG5C5iB03X2X6q6RkQeAnJUdRrwKNAaeNMt8pV3ke0H/ENE/DgJ7Y+BvaiMMcY0LbspzxhjWrBw6DprjDGmmbBkYYwxplaWLIwxxtTKkoUxxphaWbIwxhhTq2bTG0pECoHtp3GKVGBfI4UTLOEeY7jHBxZjY7EYG0c4xNhNVU+dELySZpMsTpeI5NSl+1gohXuM4R4fWIyNxWJsHGdCjOWsGsoYY0ytLFkYY4yplSWLE54LdQB1EO4xhnt8YDE2FouxcZwJMQLWZmGMMaYOrGRhjDGmVi0+WYjIBBHZICK5InJfqOMBEJEMEZklImtFZI2I/NBdnyIiM0Vkk/szOQxijRSRZSLyvrucKSKL3Ov5ujs8fSjjaysib4nIehFZJyJjwuk6isiP3d/xahF5TUTiwuEaisi/RGSviKwOWFfldRPH39x4V4rI8BDF96j7e14pIu+ISNuAbfe78W0QkUuCHV91MQZsu1dE1J2CISTXsL5adLIQkUjgKeBSoD9wnYiEwyS7XuBeVe0PjAbudOO6D/hMVbOAz9zlUPshsC5g+U/AX1W1F3AQuDUkUZ3wBPCxqvYFhuDEGhbXUUS6APcA2ao6EGco/ymExzX8DzCh0rrqrtulQJb7uAN4JkTxzQQGqupgYCNwP4D7vzMFGOAe87T7vx+KGBGRDOBiYEfA6lBcw3pp0ckCGAnkquoWVS0DpgKTQxwTqrpLVZe6z4/gfMB1wYntRXe3F4ErQhOhQ0TSgcuBF9xlAc4H3nJ3CWmMIpIEnAv8E0BVy1T1EOF1HaOAeBGJAloBuwiDa6iqc4ADlVZXd90mAy+pYyHQVkQ6NXV8qvqJqnrdxYU4s3OWxzdVVUtVdSuQi/O/H1TVXEOAvwL/BwQ2GDf5Nayvlp4sugB5Acv57rqwISLdcaaUXQR0UNVd7qbdQKjngHwc54/e7y63Aw4F/MOG+npmAoXAv92qshdEJIEwuY7u1MF/xvmGuQsoApYQXtcwUHXXLRz/j74DfOQ+D5v4RGQyUKCqKyptCpsYq9PSk0VYE5HWwP+AH6nq4cBt6nRjC1lXNhGZCOxV1SWhiqEOooDhwDOqOgw4RqUqp1BeR7fOfzJOUusMJFBFtUU4CvXfX01E5Jc4VbmvhDqWQCLSCvgF8ECoY2mIlp4sCoCMgOV0d13IiUg0TqJ4RVXfdlfvKS+auj/3Vnd8EzgHmCQi23Cq787HaR9o61apQOivZz6Qr6qL3OW3cJJHuFzHC4Gtqlqoqh7gbZzrGk7XMFB11y1s/o9E5BZgInCDnrgvIFzi64nzxWCF+3+TDiwVkY6ET4zVaunJYjGQ5fY+icFpBJsW4pjK6/7/CaxT1ccCNk0Dbnaf3wy819SxlVPV+1U1XVW741y3z1X1BmAWcLW7W6hj3A3kiUgfd9UFwFrC5zruAEaLSCv3d14eX9hcw0qqu27TgG+5PXpGA0UB1VVNRkQm4FSLTlLV4wGbpgFTRCRWRDJxGpG/aur4VHWVqrZX1e7u/00+MNz9Ow2La1gjVW3RD+AynJ4Tm4FfhjoeN6axOEX8lcBy93EZTpvAZ8Am4FMgJdSxuvGeB7zvPu+B84+YC7wJxIY4tqFAjnst3wWSw+k6Ag8C64HVwMtAbDhcQ+A1nHYUD86H2q3VXTdAcHoVbgZW4fTuCkV8uTj1/uX/M88G7P9LN74NwKWhuoaVtm8DUkN1Dev7sDu4jTHG1KqlV0MZY4ypA0sWxhhjamXJwhhjTK0sWRhjjKmVJQtjjDG1smRhGp2I+ERkuTuS6vTA0T+D+JrbykfwrOP+j4vIue7z80VkqRvviwE3xDUZEfl/IvLTpn7dplL++xGRGBGZE4prbE6PJQsTDMWqOlSdkVQPAHeGOqBAItIOGK2qc0QkAmdQvCluvNs5ceNZs1LTB3RTfXirM2DnZ8A3m+L1TOOxZGGCbQHugGju3amPut/gV4nIN93154k7H4a7/KQ7bEP5N9IH3W/+q0Skr7u+nYh8Is5cEC/g3NSEiCSIyAcissJ9nao+lK4CPnaftwPKVHWjuzzT3X4ScebteFREFrvzDXw3IPY57mtuEJFn3QSEiFznxrxaRP4UcK4J7vtZISKfBbxMfxGZLSJbROSeqi6miBwVkb+67/szEUlz1/cUkY9FZImIzA24Tv9xY1oEPFLpXLeIyDQR+Rz4rCl+P653gRuqen8mfFmyMEEjzpwBF3BiCJUrce6oHoIzLtKjUrdhmPep6nCcMf7Lq2p+A8xT1QHAO0BXd/0EYKeqDnFLCh+fcjZn/KXyARD3AVEiku0uX83JY/SUuxVnCIazgLOA292hI8AZ7vpunDlRegJXikhnnHkpznff81kicoX74f48cJWqDgGuCXiNvsAl7vl+I874YJUlADnu+/7CvQ7gzOV8t6qOcK/R0wHHpANnq+pPqjjfcOBqVR1P0/x+wLlb/aw6nNeEEas3NMEQLyLLcUoU63C+rYMzjMlrqurDGZTuC5wPjcNVn6ZC+UCKS3A+0MCZp+JKAFX9QEQOuutXAX9xv8m/r6pzqzhfJ5yhy1FVFZEpwF9FJBb4BPBVcczFwGARKR+zKQlnjKEy4CtV3QIgIq+579MDzFbVQnf9K27MPmCOOvMqoKqB8x18oKqlQKmI7MUZAjy/Uhx+4HX3+X+Bt8UZnfhs4E2Rii/wsQHHvOle86rMDIihKX4/qKpPRMpEJFGd+VrMGcCShQmGYlUdKs6QzDNw2iz+VsP+Xk4u5cZV2l7q/vRRy9+sqm4UZ0rKy4DfishnqvpQ5fgCX0NVFwDjAETkYqB3FacWnG/uM05aKXIepw7V3dAxdEoDntf6XgNeKwJnDoyh1exzrIbja9pWrtF+PwFigZI67mvCgFVDmaBRZ+TPe4B73QbUucA33fr/NJxvn1/hNCr3F2dU0LY4VVe1mQNcDyAil+IMEIhb/XNcVf8LPIpTzVLZOqBX+YKItHd/xgI/B56t4pgZwPfLq4ZEpLc4EykBjBRn5OIInIbbee77Gi9OD6BI4DqcaqOFwLnlVVgiklKH9xooghMj0l6PU9VzGNgqIte45xQRGVLP80IT/H7c5XY4VVeeBsRoQsRKFiaoVHWZiKzE+bD8LzAGWIHzjfj/1BmeGRF5A6cueyuwrA6nfhB4TUTWAPM5MZ/xIJy6dj9OVdD3qzj2A+C7uNPBAj8TZzKnCJyJkj6v4pgXgO448w8ITjVW+bSii4EncRLQLOAdVfWLyH3usuBUMb3nvtc7cKqPInDmhLioDu+33DGc5PQr99jyBvwbgGfc9dE4c4xUno2tNu8Q/N8PwNdwfgfmDGKjzpoWSUTmARPVmZP7dM5zHvBTVZ3YKIHV/npHVbV1U7xWsIjI28B9AT3QzBnAqqFMS3UvJ/fQMU1AnEnG3rVEceaxkoUxxphaWcnCGGNMrSxZGGOMqZUlC2OMMbWyZGGMMaZWliyMMcbUypKFMcaYWv1/yMd0LG7XjZ8AAAAASUVORK5CYII=\n", 213 | "text/plain": [ 214 | "" 215 | ] 216 | }, 217 | "metadata": {}, 218 | "output_type": "display_data" 219 | } 220 | ], 221 | "source": [ 222 | "x5 = range(len(dis_adversarial_eval_loss))\n", 223 | "plt.plot(x5, dis_adversarial_eval_loss)\n", 224 | "plt.xlabel('Rounds (9 epoch per round)')\n", 225 | "plt.ylabel('Oracle NLL')" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": 9, 231 | "metadata": {}, 232 | "outputs": [ 233 | { 234 | "data": { 235 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEKCAYAAADXdbjqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd4HNX18PHv2VWxim1VV8mW3MEFG4RtMGBMiymhhB4SWgiBQICEFPLCj0AIKZBAgJAChN5NdehgsE0x2DLuvduSi+Qqy7LqnvePmZVX0qp7i6XzeR492p2dnT0zOztn7tw794qqYowxxrSHJ9IBGGOMOfRZMjHGGNNulkyMMca0myUTY4wx7WbJxBhjTLtZMjHGGNNulkyMMca0myUTY4wx7WbJxBhjTLvFRDqA1srIyNCcnJxIh2GMMYeUuXPnblfVzFAt/5BLJjk5OeTn50c6DGOMOaSIyIZQLt8ucxljjGk3SybGGGPazZKJMcaYdrNkYowxpt0smRhjjGm3kCYTEZksIitEZLWI3Bbk9QdFZL77t1JEdocyHmOMMaERsqbBIuIFHgVOBQqAOSIyVVWX+udR1Z8HzP8zYEyo4jHGGBM6oSyZjAVWq+paVa0EXgbOaWL+S4GXQhXM0s0lPPTJKnaXVYbqI4wxptMKZTLpC2wKeF7gTmtARPoDucCnjbx+rYjki0h+cXFxm4L5fFUxD36ykgl//pQ/vb+Mor3lbVqOMcaYhqKlAv4S4DVVrQn2oqo+pqp5qpqXmdm23gB+MnEg7998PCcf1pPHZ65l4n3TmZK/CVWlqsbXntiNMabTC2UyKQSyA55nudOCuYQQXuLyO6x3Nx6+dAzTbj2R0dkp/Oq1hQy94wNG/O5DXpmzkZ37KvlqzXaqLbkYY0yriKqGZsEiMcBK4GScJDIH+L6qLqk33zDgAyBXWxBMXl6eHoy+uWp8ynOz1rOlpJxFBXv4as0OvB6hxqdMGJTO3WePoEush537KomP8TKoRzJej7T7c40xJhJEZK6q5oVq+SFrzaWq1SJyI/Ah4AWeVNUlIvJ7IF9Vp7qzXgK83JJEcjB5PcKVE3IBqK7x8fjn69izv4qM5Dju+3AFpzwwo878XWI99ElJoGfXLvTsFs/AzGTG5qYxbkA6ny0vYuaqYq46Npd+6YnhXA1jjIkKISuZhMrBKpk0ZXXRXuZv2kONz0dqYhz7KqtZXFjC1j3lbCspZ8uecgp37wfgiKzuLCjYA0CMRxjcsyuj+nbnrrOHkxDnDWmcxhjTUodsyeRQNqhHVwb16Fpn2nn17oAprajmxW828Mi01Vx4VBY/O2kwL83ZyPItJbw6dxMl5VVkJMfz1rxCRvdLYdLQHpw0rAc5GUlhXBNjjAkPK5m0k8+neOrVpTw+cy33vrcMgNNH9GJVUSmri0oBOLJfCnk5aSTHx3DFsTl0T4gNe8zGmM7HSiZRrn4iAbjm+FwS4rzkZiQxYVAGAJt2lvH+4i1MyS/g2Vnrqaj28db8Qu49dyQpibF8uryItcX7iI/1cNWxOQzu2ZXNu/fzyKer2LKnnL9eeAQZyfG1n7Flz3427y7nqP6p4VpVY4xplJVMIuSbtTu47vm57Cqrqp3Wu3sX9uyvQhWOHZjOzFXFCIII9OrehUlDe7BzXyVbS8rJX78Tn8Lr1x9rCcUY06xQl0wsmUTQjtIKFhTsZndZFeMGpNM3JYFtJeX84tX5rNi6l3NG9+Xq43LZVlLODS98S2l5NWnJcaQlxXHMgHRe/7aAXt0TePaqsWzes5/DeneL9CoZY6KUJZN6OlIyaa/X5xZw65QFxHqFqhrlgqOyuOecEdaKzBjTgNWZmEadN6Yvs9buINbroVtCDI/NXMvXa3dw88mDSUmMY1tJOSXlVVx6dD9Sk+IiHa4xpgOzkkkHMmvNDu6auoQV2/bWmd6rWxfyclKZvW4nZ4zszY0nDSIjOZ5NO8tYULCb5PgYxuWmW4nGmA7MLnPVY8mkadU1PhYV7iHG4yGjaxzb91by81fns720gjHZKcxctZ34GA+Th/fi3UVbqKh2+iHr1a0Ll47tR1K8F48IA3skM3FI2zrVNMZEH0sm9VgyaRv//TBrikt58OOVvLNwC6ce3pObThrM9tIKHv50FfM2HhjoMs7rYeW9p0cwYmPMwWR1Juag8N8PMzAzmX98/0j+dlENcV4PIs70ScN6UFZZTbVPefSz1fxnxlpUtfZ1Y4xpiiWTTio+pmH9SGKcszskuf9rfEqM15KJMaZ50TI4loki/q72q32H1iVQY0zkWDIxDfiTie8Qq08zxkSOJRPTgFesZGKMaR1LJqaB2pKJJRNjTAtZMjEN+JNJjSUTY0wLWTIxDVgyMca0liUT00BtMrEKeGNMC1kyMQ3UNg2usWRijGkZSyamAX9rLmsabIxpKUsmpgG7adEY01qWTEwD1jTYGNNalkxMA1YBb4xpLUsmpgGrgDfGtJYlE9OAVcAbY1rLkolpwOu1CnhjTOtYMjEN1JZMLJkYY1rIkolpwJoGG2Nay5KJacCaBhtjWsuSiWnAmgYbY1rLkolpwC5zGWNay5KJacAq4I0xrWXJxDRgJRNjTGuFNJmIyGQRWSEiq0XktkbmuUhElorIEhF5MZTxmJaxCnhjTGvFhGrBIuIFHgVOBQqAOSIyVVWXBswzGPgtMEFVd4lIj1DFY1rOSibGmNYKZclkLLBaVdeqaiXwMnBOvXl+DDyqqrsAVLUohPGYFqotmVhrLmNMC4UymfQFNgU8L3CnBRoCDBGRL0XkaxGZHMJ4TAv5K+BtDHhjTEuF7DJXKz5/MHAikAXMFJGRqro7cCYRuRa4FqBfv37hjrHTsctcxpjWCmXJpBDIDnie5U4LVABMVdUqVV0HrMRJLnWo6mOqmqeqeZmZmSEL2DisAt4Y01qhTCZzgMEikisiccAlwNR687yFUypBRDJwLnutDWFMpgVirGRijGmlkCUTVa0GbgQ+BJYBr6rqEhH5vYic7c72IbBDRJYCnwG/UtUdoYrJtIzHKuCNMa0U0joTVX0PeK/etDsDHivwC/fPRAl/BbyNtGiMaSm7A9404B8cy6fK4sI9/PaNhdayyxjTJEsmpoHakolPmbmqmJdmb2Ld9n0RjsoYE80smZgGarug92ntpa4lm/dEMiRjTJSzZGIaCGwa7G/RtXRzSSRDMsZEOUsmpoHAy1zVNT4Alm6xZGKMaZwlE9OAxyOIOBXw/pLJks0lqDUVNsY0wpKJCcorQrVPqXJLJjv3VbKtpCLCURljopUlExOU1yNOnUnAvSZWCW+MaYwlExOU1+OUTKp9PpLjYxCBBQWWTIwxwVkyMUF5PUKNT6mqUbp1iWF8bjpPf7mOopLySIdmjIlClkxMUF6POBXwNT5ivB7uPW8EFdU+7nhrcaRDM8ZEIUsmJqgY9zJXlU+J8QoDMpO56eTBfLR0G6u27Y10eMaYKGPJxATlEX8FvI9Yj7ObnDvGGShz5qrtkQzNGBOFLJmYoGor4GuckglA35QEBmQmMXNlcYSjM8ZEG0smJih/02DnMteB3eSEwZl8s24H5VU1EYzOGBNtLJmYoA6UTHzEun11ARw/OIPyKh9zN+yKYHTGmGhjycQE5fUINW53Kv7LXADjB6QT6xXu+2A5z3+9wcY5McYAlkxMI7yBFfABl7mS4mP4zeRh7NhXyR1vLebJL9ZFMEpjTLSwZGKCOnAHvBITcJkL4JrjB/D5rydx2uE9uf+jFdZU2BhjycQEV1sBX1O3At5PRLj3vJEkxnn5+yerIhChMSaaWDIxQdWpgPdK0Hkyu8aT1z+VNcWlYY7OGBNtLJmYoGq7U/EpXk/ju0lWaiIbd5bZWCfGdHKWTExQXhGqa5zxTGI9wUsmAP3SEimrrGHnvsowRmeMiTZtSiYi0udgB2KiS23T4Jq6TYPry05LBGDTrv3hCs0YE4XaWjL5+qBGYaJO7eBYPl/QCni/7LQEADbuLAtXaMaYKNTWZNL4qarpEPwV8FU12uRlruxUt2RiycSYTq2tycRqWzu4+uOZNCYpPob0pDgKdlkyMaYzi2nsBRF5hOBJQ4CUkEVkokKMx62A9zVdZwKQlZZol7mM6eQaTSZAfhtfMx2AR5ySSY1Pa8czaUx2agILbXx4Yzq1RpOJqj7T2Gsi8tfQhGOihdcjVNX4qGlBySQ7LZEPFm9t9pKYMabjausv/6KDGoWJOl6PUFnjA6jT0WMw/dISqfYpW/aUhyM0Y0wUstZcJiivRyivcpJJ/Y4e6+uf7rToWrt9X8jjMsZEp6Yq4NMaewlLJh2e1yNUuKMpNnfpanjv7gAsLtzDxCGZIY/NGBN9mqqAn4vTmitY4qgKTTgmWnhFKK/2X+Zq+tyhe2IsOemJLCzYHY7QjDFRqKkK+NxwBmKiS4xXqKz2X+Zq/mroyKwU5q7fGeqwjDFRqtGjhIj0a+qvJQsXkckiskJEVovIbUFev1JEikVkvvt3TXtWxhw8HjlQGmmuzgRgVN/ubN5TzvbSilCGZYyJUk1d5nqXhpe5FMgEegDephYsIl7gUeBUoACYIyJTVXVpvVlfUdUbWxu4CS1vQAJprmkwwMgsp95kUeEeJg3tEbK4jDHRqdGSiaqOVNVR7v+RwHeBL4FS4JYWLHsssFpV16pqJfAycM7BCNqEXt1k0vxlruF9uiECi+zmRWM6pWaPEiIyWESeBt7HqZQ/XFUfacGy+wKbAp4XuNPqO19EForIayKS3YLlmjDwBlzmaqqjR7+uXWIZkJFklfDGdFJN1ZmMEJGXgNeBT4ARqvqEqh7Mllz/A3JUdRTwMRD0rnsRuVZE8kUkv7i4+CB+vGmM19u6kgnA8D7dWbZlb6hCMsZEsabqTBbglCzexblkNVYCzlZV9aZmll0IBJY0stxpgcvYEfD0CeC+YAtS1ceAxwDy8vKsx+IwCCyZtKTOBGBY765MXbCZkvIqunWJDVVoxpgo1FQyubqdy54DDBaRXJwkcgnw/cAZRKS3qm5xn54NLGvnZ5qDJLAFV3MdPfod1qsbACu27uXonMbueTXGdERt6uixJVS1WkRuBD7Eafn1pKouEZHfA/mqOhW4SUTOBqqBncCV7flMc/B4WtmaC5ySCcDyLSWWTIzpZJoqmbSbqr4HvFdv2p0Bj38L/DaUMZi2qVMB38Jk0qtbF7onxLJsq9WbGNPZWH/hJqg6FfAtvMwlIgzr1ZXlW0pCFZYxJkpZMjFBtaUCHuCw3t1YsXUvPp+1kzCmM2nJfSZDRGSaiCx2n48SkTtCH5qJpMCbFpsbzyTQ0F5d2VdZQ8Gu/aEIyxgTpVpylHgcp16jCkBVF+K0zDIdWJ074Ftw06LfkJ5OJfzKbVZvYkxn0pJkkqiqs+tNqw5FMCZ6xLSxZOIfKGvTrrKDHpMxJnq15CixXUQG4nTyiIhcAGxp+i3mUBfYNNjbipJJelIcCbFeNu20y1zGdCYtaRp8A87d58NEpBBYB/wgpFGZiGtrBbyIkJWaQIGVTIzpVJpNJqq6FjhFRJIAj6raxfBOwNuGO+D9stMS2WQV8MZ0Kk2NAf+LRqYDoKoPhCgmEwVaO55JoOzUBObYqIvGdCpNlUy6hi0KE3Xa2jQYICs1kb3l1ewpq6J7onX4aExn0FTfXHeHMxATXdraNBggOy0BcFp0dU/sflDjMsZEp5bctPiMiKQEPE8VkSdDG5aJtJg2tuYCp2QCWCW8MZ1IS65fjFLV2uHzVHUXMCZ0IZlo4HHrxmK9QuA4Ni2R7SYTax5sTOfRkmTiEZFU/xMRSSPEvQ2byPOXRlrayWOg7omxdO0SYzcuGtOJtCQp/A2YJSJTAAEuAO4NaVQm4mqTSStbcvllpya2u3+uJZv3EB/jZVCP5HYtxxgTei25z+RZEZkLTHInfU9Vl4Y2LBNp/mTS2pZcfrkZSczftBtVbfVlMoCK6houfexrSiuqufyYHO4487AWj0VvjAm/Fv06VXUJ8CowFSgVkX4hjcpE3IHLXG0rmYwfkEbh7v1s2NG2S10zV26npLya8QPSefqr9by/eGublmOMCY+WtOY6W0RW4XSjMgNYD7wf4rhMhHmlfSWT4wZnAvD56u2tet+O0goqqmv434LNpCbG8tRVR9MvLZFnZ61vUxzGmPBoyZHiHmA8sFJVc4GTga9DGpWJOH9dSWubBfvlpCfSNyWBL1e1PJnU+JTTH/qcsx7+gk+WbWPyiN7Ex3i5/Jj+zFm/iyWb97QpFmNM6LUkmVSp6g6cVl0eVf0MyAtxXCbC/E2D21oBLyIcNyiDr9Zsp6aFoy4u3VxC0d4K1hSXUlZZw3eP6A3AhXnZJMR6efSz1ajaCI7GRKOWJJPdIpIMzAReEJGHgH2hDctEWm0FfBuaBvsdNziDkvJqFhTsbn5mYNZapxTz8rXHcM+5Ixifmw5A94RYfnriQN5btJVnZ21oczzGmNBpyZHiHKAM+DnwAbAG+G4ogzKR196mwQDHD84gLsbDm98Wtmj+r9fuZEBmEmNz0/jh+P51xlS5YdIgTjmsB/e8s5S1xaVtjskYExpNJhMR8QLvqKpPVatV9RlVfdi97GU6sAPJpO0lk5TEOL47qg9vfFvA3vKqOq/Vv1xVXeNj9rqdHDMgPeiyPB7hnnNHUO1TPl66rc0xGWNCo8kjharWAD4Rsd76OpmY2stcbS+ZAPzwmP7sq6zhzXkHSicfL93GmHs+5pU5G2unLd5cQmmF0xS4Mb27JzCsV1dmrCxuV0zGmIOvJaedpcAiEfmviDzs/wt1YCay2lsB7zc6O4VRWd158ot1lFfV8MI3G/jxs/ns2V/FP6evwedWzs9Y4SSIppIJwMShmcxZv5PSiup2xWWMObhakkzeAP4PpwJ+bsCf6cD8fXK19T6TQLeeNpT1O8q4dcoC7p66lIlDMrn/giPYsKOM6SuLUFXemFfAMQPSyewa3+SyJg7JpKpGmbXGrrQaE01a0jfXK8Ag9/FqVS0PYTwmSvgbcbX1DvhAE4dkclFeFq/mF9CzWzwPXjyarl1iuP/D5fz3i3UkxcWwYUcZN500uNll5fVPIzHOy6fLt3Hq4T3bHZsx5uBoatjeGOCPwNXABpxOHrNF5CngdlWtauy95tB3MCrgA91+5uGUVdZw5bE5pCXFAfCj43L543vLWb5lL8nxMZw+slezy4mL8XD6iN68MmcTJw7twXeGN/8eY0zoNXWkuB9IA3JV9ShVPRIYCKQAfw1HcCZyDnT02P6SCTj3ivzj+0eSl5NWO+3Hxw/g5pMHs2NfJWeN6k1iXMtGNrjn3OGMykrhppfmsWrb3oMSnzGmfZpKJmcBP1bV2l+rqpYA1wNnhDowE1n+vrnaMp5JS4kIPz91CG/fMIH/O+vwFr8vMS6GJ65wOmF4Ztb60ARnjGmVpo4UqkH6rnCbC1ufFh2cP4m0tzVXSxyRnUJSfOvGW8tIjueMkb15e95myiqtZZcxkdZUMlkqIpfXnygiPwCWhy4kEw38BZL2dKcSapeO7cfeimreXbgl0qEY0+k1dTp4A/CGiFzNgabAeUACcF6oAzOR5S+ZeMNQMmmro3NSGZiZxJT8Ai7My450OMZ0ao0mE1UtBMaJyEnAcHfye6o6LSyRmYg6UDKJ3mQiIpw2vBePz1zL/soaEuK8kQ7JmE6r2WsYqvqpqj7i/lki6SRqK+CjfKjcsblpVPuUeRt3RToUYzq1kB4pRGSyiKwQkdUiclsT850vIioiNk5KlDgYvQaHw1H9U/EIfLNuZ6RDMaZTC1kycXscfhQ4HTgcuFREGrT/FJGuwM3AN6GKxbSeiJCRHE+Prl0iHUqTunWJ5fA+3fhmnXWvYkwkhbJkMhan+5W1qloJvIwzNkp99wB/AayblijzyS9O4PJj+kc6jGaNy01n3sbdVFTXRDoUYzqtUCaTvsCmgOcF7rRaInIkkK2q74YwDtNGKYlxB6Wjx1Abm5tGRbWPhQU2RrwxkRKxI4WIeIAHgFtbMO+1IpIvIvnFxTaWhalrXG4aHoHPV22PdCjGdFqhTCaFQGDj/yx3ml9XYAQwXUTWA+OBqcEq4VX1MVXNU9W8zMzMEIZsDkUpiXGMzk5hxoqiSIdiTKcVymQyBxgsIrkiEgdcAkz1v6iqe1Q1Q1VzVDUH+Bo4W1XzQxiT6aAmDe3BgoI9bC+tiHQoxnRKIUsmqloN3Ah8CCwDXlXVJSLyexE5O1SfazqnScN6ADDThvQ1JiJa17teK6nqe8B79abd2ci8J4YyFtOxHd67GxnJ8Xy6vIjvHZkV6XCM6XSiv6mOMS3g8QjfGd6T9xZt4c15BZEOx5hOx5KJ6TBuP/MwxuWm84tXF/DVGmvZZUw4WTIxHUZiXAxPXnk0yfExTJ2/OdLhGNOpWDIxHUpCnJcJAzOYsbKYIGO7GWNCxJKJ6XAmDs1ky55yVheVRjoUYzoNSyamwzlhiHNj6wxrJmxM2FgyMR1O35QEBvVItmRiTBhZMjEd0gmDM5m9biflVdaTsDHhYMnEdEgnDMmgotpng2YZEyaWTEyHNH5AOvExHmassEtdxoSDJRPTIXWJ9TI2N42Zq4pZvrWEl2dvBKCqxsfaYmvlZczBFtK+uYyJpIlDMvnDu8u46N+zKCmvJrNrPFMXbOadhVv45BcTyc1IinSIxnQYVjIxHdZEt4lwfKyX3Iwkbp2ygLfnb6bGpzw7a32D+fdXWmW9MW1lycR0WIN6JPOn743kpR+P555zRrC7rIphvbpy5sjevJZfwL6K6tp5p68o4oi7P2L99n0RjNiYQ5clE9NhiQiXju3HoB7JHDc4g39ediRPXJHHj47PZW9FNfd/uKK2NPLhkq1U1vj4ZNm2CEdtzKHJkonpNM4Y2Zus1ETGZKdw7ug+PP3Vek55YAa79lUyc6XTy7Dd6GhM21gyMZ2OiPD3S8bwzNVjKdy9nz++t4zC3ftJT4rjm7U7KausbvAeVcXns44jjWmMJRPTaU0cksnxgzOYMtcZTOuWU4dQWePj67U7Gsx766sLuPA/s6is9oU7TGMOCZZMTKf2o+NyAchKTeDCo7JIiPXy0CereHbWekrKqwCYu2EXb8wrZO6GXTz62eoIRmtM9LL7TEynNnFIJnn9Uxk3II0usV6uP3Egz3+9gTvfXsIDH6/ksnH9mLVmBxnJcYzNTePRz1YzeUQvDuvdLdKhGxNV5FAbQCgvL0/z8/MjHYbp4BYV7OHBT1YyfUURPoW7zx7O2Uf04aS/TWdwz668cu14RCTSYRrTYiIyV1XzQrV8K5kYE8TIrO48eeXR7NpXydItJRwzIB2PR7j1tKHc8dZi3lu0lTNH9a7zHlVlf1UNiXH2szKdj9WZGNOE1KQ4JgzKwONxSiGXju3HYb27cdvrC3nmq/XUBLTwuu/DFRz9h0/IX289FZvOx5KJMa3g9QiP/fAojshO4XdTl/CT5/Ipq6xmy579/PeLdZRV1XDlU3NYXLgn0qEaE1aWTIxppey0RJ770VjuPns4ny4v4txHv+SXUxagqkz5yTF06xLD9S/MrW0NZkxnYMnEmDYQEa44NocnrsjDp/Dl6h1cNq4/eTlpPPL9MWzeXc4dby6OdJjGhI3VFBrTDicN68mkoT1YXFjCkF7JABzVP42bTx7MAx+v5OKjsxmXm0bh7v30T7cu703HZSUTY9pJRBiZ1Z34GG/ttJ9MHEDflAT+9P4yrn/hWyb9dTqrtu2NYJTGhJYlE2NCID7Gyy+/M4TFhSV8vHQbPoW35hdGOixjQsaSiTEhcs4RfTlndB/uOPMwjh+cwdvzN3Oo3SRsTEtZnYkxIeLxCA9dMgaA1MQ4bp2ygLkbdpGXkxbhyIw5+KxkYkwYfGdEL7rEenhznl3qMh2TJRNjwiA5PoYzRvTm7fmb6wwXDFBWWR10DBVjDiWWTIwJk++P60dpRTX/W7C5dlpFdQ3f++dXXP30nAhGZkz7WZ2JMWFyVP9Uhvbsyr9nrOHpr9aTkRxPbkYSy7c6TYY37SwjOy2R4r0VXP/8XG6YNIhJw3pEOGpjWiakJRMRmSwiK0RktYjcFuT160RkkYjMF5EvROTwUMZjTCSJCD8Y34/1O8qorPbx7cZdPPf1Bo4fnAHAOwu3APD6twXkb9jFjS9+y4qtdm+KOTSEbDwTEfECK4FTgQJgDnCpqi4NmKebqpa4j88Gfqqqk5taro1nYg5lPp8yb9NuRmensGHHPqbMLeC6iQO54snZVFb7ePem4zj1wZnEeISd+ypJTYzjg1uOb3bslKKScrp2iSUhztvkfKbzCvV4JqEsmYwFVqvqWlWtBF4GzgmcwZ9IXEmANcI3HZrHIxzVPxWvRxiQmcxvJg+je0IsZx/Rh6VbSvj7J6tYXVTKlcfmcNPJg1mxbS8rt5UyJX8T1z03F5+v4U9k084yTvrbDO59b2mQTzQmPEKZTPoCmwKeF7jT6hCRG0RkDXAfcFMI4zEmap03pi9Deibz0LRVdIn1cOao3pw2vCci8M7Czfzto5V8sGQrHy3dWud9NT7l1ikLKK2o5oPF24ImG2PCIeIV8Kr6KPCoiHwfuAO4ov48InItcC1Av379whugMWGQmhTHuzcdz8uzN5IYF0PXLrF07RJLXv9U/jNzLZXVPhLjvDw0bTXfGd6Lddv38X9vL2ZxYQl79ldx0rAefLq8iAUFuxnTLzXSq2M6oVCWTAqB7IDnWe60xrwMnBvsBVV9TFXzVDUvMzPzIIZoTPSI9Xr44TE5nH9UVu20ySN6U1nto396InefPZxlW0r43r++4qxHvmDJ5hLOPqIP918wigcuOgKvR5i2rAiA8qoapuRvsvtXTNiEMpnMAQaLSK6IxAGXAFMDZxCRwQFPzwRWhTAeYw45k0f0Is7r4UfH5XLemL58b0xfYr0eTj6sJ+/ffDz3nDuCC/OySUmMI69/Kp8s2wbAi99s5FevLeSsh79gyWYb9dGEXsguc6lqtYjcCHwIeIEnVXWJiPweyFfVqcCNInIKUAXsIsglLmM6s74pCXz125NIT4pDRHjg4tGNznvq4T35w7vLWFtcymdAOwPFAAAbgElEQVQriujVrQv7Kqu55pl8Pvr5CXTtEhvGyE1nE7KmwaFiTYONCa6opJxj/vwpl43rx8uzN3HFsf05Y2RvvvevrzhvdF8G9kima5cYLjm6H3Exbb8oMWNlMb//3xLeuH4C3RMbJqjb31xEXIyH3313eHtWxxxkoW4aHPEKeGPMwdGjWxcmDc3kua83oAqThvVgTL9Urp6Qy3+/WFc73zNfreeZq8eSlZrI/soaduyroGR/NeXVNYzq250Yb9OJ5uFpq1hTvI8Zq4o5+4g+dV77bEURL3yzkRiPcOOkQaQnx4dkXU30sWRiTAdyUV42nywromt8DEe7Xd3/6jtDGdqrK8cMSGdV0V5+9uI8fvvGIq6bOJBrnslnf1VN7ft/993DuWpCbp1lFu0tJyUhjrgYDwsLdjN3wy4Api8vqpNMyiqruWvqEnp2i2dbSQVvz9/M1cfVXZbpuKyjR2M6kEnDetC7excmDetBrFvC6BLr5aK8bLLTEjlpWE9+PXkYn6/azpVPzSY7LYG/nD+Sf112JEN6JjM1oBNKgOVbSzjhvs/4+ycrAXjqy/UkxXk5eVgPZqwsrr2vZXXRXs579Cs27izjwYtGMyqrO1PmFgBOknl7fqG1LOvgrGRiTAcS6/Xw9o0TSIxr/Kf9g/H9eWfhZjbt3M/TV42lT0oCAOt27OO+D1ZQsKuMrNREyiqrueGFbymv8jF1wWauOX4A7y7cwqVjsxndL4Vpy4uYtryI6SuKeGXOJrolxPL0VWM5dlAGFxyVxZ1vL+Hqp+ewZPMetpVUcNWEHKtH6cCsZGJMB9OjaxeS4xtPJl6P8Pw14/j0lxNrEwnAmSN7A/D+Iucu+zvfXsLa7fv43pF9Kdi1nz+8s5TKGh+XjuvHCYMzEYEfP5vPK3M2cenYfnxwy/FMHOLcB3b+kVlccnQ2G3bso39aEqcc1oPnZm1gTXFpCNf80KKq3P7mIh6buSbSoRwUVjIxphOKj2nYIWT/9CRG9u3OS7M3sq+ymtfmFnDTyYO56tgc3p6/mTfmFTI6O4VhvboB8MPx/SmtqObmkwfTPz2pzrKS4mP48/mjap8X761g0l+n84d3lvLklUc323FlZzAlv4AXvtlIv7RErj1hYKTDaTcrmRhjat0waRBFeyv4+yerGJebxs0nDyY1KY7xA5zK/EuOPtCpxe/PGcEDF41ukEiCyewaz89PHcJnK4p56sv1oQr/kFBSXsWr+Zu4+39L6BLrYePOMraVlEc6rHazZGKMqTV5RC9m334y/7rsSP79g6PwepwSxGXj+jMwM4mz6jUFbo2rJ+Rw6uE9+eN7y5ixsrh2+triUl6evbHFy1lcuIcxv/+IVdsOvbFe9lVU891HvuDXry2kZ7cuPHCRcxPqnPU7IxxZ+1kyMcbUkRgXw+kje5OaFFc77YyRvZl264lN1sU0R0T464VHMKhHMlc/PYfnv94AwF8+WM5tbyyicPf+Fi3nnYVb2FVWVdta7FDy4Mcr2bCjjMcvz2ParRM57fCeJMZ5mbPOkokxxrRY94RYXrv+WI4blMGdby9m7oZdfLrc6Zxy+oqioO9ZtqWEtQEV9/753lmwubZp8raS8maT0e6ySt5ZuJlPlm6jtCL8zZQXbNrNk1+u49Kx/Tj18J6ICDFeD2P6pTB7/S5enbOJJz5fS1llNVv27Gf99n0cSj2UWHcqxpiwK95bwQn3fUaXWA+7yqroGh/D+IHpPH6509vHmuJSMrvGU15Zw6S/TqesqoZzjujDjScN4pQHZjIqqzsLC/bw3yvymL1uJ099tZ6EWC9v3zCBnIzgdTjXPpvPR0udjjAvG9ePe88bGbL18/mUWWt30D0hluF9ulGyv5ozH/mcGp/ywc0n1OmG5sGPV/LQtAN93MbHeKio9gHQo2s8/dMTGdG3Oz89cRCZXdveo4B1p2KM6XAyu8Zz+bH9+c+MtQzv040x/VJ449tCKqpreG7WBv70/nJyM5IYlJlMVY1yxTE5PDtrPV+u2QHAPeeM4OLHZvGjZ/IRgXNH92X6iiKueTaf6yYOpKyymkUFe1hUuIek+Bguysvio6XbuG7iQDbs2MfU+Zu5/czDmrwfpyUWFezhuufnctWEHK45fgCqyrRlRfzt45Us2+IMJJueFEeXWC9Fe8t55SfHNOjP7KRhPfjn9NVcP3Egxw/J5M15hQzMTCY+xsPcDbso3L2f52Zt4NU5m/jT+aMadGETLaxkYoyJiF37Kpn80ExuPW0o6Ulx/OiZfAb3SGZVUSnHDcogf8NOyqt8XH/iQH4zeRhPfL6WP7y7jKzUBD7/9SQe/Hgly7fu5eenDuGw3t34as12rn56DuVVzll9elIcI/p2Z0HBbnaXVdE3JYFpt05kwabdXPzY19xz7giWFO6hd/cEfnxCbosSy5ert7NzXyXfPaIPK7ft5eL/zKK0opqqGuXc0X1Ysa2UZVtKyElP5GcnDcanyux1O9myp5yLjs5uNBFUVvua7HxzbXEpf35/OTedPJgRfbu3aXuHumRiycQYE3FlldWMvXca8TEebjllMJeN68/X63YwJb+Ae84dQXJ8DKrKw9NW0zc1gQsCBhALtK+imh2llcTGCL26dUFE2LBjH3e+vYSrJuRw4tAeqCon/W0G67bvQwRUna7+X7v+GHp168Kny4t4c14h3RNiuf7EgWSlJgKwcUcZkx+aSUW1j7dvmMAtr8xnz/4qXvrxeB6atooPFm9hZN/uXHx0NucfmdVsh5nhZsmkHksmxnRMW/bsp1uXWJLa0WKspZ6dtZ4/vLuMhy8ZTWpiHFc9PYcRfbuTlZLAG/MKSU+KY2+5U0n//DXjyOufyqWPf82SzSV4PUKNTymtqOapK49m0rAeANT4tLYpdTSyZFKPJRNjTHupKuVVPhLinJ4AXp9bwK1TFgDw81OG8NNJA9leWsGlj31NVY0ybkAab3xbyH3nj6Kixsf/vbWY88b05cEmBiuLNlYBb4wxB5mI1CYSgPOPyqJobwW9u3fh3DF9AejdPYEHLh7Nhf+exRvfFnLzyYO5MC8Ln0JaYhwTh2ZGKvyoZCUTY4xpwpvzCthf6eP74/pFOpR2sZKJMcZE0Hljglf2m7qiq7mBMcaYQ5IlE2OMMe1mycQYY0y7WTIxxhjTbpZMjDHGtJslE2OMMe1mycQYY0y7WTIxxhjTbofcHfAiUgxsaOPbM4DtBzGcgylaY7O4Wsfiar1oja2jxdVfVUPWB8whl0zaQ0TyQ9mdQHtEa2wWV+tYXK0XrbFZXK1jl7mMMca0myUTY4wx7dbZksljkQ6gCdEam8XVOhZX60VrbBZXK3SqOhNjjDGh0dlKJsYYY0Kg0yQTEZksIitEZLWI3BbBOLJF5DMRWSoiS0TkZnf6XSJSKCLz3b8zIhDbehFZ5H5+vjstTUQ+FpFV7v/UMMc0NGCbzBeREhG5JVLbS0SeFJEiEVkcMC3oNhLHw+4+t1BEjgxzXPeLyHL3s98UkRR3eo6I7A/Ydv8Oc1yNfnci8lt3e60Qke+EKq4mYnslIK71IjLfnR6WbdbE8SHi+1izVLXD/wFeYA0wAIgDFgCHRyiW3sCR7uOuwErgcOAu4JcR3k7rgYx60+4DbnMf3wb8JcLf41agf6S2F3ACcCSwuLltBJwBvA8IMB74JsxxnQbEuI//EhBXTuB8EdheQb8793ewAIgHct3frDecsdV7/W/AneHcZk0cHyK+jzX311lKJmOB1aq6VlUrgZeBcyIRiKpuUdVv3cd7gWVA30jE0kLnAM+4j58Bzo1gLCcDa1S1rTettpuqzgR21pvc2DY6B3hWHV8DKSLSO1xxqepHqlrtPv0aCPuQgY1sr8acA7ysqhWqug5YjfPbDXtsIiLARcBLofr8RmJq7PgQ8X2sOZ0lmfQFNgU8LyAKDuAikgOMAb5xJ93oFlWfDPflJJcCH4nIXBG51p3WU1W3uI+3Aj0jEJffJdT9cUd6e/k1to2iab+7GucM1i9XROaJyAwROT4C8QT77qJpex0PbFPVVQHTwrrN6h0fon4f6yzJJOqISDLwOnCLqpYA/wIGAqOBLThF7HA7TlWPBE4HbhCREwJfVKdcHZHmfyISB5wNTHEnRcP2aiCS26gxInI7UA284E7aAvRT1THAL4AXRaRbGEOKyu+unkupe+IS1m0W5PhQKxr3Meg8yaQQyA54nuVOiwgRicXZUV5Q1TcAVHWbqtaoqg94nBAW7xujqoXu/yLgTTeGbf5is/u/KNxxuU4HvlXVbW6MEd9eARrbRhHf70TkSuAs4DL3IIR7GWmH+3guTt3EkHDF1MR3F/HtBSAiMcD3gFf808K5zYIdH4jifcyvsySTOcBgEcl1z3AvAaZGIhD3Wux/gWWq+kDA9MDrnOcBi+u/N8RxJYlIV/9jnMrbxTjb6Qp3tiuAt8MZV4A6Z4qR3l71NLaNpgKXuy1uxgN7Ai5VhJyITAZ+DZytqmUB0zNFxOs+HgAMBtaGMa7GvrupwCUiEi8iuW5cs8MVV4BTgOWqWuCfEK5t1tjxgSjdx+qIVM1/uP9wWj2sxDmjuD2CcRyHU0RdCMx3/84AngMWudOnAr3DHNcAnJY0C4Al/m0EpAPTgFXAJ0BaBLZZErAD6B4wLSLbCyehbQGqcK5P/6ixbYTTwuZRd59bBOSFOa7VONfT/fvZv915z3e/4/nAt8B3wxxXo98dcLu7vVYAp4f7u3SnPw1cV2/esGyzJo4PEd/HmvuzO+CNMca0W2e5zGWMMSaELJkYY4xpN0smxhhj2s2SiTHGmHazZGKMMabdwppMROR2tyfMhW7Pm+Pc6beISGKIPnOu227d3yPuQrc7hP6h+Lx6nz1dRFo8VrO7HS53H98TsJ0+EpE+oYu00XjuEpFfhvtzg8SRIiI/PcjL/Hv9O/xb8d54cXqXXS0i37jdXjQ1/60ioiKS4T5PFacX34UiMltERrjT40RkpnvTXHMxXOy+f4mI/KUt69FeIvK0iFwQic+OlKZ+0yLymnsPSluW266elEVkmIjMEpGK+r9ZEblZRBa7+8otAdOPcN+zSET+57+jX0RGisjTAfOdJSK/b24dwpZMROQYnDtxj1TVUTg3Bvn7lLkFOOjJxL3xqVBVK9xJk9zPng7ccbA/rz3cA8jVwIvupPtVdZSqjgbeAe6MWHAh1JIDJ5ACHLRkIiLpwHh1Ovprix8Bu1R1EPAgTo+8jX1WNs4NoBsDJv8/YL67L14OPASgTiek04CLWxD//cDJqjoc6CUiJ7dxXQ45Te0zLdyfDjoRGY7Tw3F7bmR8UFVHu3/vucs9HOcm6+HAZOCf/psn69kJ3AT8tV5cI4Af4/QycARwlogMcl9+Aqcn4pE4PV78CkBVFwFZItLPne9d4LvNnfCHs2TSG9juP7Cr6nZV3SwiNwF9gM9E5DMAETnNzZjfisgUcfqp8Y+3cZ+bSWf7N4qIXOhm3gUiEniAmAx8ECSWWQR0hiYiv3Dfv9ifucUZvyBwnINfishd7uPpIvIXN4aV4nb6JiIJIvKyiCwTkTeBBHe61z2LW+zG/vMgMZ2E011Itbt9AvvjSSJIXzzucu8XkTnuWepP3Oknume477pnM/8WEY/72qVuDIsDz2jFGe/lW3cbTgv4mMPd9V3rflcNiEipiDzonvlME5FMd/pAEflAnNLh5yIyzJ3+tBvTNzhdawcua7i7Xee76zQY+DMw0J12vzvfrwLW++6A72y5iLzgfgevNfIDOB93vxCR7u42Guo+f0lEfhxsPQME9uD6GnCyiEgj8z6Icxd64Pd3OPApgKouB3JExN9x31vAZc18/gBglaoWu88/cdepDnF6NXjS3Z7zROQcd/qVIvK2+72uEpHfBbynwW/BnX65u60XiMhzAR9zgoh85e4fF7jz9nb3v/nuchp0itjEbzlTRF53v9s5IjLBnX6XiDwnIl/i3PQYuKwT3f1rKrC0sfU4mL/pIC7DvStdRPq72zVDRDxubKc18r7mtKgnZVUtUtU5ODdgBjoMp1v6MvfYMgOnqxhwuoPxHy8/pu4+9D+cJIY6NyNOxykMNC5cd0cCyTh3c64E/glMDHhtPe44GkCGu4JJ7vPfcGBMgfUcuDP7cuAd9/EioK/7OCVguW8DA4J8xt+Ba93HR7nvT3JjXILTU2cOdcdg+CVwl/t4OvA39/EZwCfu418AT7qPR+F0rpfnfsbHActKCbJ97gZ+Vm/avTilt8VAZpD3XAvc4T6OB/JxxoE4ESjHOeh4cXaUC3CS9kYgE4jBOaCd6z7fBOS6y/LfXXsX8JW77Aycu9Bjg8ShOH0/gVOC+of7eBow2H08DvhUD9xh/A5BxqoAHglYVhzOj7f+d3EazjjYgnNC9A7O2BQ5biwT3PmeJPi4Gc8QcAczcCrOCcYlwAcB058gyB3F7veRFfB8DfXGgXGnnwM8FGT/+yPOWSg4B4Zq4Cj3uRcobua3lIpzx3aO+z2+DvwvyHx/BH7g3+dwfntJwJU4d36nu9t3MQf202C/heHue/3x+/ePp3E63vTgJMjV7vRbOfA79QJdg8S2nuC/5RdxOhwF6IfTrQg4++JcICHIsk4E9nFg/w35bzpIDDOAkQHPr3G3za+A/wRMfw/oE+T9d7nbZCHOfpvqTv+H/zt0n/8XuKCJfeMuAvZ5nGSy0v2uE3H280fc174Czg1Yz70B75tAwD6FkywfaWq/DFvJRFVLcb7ka4Fi4BVxOqGrbzzOjvmlOKOcXYEzGJLfSwH/j3Effwk87Z5R+vvPicP5wQcWOz8TkUKcTgP9yzkOeFNV97kxvoHT/XRz/B2wzcXZScE5oD3vru9CnB0DnD58BojII+L0l1SnF1BXb5ztUktVb1fVbJzeXm8M8p7TcPrlmY/TTXU6Tp9BALPVGb+lxl3X44CjgemqWqzOWcoLbszjgZnqnPmgqoFjPLyrzlnRdpzO5YJ1Qe/jQKd4zwPHiVOaPBaY4sb3H3cd/aa4sdU3C/h/IvIboL+q7m9kvU8D5uF0bTEsYL03qeqXgbEEeX+dba2qH+McfB7FOQj4p1+jqvlB3t8st0T0/wh+efLPOONOzAd+5q5HjfuZNUCluP2kBaOqu4Drcbb55zgHoWDb8jTgNvdzpgNdcA7Q4Jzc7HC37xs426mx38JJON/XdvfzA/ePt1TVp6pLObBvzAGucs/6R6ozLkcwwX7LpwD/cGOeCnRz9yWAqY3sD+Ds7+vcx+H4TddXf596AugGXIeTtPzTz1DVzUHeH5KelFV1Gc5l2I9wSuPzObCvXA38VETm4gzEVRnw1iKck8/GnjcQ1uuL7g9lOjBdRBbhJIqn680mODv6pY0tpv5jVb1OnMr8M4G5InIUzpfyRb33TgJ24xxE78bJxo2ppu5lwC71XvfXw9TQzHZU1V0icgTwHZyd6yKcLzLQ/iCf4fcCzhnN7+pNF5zSzId1JoqcSMPLYm3tN6ci4HGz6xrwWR5gtzp1PsHsC/pG1Rfdy19nAu+Jc+mu/nVoAf6kqv+pM9GpCG/JetfZ1uJcAjwMKOPAWX9T/D21Fohzjb47sENEnsI5A96MU6LOBRa4V8CygG9FZKyqbgWucj9bgHX11jEep2TZKFX9H86lCMQZeyZYMhHgfFVdUWei81sJxf4hbmwzxWnccCbOSd4DqvpskPc2+C3j7DfjVbXO+rvbMOg+42rqNb+D9psOov4+lciBwciSgcYSKuD0pBzw3sdxStvQSK/AInIDTl0IQGMJyr/s/+KUaBCRP+Lu3+pcYj3NnT4E5/vy6+KuU2PPGwhnBfxQca5/+40G/CPm7cXJjOCMCDch4BpqkruifhcH/J/lzjNQVb9R1Ttxzg6ycepLAgcDAsA9I78F54w+DefM7lwRSRSnt9zz3GnbgB4iki4i8TR3vdAxE/i+G9MInGIx4rTi8ajq6zgV/8HGaV4G+CvGqLetzgGWB3nPh8D14nRZjYgMcdcBYKw4vSR7cLbVFzg9sE50r+V6cXrinYGzzU8Qp8EC7nZpDQ/OZTTc9f9CnTqfdSJyobtMcRNqk8RpDbNWVR/GuUw5irr7h3+9r5YDdWl9RaSH+1o/cRp71MYS5GPqbGvg5+607wNP+bdnEwJ7cL0A5/KdqupV6lSenqGqi1S1h6rmqGoOzg/4SFXdKk7rtDj3/dfglApL3HVJx6lbrHKfB/ve8a+vOANL/RTnklx9HwI/cxMWIjIm4LVTxRlXPAHnUueXNP5b+BS40I2t2f1DnJaS21T1cTeuxsYlb/BbxjmD/lnAsho7GWlKyH/TQdTfp/6CcxJ4J043+02SVvakrKqP6oHK+kYTibts/77SD6e+5MV60z04x6XAce2HULcn7vrPGwhnySQZeEREUnDOEFbjXPIC5/r3ByKyWVUnuZe/XnK/cHBWdKX7OFVEFuKcRfhLL/e7B1/BuU6/AOcLDNoCSlW3iMhLwA2qeo84zeD8XV0/oarzAMRpDjcb5+wg6I+6nn/hHIyW4excc93pfd3p/uT92yDvfZ+6FYt/FqdS2IeTdK8L8p4ncIrj37oHjGIODOc5B+d66yDgM5xiv09EbnOfC84lLH+l4bXAG26MRTj1CC21Dyd53eG+13+QuAz4lzs9Fme45AXNLOsi4IciUoUzotwfVXWniHwpTuXp+6r6KxE5DJjlHidLgR/gnFGuwBnY60mcyth/BfmMd4GfAE+42/gaYKyq7hWnAccdwO9E5AmcnnbrX+r6L/CciKzGaUVzSUs2UoDDgGdERHGu5/8o4LVJbnz+k5DGKvYfCkjOv1fVlUHmuQenfnCh+72u48ABdDZOXUsW8Lx/HZv4LdwLzBCRGpzLclc2sX4nAr9yv8NSnDqRYIL9lm8CHnWnx+AczIPt+41S1W/D8Juu712c9f5ERCbiXFKeoKo1InK+iFylqk+JyHvANUESwH1u4lScy5Y/cddliYi8irMvV+McsxqUQkWkF06daTfAJ06jg8Pdk5TX3ROBKvf9u923XeqWcMC5xPdUwCJr98OA58GOWwdicCtXDgkish6n8mt7M/NlAY+r6ulhCewgEae1yK+17lChbVnOiTiVcC0582o3ESlV1eTm5wx5HDk4FbkjWjDvF8BZAT+sqCAib+A011wpImfhNCB5+CB/xpU4v6Ng9XBh0dLf8qHCLeF9hptAIh1Pe7gn8TNwGkJUi9PS8EVVbbL5eUTaZIeaOoPaHFKJxHUbTkVeu5KJaZFbcSqjoyaZuJe+3vKXMlT1nWbeYqKEqu4Xp4l1X+reU3Qo6odzQlMd8PzW5t50SJVMjDHGRCfrm8sYY0y7WTIxxhjTbpZMjDHGtJslE2OMMe1mycQYY0y7WTIxxhjTbv8f78Fyf86DjEAAAAAASUVORK5CYII=\n", 236 | "text/plain": [ 237 | "" 238 | ] 239 | }, 240 | "metadata": {}, 241 | "output_type": "display_data" 242 | } 243 | ], 244 | "source": [ 245 | "x6 = range(len(dis_pretrain_eval_loss) + len(dis_adversarial_eval_loss))\n", 246 | "dis_loss = dis_pretrain_eval_loss + dis_adversarial_eval_loss\n", 247 | "plt.plot(x6, dis_loss)\n", 248 | "plt.xlabel('Steps/Rounds (3 epoch per step (x: 0-49), 9 epochs per round (x: 50-199))')\n", 249 | "plt.ylabel('Oracle NLL')\n", 250 | "plt.savefig('dis_loss.png',dpi=600)" 251 | ] 252 | } 253 | ], 254 | "metadata": { 255 | "kernelspec": { 256 | "display_name": "Python 3", 257 | "language": "python", 258 | "name": "python3" 259 | }, 260 | "language_info": { 261 | "codemirror_mode": { 262 | "name": "ipython", 263 | "version": 3 264 | }, 265 | "file_extension": ".py", 266 | "mimetype": "text/x-python", 267 | "name": "python", 268 | "nbconvert_exporter": "python", 269 | "pygments_lexer": "ipython3", 270 | "version": "3.5.2" 271 | } 272 | }, 273 | "nbformat": 4, 274 | "nbformat_minor": 2 275 | } 276 | --------------------------------------------------------------------------------