├── lib ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-35.pyc │ ├── data_split.cpython-35.pyc │ └── features_word2vec.cpython-35.pyc ├── data_split.py ├── batch_generator.py ├── read_article.py ├── features_word2vec.py └── model_lstm_tf.py ├── model ├── .gitignore ├── embedding_weights.pkl ├── imdb_indices.pickle ├── 300features_40minwords_10context └── readme.txt ├── tensorboard └── .gitignore ├── data ├── labeledTrainData.tsv └── unlabeledTrainData.tsv ├── figures ├── .DS_Store └── attention_map.png ├── README.md ├── main_tf.py └── main_lstm.ipynb /lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /model/.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorboard/.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/labeledTrainData.tsv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/unlabeledTrainData.tsv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /model/embedding_weights.pkl: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /model/imdb_indices.pickle: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /model/300features_40minwords_10context: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /figures/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rantsandruse/lstm_attention_tf/HEAD/figures/.DS_Store -------------------------------------------------------------------------------- /figures/attention_map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rantsandruse/lstm_attention_tf/HEAD/figures/attention_map.png -------------------------------------------------------------------------------- /model/readme.txt: -------------------------------------------------------------------------------- 1 | The existing model files are dummies. The proper files will be generated by running main_lstm.ipynb or main_tf.py 2 | -------------------------------------------------------------------------------- /lib/__pycache__/__init__.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rantsandruse/lstm_attention_tf/HEAD/lib/__pycache__/__init__.cpython-35.pyc -------------------------------------------------------------------------------- /lib/__pycache__/data_split.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rantsandruse/lstm_attention_tf/HEAD/lib/__pycache__/data_split.cpython-35.pyc -------------------------------------------------------------------------------- /lib/__pycache__/features_word2vec.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rantsandruse/lstm_attention_tf/HEAD/lib/__pycache__/features_word2vec.cpython-35.pyc -------------------------------------------------------------------------------- /lib/data_split.py: -------------------------------------------------------------------------------- 1 | from sklearn.cross_validation import StratifiedShuffleSplit 2 | 3 | 4 | def train_test_split_shuffle(target, features, test_size = 0.1): 5 | sss = StratifiedShuffleSplit(target, 1, test_size = test_size, random_state=0) 6 | for train_index, test_index in sss: 7 | X_train, X_test = features[train_index], features[test_index] 8 | y_train, y_test = target[train_index], target[test_index] 9 | y_test = y_test.values 10 | y_train = y_train.values 11 | 12 | return X_train, y_train, X_test, y_test 13 | 14 | 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tensorflow implementation of LSTM based sentiment analysis with attention layer 2 | Bag of Words vs. Bag of Popcorn as an example 3 | 4 | This is an extension of the previous project (lstm_word2vec). Now we have a tensorflow implementation of LSTM for the purpose of sentiment analysis, with the option of adding an attention layer. 5 | 6 | In this project, I reused the I/O function and data prep functions from lstm_word2vec. 7 | 8 | The major improvements are: 9 | 1. Migration of code from keras to TF. 10 | 2. Added an optional attention layer plus visualization. 11 | 3. Added a batch generator class that can be easily used for different datasets. 12 | 13 | The main_lstm.ipynb show cases how to leverage existing functions to train your own LSTM + attention layer model. 14 | 15 | Addition of the attention layer appears to marginally improve the results of the testing set. But further testing is needed to establish the significance of the improvement... 16 | 17 | -------------------------------------------------------------------------------- /main_tf.py: -------------------------------------------------------------------------------- 1 | from lib import features_word2vec, model_lstm_tf 2 | import pandas as pd 3 | import os 4 | 5 | 6 | # The next steps: 7 | # 1. Check 8 | 9 | # Will ingest/clean data and save the following: 10 | # 1. cleaned text translated to array of word indices: imdb_indices.pickle 11 | # 2. word2vec model, where the indices/word vecs are stored: 300features_40minwords_10context 12 | # 3. word embeddings: this is the index to wordvec mapping derived from 2. 13 | 14 | # ingestion clean data 15 | # create word embedding 16 | # create word indices that can be mapped to word embedding 17 | labeled_data_path = "./data/labeledTrainData.tsv" 18 | unlabeled_data_path = "./data/unlabeledTrainData.tsv" 19 | model_path = "./model/300features_40minwords_10context" 20 | embedding_path = "./model/embedding_weights.pkl" 21 | text2indices_path = "./model/imdb_indices.pickle" 22 | 23 | 24 | 25 | def data_prep(): 26 | # Read data 27 | # Use the kaggle Bag of words vs Bag of popcorn data: 28 | # https://www.kaggle.com/c/word2vec-nlp-tutorial/data 29 | 30 | data = pd.read_csv(labeled_data_path, header=0, 31 | delimiter="\t", quoting=3, encoding="utf-8") 32 | 33 | data2 = pd.read_csv(unlabeled_data_path, header=0, 34 | delimiter="\t", quoting=3, encoding="utf-8") 35 | 36 | # data2 and data are combined to train word2vec model 37 | data2 = data.append(data2) 38 | 39 | model = features_word2vec.get_word2vec_model(data2, "review", num_features=300, downsampling=1e-3, model_path=model_path) 40 | embedding_weights = features_word2vec.create_embedding_weights(model) 41 | features = features_word2vec.get_indices_word2vec(data, "review", model, maxLength=500, 42 | writeIndexFileName="./model/imdb_indices.pickle", padLeft=True) 43 | return model, embedding_weights, features 44 | 45 | if __name__ == '__main__': 46 | 47 | 48 | # Run data prep routine if some files are not found 49 | if not(os.path.isfile(model_path) and os.path.isfile(embedding_path) and os.path.isfile(text2indices_path)): 50 | data_prep() 51 | 52 | #model1 = model_lstm_tf.LstmTFModel(useAttention=True, restore = False) 53 | #model1.train_epochs(1) 54 | #model1.test() 55 | 56 | 57 | model2 = model_lstm_tf.LstmTFModel(useAttention=True, restore = True) 58 | model2.train_epochs(5) 59 | model2.test() 60 | 61 | model2.plot_attention() 62 | 63 | 64 | 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /lib/batch_generator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import random 3 | 4 | class BatchGenerator: 5 | def __init__(self, X_train, y_train, X_test, y_test, batchSize, shuffle=True): 6 | self.X_train = X_train 7 | self.y_train = y_train 8 | self.X_test = X_test 9 | self.y_test = y_test 10 | self.X_train_offset = 0 11 | self.X_test_offset = 0 12 | self.shuffle = shuffle 13 | self.batchSize = batchSize 14 | 15 | def shuffleIfTrue(self): 16 | if self.shuffle: 17 | arr = np.arange(len(self.X_train)) 18 | np.random.shuffle(arr) 19 | self.X_train = self.X_train[arr] 20 | self.y_train = self.y_train[arr] 21 | 22 | 23 | def nextTrainBatch(self): 24 | start = self.X_train_offset 25 | end = self.X_train_offset + self.batchSize 26 | 27 | self.X_train_offset = end 28 | 29 | # handle wrap around , 30 | if end > len(self.X_train): 31 | spillover = end - len(self.X_train) 32 | self.X_train_offset = spillover 33 | X = np.concatenate((self.X_train[start:], self.X_train[:spillover]), axis = 0) 34 | y = np.transpose([ np.concatenate((self.y_train[start:], self.y_train[:spillover]), axis = 0), 35 | 1 - np.concatenate((self.y_train[start:], self.y_train[:spillover]), axis = 0)]) 36 | 37 | self.X_train_offset = 0 38 | self.shuffleIfTrue() 39 | 40 | else: 41 | X = self.X_train[start:end] 42 | y = np.transpose([self.y_train[start:end], 1- self.y_train[start:end]]) 43 | 44 | X = X.astype(np.int32, copy = False) 45 | y = y.astype(np.float32, copy = False) 46 | 47 | return X,y 48 | 49 | def nextTestBatch(self): 50 | start = self.X_test_offset 51 | end = self.X_test_offset + self.batchSize 52 | self.X_test_offset = end 53 | 54 | # handle wrap around , 55 | if end > len(self.X_test): 56 | spillover = end - len(self.X_test) 57 | self.X_test_offset = spillover 58 | X = np.concatenate((self.X_test[start:], self.X_test[:spillover]), axis=0) 59 | y = np.transpose([np.concatenate((self.y_test[start:], self.y_test[:spillover]), axis=0), 60 | 1 - np.concatenate((self.y_test[start:], self.y_test[:spillover]), axis=0)]) 61 | 62 | self.X_test_offset = 0 63 | 64 | self.shuffleIfTrue() 65 | 66 | else: 67 | X = self.X_test[start:end] 68 | y = np.transpose([self.y_test[start:end], 1 - self.y_test[start:end]]) 69 | 70 | X = X.astype(np.int32, copy=False) 71 | y = y.astype(np.float32, copy=False) 72 | 73 | return X, y 74 | 75 | 76 | 77 | -------------------------------------------------------------------------------- /lib/read_article.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | from nltk.corpus import stopwords 3 | import nltk ,re 4 | 5 | 6 | def data_to_reviews( data, column, toLower = True, remove_stopwords = False, keep_freqwords = []): 7 | 8 | reviews = [] 9 | for review in data[column]: 10 | reviews.append( review_to_wordlist( review, remove_stopwords = remove_stopwords, toLower = toLower)) 11 | 12 | return reviews 13 | 14 | def review_to_wordlist( review_text, remove_stopwords=False, remove_nonletters = True, toLower = True, remove_html = True ): 15 | # Function to convert a document to a sequence of words, 16 | # optionally removing stop words. Returns a list of words. 17 | # 18 | # 1. Remove HTML 19 | # But if we have clean formatted 20 | # There is no need in doing so. 21 | 22 | if remove_html: 23 | review_text = BeautifulSoup(review_text, "lxml").get_text() #BeautifulSoup([your markup], "lxml") 24 | # 25 | # 2. Remove non-letters 26 | # For tweets, we do not wish to remove non-letters. 27 | if remove_nonletters: 28 | review_text = re.sub("[^a-zA-Z]"," ", review_text) 29 | # 30 | # 3. Convert words to lower case and split them 31 | if(toLower): 32 | words = review_text.lower().split() 33 | else: 34 | words = review_text.split() 35 | # 36 | # 4. Optionally remove stop words (false by default) 37 | if remove_stopwords: 38 | stops = set(stopwords.words("english")) 39 | # You can also create 40 | # stops = FINAL_STOPWORDS 41 | words = [w for w in words if not w in stops] 42 | # 43 | 44 | # 5. Return a list of words 45 | return(words) 46 | 47 | # Get all sentences for all articles so that we can train word2vec model 48 | def data_to_sentences( data, column, remove_stopwords = False ): 49 | sentences = [] 50 | tokenizer = nltk.data.load('tokenizers/punkt/english.pickle') 51 | for review in data[column]: 52 | sentences += review_to_sentences(review, tokenizer = tokenizer, remove_stopwords = remove_stopwords) 53 | 54 | return sentences 55 | 56 | # Define a function to split a review into parsed sentences 57 | def review_to_sentences( review, tokenizer = nltk.data.load('tokenizers/punkt/english.pickle'), remove_stopwords = False, remove_nonletters = True, toLower=True): 58 | # Function to split a review into parsed sentences. Returns a 59 | # list of sentences, where each sentence is a list of words 60 | # 61 | # 1. Use the NLTK tokenizer to split the paragraph into sentences 62 | # Split things into a list of sentences 63 | #tokenizer = RegexpTokenizer(r'(\w|\')+') 64 | 65 | #use this for regular reviews 66 | #tokenizer = nltk.data.load('tokenizers/punkt/english.pickle') 67 | 68 | #Use this from twitter 69 | #tokenizer = TweetTokenizer() 70 | 71 | #hi = review.strip().decode('utf-8') 72 | raw_sentences = tokenizer.tokenize(review) 73 | 74 | # 75 | # 2. Loop over each sentence 76 | sentences = [] 77 | for raw_sentence in raw_sentences: 78 | # If a sentence is empty, skip it 79 | if len(raw_sentence) > 0: 80 | # do not mark negation for now. 81 | # Otherwise, call review_to_wordlist to get a list of words 82 | #sentences.append( review_to_wordlist( mark_negation(raw_sentence), \ 83 | # remove_stopwords, remove_nonletters, toLower ) ) 84 | sentences.append( review_to_wordlist( raw_sentence, \ 85 | remove_stopwords = remove_stopwords, remove_nonletters = remove_nonletters, toLower = toLower ) ) 86 | 87 | # 88 | # Return the list of sentences (each sentence is a list of words, 89 | # so this returns a list of lists 90 | return sentences 91 | 92 | -------------------------------------------------------------------------------- /lib/features_word2vec.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cPickle 3 | import read_article 4 | import os 5 | from gensim.models import word2vec 6 | from gensim.models import Word2Vec 7 | 8 | 9 | # create an index vec, something like 10 | # [ 2,5,27,9,0,0....] 11 | # adapt code so that if we are 12 | def makeIndexVec(words, model, maxLength=50, withZeros=False, padLeft=True): 13 | indexVec = np.zeros((maxLength,), dtype="float32") 14 | 15 | vocab = model.vocab 16 | index2word_set = set(model.index2word) 17 | 18 | if padLeft: 19 | # we shall always choose maxLength now. 20 | i = maxLength - 1 21 | for word in reversed(words): 22 | if i == 0: 23 | break 24 | if word in index2word_set: 25 | indexVec[i] = vocab[word].index 26 | i = i - 1 27 | else: 28 | i = 0 29 | for word in words: 30 | if i == maxLength: 31 | break 32 | if word in index2word_set: 33 | indexVec[i] = vocab[word].index 34 | i = i + 1 35 | 36 | return indexVec 37 | 38 | 39 | def indices_to_words( indices, model, maxLength=50, padLeft = True): 40 | wordVec = [""] * maxLength 41 | index2word = model.index2word 42 | 43 | if padLeft: 44 | # we shall always choose maxLength now. 45 | i = maxLength - 1 46 | for index in reversed(indices): 47 | if i == 0: 48 | break 49 | 50 | if i < 0: 51 | wordVec[i] = "" 52 | else: 53 | wordVec[i] = index2word[index] 54 | i = i - 1 55 | else: 56 | i = 0 57 | for index in indices: 58 | if i == maxLength: 59 | break 60 | 61 | if i < 0: 62 | wordVec[i] = "" 63 | else: 64 | wordVec[i] = index2word[index] 65 | i = i + 1 66 | 67 | return wordVec 68 | 69 | def get_indices_word2vec(data, column, model, maxLength=50, writeIndexFileName="./model/word2vec_indices.pickle", 70 | padLeft=True, keep_freqwords=[]): 71 | 72 | if (os.path.isfile(writeIndexFileName)): 73 | reviewIndexVecs = cPickle.load(open(writeIndexFileName)) 74 | return reviewIndexVecs 75 | # 76 | reviews = read_article.data_to_reviews(data, column, keep_freqwords=keep_freqwords) 77 | # Initialize a counter 78 | counter = 0 79 | # 80 | # Preallocate a 2D numpy array, for speed 81 | reviewIndexVecs = np.zeros((len(reviews), maxLength), dtype="int32") 82 | # 83 | # Loop through the reviews 84 | for review in reviews: 85 | # Print a status message every 1000th review 86 | if counter % 1000 == 0: 87 | print("Review %d of %d" % (counter, len(reviews))) 88 | # Call the function (defined above) that makes average feature vectors 89 | reviewIndexVecs[counter] = makeIndexVec(review, model, maxLength, padLeft=padLeft) 90 | # Increment the counter 91 | counter = counter + 1 92 | 93 | cPickle.dump(reviewIndexVecs, open(writeIndexFileName, 'w')) 94 | return reviewIndexVecs 95 | 96 | 97 | def create_embedding_weights(model, max_index=0, writeEmbeddingFileName = "./model/embedding_weights.pkl"): 98 | 99 | if (os.path.isfile(writeEmbeddingFileName)): 100 | reviewIndexVecs = cPickle.load(open(writeEmbeddingFileName)) 101 | return reviewIndexVecs 102 | 103 | # dimensionality of your word vectors 104 | num_features = len(model[model.vocab.keys()[0]]) 105 | n_symbols = len(model.vocab) + 1 # adding 1 to account for 0th index (for masking) 106 | 107 | # Only word2vec feature set 108 | embedding_weights = np.zeros((max(n_symbols + 1, max_index + 1), num_features)) 109 | for word, value in model.vocab.items(): 110 | embedding_weights[value.index, :] = model[word] 111 | 112 | cPickle.dump(embedding_weights, open(writeEmbeddingFileName, 'w')) 113 | 114 | return embedding_weights 115 | 116 | def get_word2vec_model(data, column, num_features=300, min_word_count=10, num_workers=4, context=10, 117 | downsampling=1e-3, 118 | model_path="./model/300features_40minwords_10context", remove_stopwords=False): 119 | 120 | if os.path.isfile(model_path): 121 | return load_word2vec_model(model_path) 122 | 123 | # Set values for various parameters 124 | # num_features = 300 # Word vector dimensionality 125 | # min_word_count = 10 # Minimum word count 126 | # num_workers = 4 # Number of threads to run in parallel 127 | # context = 10 # Context window size 128 | # downsampling = 1e-3 # Downsample setting for frequent words 129 | 130 | sentences = read_article.data_to_sentences(data, column, remove_stopwords=remove_stopwords) 131 | # Initialize and train the model (this will take some time -- we are using everything here) 132 | print( "Training model...") 133 | model = word2vec.Word2Vec(sentences, workers=num_workers, \ 134 | size=num_features, min_count=min_word_count, \ 135 | window=context, sample=downsampling) 136 | 137 | # If you don't plan to train the model any further, calling 138 | # init_sims will make the model much more memory-efficient. 139 | model.init_sims(replace=True) 140 | 141 | # It can be helpful to create a meaningful model name and 142 | # save the model for later use. You can load it later using Word2Vec.load() 143 | model.save(model_name) 144 | 145 | return model 146 | 147 | def load_word2vec_model(model_name="300features_40minwords_10context"): 148 | model = Word2Vec.load(model_name) 149 | return model 150 | 151 | 152 | 153 | # Function to produce average 154 | # for all of the word vectors in a given paragraph 155 | # This code may be refactors later... 156 | def makeAvgVec(words, model): 157 | # 158 | # Pre-initialize an empty numpy array (for speed) 159 | # 160 | # 161 | nwords = 0. 162 | # 163 | # Index2word is a list that contains the names of the words in 164 | # the model's vocabulary. Convert it to a set, for speed 165 | index2word_set = set(model.index2word) 166 | 167 | num_features = len(model[model.vocab.keys()[0]]) 168 | # 169 | # Loop over each word in the review and, if it is in the model's 170 | # vocaublary, add its feature vector to the total 171 | # This is if we are running sentence level 172 | featureVec = np.zeros((num_features,), dtype="float32") 173 | i = 0 174 | for word in words: 175 | if word in index2word_set: 176 | nwords = nwords + 1. 177 | featureVec = np.add(featureVec, model[word]) 178 | 179 | 180 | if nwords > 0: 181 | # return np.average(featureVec, axis = 0) 182 | featureVec = np.divide(featureVec, nwords) 183 | 184 | return featureVec 185 | 186 | # Given a set of reviews (each one a list of words), calculate 187 | # the average feature vector for each one and return a 2D numpy array 188 | # This is pretty much lifted from the Kaggle tutorial. 189 | def get_avgfeatures_word2vec(data, column, model, num_features = 300, writeFeaturesFileName = "./model/imdb_avgfeatures.pickle" ): 190 | if(os.path.isfile(writeFeaturesFileName)): 191 | reviewFeatureVecs = cPickle.load(open(writeFeaturesFileName)) 192 | return reviewFeatureVecs 193 | # 194 | reviews = read_article.data_to_reviews( data, column ) 195 | # Initialize a counter 196 | counter = 0 197 | # 198 | # Preallocate a 2D numpy array, for speed 199 | reviewFeatureVecs = np.zeros((len(reviews),num_features),dtype="float32") 200 | # 201 | # Loop through the reviews 202 | for review in reviews: 203 | # Print a status message every 1000th review 204 | if counter%1000. == 0.: 205 | print("Review %d of %d" % (counter, len(reviews))) 206 | # Call the function (defined above) that makes average feature vectors 207 | 208 | reviewFeatureVecs[counter] = makeAvgVec(review, model) 209 | # Increment the counter 210 | counter = counter + 1 211 | 212 | cPickle.dump(reviewFeatureVecs, open(writeFeaturesFileName, 'w')) 213 | return reviewFeatureVecs 214 | 215 | 216 | 217 | 218 | -------------------------------------------------------------------------------- /lib/model_lstm_tf.py: -------------------------------------------------------------------------------- 1 | from batch_generator import BatchGenerator 2 | import tensorflow as tf 3 | from lib import features_word2vec 4 | import data_split 5 | import pandas as pd 6 | import datetime 7 | import seaborn as sns 8 | import numpy as np 9 | import matplotlib.pyplot as plt 10 | 11 | #word2vecmodel, embedding_weights, word_features, data = data_prep() 12 | # To do tomorrow: 13 | # 1. rename params for the alpha part 14 | # 2. find out 15 | 16 | class LstmTFModel: 17 | 18 | # initiate everything 19 | # embedding weights: map index to word vecs 20 | # word_features: map 21 | 22 | def __init__(self, useAttention = True, restore = False): 23 | 24 | self.useAttention = useAttention 25 | tf.reset_default_graph() 26 | self.session = tf.Session() 27 | self.restore = restore 28 | 29 | #if restore: 30 | # self.restore_model() 31 | self.initialize_params() 32 | self.initialize_filepaths() 33 | self.initialize_inputs() 34 | self.initialize_train_test() 35 | self.initialize_model() 36 | self.initialize_tboard() 37 | 38 | 39 | # initialize all hyperparameters 40 | # including batch size, network size etc. 41 | def initialize_params(self): 42 | self.batchSize = 128 43 | self.lstmUnits = 64 44 | self.numClasses = 2 45 | self.maxSeqLength = 500 46 | self.numDimensions = 300 47 | self.dropOutRate = 0.20 48 | self.attentionSize = 128 49 | 50 | 51 | def initialize_filepaths(self): 52 | self.word2vecmodel_path = "./model/300features_40minwords_10context" 53 | self.embedding_path = "./model/embedding_weights.pkl" 54 | self.text2indices_path = "./model/imdb_indices.pickle" 55 | self.labeled_data_path = "./data/labeledTrainData.tsv" 56 | self.unlabeled_data_path = "./data/unlabeledTrainData.tsv" 57 | self.lstm_model_path = "./model/pretrained_lstm_tf.model" 58 | self.attention_map_path = "./figures/attention_map.png" 59 | 60 | 61 | # initialize inputs 62 | # word2vec model 63 | # word to indices vectors 64 | # word embeddings 65 | def initialize_inputs(self): 66 | # Read data 67 | # Use the kaggle Bag of words vs Bag of popcorn data: 68 | # https://www.kaggle.com/c/word2vec-nlp-tutorial/data 69 | data_labeled = pd.read_csv(self.labeled_data_path, header=0, delimiter="\t", quoting=3, encoding="utf-8") 70 | data_unlabeled = pd.read_csv(self.unlabeled_data_path, header=0, delimiter="\t", quoting=3, encoding="utf-8") 71 | # data2 and data are combined to train word2vec model 72 | data_combined = data_labeled.append(data_unlabeled) 73 | 74 | # Load or create(if not exists) word2vec model 75 | self.word2vecmodel = features_word2vec.get_word2vec_model(data_combined, "review", num_features=300, downsampling=1e-3, 76 | model_path=self.word2vecmodel_path) 77 | # Create word embeddings, which is essentially a dictionary that maps indices to features 78 | self.embedding_weights = features_word2vec.create_embedding_weights(self.word2vecmodel, writeEmbeddingFileName=self.embedding_path) 79 | 80 | 81 | # Map words to indices 82 | self.X = features_word2vec.get_indices_word2vec(data_labeled, "review", self.word2vecmodel, maxLength=500, 83 | writeIndexFileName=self.text2indices_path, padLeft=True) 84 | self.y = data_labeled["sentiment"] 85 | 86 | # convert types 87 | self.embedding_weights = self.embedding_weights.astype("float32") 88 | self.X = self.X.astype("int32") 89 | 90 | # Split train and test 91 | def initialize_train_test(self): 92 | self.X_train, self.y_train, self.X_test, self.y_test = data_split.train_test_split_shuffle(self.y, self.X, test_size=0.1) 93 | self.myBatchGenerator = BatchGenerator(self.X_train, self.y_train, self.X_test, self.y_test, self.batchSize) 94 | 95 | # http://aclweb.org/anthology/E17-2091 96 | # https://arxiv.org/pdf/1409.0473.pdf 97 | # 98 | def addAttentionToModel(self, hidden_layer): 99 | 100 | v_att = tf.tanh(tf.matmul(tf.reshape(hidden_layer, [-1, self.lstmUnits]), self.w_att) \ 101 | + tf.reshape(self.b_att, [1, -1])) 102 | betas = tf.matmul(v_att, tf.reshape(self.u_att, [-1, 1])) 103 | 104 | exp_betas = tf.reshape(tf.exp(betas), [-1, self.maxSeqLength]) 105 | alphas = exp_betas / tf.reshape(tf.reduce_sum(exp_betas, 1), [-1, 1]) 106 | 107 | output = tf.reduce_sum(hidden_layer * tf.reshape(alphas, 108 | [-1, self.maxSeqLength, 1]), 1) 109 | 110 | return output, alphas 111 | 112 | # input a sentence 113 | # calculate alpha 114 | # visualize alpha. 115 | #def visualize_attention(self, sentence): 116 | # alphas_test = self.sess.run([y_hat, alphas], {batch_ph: batchtest, target_ph: batchlabel}) 117 | # for word, coef in zip(sentence.split()[:37], alphas_test[0, :37] * 1000 / 1.7): 118 | # print "\colorbox{yellow!%d}{%s}" % (int(coef), word) 119 | # print("yes") 120 | 121 | # initialize model weights, placeholders etc. 122 | # And model cell itself 123 | def initialize_model(self): 124 | 125 | if not self.restore: 126 | self.labels = tf.placeholder(tf.float32, [self.batchSize, self.numClasses], name = "labels") 127 | self.input_data = tf.placeholder(tf.int32, [self.batchSize, self.maxSeqLength], name = "input_data" ) 128 | 129 | self.data = tf.Variable(tf.zeros([self.batchSize, self.maxSeqLength, self.numDimensions]), dtype=tf.float32) 130 | self.data = tf.nn.embedding_lookup(self.embedding_weights, self.input_data, name = "data") 131 | 132 | self.weight = tf.Variable(tf.random_normal([self.lstmUnits, self.numClasses], stddev=0.1), \ 133 | name = "weight") 134 | self.bias = tf.Variable(tf.random_normal([self.numClasses], stddev=0.1), \ 135 | name = "bias" ) 136 | 137 | lstmCell = tf.contrib.rnn.BasicLSTMCell(self.lstmUnits) 138 | self.lstmCell = tf.contrib.rnn.DropoutWrapper(cell=lstmCell, output_keep_prob=1 - self.dropOutRate) 139 | self.output, _ = tf.nn.dynamic_rnn(self.lstmCell, self.data, dtype=tf.float32) 140 | 141 | 142 | if self.useAttention: 143 | self.w_att = tf.Variable(tf.random_normal([self.lstmUnits, self.attentionSize], stddev=0.1), \ 144 | name="w_att") 145 | self.b_att = tf.Variable(tf.random_normal([self.attentionSize], stddev=0.1), \ 146 | name="b_att") 147 | self.u_att = tf.Variable(tf.random_normal([self.attentionSize], stddev=0.1), \ 148 | name="u_att") 149 | self.last, self.alphas = self.addAttentionToModel(self.output) 150 | 151 | else: 152 | self.output = tf.transpose(self.output, [1, 0, 2]) 153 | self.last = tf.gather(self.output, int(self.output.get_shape()[0]) - 1) 154 | 155 | print(self.last.get_shape()) 156 | 157 | self.prediction = (tf.matmul(self.last, self.weight) + self.bias) 158 | self.correctPred = tf.equal(tf.argmax(self.prediction, 1), tf.argmax(self.labels, 1)) 159 | self.accuracy = tf.reduce_mean(tf.cast(self.correctPred, tf.float32)) 160 | self.loss = tf.reduce_mean( 161 | tf.nn.softmax_cross_entropy_with_logits(logits=self.prediction, labels=self.labels)) 162 | self.optimizer = tf.train.AdamOptimizer().minimize(self.loss) 163 | 164 | tf.add_to_collection('output', self.output) 165 | tf.add_to_collection('last', self.last) 166 | tf.add_to_collection('prediction', self.prediction) 167 | tf.add_to_collection('correctPred', self.correctPred) 168 | tf.add_to_collection('accuracy', self.accuracy) 169 | tf.add_to_collection('loss', self.loss) 170 | tf.add_to_collection('optimizer', self.optimizer) 171 | 172 | if self.useAttention: 173 | tf.add_to_collection('alphas', self.alphas) 174 | 175 | 176 | else: 177 | self.saver = tf.train.import_meta_graph('./model/pretrained_lstm_tf.model-0.meta') 178 | self.saver.restore(self.session, tf.train.latest_checkpoint('./model')) 179 | 180 | graph = tf.get_default_graph() 181 | 182 | self.weight = graph.get_tensor_by_name("weight:0") 183 | self.bias = graph.get_tensor_by_name("bias:0") 184 | # self.w_att = graph.get_tensor_by_name("w_att:0") 185 | # self.b_att = graph.get_tensor_by_name("b_att:0") 186 | # self.u_att = graph.get_tensor_by_name("u_att:0") 187 | 188 | self.labels = graph.get_tensor_by_name('labels:0') 189 | self.input_data = graph.get_tensor_by_name('input_data:0') 190 | self.data = graph.get_tensor_by_name('data:0') 191 | 192 | #self.lstmCell = tf.get_collection('lstmCell') 193 | self.output = tf.get_collection('output')[0] 194 | if self.useAttention: 195 | self.w_att = tf.Variable(tf.random_normal([self.lstmUnits, self.attentionSize], stddev=0.1), \ 196 | name="w_att") 197 | self.b_att = tf.Variable(tf.random_normal([self.attentionSize], stddev=0.1), \ 198 | name="b_att") 199 | self.u_att = tf.Variable(tf.random_normal([self.attentionSize], stddev=0.1), \ 200 | name="u_att") 201 | self.alphas = tf.get_collection('alphas')[0] 202 | 203 | self.last = tf.get_collection('last')[0] 204 | self.prediction = tf.get_collection('prediction')[0] 205 | self.correctPred = tf.get_collection('correctPred')[0] 206 | self.accuracy = tf.get_collection('accuracy')[0] 207 | self.loss = tf.get_collection('loss')[0] 208 | self.optimizer = tf.get_collection('optimizer')[0] 209 | 210 | 211 | # self.weight = tf.get_variable("softmax_w", [self.lstmUnits, self.numClasses]) 212 | # self.bias = tf.get_variable("softmax_b", [self.numClasses]) 213 | 214 | 215 | # initialize tensor board for monitoring 216 | def initialize_tboard(self): 217 | tf.summary.scalar('Loss', self.loss) 218 | tf.summary.scalar('Accuracy', self.accuracy) 219 | 220 | self.merged = tf.summary.merge_all() 221 | self.logdir = "tensorboard/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + "/" 222 | self.writer = tf.summary.FileWriter(self.logdir, self.session.graph) 223 | if not self.restore: 224 | self.saver = tf.train.Saver() 225 | 226 | def train_single_epoch(self, epoch_num = 0): 227 | i = 0 228 | while True: 229 | # Next Batch of reviews 230 | nextBatch, nextBatchLabels = self.myBatchGenerator.nextTrainBatch() 231 | if len(nextBatch) * (i+1) > len(self.X_train): break 232 | 233 | self.session.run(self.optimizer, {self.input_data: nextBatch, self.labels: nextBatchLabels}) 234 | 235 | # Write summary to Tensorboard 236 | if (i % 50 == 0): 237 | summary, acc, cost = self.session.run([ self.merged, self.accuracy, self.loss], {self.input_data: nextBatch, self.labels: nextBatchLabels}) 238 | print "Iter " + str(i) + ", Minibatch Loss= " + "{:.6f}".format(cost) + \ 239 | ", Training Accuracy= " + "{:.5f}".format(acc) 240 | 241 | i += 1 242 | 243 | def train_epochs(self, n_epochs): 244 | if not self.restore: 245 | tf.global_variables_initializer().run(session=self.session) 246 | 247 | num = 0 248 | while num < n_epochs: 249 | print("Epoch " + str(num) + ":\n") 250 | self.train_single_epoch(num) 251 | self.test() 252 | 253 | if num % 5 == 0: 254 | self.save_model(num) 255 | print("saved to %s" % self.lstm_model_path) 256 | 257 | num += 1 258 | 259 | self.writer.flush() 260 | self.writer.close() 261 | print('training finished.') 262 | 263 | def test(self): 264 | i = correct = total = 0 265 | 266 | while True: 267 | nextBatch, nextBatchLabels = self.myBatchGenerator.nextTestBatch() 268 | if len(nextBatch) * (i + 1) > len(self.X_test): break 269 | 270 | acc = self.session.run(self.accuracy, {self.input_data: nextBatch, self.labels: nextBatchLabels}) 271 | correct += acc 272 | total += len(nextBatch) 273 | 274 | i += 1 275 | 276 | total_accuracy = correct/(i-1) 277 | print("Testing accuracy = " + "{:.5f}".format(total_accuracy)) 278 | 279 | # Shows attention for each word 280 | def plot_attention(self, ind = 0 ): 281 | nextBatch, nextBatchLabels = self.myBatchGenerator.nextTestBatch() 282 | 283 | my_sentence = nextBatch[ind] 284 | my_alphas = self.session.run(self.alphas, {self.input_data: nextBatch, self.labels:nextBatchLabels}) 285 | my_words = features_word2vec.indices_to_words(my_sentence, self.word2vecmodel, maxLength=self.maxSeqLength) 286 | my_indices = xrange(self.maxSeqLength) 287 | my_dummies = [""] * self.maxSeqLength 288 | 289 | # find the first none-zero element 290 | for j in xrange(len(my_sentence)): 291 | if my_sentence[j] != 0: break 292 | 293 | words_with_index = ["{:03}_{}".format(i, w) for i, w in zip(my_indices, my_words)] 294 | words_for_annot = np.array([[w] for w in my_words ]) 295 | 296 | # get the attention of 35 words 297 | # customizable at this point. 298 | wordmap = pd.DataFrame({"words": words_with_index[j:j+35], "alphas": my_alphas[ind][j:j+35], "dummy": my_dummies[j:j+35]}) 299 | mymap = wordmap.pivot("words", "dummy", "alphas") 300 | 301 | 302 | 303 | print(words_for_annot.shape) 304 | my_plot = sns.heatmap(mymap, annot=words_for_annot[j:j+35], yticklabels=my_dummies, fmt="") 305 | 306 | # This sets the yticks "upright" with 0, as opposed to sideways with 90. 307 | plt.yticks(rotation=0) 308 | fig = my_plot.get_figure() 309 | fig.savefig(self.attention_map_path) 310 | 311 | def save_model(self, step_num ): 312 | self.saver.save(self.session, 313 | self.lstm_model_path, global_step=step_num) 314 | 315 | -------------------------------------------------------------------------------- /main_lstm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [ 10 | { 11 | "name": "stdout", 12 | "output_type": "stream", 13 | "text": [ 14 | "Couldn't import dot_parser, loading of dot files will not be possible.\n" 15 | ] 16 | } 17 | ], 18 | "source": [ 19 | "# The first steps:\n", 20 | "# Will ingest/clean data and create/load the following:\n", 21 | "# 1. cleaned text translated to array of word indices: imdb_indices.pickle\n", 22 | "# 2. word2vec model, where the indices/word vecs are stored: 300features_40minwords_10context\n", 23 | "# 3. word embeddings: this is the index to wordvec mapping derived from 2.\n", 24 | "\n", 25 | "\n", 26 | "from lib import features_word2vec, model_lstm_tf\n", 27 | "import pandas as pd\n", 28 | "import os\n", 29 | "\n", 30 | "labeled_data_path = \"./data/labeledTrainData.tsv\"\n", 31 | "unlabeled_data_path = \"./data/unlabeledTrainData.tsv\"\n", 32 | "model_path = \"./model/300features_40minwords_10context\"\n", 33 | "embedding_path = \"./model/embedding_weights.pkl\"\n", 34 | "text2indices_path = \"./model/imdb_indices.pickle\"\n", 35 | "\n", 36 | "\n", 37 | "def data_prep():\n", 38 | " # Read data\n", 39 | " # Use the kaggle Bag of words vs Bag of popcorn data:\n", 40 | " # https://www.kaggle.com/c/word2vec-nlp-tutorial/data\n", 41 | "\n", 42 | " data = pd.read_csv(labeled_data_path, header=0,\n", 43 | " delimiter=\"\\t\", quoting=3, encoding=\"utf-8\")\n", 44 | "\n", 45 | " data2 = pd.read_csv(unlabeled_data_path, header=0,\n", 46 | " delimiter=\"\\t\", quoting=3, encoding=\"utf-8\")\n", 47 | "\n", 48 | " # data2 and data are combined to train word2vec model\n", 49 | " data2 = data.append(data2)\n", 50 | "\n", 51 | " model = features_word2vec.get_word2vec_model(data2, \"review\", num_features=300, downsampling=1e-3, model_path=model_path)\n", 52 | " embedding_weights = features_word2vec.create_embedding_weights(model)\n", 53 | " features = features_word2vec.get_indices_word2vec(data, \"review\", model, maxLength=500,\n", 54 | " writeIndexFileName=\"./model/imdb_indices.pickle\", padLeft=True)\n", 55 | " return model, embedding_weights, features" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 3, 61 | "metadata": { 62 | "collapsed": false 63 | }, 64 | "outputs": [ 65 | { 66 | "data": { 67 | "text/plain": [ 68 | "(,\n", 69 | " array([[ 0.10585845, -0.05332218, 0.02788348, ..., -0.05418135,\n", 70 | " 0.06534412, -0.0199977 ],\n", 71 | " [-0.01134798, 0.00097455, -0.01262519, ..., 0.00096706,\n", 72 | " -0.01948956, -0.00314009],\n", 73 | " [-0.0214931 , -0.1459033 , 0.07035182, ..., -0.04496352,\n", 74 | " 0.04403584, -0.06980642],\n", 75 | " ..., \n", 76 | " [ 0.06123403, -0.04089059, -0.01340766, ..., -0.03233384,\n", 77 | " -0.10336776, -0.0290222 ],\n", 78 | " [ 0. , 0. , 0. , ..., 0. ,\n", 79 | " 0. , 0. ],\n", 80 | " [ 0. , 0. , 0. , ..., 0. ,\n", 81 | " 0. , 0. ]]),\n", 82 | " array([[ 0, 0, 0, ..., 22, 0, 1645],\n", 83 | " [ 0, 0, 0, ..., 27, 91, 5757],\n", 84 | " [ 0, 0, 0, ..., 1352, 3, 5403],\n", 85 | " ..., \n", 86 | " [ 0, 0, 0, ..., 231, 27, 333],\n", 87 | " [ 0, 0, 0, ..., 14, 81, 76],\n", 88 | " [ 0, 0, 0, ..., 13, 2, 487]], dtype=int32))" 89 | ] 90 | }, 91 | "execution_count": 3, 92 | "metadata": {}, 93 | "output_type": "execute_result" 94 | } 95 | ], 96 | "source": [ 97 | "data_prep()" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 5, 103 | "metadata": { 104 | "collapsed": false 105 | }, 106 | "outputs": [ 107 | { 108 | "name": "stdout", 109 | "output_type": "stream", 110 | "text": [ 111 | "(128, 64)\n", 112 | "Epoch 0:\n", 113 | "\n", 114 | "Iter 0, Minibatch Loss= 0.690229, Training Accuracy= 0.57031\n", 115 | "Iter 50, Minibatch Loss= 0.592788, Training Accuracy= 0.64844\n", 116 | "Iter 100, Minibatch Loss= 0.462448, Training Accuracy= 0.81250\n", 117 | "Iter 150, Minibatch Loss= 0.456239, Training Accuracy= 0.79688\n", 118 | "Testing accuracy = 0.52995\n", 119 | "saved to ./model/pretrained_lstm_tf.model\n", 120 | "Epoch 1:\n", 121 | "\n", 122 | "Iter 0, Minibatch Loss= 1.430585, Training Accuracy= 0.56250\n", 123 | "Iter 50, Minibatch Loss= 0.674530, Training Accuracy= 0.57812\n", 124 | "Iter 100, Minibatch Loss= 0.638553, Training Accuracy= 0.67969\n", 125 | "Iter 150, Minibatch Loss= 0.520375, Training Accuracy= 0.75000\n", 126 | "Testing accuracy = 0.82378\n", 127 | "Epoch 2:\n", 128 | "\n", 129 | "Iter 0, Minibatch Loss= 0.399270, Training Accuracy= 0.84375\n", 130 | "Iter 50, Minibatch Loss= 0.443942, Training Accuracy= 0.79688\n", 131 | "Iter 100, Minibatch Loss= 0.360969, Training Accuracy= 0.83594\n", 132 | "Iter 150, Minibatch Loss= 0.389484, Training Accuracy= 0.85938\n", 133 | "Testing accuracy = 0.89844\n", 134 | "Epoch 3:\n", 135 | "\n", 136 | "Iter 0, Minibatch Loss= 0.349237, Training Accuracy= 0.86719\n", 137 | "Iter 50, Minibatch Loss= 0.375731, Training Accuracy= 0.86719\n", 138 | "Iter 100, Minibatch Loss= 0.333954, Training Accuracy= 0.84375\n", 139 | "Iter 150, Minibatch Loss= 0.390951, Training Accuracy= 0.85156\n", 140 | "Testing accuracy = 0.90234\n", 141 | "Epoch 4:\n", 142 | "\n", 143 | "Iter 0, Minibatch Loss= 0.329136, Training Accuracy= 0.85938\n", 144 | "Iter 50, Minibatch Loss= 0.381535, Training Accuracy= 0.84375\n", 145 | "Iter 100, Minibatch Loss= 0.333050, Training Accuracy= 0.87500\n", 146 | "Iter 150, Minibatch Loss= 0.323877, Training Accuracy= 0.87500\n", 147 | "Testing accuracy = 0.89800\n", 148 | "training finished.\n", 149 | "Testing accuracy = 0.89280\n" 150 | ] 151 | } 152 | ], 153 | "source": [ 154 | "# Use no attention first \n", 155 | "# Train with five epochs \n", 156 | "# For each epoch, print out the accuracy on the hold-out set\n", 157 | "model1 = model_lstm_tf.LstmTFModel(useAttention=False, restore=False) \n", 158 | "model1.train_epochs(5)\n", 159 | "model1.test()" 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": 6, 165 | "metadata": { 166 | "collapsed": false 167 | }, 168 | "outputs": [ 169 | { 170 | "name": "stdout", 171 | "output_type": "stream", 172 | "text": [ 173 | "(128, 64)\n", 174 | "Epoch 0:\n", 175 | "\n", 176 | "Iter 0, Minibatch Loss= 0.690287, Training Accuracy= 0.53125\n", 177 | "Iter 50, Minibatch Loss= 0.518388, Training Accuracy= 0.75000\n", 178 | "Iter 100, Minibatch Loss= 0.447742, Training Accuracy= 0.81250\n", 179 | "Iter 150, Minibatch Loss= 0.456062, Training Accuracy= 0.77344\n", 180 | "Testing accuracy = 0.87066\n", 181 | "saved to ./model/pretrained_lstm_tf.model\n", 182 | "Epoch 1:\n", 183 | "\n", 184 | "Iter 0, Minibatch Loss= 0.376639, Training Accuracy= 0.83594\n", 185 | "Iter 50, Minibatch Loss= 0.356565, Training Accuracy= 0.85938\n", 186 | "Iter 100, Minibatch Loss= 0.359166, Training Accuracy= 0.83594\n", 187 | "Iter 150, Minibatch Loss= 0.248117, Training Accuracy= 0.92969\n", 188 | "Testing accuracy = 0.91884\n", 189 | "Epoch 2:\n", 190 | "\n", 191 | "Iter 0, Minibatch Loss= 0.266340, Training Accuracy= 0.89062\n", 192 | "Iter 50, Minibatch Loss= 0.342623, Training Accuracy= 0.85156\n", 193 | "Iter 100, Minibatch Loss= 0.279518, Training Accuracy= 0.86719\n", 194 | "Iter 150, Minibatch Loss= 0.328646, Training Accuracy= 0.86719\n", 195 | "Testing accuracy = 0.90061\n", 196 | "Epoch 3:\n", 197 | "\n", 198 | "Iter 0, Minibatch Loss= 0.321027, Training Accuracy= 0.87500\n", 199 | "Iter 50, Minibatch Loss= 0.343837, Training Accuracy= 0.84375\n", 200 | "Iter 100, Minibatch Loss= 0.214664, Training Accuracy= 0.91406\n", 201 | "Iter 150, Minibatch Loss= 0.256317, Training Accuracy= 0.90625\n", 202 | "Testing accuracy = 0.93663\n", 203 | "Epoch 4:\n", 204 | "\n", 205 | "Iter 0, Minibatch Loss= 0.385247, Training Accuracy= 0.85156\n", 206 | "Iter 50, Minibatch Loss= 0.331194, Training Accuracy= 0.87500\n", 207 | "Iter 100, Minibatch Loss= 0.209367, Training Accuracy= 0.92969\n", 208 | "Iter 150, Minibatch Loss= 0.245976, Training Accuracy= 0.89062\n", 209 | "Testing accuracy = 0.93403\n", 210 | "training finished.\n", 211 | "Testing accuracy = 0.93273\n" 212 | ] 213 | } 214 | ], 215 | "source": [ 216 | "# Now run model with attention \n", 217 | "model2 = model_lstm_tf.LstmTFModel(useAttention=True, restore=False ) \n", 218 | "model2.train_epochs(5)\n", 219 | "model2.test()" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": 7, 225 | "metadata": { 226 | "collapsed": false 227 | }, 228 | "outputs": [ 229 | { 230 | "name": "stdout", 231 | "output_type": "stream", 232 | "text": [ 233 | "INFO:tensorflow:Restoring parameters from ./model/pretrained_lstm_tf.model-0\n" 234 | ] 235 | }, 236 | { 237 | "name": "stderr", 238 | "output_type": "stream", 239 | "text": [ 240 | "INFO:tensorflow:Restoring parameters from ./model/pretrained_lstm_tf.model-0\n" 241 | ] 242 | }, 243 | { 244 | "name": "stdout", 245 | "output_type": "stream", 246 | "text": [ 247 | "Epoch 0:\n", 248 | "\n", 249 | "Iter 0, Minibatch Loss= 0.348454, Training Accuracy= 0.85938\n", 250 | "Iter 50, Minibatch Loss= 0.386253, Training Accuracy= 0.82031\n", 251 | "Iter 100, Minibatch Loss= 0.348698, Training Accuracy= 0.87500\n", 252 | "Iter 150, Minibatch Loss= 0.391788, Training Accuracy= 0.78906\n", 253 | "Testing accuracy = 0.91363\n", 254 | "saved to ./model/pretrained_lstm_tf.model\n", 255 | "training finished.\n", 256 | "Testing accuracy = 0.91536\n" 257 | ] 258 | } 259 | ], 260 | "source": [ 261 | "# Now restore checkpoint from model2 \n", 262 | "# And continue to train for a single epoch \n", 263 | "model3 = model_lstm_tf.LstmTFModel(useAttention=True, restore=True ) \n", 264 | "model3.train_epoch(1)\n", 265 | "model3.test()" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": 25, 271 | "metadata": { 272 | "collapsed": false 273 | }, 274 | "outputs": [ 275 | { 276 | "name": "stdout", 277 | "output_type": "stream", 278 | "text": [ 279 | "(500, 1)\n" 280 | ] 281 | }, 282 | { 283 | "data": { 284 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAEECAYAAADu5BX3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8U1UbwPFf0j1pKRQom1IuLXvvPUWWDAVFXICAokwB\nFQEREFBAQZCp4gAVZQoivCBLliCzcNkt0JbuvZu8f6RgUUaQtjcJz5dPPm1yc3OfU5Knp8899xyd\n0WhECCGEZdNrHYAQQogHk2QthBBWQJK1EEJYAUnWQghhBSRZCyGEFZBkLYQQVkCStbAYiqJ0UhRl\nsNZxCGGJdFqMs85MjJHB3UIIszh6+uge9TVqlm9lds45GbL7kY9XEKRnLSzG+k2/MH/hYq3DEDZI\np9OZfbNU9loHIMQtlvxBEdZNp7P+fqn1t0AIIR4D0rMWQtg8OxvoWUuyFhZFSiGiIOglWQvxYOs3\n/cLVkFBGvj7svs/r0bVLIUUkHje20Amw/l83wuLZwgdFCK1Jz1oUipOnz/DqiJHExcXzdO+n8PT0\n4Pu1P5OdnQM6+GT2hyz74iuUKgF0f/IJoqNjeG30OL5ftZL5Cxfz14mT5OTkMPC5fnRs11br5ggr\no8P6OwzSsxYFzmg0Ym9vx5IF85k/ZybfrP6B0GvX+Wzex3y1bDH+FSuy/+AhevfszsbNWwDYtPVX\nnur2JHv3HyAsPJyvli1mxeIFLFv5FUnJyRq3SFgbvU5v9s1SSc9aFDidTkegogDgU7Qo6RnpFPX2\n5p2p03B1ceHK1RBq16xBpYoVyM7JITwigm3b/8fyRQv44ed1BJ9TeXno6wBk5+QQFh6BElBZwxYJ\na2MLpThJ1qJQ5P2wJCYl89nS5ezYvB6DwcCrr4/EYDAA0KtHNz7+5DP8K1XC3d2NShUq0KBeXSa/\nPZ7s7GyWfbGKsqX9tGqGsFJ6SdZCPNix4ye4cSPs9n0PdzdqVq/GgJeHUNTbmwrlyxEdHQNAh3Zt\n+PDj+SycOxuA1i2bc+ToMV4YMoy01DTatWmFq6urJu0Q1ktnAxVfmchJFLgNm7cQHx/PCwOe1ToU\nYYXyYyKnllV7mJ1z9pzbYJHdcOlZi0Kx/+Ah9u4/QGpaGsMGv4KzszMLP1+KXq+nbJnSvDfxLYzA\ntJmzCb12HaPRyOtDh9CgXh169X+eBvXqcP7CJXQ6HZ9+NAt3dzetmySsiJRBhDCD0WikqLc3H06b\nQkxsLM+9NBgHewdWrfgcby8vFn6+jA2bt5CdnY23lxfvT3qb+PgEXnp1OOu+/5bU1FS6dOrIxLHV\nmTBpCvv+OEDnju21bpawIrYwdE+StShwOp2OenVqA6bRII6OjoSHRzBmwrsAZGRk0LhhAxISEzl2\n/ASnzgQDkGMwEB+fAECgUgWAkiVKkJGZqUErhNCWJGtR4IxGIydPn6Fvr57cjIzCYDBQ2s/vdjnj\nf7/vxtPDk/MXL1LS15dBLw0kOTmFr75dTZEinoBtDL0S2rHk8dPmkmQtCpxOpyM+IYFBw0aQlJJC\n5w7tqFunNq+NGovBYMDd3Z0ZUydRu2Z1uvd9li2/bcfezo5+fXuj0+nIyTEw46N5TJ741u3XE+Jh\n2OmtP1lrMhokIz5SRoM8po4c/Ysf121g9gdT7rp90vsz6NyxHc0aNyrUuITlcvLyfeTfzp2qP212\nztl2+geL7A1Iz1o8kn4vDGLxJx/h7u5Oyw5P8sWShVStEsDTA1+mWeNGBJ9ViU9IQAmozPuTJrLs\ny1Wcv3CJnzZson6d2kyZMYvs7BycnZ1uJ/C16zbyxdffkZycwrvjx+Dt5cX4SVP5ZsXn9H7uBRrU\nrcP5i5fQoeOTOTNxc3Nl+py5BJ9TKVbUhxvhYSz4aBZ+pUpq+rMRIj9JshaPpE3L5uw/cAjf4sUp\nU9qPA4eO4OjgQOlSfhTx9GTJgrkYDAZ69R9IZFQ0Q14ayI/rNtK7RzfeGDuRwS8OpGnjhvy+dz/n\n1AsABAUqDH5xIBt+2cqGzVt56fm/x2enpKTxRMcOTBgzkomT32ffgYM4OTqSmJDIdyuXEhcfT9c+\n/aVUIu4gNWvx2GvXphVLV36FX6mSjBg6mO9+WIvRaOSJju04dSaY8ZOm4uriQmpaGjk5ORiNphOO\nACHXrlGrRnUAWrdoBsCWbdsJqppnHpH09H8dM1AJAKCkry8ZmZmEhUdQM/d1vL28qFi+fIG3W1gX\nW/jlbf2/boSmKleqyPWwME4Hn6VFsyakpKaxa88+HBwcuHkzklnTJjNi2GAyMjIwGo3o9frbybpS\nhfKcCjYN09uybQdrfvzZrGP+84NXuVJFTp46DUBiYhIhodfysYXCFuh1OrNvlkp61uKRNaxXl7Cw\ncHQ6HfXr1ubK1RBqVgti2RereGX4mxTzKUqNakFERUdTtkxpLly8xLffr2X0iOG8/+Eclq1chYuL\nMzOmTOLM2XO3L2DQ6f5OzPfqGel0Olo2b8q+A4cYOHgYPkV9cHZ2xt7ertDaLyyfLVwUI6NBRKHK\nzMxk09ZtREZFU6xoUfr26nHH9lHj32XerA8e6jWvhISinr9A5w7tiE9IoFf/gWzb+BMO9tIXsQX5\nMRqke+0BZuecjce/scjMrsm72Zhj0OKwwgJERkbx84bNNG/cCKPR+K/3wtwZ7z/0+6NEsWLMX7CY\nb1b/QI7BwMjhQ7HX6eV9Jm6z5PKGuaTrIQrV8i+/5srVEM6cPUfTRg3YvvN34hMSeG3IK7Rq1pS2\nXZ9i5+Z1fP/TOjb9+ht6nY5qgVUZP+qNe76mi7Mz82dNL8RWCGtjC2UQSdaiUA1+cSAXL1+hWeNG\nREZF8d6Ecfz513G+/GY1rZo15VYHaMOWX3l33GiCqir8sG4DOTk52NlJHVr8NzJ0T4iHZOTv0mHV\n3MmZfLy9Scu4c4je++9M4Kvv1piG5VWvhhbnVoSwJNb/60ZYFTu9HYbcxHu/sa8/b9zMpLfGsOKz\nTzh3/gInT58prBCFDdLpdGbfLJX0rEWhunw1hJDQazSsV/eOKuLfw/VMXyv7V+LFYSNwc3WlhG9x\nalQL0iBaYSvsbKAMosnQvfToMPmb9jF15Nhx1m7YxKypk7QORVgJ52J+j9zd7d9gkNk5Z/WR5RbZ\nvZaetShQV0OvMXnGLOzt7TEYjPTu0ZXQa9d5bcwEYuPiaNW8KUNffoGz5y8wa94C7Oz0ODo6Mnn8\nWL7+/kfq1KhO+zatGDb6LZo2bMDz/foy9cOP6Nn1CWpVr6Z184QoNJKsRYE69OdRalQLYuSwVzl2\n4iSXr14lIzOT+bM+ICcnh869nmHoyy/w/ocfMfXtt6hS2Z/f9+7nowWLeLZPLzZu3UaLpk1ISkrm\n8LG/eL5fX86eP8/k6mO1bpqwIvlVi1YURQ8sAmoCGcAgVVUv5dneDZgEZAMrVVVdfq99FEWpCiwH\njMD53Mfv+ReA9RdyhEV7qmsX3N3cGD7mLdb8tA47OzsqV6qIg709zk5Ot4fjRcXEUqWyPwB1a9Xk\n0pWr1KlVg7PnL3Dk2F+0b92S2Lg4jh0/Sc1q0qMWDycf5wbpCTiqqtoUmAB8fGuDoigOwFygA9AK\nGKIoim/uPk532WcK8IGqqi0AJ+DJ+7bhYRstxMPYtXc/dWvVZOknH9OhTSu++HbNXXs5vsV8uHDp\nMgB/Hj9B+XJl0el0VKtahS++XUOThvWpU7MG8xYtoX3rloXdDGHldA/x7wGaAb8CqKp6CKifZ1sg\ncFFV1QRVVbOAfUDL3H223mWfNMBHURQd4AHcd3FRKYOIAlWtqsK7H3zIsq++xmAw0L5VS/44fOT2\n9lsfjvfGj2Xm3E8wGsHe3o4puUt4tWvVkvdmzEIJqEx0bCybf91O/Tq1NGmLsF75eLm5J5CY536O\noih6VVUNudsS8mxLAorcax9gAfAb8C4QD+y+34E1SdaGzAwtDis04Ffch5WfzLnjsZFDXrr9Hvht\n7bcYMjOoUqEsy+fNvuN5hswMmtStxfa135m+r1OLnevWQHYWMuuHeBj5OH46EVMv+JZbiRpMiTrv\nNg9MSfiu+yiK8g3QQlXVs4qiDMdUHnn9XgeWnrUoUCHXrjN51lzs7U0Xw/Tu+gR7Dx7mw0kT6T7g\nZerUqMbVa9fx8fbmo6nvkpmVxeRZc4mIjCQrK5sJbw4nsEoA0+d+yrUb4RiNBoa//AL1a9fUumnC\niuRjz3o/0A34UVGUxsDJPNvOAQGKongDKZhKIHMwnUC82z6umHrfAOFA0/sdWJK1KFCHjv5FjaCq\nvPnqK/x18jSXr4bcLn2EhUewfN5sfIsX46URozlz7jwnzgRTplRJZr03kdAbYew7cBj14mW8vbyY\n8tZo4hMSGTRyHGu/WKJxy4Q1yceJnNYBHRRF2Z97/yVFUfoD7qqqLlMUZTSwDdP5wBWqqoYrivKv\nfXK/DgLWKoqSjmmUyOD7HViStShQPbt04ovVP/LaW+/g7uZGk/p1b88P4lWkCL7FiwFQwrc4GZmZ\nhFy/QbOGpvMv5Ur78WyfnsyYt5Djp09z+uw5AAwGAwmJSRTx9Lj7QYX4h/zqWecOrRv2j4fP59m+\nGdhsxj6oqroD2GHusSVZiwL1+/6D1K1ZnVdfeI6t/9vFZyu+okZgVQDu9vmpWK4sZ9TztG7WhOth\n4Xz+5ddUq6pQwrcYrzzXj+SUFL7+4Wc8PdwLuSVCaEuStShQQUoA7334Ecu/diDHkEP/Xj04fVY1\nbfxHttbpdPTp1oUps+cxaOQ4DAYD414fSuWKFXj/o08YNHIcKampPN2jm0VPuCMsjy28XzSZGyQ1\n7IrMDSL+5Y/DfxIRGUWvrk9oHYqwIK5+FR850w5t8YbZOefzvZ9aZGbXpmctk8iLu2japJHWIQgb\nZQuLD1h/C4TN2Lh1G58uWaF1GEJYJKlZCwtikX99Chugt4G3liRrIYTNs4UTjJKshRA2Lx+vYNSM\nJGthUWzgMyUskPSs/6Ps5BQtDissXJcWzQB5f4h/KKF1AJZBetaiUAx8YwyfTpuMu5sr7Z95niWz\np6P4V2LAiNE0qVeHsxcukpCUREDFirw3agQnzpxl/vIvcLC3x8nZiVlvv4Wri4vWzRBWyhYWzJVk\nLQpFy8YN+ePoMXx9fPArVYLDf53A0cEBvxK+eHp4sHD6VAwGA/2GvUFUTCy7Dx6iQ6vm9O/RjT0H\nD5OYnCzJWvxnUrMWwkxtmjVh5ZofKelbnOEDB/D9xs0YjEY6tW7JafU87876GBcXZ1LT08nJyeGl\nZ/qw8vu1DJ/4HsV9ilJNqaJ1E4QVs4FcLRfFiMLhX74cN8IjCD5/gWYN6pGSlsbug4ewt7fnZlQ0\nH4wfw/AXBpCRkYnBaGDrrt10bd+WxR9Oo2K5sqz79TetmyCEpqRnLQpNvVo1CL8ZiU6no16N6ly9\ndp3qShVWrvmRYRMm4ePtRXWlClExcVSrEsD0Tz7D2dkJO72et0cM1zp8YcVsoQyiyUROySHnZSIn\nYbaRk95n/OtDKVXCV+tQhAbcy1d55Ew7seMEs3POzN8+tMjMLj1rYR0s8uMjrIWMsxbiISSnpPLB\n/AUkJacQFRNL325d2L57L4p/JS5dDSU5NZVZk8ZTyteXz1d9y75Df1KsqDc3o6K1Dl1YOTsbmBxE\nkrUoNNfDw+nYqiVtmzchKiaWwWMm4Fu8GNWrKowZNphFX3zNtl17aFS3Nn8eP8U3n80jIzOTp4fc\nc8FnIR4bkqxFoSnq5cV3P29k1/4/cHN1JScnBwClciUAShQvRkxcPCHXbxAY4A+Ak6Mj1aoEgJzl\nEI/AFk4wytA9UWi++Wk9NYOqMm38GNq1aHbP/FupXDlOq+cxGAxkZWVx7uIlqVmLR6J7iH+WSnrW\notC0bNyQ2Z8tYdf+A/iXL4erszPZWdm3T/5cCgkl/GYkQ57vT6smjRg4YgxFvYrgVcRT48iFtbOF\nnrU2Q/dCL8oftUIIs7iXq/zImXZa10lm55xJm6dZZGaXMoiwGBu3bWfBii+1DkPYIJ1OZ/bNUkkZ\nRFgMS/6gCOtmC2UQSdZCCJtnA7lakrUQwvbZQs9aatbColjy0CkhtKRJz9qQmaHFYYWFe7J1C0De\nHyL/2UInQMogQhNvTZ9Nvx5PUrd6NYIvXGTZt9/j4+1FaFg4RqORoc8/S70a1bQOU9gIWzh5LWUQ\noYmendrzy47fAdi0fSdN6tWhiKcnS2d9wJx3JzB78VJtAxQ2xU6vM/tmqaRnLTTRuG5tPl25isSk\nZE4En8VoNHIi+Bxn1AsAGHIMJCQlUcTDQ+NIhbAMkqyFJvR6Pe2aN2XmZ5/Tukkjinh4UKJ4MV56\nujfJqal8+/NGPN3dtQ5T2AgpgwjxCLp1aMvuA4fp3qEdvZ7oSMj1G7w6YRJDJ0yiZPFiNvEBE5ZB\nrzP/ZqmkZy0K1KbtOwm5EcbrLw7417aSxYvxx4Yfbt+fMvoNs15z9uJltGveVE5ACrPZwi9+TZK1\n0WDQ4rBCK0Zj/v+fGwzyPhJms4FcLT1rUTi+XbeR7Xv/wM5OT51qQbz+4gBeGDWeDyeOpZRvcf63\n/wAngs8x5Lln+OCTRSQkJwMwdsjL+Jcvx09bt7H+1x14exUhLT2ddk0ba9wiIQqX1KxFgQu9Ecb/\n9h9gxZzprJgzg2th4ew7cpTuHduyZefvAGzesYuendrzxQ8/0aB2DRZPn8LE117lw0VLiUtIYM2G\nX/ji45nMn/y26U9aW+gqiUJjp9ObfbNU0rMWBcto5MLVEJo3qIednR0AtasFcjn0Gr2e6MiQ8ZPo\n0bE9KWlpVCpXlktXQzl68gw79v4BQFJyCtfCI6hQtgz29qa3a63AqqDBPOzCeuXX73ZFUfTAIqAm\nkAEMUlX1Up7t3YBJQDawUlXV5ffaR1EUX2AZ4IVpLaSBqqpevdexJVmLgqXTUaViBU6rF8jJyUGv\n1/PXmWCebNsad1dXAitXYu6yL+jevi0AFcqWIbCyP51aNScyJoZtu/dRrlQpLodeIz0jAydHR86c\nv0CTenU0bZawLvk4kVNPwFFV1aaKojQCPs59DEVRHIC5QH0gFdivKMpGoDngdJd9ZgNfq6q6VlGU\n1kB14Oq9DizJWhS4sn6lqBmoMHj8uxgMRmpXC6RV44YA9OjUnhGT3qdO9UDWb9vOwN49mfnZEtZv\n205KahqDn30aryKevPxMb4aMn4Snh/vtHrYQGmgG/AqgquohRVHq59kWCFxUVTUBQFGUfUBLoAmw\n9S77NAVOKIqyHVOSfvN+B5Z3vShQXdu1vv39sz27/Wt7zaoKu3/8FoAerwynS9vWzHnnrX8978m2\nrXmybet/PS6EOfJx6J4nkJjnfo6iKHpVVQ252xLybEsCitxjHzugAhCrqmoHRVEmAeOByfc6sOVW\n08VjY/OOXXR87mVi4+N5d848rcMRNujWOWlzbg+QCOSdA+FWogZTos67zQOIv8c+OUAMsDH3sU2Y\nyif3JMlaaE+no0fHdhT18mL6uFFaRyNsUD6uwbgf6AKgKEpj4GSebeeAAEVRvBVFccRUAvnjPvvs\nA57M/b4VcPp+B5YyiBDC5uXjZeTrgA6KouzPvf+Soij9AXdVVZcpijIa2IapI7xCVdVwRVH+tU/u\n1zHAckVRhmHqgT97vwNLshYWQ6/XkWMw4KB1IMLm5FfNWlVVIzDsHw+fz7N9M7DZjH1QVTUU6Gju\nsaUMIixG7aBARk2doXUYwgblY81aM9os65WRqcVhhYXq0rzpHffl/SHymy0smCtlEFEosrOzmbV0\nJdcjIjAYjLRt0og9R/7ksynvAjBmxhxe7d+X5NQ0lqz+Ab1eT5mSJRj/6iv8umcfm3fuxmg0MviZ\nPtSX2fbEQ5JZ94Qw04b/7cLL04N3hg8hISmJoZPex93NjYioaOzt7UhITqJKxQr0HTGaZdOn4OXp\nydI1P/LLrj3Y29vh6e7G7PFjtG6GEJqRZC0KxaXQa5w4q3LmgmkaBYPBQJvGDdiyey+ODvZ0bdOa\nuIREYuITePvjTwHIyMykYc3qlClVknJ+flqGL6ycDXSsJVmLwlGhdGlK+PjwQq8epKSm8t2mLfRo\n35YRU2dgp9fz6XsTcXJ0xNenKHMmjMHNxYXdh4/g4eZOeFQUektewkNYPFt4/0iyFoXiqQ7tmPn5\nMoa9N42UtDT6dO6Am4sLVSqWJ8dgwMXZGYBRLw1k9PTZGIxG3F1deO/1YYRHRWH9HzWhJVs4wagz\najDVZNypozK/5WPqwtUQ9h45yst9e2kdirAS3jXqPXKmXTNkntk5p9/SURaZ2TXpWesd5LKHx5US\nUBkloLLWYYjHjA10rKUMIgpWyI0wpn26CHt7OwwGI091as/+P4/xwdiR9B46glqBVQm5EUZRryLM\nmjCWzKwspn26iIioaLKysxn36isE+ldi5uKlXA+PwGA0Muy5ftStLsP3hPlk6J4QD3DkxCmqKwG8\n/sIAjp85y5Vr17lVgA67Gcni6VPw9fFh8IR3Cb5wiZPnVEqXLMH0caO4Fh7O/j+Pcf7yVbw9PZk0\nYjjxiUkMfWcyaxbM1bZhwqrYQK6WZC0KVvcObVn103renDIddzdXGtWuBbnVwyKeHvj6+ABQolgx\nMrMyCQ0Lo2ld0yowZUuVol+3J5n1+TJOBJ/jzPmLgGnYX0JSEkU8PO56TCH+6bHoWecuQ9McWIhp\nztW6wFBVVdcWcGzCBuw+dITaQYEM6teXbXv2sfib1VRXAoC7f4AqlilD8MVLtGzUgBsRN1m6+nuC\nAgIoUawYL/Z5iuTUVL5bvwlPd/fCboqwYjaQq83qWX8KvAX0BtIwJeufAUnW4oGCKvsz5ZOFrPzx\nJwwGA8907cKZCxcA0P1jQN6ZC5dwcXbiRsRNhr4zmRyDgTGDXsK/XFmmf7aEoe9MJiU1jT5dOtlE\nT0kUnsdi6J6iKEdUVW2gKMq3wDZVVVcpivKXqqr/ecXSxAunZeieEMIsngHVHznTrh/xqdk5p+eC\nNywys5szRWqqoihjgXbAZkVR3sS0tpgQ+WrTjp0s/PIbrcMQNigfV4rRjDnJ+jnAFeilqmosUJIH\nrGggxH9hyR8UIbR2z5q1oiituH3ent2Ag6IoLTEtw14JuF7w4QkhxKOzhX7A/U4wjsOUrEsBVYCd\nQDbQGtOCj20LOjghhMgPNj2Rk6qqXQEURdkG1FRV9Wru/VKAFBZFgZBSiCgItvC+MmfoXrlbiTpX\nBFC6YMIRtmrTjp2EXA/j9RcH3PM5Xdu1KcSIhLAu5iTrw4qifAOswXRC8nlg16McNCdT1th73Bhz\ncjAacuT/XmjCBjrWZiXrwcAI4FVMNeztwOKCDErYrm83bOZ/+w9iZ6endlAgrz3fn5fGvcOMcSMp\n5VucnX8c4sTZcwzp35cPFi4hMTkZgNGvvIh/+bLaBi+s1uNSBtmsqmpH4OOCDkbYttCwCKJiglk2\n833s7PRMmD2P/X8eo1v71mz9fS8vP92LX3bt5vWBz/LF2vU0qFWdXp06EBoWzvTPlrBk+hStmyCs\nlA3karOStYuiKOVUVQ0t8GiEzTIa4eLVEJrXr4udnWl4f+1AhcvXrtOrU3tefWcq3du3ISUtjYpl\ny3Ap9BrHTgezY/9BAJKSU7QMX1g5W7jc3JxkXRy4qihKJKa5QQCMqqpWKriwhK3R6SCgQnlOn79I\nTo4BvV7HX8HneLJNS9xcXanqX5F5K1fRtW1rACqU9iOwVSU6tmhGZEwsv+3dr2n8wrrZQK42K1l3\nzv2a99p6G2i6KGxl/UpSrUpl+r42iqzsLCqUKU3LhvUB6NGhLaOmzSIzO5uubVvxYp+nmPHZEtZv\n30lKahqD+/XROHphzR6XmnUoMBTT3CD2mC6OWfAoB9XbyTTaj5tu7dsBpgUHfj94hCb16uDj5XX7\nvVArKIidq7+6/XxvLy/mvDNek1iF7bGBXG1Wsp4NVAZWYhq69xJQERhZgHEJG/XFjz9z5fp1gi9e\npHGd2vxv/wESkpJ49bl+tGhQj84vDOLXr5azdss2tuzajU6nIyjAnzGDX9Y6dGHFdLZ8BWMeHYE6\nqqrmACiKshk4XaBRCZv18tO9uRQaSpO6dYiKieHt14Zy7PQZvv55Ay0a1Lv95+rmnbsYP3QwgZX9\n+Wnrb+Tk5GBnZ6dx9MJaPS49a7vc5+Xk2Se7wCISNi3v/OlKJdM56qJeXqRn3HmxzKQ3XuPbdRsJ\ni4ykhlIFmQBdPO7MSdbfAr8rivIdphOL/YHVBRqVsFl2ej1Ggyn13u+kz4bfdjBh+BAcHRx4Y8oH\nnDp3njrVAgsrTGFjHpcTjM2AVUB9IB74QFXVXwo0KmGzrly/QWhYOPUzM+/801R364vpG//y5Rgy\ncRKuLi74+vhQvUrlwg9W2AxbmHXPnGW9GgNP5N4cgF8wXdV48L8eNO7MMfmr9jF19HQw63/bwbTR\nb2gdirAS3tXqPnKm3Ttlmdk5p8WUwRaZ2R/Ys85NygcVRVkI9AXewbSArmMBxyZsQGhYONMWfI69\nvR1Go5EeHdpyLTyCUR/MIi4hgeb16zLomT6ol68wd8VX6PV6nBwcmDh8CKs3baFm1Sq0bdKIke/P\npFGdWvTv1oUZi5bSrV1raihVtG6eEIXmgct6KYqySFGUE8A2TIsQDAN8CzowYRsOnzhF9SqVWTDl\nHQY/04eU1DQyM7OYM2EMn38whbVbfwNg5uJljB38EounvUevzh345Iuvad2oAQeOnSAjM5OklFT+\nPHUGAPXyVUnU4uHodObfLJQ5azAWyX2eCpwFzqmqGl+gUQmb0b19G9zdXBk57UN+3Pobdno9lcqV\nxd7eHmcnR+z0puF4MXHxBFQoD0DtoKpcvnadWoEK6uUrHD0dTJsmDYlLSOB48DlqKAFaNklYocdi\nwVxVVZ9TVbUG8D6m0scviqLcKPDIhE3Yc/hPagdWZeGUd2jbpCFfr990185LMW9vLoaY5gr768xZ\nyvuVQqfTEVi5Et+s30SjWjWpVVVh4arvaNO4YSG3Qlg7G+hYP7hmrShKVUyXmrcDagOHMJ1kFOKB\nAv0r8f6MHP6HAAAZqklEQVSCxTisXUeOwcDTXToRfPHS7e23PhwThw/m42VfYATs7ex4+7VXAWjd\nqAEfLFxCQMXyNIqvya+798kQPvHQbOEKRnNGg5zElJx/AQ7cupLxUchoENuyeeduEpKSeK5HV61D\nETYoP0aDHJix0uyc0+Ttl+95PEVR9MAioCaQAQxSVfVSnu3dgEmYLhxcqarqcjP2eRZ4XVXVpveL\ny5zRIDUf9JyHlZ2Umt8vKTRkzMjEkJEl/6/CYuVjLbon4KiqalNFURphWpSlJ4CiKA7AXEzXpKQC\n+xVF2Qg0B5zusU8dwKyJb2T6O5EvDp86zcGTJ0lNz+Dlnj1wdnJk2U/r0Ov1lPYtzrgXB5KTk8OM\n5Su5GRNLVnY2o55/joql/fhwxZekpKURHR9Pr3Zt6dm2Na/PnM1bLw2kXMmSrN/5O7EJCQzo2oVJ\nCxeTkp5ORkYmQ/o8RYPq1dh5+Ag/bNuOXq+nZkAAQ5/urfWPQ1iYfKxFNwN+BVBV9ZCiKPXzbAsE\nLqqqmgCgKMo+oCXQBNj6z30URfEBpmOaFG/Zgw4syVo8MqPRiLenJ+8NHUxcYiJDpk7H3t6Oxe9O\nxMvDg+U/r2fr3v2kpqfjV7w4U4cP5frNm/xx4iSODva0b9yIVvXrEh0Xx+sz59Czbes7J0zPvXMj\nMorE5BQ+HjuSuMQkQiNukpiczMp1G1kx9T2cHB2YtmQ5R84E06BakAY/CWGp8rFn7Qkk5rmfoyiK\nXlVVQ+62hDzbkjCNprvbPo7ACmA0kG7OgSVZi0em0+molTvu2dvTE0cHByJiYpi00LSuckZWFg2q\nBZGQlEzjmjUAKFOiBE937EBUbBw/bNvBnqNHcXVxIcfw71Mit+YSqVjaj+5tWjFl8VKyc3Lo06Ed\nNyKjiE9KYuzH8wBITU8nLDIKqhVGy8VjKBHwyHP/VqIGU6LOu80D0xQd/9oHqIVp6unFgDMQpCjK\nXFVVR9/rwJKsxSMzGo2cuXSJHm1aERUbR47BQKlixfhw5AjcXFzYc/QYHm5uXAy9xtkrV2hetzY3\nIqNYuW49Xp6eVK/sT8+2rTl29hwHTpwEwNHBgei4eMqVLMn5kBCKe3tz+fp1UtPTmT36TaLj4xn2\nwUyWTX4X36JFmT9+LHZ6PZv37CWwYkVtfyDC4uRjGWQ/0A34MXcqjpN5tp0DAhRF8QZSMJVA5mBa\nZeuOfVRVPQJUB1AUpTyw5n6JGiRZi3yg0+lITE7hzVlzSMvIZMLLL5CVnc24uZ9gNBhxc3Xh3SGv\nUM2/EjNXfMHrM2djMBh487n+pKSlMWP5StZu30Exby9cnJzIys6mT4f2zF31Db4+PhT39kKn01Gm\nRAlWrt/IrsNHMBiNDO71FF4eHvTr3JHXp88ix2j6JdGhcSOtfyTCwujs8i1brwM6KIpya1HQlxRF\n6Q+4q6q6TFGU0Ziu9tYDK1RVDVcU5V/7/DM8ePAswA8culcQog7tl6F74l/6jn6Lb2dNx9HBQetQ\nhAUp3qjZI2fao3NXmZ1z6o0eaJGDss253FyIArVl7z66vvYGMQkJTFm0ROtwhLBIkqyF5nTo6Nqq\nJT5FijD1taFahyNs0GNxubkQQlg7S56gyVySrIXF0Ol1GAyGBz9RiIdkA7layiDCctSqUoWxH8/X\nOgxhi2ygDqJJz9qY/chzQQkb0rlJ4zvuy/tD5DdbmHVPyiBCc6EREcxc+SX2dqalv94bMgjfokW1\nDkvYEAvuMJtNkrXQ3NHgs1SrVImhfXtz8vwFUtLStA5J2BhbOMEoNWuhuSdbNMfN1YWxcz/h5527\nbi/1JUR+sYGStSRrob19fx2nVkAA88eNpnW9uny79VetQxLC4kgZRGiuaoUKTF+xkq82/4LRaGRE\nv6e1DknYGkvuMptJkrUoVIdOn+ZmTCzdW7W8/Zifb3E+mzj+rs/PzMritwMH6dqyRWGFKGyQjAb5\nj3IysrQ4rLAA9QMUCDD/PRAZE8Om3Xt5olHjBz9ZiHuQZC3EQ9p64ACHzwRzMzaWRW+NA2DYrNlM\nHvQKUfHxLFr7Ew729jg5OjJ18CC+2forIeHhrNqylYFdntA4eiG0I8laFCodOnQ63R0lxFvDqvaf\nOEHb+vXp07YN+0+eJDk1leef6MyVsDBJ1OKR2EDJWkaDCG3knUbdaDSi0+l4rnNnouPjGT3/E3Yf\n+ws7O7sHz8guhBl0ep3ZN0slyVoUOncXF+KTkjAYDCSlphIRE4PRaGT74cN0btKYeaNGUqFUSTbv\n24+dTocWC2QI22L6a868m6WSMogodO6uLjRv3Zqhs2bhV6w4pX190el0BFaowJxvvsXZ0REnT0/e\n7NEdLw8PsrKzWbp+PUN69tQ6dGGtLDcHm02TZH3zdLgWhxUWIDY0hozkNCZPm8m5rzdivDUlalgG\nRXFicpcBfz85PJO48Gim9zAtWSfvm8dTqbZaR2AZpGctClzRwEp4lPfj0Mnj7Nx6luFd+t7e5ly0\nCH4t6qHT6bB3ceLarsOkRkRT7ZXenFnxE5V7dSA1KhYXHy/0jg5c3bKHrORUDVsjrJFeb/0VX0nW\nosAZjYBOR8mwZGb1foUqT3fOnYgBnH2KELb3KOmxCXhVqUDRIH9SI6L/3hcjqRHRhO09SsnGtfBW\nKhB5NFi7xgjrZP25WpK1KBzJ1yIAyE5NJycjCycvDzBCVnIaJRrWwJidg97RgZyMzH/tmxYVB0BW\ncir2rs6FGrewDZZ84tBcNvD7RlgD1xI+ANi7OKN3sCM7LR10ULpVfSIOnSR0xwHSY+Lv8aGS0SBC\nSM9aFAonLw/8e7ZD7+jAtV2HKdu2EQBx565Q4YkWZCWnknozBns3F9MOMlxP5CNb6FnrtBjDevzT\nb+ST+DjQ6fB/qh0Obq7Enr1M5J+ntY5IWKHabwx45Ex7ac06s3OOf7+nLDKza9KzHvjxMi0OKwpZ\nST9f5rWqx+q5KyniXYSvl/+gdUjCCp18Y8CDn/QAlnxlormkZi0KzKQZYyhXoQw161YjLdW0VFcF\n/3IsX2NawXztrysZP+UNVqyZz/I183Fzd6V+49os+mo285d+wNpfVzLoNdMHddPv3+Lh6Q7A0wN6\n8OKr/bRplLBONrBUjCRrUWA+eGculy+GEB0Zc9ftbu6ubN2wg1f6jSQyIormrRthNBop6efLqFcn\nMaDnsNtJecv67TzRvR0AXXq2Z8OPWwutHcL62UCulmQtCs7dTur887GzZy4AEBEWiZOTIwAXzl3G\naDSSnp5BRu5QvnU/bKFrr474B1QgNjqOuNiEAo5e2BKZG0QIM2RkZFLc1zR0L7B6wB3b7nqC+y4P\nRYRFkpSYzOARz/Pzml8KIkxhy6RmLcT9GY1Gtm3eRfM2jVm+Zj6B1QLuOSzv1sPGPNk6bzL/afVm\n6jaoyb7fDxVozML22ELPWpOhe/0bDJKhe+KhNWxbj7L+pflp2UatQxGFaPWR5Y+cQUPWbzY755Tv\n2dUiM7aUQUSheW3aIPZvPcTxP07hV6EUA97sS3JiCr6li6PX69jy3XYO7viTSZ+PY9mMVUSE3qR9\nr1YUKeqJvaM9gXUV5oz6VOtmCCtkC0P3JFmLQrNz3V469GnN8T9O0bp7My6cvoybhyuLJq/AycWJ\nmV9P4vSRs3eUPm59//2idVqFLWyALSRrqVmLQnP2mErpiqXwKOJOjUZBFCnqybm/zgOQkZbB9Svh\nlCjte8c+OhuY2lJYABsYuyefBFGo9m05yIvj+nPywBluXA2nap0qADi7OlGucmkiw6LIyszCu5gX\nABWrltMyXGEjbOEEo5RBRKHavXk/fYfO5q3+U4i6Ec3gdwcyeelbuLi5EHkjmqT4ZCpUKccrEwbg\n4ubMtUs3iLkZq3XYQmhOk2QdVKKUFocVFsDd251rZ0IomqGnaDFf9nz+6x3bg0qUwpht4KtRS+jy\nWnfO7D3F5WMX5T0jHo3ldpjNJj1rUWiqNgmi59g+/DBjNXo7PWNXT+Sr8Su4eSWCsasnEn8znuUj\nF2sdprBB+XWCUVEUPbAIqAlkAINUVb2UZ3s3YBKQDaxUVXX5vfZRFKU28CmQk/v4QFVVI+91bKlZ\ni0Jz7kAw+3/cg5uHK2WDyhMXEUelOv4UK1OcS8cukp2VrXWIwkbp9Hqzbw/QE3BUVbUpMAH4+NYG\nRVEcgLlAB6AVMERRFN/cfZzuss984HVVVdsAPwPj73dgSdaiUJ37I5jKDargX7cyu1btoGJtf6o0\nqkr4xTBLPhEvxC3NgF8BVFU9BNTPsy0QuKiqaoKqqlnAPqBl7j5b77JPP1VVT+Z+7wCk3e/AkqxF\noYoKjcS7pDd+VUpz8c/zOLo4UaVxVS7+eV7r0IQt0+vMv92fJ5CY535Obpnj1ra8M4wlAUXutY+q\nqhEAiqI0BV4D5t23CQ9spBD57OrJK6QmpAIQcuoKKfHJZKVn5pkyJM+VwbK8l8gH+Th0LxHwyHNf\nr6qqIff7hH9s8wDi77ePoijPAIuBLqqq3n0u4VxyglEUup1fbb/9/a5VOyhRsSQ129Xhy3GmFYQW\nDjJ1MDZ9IlctinySfyW2/UA34EdFURoDJ/NsOwcEKIriDaRgKoHMwdT7+Nc+iqIMAIYArVVVjXtg\nE7SYyGlI8xHSXRJCmGXpvgWPnGojft9pds4p2brtPY+nKIqOv0d2ALwE1APcVVVdpihKV+A9TFWL\nFaqqLr7HPpeASCCEv0snu1VVnXKvY0vPWhQYOzs9z43rh2/pYqDTsXHFL/R7sw/qXxcoU7k0GI18\nNmEZ5ZQytOzRnOVTvqRhh/q069ua7KxsIq9H8vXsNbz0zgAO/fYnpw8GU7J8CfoM78nC8Uu0bp54\nDKmqagSG/ePh83m2bwY2m7EPgM/DHFtq1qLANO/WlKT4ZD4a8SmL317Gs6OfxtnVmcM7jvLxiE+J\ni0qgeuOg2yVqVw9Xur38BB+/8SlzXptPalIaLXs0Y++mP2jyRCMAmj3ZmH2bD2jYKmGNdHZ6s2+W\nSnrWosCU9vejco1KVAoqD4Ber8etiCvXzl8HIC4yDgfHv9+Cxf18CL8SQWa6aSmvCycuEdSwKr//\nvJf+I/viXsSNoAZV+flzmc9aPCQbGBcqyVoUmPCrEcTejOPXb7bj5OJEx/7taNql0d2X8gKiw2Mo\nVbEkjk4OZGZkUaVOZSJCbwJwcNth+o3qy5nDZzEa5JSHeDiWPEGTuSRZiwKzZ8N+nh/fnzEL3sDF\n1Znf1+3FaDDSrEtjPH08KRtQhvCQm7efn5KYSmpSKmMWvIHBYCTyehQ/LdoAwB9bDvHhoCeZ+sJM\nrZojhKY0SdaxaclaHFZoYO67n99x/8C+Y4yaOYy/9p9i//+OsH3d79RsFERqejqxacmMe37qXV/H\n4GZH8LHznDt/pTDCFrbGBhYfkJ61KFS9X+lG6Yql8A+qyIkDp2nbvTnlAsryy+rfAFi2bT6DO42k\nU982tHqyGUaDkeTEFIqVLMqS6V9pHL2wVlIGEeIhrV2+ibL+pTn+x2mK+nox4815BNVV6PHCE7nP\nMNWjW3dtzrKZq7h8LoSOvVuzY91uDFKrFv+VJGshHk7ez8yVcyEAxMcm4OTseMfzPpu6gu7Pd6aE\nX3HUU5dyd5RkLf4bWYNRiIdkMBjR535w7nfxbPunWrF0xiomvzqLiko5lBr+hRShsEk2sAaj9KxF\noUqITcTe3h4HJ4d/rGJ+66vpm9CL15m2fCLpqenERMZx4fRlLcIVNsIWataazA2yd8oy+Xv2MVWk\nQilK1quK+tMurUMRVqLFlMGPnGljjh0yO+f41G1kkZldetZCCJtnCzVrSdaiQLn4FCGgR0uMBgM6\nnY7wo+dw8SlCtec64eDmQqwaSujuY7iV9MH/iaYYjQYM2Tlc2LiX0k1qkBgaQczZq1Qb0Jm4i9cJ\nO3iayt1acPMvlaTr91yuTgibIycYRYHyquRH0vVITq3aQsiuo9g7OaK3tyN49W+cXLkJv4ZBAAR0\nb8HFLfs59eUvhB8JplKnxsScvUrRgLLo7O2wd3bEq6IfAO6lfCRRi4djAycYJVmLAhVxTCUnI5Pq\nA57Ar2E1jAYDqZFxGA1GDNk5t+f5cHR3JfVmLACJIRG4+XqTGBqBe6lieFUoRXTwVRzcXPAsV1IS\ntXho+bhgrmYsNzJhE3yqViAhJILTq7YQHXyFMs1r3XUip8ykVFx9vQHTScjUaNN87ElhUZRpVov4\nS9dJDI2gYoeGRAfLJefiIeXfGoyakZq1KFDJYVFU6dkaY04O6HWEHTqDR+nit7cbcy90ubBpD5W7\nNAMdGHMMnN+4B4CYs1ep0qMVKTdjibt0A99aASSEhGvSFiG0pMnQvTVD5snQPSGEWfotHfXI3d34\n4ONm5xyvoNoW2b2WnrXQnL2zIw2fb4+DqxMuXu5c2HWCS3tOPnhHIcxlwScOzSXJWmjOvXgRQo6o\n3Dh+CecibrQd21eStchXtnAFoyRrobmMpFSUdnUpWzeArLQM9Ba8Dp6wUhZ84tBckqyF5pT29Yi+\nHM6lPSfxVcrgV6Oi1iEJG2MLPWvpwghNVGwSRM2nmgEQdvIyAW1q0erNp/CrWYnszGzKN6yqcYTC\nptjARTGa9KyDavlqcVhhQbzLeuDk7Zb7XsgkdP2229uik2MJbBmIW0asdgEK26Kz/n6plEGEporV\nropX5fIYDUZSwiKJOHgc3/rVcfbxomigP7FnL2kdorABMpGTEI/AycsDN7cSXPzpNzAaKd+5BR7l\n/Yj88zQ+1QIkUQuRh/X/bSCsk06HczFvUiOibq88kBIWiXNRL40DEzZJatZC/EdGI2lRcbiWLGb6\ngBiNuPn5EqdeMc0dYsEfGmF9dHo7rUN4ZJKshWYyE5JIjYjCv1dHdDpICY8i8cp17N1ccPbxwqem\nQsxJVeswhQ2QmrUQgHfVStg5OxJ9/JzZ+8Spf8+cF33izv2yU9I4v3pzvsUnhC3QJFnfuJKgxWFF\nQfFJxcHNIP+vokDUzI8XsYGymvSsRb4oXqUcxauWx97JgQu/HcGQmU1A54YYjUZSoxM589Pv6PR6\najzTFhdvD/R2eoLX7SX5ZizVn26DvbMjzp5uhPxxmmsHztBwWA9O//g7qdEJlG1SDSd3Fy7vPEbt\ngZ2wd3bEzsGe81sPEXPhOiVr+lOhZS2MBgNxVyI4v/Wg1j8OYWFs4QpGSdYiX2Qkp3Fy9Q4c3V1o\n8kZvDNk5HFy4jqzUdAI6NaB0/arYOzmQFpvIiW+34+rjSfHAChiycwj/6wI3T1/BydOVRsN6cu3A\nGcg7oWXuaBHXYkVwdHXmyPLNOLm74FrcCwcXJyp3bMAf83/EkJ1DzX7t8AkoQ8yF69r8IIRlkoti\nhDCJuxwGQGZyGoasHFy83akzsBMAdg52RJ+/jqObM1HnQgFIjUkkZN9JnDzdKN+yJiVqVCI7PfOu\nJ4Ju9YqSb8YRejCY2s91QGenJ2TfKVMCd3em/qCuANg7OeBa1JOYwmi0sB5yglEIE6/yJbh2KBgn\nTzd0eh2psUkc+2IL2RlZ+FarSHZaBh5+PhQp60tk8FVcinoS0KkBmclpxIfc5NqBMxT196N4YHkA\nDNnZOBdxIzU6Ac/SxUlPSMa9ZFHsnRw4unILTh6uNH79Kf745CfS45M5vGQjGI2UaVCV+GuyRqO4\nk5RBhMjl4OpMg1e7Y+9oz6kfd6G3s6PeK0+i0+nISs/k5Or/ER8SQY1n2tJwWA90Oh1nN+zD3tmR\nGs+0pXyzGmQkppCTkYXOTs/Vfaeo1qslaXHJpCekAJAaFU/lDvUpVcsfdDou/HqYrNR0ru4+QePh\nPW//kgj764LGPw1hcWygDKLJsl5H566SZb3Ev1R/5SnOfLkBY45B61CEBak3euAjd4vTo8PMzjnO\nxfwsshtu/b9uhNXzCfKn5tCncXBzoWKXFlqHI2yRTm/+zUJZbmTisWHESPTpC2SlpHHll71ahyOE\nRZKatRDC5snl5kLkJyOmIVZSshb5zQZGg0gZRFiMpBs3CXiqndZhCBuk09uZfbNUmowGEUII8XCk\nZy2EEFZAkrUQQlgBSdZCCGEFJFkLIYQVkGQthBBWQJK1EEJYAUnWQghhBSRZCyGEFZBkLYQQVkCS\ntRBCWAFJ1kIIYQUkWQshhBWQZC3ynaIoCxVFeUHrOISwJZKsRUGQqRyFyGey+IDIF4qifAR0A24C\nmcCfiqJcUVW1Yu72KYBRVdWpiqJEABuBFkA4sAh4AygDvKiq6h5FUX4HjgHtARdgBPAmEATMAz4B\nrgAdVFW9oCiKG3AWqKyqambhtFqIwiM9a/HIFEXpDdTHlEh7AJXv8jQjf/e4fYFNqqoG5t7vqapq\nS2AKMDLv81VVrQl8DSwAnsKU4N9TVdUIfAkMyH1+79zXlEQtbJIka5EfWgNrVVXNUVU1Dlhvxj5b\nc7+GADtzvw8FvO/ynFDgoKqq6aqqhgJeuY9/CTyb+/0LufeFsEmSrEV+MHLneykbKA/kXfjOMe8O\nqqpm57mbc4/XzdtLzv7nRlVVrwIhiqL0AnxVVT3yEDELYVUkWYv8sB3opyiKo6IonkBXIB7wVhSl\nmKIoTkDnAjr2Skz161UF9PpCWARJ1uKRqaq6CVPCPo2pdHEOSADmAEdytx3Ms8s/R4sY77Pt1mP3\nes46oCimurYQNksWzBVWS1EUHfAEMERV1Z5axyNEQZKhe8KazQOexJSwhbBp0rMWQggrIDVrIYSw\nApKshRDCCkiyFkIIKyDJWgghrIAkayGEsAL/BzbQU8zKhYbyAAAAAElFTkSuQmCC\n", 285 | "text/plain": [ 286 | "" 287 | ] 288 | }, 289 | "metadata": {}, 290 | "output_type": "display_data" 291 | } 292 | ], 293 | "source": [ 294 | "# For this particular text segment, attention is focused on \n", 295 | "# words indicating positive sentiment, such as \"funny\", and \"enjoy\" \n", 296 | "model3.plot_attention(ind=16)" 297 | ] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "execution_count": 42, 302 | "metadata": { 303 | "collapsed": false 304 | }, 305 | "outputs": [ 306 | { 307 | "name": "stdout", 308 | "output_type": "stream", 309 | "text": [ 310 | "(500, 1)\n" 311 | ] 312 | }, 313 | { 314 | "data": { 315 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAEECAYAAADu5BX3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8FEUfx/HPXhrpCUkIkJBQAkvvvQuCVCmCoCgKghQF\nEVRQAbFXUIqgKDyCoAgCShEQkBrpvS4QSoAkhPSQfuX54wIGaYckucvxe/u6V3I3N7ezePfN3Ozs\nrGIymRBCCGHbdNZugBBCiHuTsBZCiCJAwloIIYoACWshhCgCJKyFEKIIkLAWQogiwNHaDRBCVdXH\ngBAgDOgAjNA0bat1WyWEbVGsMc86OyVeJneLW3Ts3oulP/+Im6urtZsibIizl5/yoK9RM7SVxZlz\n+MKWB95eQZBhEGF1v61cTc2GzbgaF8dLo14jKyvL2k0SdkZRFItvtkrCWlidoigM7P8M/n5+zJ7x\nFS4uLtZukrAziqKz+GarbLdl4qEkyx8IcXtygFHYhOshbctfQ0XR5WDDPWZLSVgLm2AOaQlqUTB0\n+RTWqqrqgJlATSALGKRpWkSe8q7ABEAPzNU07fs71VFVtQ6wEjidW32WpmmL77RtCWthdd26dAJg\n1MvDrNwSYa/y8Rtbd8BZ07Smqqo2AibnPoaqqk7AFKA+kA6Eq6q6AmgOuNymTj1giqZpUyzZcNH/\nbiCEEIWnGbAWQNO0XZiD+boqwBlN05I1TcsBtgMtc+usuU2dekBnVVW3qKr6vaqqHnfbsIS1KDRj\nx09ia/jfAJw9d57ho8bw5sR3efaFIfQbMJi16zcCMGDIS5y/EAnA4qXLmfXdHKu1WdgH5T7+uwcv\nICXPfUPuMMf1suQ8ZamA9x3qOAC7gNc0TWsFnAXeuduGJaxFoenV43FWrFoDwPIVq6hZvTrFfX35\ncc63fPf1VGZ8M5ukpOSbv7LKAUeRD3SKzuLbPaQAnnlfWtM0Y+7vyf8q8wSS7lDHAPymadqB3Md+\nA+rcdR/u1TIh8kv9unWIOHeOxKQkduzaQ3xCAnXr1ALAzc2N8uXKcvHy5ZvqmIzG27ySEPcnH0+K\nCQc6Aaiq2hg4nKfsJFBRVVVfVVWdMQ+B/H2XOmtUVW2Q+3tbYO/dNixhLQqNoih07dSBjz+fQtMm\nDSlftiz7DxwCIC0tjdNnIggqXQpnZ2euxsUBcOKkZs0mCzuhUxSLb/ewHMhUVTUc84HCV1VVfUpV\n1cG549SjgXWYQ3qOpmnRt6uT+1pDgS9VVd0ENAE+uNuGZW0QUajiExJo16UHy37+kaCg0kz64BMu\nXb5MZlYWz/TtTddOHdkWvoMvpk6nVGAgJUoEUKpkSYYNHmjtpgsryY+1QVqoj1ucOdu0FTY59maV\nqXuGzHRrbFbYgJz0NOrWqkGZQH/QZ/PeuNE3lRsy02larxbL5n9/y+PiIeXl98AvYQ8nW8k8a1Fo\nNm7Zxszvf8DPz5dho8dyNT6ePj26sXbjJvyK+5KSeo3pn37Ah5OncvFSFEaTkZcGDaB+7ri2EP+V\nBcMbNk/CWhSatq1aEFSqJFExV2jTsjlX4+IZOGI0gQH+dHy0DY+0aMbi5Svw9fFm0rjXSEpO5oUR\nY1j6r162EPfLgil5Nk/CWhQqX19fFixZxsYt23F3d8NgMABQNqQMAKfPnuPg4aMcOX4SAKPRSHJK\nCt5eXlZrsxC2QMJaFKoFv/xKrWpV6d29K3v2H2Tbjl3AP2OK5UNDKFkigBeefZpraWnMX/QrXp6e\nd3tJIe4pv9YGsaaivwfCJnXs3Y+cnJxbHm/ZrDG/LP+dYaPHsiV8B26uruTk6G98Te3VrQvnIi/y\nwojRDBoxhlKBJfhi+ixirsQW9i4IO+Kg01l8s1VW6VnrnGVxeXunKDp0zi7onJxuerxRo0Ys+7nR\nHeu5OLvw0bsTC7p54iEjY9ZCAL+v+oPwHbtITE4mKSmZoYMG3Cg7HXGWyVNnYDAaSUpKZvzYMdSq\nUZ0uTzxFnVo1OB95Eb/ivkz55ANW/rGWbX/vJDMri0uXLjOgfz+6de7IwGEjmDDuddb8uYGo6GgS\nEpKIjonh9VEjaNq4IVu2hzNz9lw8PDzw8vSkUsUKDMvTBiHsgYS1eHCKgtFk4rsZXxEXH0+/gUMw\nmUyYTCbOnjvPmFdepmKF8vyxbgO/rfyDWjWqczk6mjmzphFYIoDnBg/n6PEToChcS0vjm6mTibx4\niRGvjaNb5443ekWKouDi7MLMrz5n5+69zPtpEY0b1ufTKdNYMOcbivv68uY779lBH0rkN3sYs5aw\nFvmiUf16APj7+eHp4UHkxUsoikKAvz+z587DxcWZtLQMPD3cAfDx9iawRAAAgYElyM7OBqBypYrm\nx0oEkJWVfct21EphN8qzs7JJTEzCw92d4r6+ANStVYu4hISC3VlR5NjDSTFF/8+NsAlHT5wAID4+\ngczMLHx9fTCZTHw6ZSrDBw/kg4lvUzGsPEaTeWGmO3127vWR+vfYY/HivqSlp5OYlATAoaNHH2g/\nhH3Kx7VBrEZ61iJfRF68xOCXR5GWns74sWN49+PPAOjSsT1j3ppIyRIlqFZFJS4u/u4vlOfDcrvP\nzb+XT1UUhTdfe5WXXn0DDw93jEYjoblztoW4zh4OMFplIaespFhZyMmO/L56DUlJyTzXr+991cvO\nzmbV2j/p+XiXB9r+nHkL6P90H5ycnHjrnfdp2rghXTo+9kCvKWyHi0+JB07ax2s/Y3HmrDi4wCaT\nXXrWIl/8l2+PV+PiWfb7qgcOa3c3N/q9MARXl2IElS7FY+3aPtDrCftjy8MblpKetbCaSR99yp8b\nNvFcv74cOXactPR0DAYDLw8ZTMP6da3dPGEj8qNn3aNOf4szZ/mB+TaZ7NKzFlbz4oDnOBNxjmtp\naTRt3JCnn+xF7NU4nntxOGuWL7Z284QdsYepe0V/D0SRdf1b3bnzF6hb27wMaokAfzzc3UlITLRm\n04SwORLWwmocHHQYTUbKlQ1l/0Hz5b2uxF4lJTUVH29vK7dO2JN8vAaj1cgwiLCa4r6+6HP0pKWl\ns3vvftb/tZmsrGzeefMNdDa8oI4oehzsYBjEKmFt0uutsVlhY5x0On753+zblsl7ROQne5gNIj1r\nUWjOR17knY8+xdHREaPRxCeTxrNo6W8cOHwEo9HIs3170+6RVpyOOMunX83AZDLh4+3Fu2+9gYe7\nu7WbL4RVSViLQrNr7z5qVKvKqGFD2H/oMH9t3U5UTAw/zJpGVlY2zw55icYN6vHuJ1/w/vhxlAsN\nYdnK1fywcBEvv/iCtZsvijBbHou2lIS1KDQ9unRi7oKfGT7G3FNWK4ZxXDvFCy+/CoDBYCAqOobz\nkZF88PmXAOgNekLLBFuz2cIOyDCIEPdh07Zw6taqydCBz7Fm/Uamz55Dkwb1mfDGaPR6A9/PX0CZ\noCBCQ8rw0cS3CCwRwL6Dh0hKTrF200URJ2uD/EeZcVFyBuND6NLlKN5+/yOSU1LJys6mU7u2ZGVn\nc+yERnpGBm1bteDkqdMMfu4Zvpg+C4PBgKIovPvWG4QEB1m7+cJKivmXfuCkfa7xUIszZ97Ob2wy\n2a0T1vExEtYPqcvR0Yyd+B7NGzfCz684vbs/bu0mCRtXzK/kA4fn802GWZw5P+yYZZNhLcMgolB9\n/8OPnDt/gWMnTtK0UQPW/7WZpORkXnrxBVo1a0qbLj34a9Vyflm6nJVr/0SnKFSrUpmxr460dtNF\nESZj1kLcp8HP9+fM2XM0a9yI2KtXmTjudfYeOMgPC36mVbOmN1bv+/2PtYx/fTRVK6ssXv47BoMB\nBwcH6zZeFFn2MGYtYS0KlYl/vo1WVisB4OfrS0ZW5k3Pe+/tccz7aRFR0THUrF4NawzXCfthDz3r\non8OpihSHHQOGHOD925zX5etWMWEN8Yw5+upnDx1msNHjxVWE4WwSdKzFoWquK8POTk5ZGVn3/TF\nNO8VzAHCKpTn+WEjcHdzI7BEADWqVbVCa4W9sIeTYmQ2iLBp4Tt3E3PlCk9062rtpggryY/ZIENb\njLQ4c77ZNs0mk90qPWtDRpo1NiuKoMa1qgHV5D0jHog9XHxAhkFEgVixdj1bd+4iOzuHuPgEnurZ\njc1/7yTi3HleHTqItPQMfl72O05OToQEl2b86JGMe/9jnurZjXo1a3BcO8X3CxbxSPOmnL94kRGD\nBrBo+QrW/rUFRYHHHmlF3x4yR1s8PCSsRYHJyMzi60/eZ92mLSxc+hvzZ3zJ3oOH+XHJMs5fvMSi\nb2fg6lqMyTNns3TlH/To1IFV6zZSr2YNfl+7np5dOpKYlAzA2fOR/Ll5G/+b9gVGo5HhY8fTpH5d\nWTdEWERnkwMb90fCWhQIRVFQK5QHwMPdnXIhZQDw9HAnKzubsLKhuLoWA6Buzers3HeAJ7t1Yers\nOaSkpnLw6HHGjRzOqj83AhBx4QIxsbEMGfMmAKlp17gYFS1hLSySXwcYVVXVATOBmkAWMEjTtIg8\n5V2BCYAemKtp2vcW1HkaeFnTtKZ327aEtSgwd/t8nL0QSUZmJq7FirH30BFCg4NQFIVHW7bgw69m\n8EjzJiiKcmN+dWhwMOVDQ5jxyfsALFiyjIrlyxXGbgg7kI/zrLsDzpqmNVVVtREwOfcxVFV1AqYA\n9YF0IFxV1RVAc8DlDnXqAAMt2of82gMhbvXPdLwNW7aTk5ODoig4Ojgw9LlnGDJmHM+9PJqU1FR6\nP94ZgMc7tGNz+A66dWh/oy4oVKpQjoZ1azPwldfoN2wkl6JjCPArbq0dE0VMPl6DsRmwFkDTtF2Y\ng/m6KsAZTdOSNU3LAbYDLXPrrPl3HVVV/YAPgVFw71MsrdKz1l+TI/v2rmOzJoD5/3XDKpXx8fJE\nfy2N8oGBfDV+HABtG+Z5n2dlo8/Kxt/NjfBff75RN+/rPN2pA0936nCjijE9A2Mh7Y8QubyAvGv2\nGlRV1WmaZswtS85Tlgp436GOMzAHGA3cfPruHcgwiMh3qzZuYsf+gySlpJKUmsKgPr1vlF25GsfH\ns2aTlZWNi4szbw4fQqC/H1/PX8jJiLMkp6ZSsWxZJowczqETJ5k6dz6Ojo4Uc3Hmk7FjcHZy4uNZ\ns7kUHYPRZGJYv77UrV7NinsrioJ8vGBuCuCZ5/71oAZzUOct8wSSblcHqAWEAbOAYkBVVVWnaJo2\n+k4blrAW+U9RMJpMfP3+ROISExn4+luYTOZVQab+bz59unSkSd067D50hK/nL2Ts0MF4eXgw/d0J\nGI1GnhoxmqvxCWzZtYd2LZrRt2sntu7eS+q1NLbt3YevlxcTRgwnKSWVoW+/w6LpU6y9x8LG5eOY\ndTjQFViiqmpj4HCespNARVVVfYE0zEMgnwOmf9fRNG0PUB1AVdVQYNHdghokrEUBaVCzBgD+vr54\nuLtzMSoagIgLkfxvyXLmL/0dEyacHB1xcXEmITmZ8ZO/wq1YMdIzMzEYDQzo1ZP/LVnK8AnvUsKv\nONUqhRFxIZJDx09y7NQZAIxGI8mpqXh7et6xLULk49nmy4F2qqqG594foKrqU4CHpmnfqao6GliH\nufc8R9O0aFVVb6nz7+YB9zzDUsJaFIjjp8/Qs0M74pOSyMrKwtfbC4CyZYLp170rNSurRFyI5Oip\n0+zYd4DYuHg+fP1VEpOT2bxzN0ajiTVbttK57SOMHNCfeb8u57d1GygbHEygvz/P9+rBtfR0fvpt\nJV4eHlbeW/Gw0DTNBAz718On8pSvAlZZUCdv+XngrtP2QMJaFJCL0dG8NOE90jMyGDt0MB/N/BYF\nGDngWT6d9R3ZOTlkZWUzZvBASgUGMGfxrwwbPwk/Xx+qVapIXEIi1SqG8eGMWbi6FEOn0/HWS0Pw\n9/Xhw6+/Zejb75CWnkGvTo/ZxSI9omDZwxKpVlnIKfHIPlnIyY6t3ryVpJRU+uVOx3tQ8UnJzF2y\njNcH3/ztceHvq/Dx9qJz65b5sh1hm3xr1HvgpH2z/TiLM+fjPz+xyWSXnrUoEPnZkfHz8b4lqPN9\nI8Ku2cO3LwlrcV9WbdrCqr+2YDKZ6NWxPYtXr0Wn01Grisrwfn1JTE5hQ/gOrqVnsGnHbiaOGIaP\nlyeTps4kPTMDvcHI0Kd6U696NfqNHkvdalU5cyESRVH4fOxosnP0jP9yGiYTZOdkM/bFF3B3c2Pi\nV9P5/qP32LJ7D3N/XY63h6d5QacWzQCYuXARh05o5tkkXTvRpkkjhk18n+I+3qRcu8bU8ePQ6eQc\nsIeVgx0sDiJhLe6bt6cHbw9/kSHj3+WHzz7ExdmZd6fNZPfhI2zfu59WDevTvV1bjminOH4mAu3s\neRrVrsmTnR7jakIiQ8ZPYtnMqaRnZvJYi6aMeeE53pn6NX8fOIS7azF8PD2ZOGIY5y5dJiMzCw93\nNwAMBiNTf1jAD599iJeHBxO/moEJ+Hv/QaJjr/LtB++QlZ3N4LfeoWHNGiiKwmPNm9Iy78k3QhRR\nEtbiviiKQplSpbgUc4WklFRe/fAzADIyM7kcE0tkVAyPt30EgBpqJWqolfhz2990bNUcgIDivri7\nuZGQbD7Rq1K5sgAE+vuRnZPDo00bczE6hjc+nYyjgyMDenXn+mGVxJRkPNzcbsz+qFO1CgBnL17k\n5NlzDH/nAwAMRiPRV68CEBJUqsD/TYTts4cDjBLW4r7pdAqlSwRQwr840ye+hYODjpUbN1O5Qnku\nREVx/EwEYaEhHDh+gh0HDlE2uDQHjp+kYtlQYuMTSE1Lw9vDPC/631ed3n/sOH6+vkyd8CZHtFPM\n+ukXxr80BABfL2+upaeTkJxMcW9vjp4+Td1qVQkNKk296lUZN2QQeoOBect+Jygw0NxWO1h0Xjw4\nubq5eCgpgI+XF0916cTQie9hNBopXSKA9i2a8nzPbnwwczZrt4ajUxTeGv4iHm6ufDhzNpt27iIr\nO5s3hwzCwUF3y8cnIvIiKanX2HHgEMvWbcBgMPDCkz1zt6ng4KDjjRcHMvrDz/Bwd8OtWDEUoEX9\neuw/doKhE94jPTOT1o0a4Ja7/KoQYB89a6tM3Yvft1Om7gkhLOJXr/EDJ+37XSZYnDkTVr1vk8ku\n3xGFzVi9ZRuzFi22djOEHcrHJVKtRoZBhM2w5Q+KKNrsYRhEwloIYffsIKslrIUQ9s8eetYyZi2s\n5pmxb9/ymD1MsRKiIFilZ52VkGqNzQobY9IbbnovtK1eC6rXkveHyHf20AmQYRBRaDKzs/lo3jyS\nr12jtL8/RpOJM5cuMXXxYhx0OpydnHjt6acxGo28/7//UaJ4caKuXqVy2bK82qePtZsvijB7OHgt\nYS0KzYpt2yhbqhQDu3Qh8soV3pw1iy9++onX+/WjQlAQ4YcPM3PZMob26MGlq1f5YsQIXJyceHrS\nJBJTU/GVq8GI/8geFnKSMWtRaC7GxlIpJASAkMBAvD08iE9JoUJQEAA1w8I4H22+/FdQQACuLi7o\ndDr8vLzIzsmxWruFsAUS1qLQhJYsydGzZwG4fPUqydeu4e/tzdnLlwE4dPo0ZUqUALCDEUZhS+Sk\nGCHuw+PNm/PZwoWMmDKFkn5+eLm7M+app5i6ZAmYTDg4OPBGv35gMtn0h0YUPXYwCmKdtUGi1q+X\ntUHETbJzcli/Zw+dm97zuqHiIVO6XbsHjtppT35oceaMXPy2TUa7VXrWOWlZ1tissGFXEhJYvT2c\n9rXqWbspwg7Zwxc1GbMWNuHnvzZw4coVFm5cb+2mCGGTZMxa2ISn2z7K+Ssx9GvbztpNEXbIwQ4u\nQiFhLWyCFQ6diIeIPQyDSFgLm6BTFIyS2KKAyEJOQuQTH09P9AY9c9asvqXsz727mbNmNVcSE3jl\n62lWaJ0Q1meVnrVRb7DGZoUNc0Th65dGAbe+P0xGEyajCaPeiMlkkvePuG/2MG9fetYiX/25fy9z\n160BzHOn+3/+Mat27WDUNzN49duvmbXqdwC++PUX9p7WANh7SmPyUvPlvFbsCGfcnNmM+mYGE+fP\nRW+4NZgvx11l5MzpN+5/tGgB2qWLBb1roghTFMtvtkrGrEW++vd7XVFg/f69vPx4DyoGBbN61w4M\nRqP51N7rz879YTKZSM1I5+OBg1EUhbd/+J5TtwnhIP8AXJydiIy9gq+HJ1cSE1GDyxTsjokizR56\n1hLWosBcP1z4as/eLN2+lZjEBKqUCb1l6ofJaL6vKAoODg588stPFHN2Ji45Gb3x9kMeHes3ZP3+\nfQT4+NC2dt2C3A1hB+zhdHMZBhH5ytnRkYTUFADORJkXaFq7dzcjuvXk80FDiYi+zPHICzg7OhL/\nr+edi4lm54njvNm3H8O6dMNkMt1xSl/z6jXZd1rj7+NHaSNhLe5BFnIS4l/qV1JZtXsnY2bPpGJQ\nMO4uxSgbWJLXZs/CzcUFf29vKpcJoZizM1OWLWHTwQME+fujAKX9/Cnm5Mxr383Cy82dCqWDbgT/\n2ejLGI2GGx8mZ0dHapQrT0p6Gh6urlbcY1EU2HAGW8wqCznFbP5LJtSK+9LnrfH8+O47ODs53Xjs\nq59/oVXdOtRRK1mxZaKglWzd5oGjdv6ALyzOnP7/e80mo1161qJQrPl7B7uPHSP52jWSr6XxfNfO\nFHN2Ye6KFTg7OeHl7s7Y5/qj1+uZ9N33mEzm2SRj+j2NduECCSkpvPf9XD4YNgSA16ZOw8fDU4Ja\nWMSWhzcsJWEtCoWigNFkYsqro4hPTmbYJ5+h0yl8/cbr+Hl78+tff/Hj6j+oo6p4e3jw1oDnuRAd\nTWZ2Fp2bN2P+H2t4Z/ALN17vi1dGWnFvxMNKVVUdMBOoCWQBgzRNi8hT3hWYAOiBuZqmfX+nOqqq\nVgVm51Y9nfv4HU8ikAOMotDUq1wZAD9vb1xdXHBydMLP2xuAmmEVORcdTaPq1ahRoQJvz5zF3BUr\n7aJHJKwvH+dZdwecNU1rCowDJl8vUFXVCZgCtANaAS+qqloit47Lbep8CIzTNK157v2ud9uwhLUo\nNCfPnwcgISWFHL0evV5PfHIyAIdOnSIkMJADp05R3NubL14ZyTMdO/Ldb+aTaHSKgtFotFbTRRGn\n0ykW3+6hGbAWQNO0XUD9PGVVgDOapiVrmpYDbAda5tZZc5s6T2iatl1VVWegJJB0tw3LMIgoNJdi\nrzL6y69Iy8xkTL+nAZjwzWx0ioKnuxtvPv8cAO9+N4fft2zFYDTwfJfOANSsGMbYGV/z1ehXrdZ+\nUXTl40JOXkBKnvsGVVV1mqYZc8uS85SlAt53q6OqagiwAXNQH77bhiWsRaFpVqsmfdo9etNj9apU\nvuV5k0fdOh59PciFsLIUwDPP/etBDeagzlvmiTmE71hH07RIoJKqqi9gHkJ5/k4btkpYKw4y+vLQ\nUXQoOkX+3wuryMdDH+GYx5aXqKramJt7wyeBiqqq+gJpmIdAPsd8Mu8tdVRVXQGM1jTtDHANuOsK\nZdKzFoWiY7MmN35fE/434YcOk52TQ3xyMr3atmX7wYOci4pieO9erN+5i3eHvgjA8I8/4/3hQ24c\niBTiv8jHA9XLgXaqqobn3h+gqupTgIemad+pqjoaWIf5eOAcTdOiVVW9pU7uz4+BH1RVzcYc7oPu\ntmEJa2EVmVlZfPHqK2zcvYclGzbyzVvjOHBSY/GGjVy6EktqejpxiUn4eHpIUIsHll9ZrWmaCRj2\nr4dP5SlfBayyoA6apu0Amv/78TuRsBZWoBBWxrxKnoebK6ElS+b+7oZer6d944Zs2LWb6Lg4urSw\n+L0sxB3ZwxTQe4a1qqqNMKf/DGAlUBcYqmnarwXcNmHH7vbh6disKe9/N4esnByGPtGzEFsl7JUd\nZLVF86ynAXuBJ4AMzGE9riAbJezf9Q+Pwr9XOlPw9/HBzdWVepUro9NZdkDy0KnTRFy6nP8NFXZB\npygW32yVJcMgOk3TtqiquhBYqmlapKqqDg+y0eyUjAepLoq4tjVqQw3z+6B2SHlqh5QnOyWDUB9/\nPhw4iOyUDIw5BtrVqWfxe2Xlpi08UqcuZbyKF3DrRVFkwxlsMUvCOl1V1deAtsAIVVVfwTzZWzyE\nsrKz+XzRz8QmJaLXGxjWvQer/g4nJiEBg9FIr9ataV27DqO/nk5Y6SDOxcTg6uJMjfIV2HvyJNcy\nMvh06FDCjxxhx7FjZOv1JKSk0LNlS/4+epSz0VFgghY1a3L64iU+WbgAnaKjerlyDOrSlXlr13Al\nMYGk1GtcSUxkWPfueLu7s/ekRsTly4QGlmTe2jVExceRlZNDzxateLR+/XvvmLBr9jBmbcl3zH6A\nG9BT07QEzKdFPl2grRI2a+WOvynl58e0kaN4+9n+HI44g6+nJ1NHvsLnw4bzvzV/kJyWhoJC5dBQ\nPh82nBy9nmLOznw6dBihJQM5FBGBoihkZGfx0eAX6dOmDSvCw5k0YCCjn+xDuVKl6Nv2UeavW8sX\nw17iqxEjiUtJZt8pDUVRcHZ04qMXhzC8Rw+WbtlMxeAyNKhcmcFdH8fTzY0jZ88yacALfPziUEtO\nHxaiSLhjz1pV1Vb8c2WmLYCTqqotMZ8XXx64VPDNE7bm0tVYGlauCkBQQADxKSnUq2ReptTVxYXQ\nwECi4+IAqBgcDICHqyuhgbkzPlzdyM7JASAsyFzuXsyV0MDAG8/N1udwOS6O5LRrvDn7WwAysrJu\nvG6FoCAAArx9yM7R39Q+VxcXhnfvwZeLfyEtM5NH69UrmH8IUaTYQcf6rsMgr2MO61JAJeAvzMv+\ntcZ8Bk6bgm6csD2hJQLRLkbStHp1ouLj2HLwAM6OjjSrUZP0zEzORUdT0s8P4J8L4t7B3UpLFS9O\ngI8Pnw0bjoNOx5pdO1HLhLD9yOHb1lNyF3pKSEnh1KWLTBowkOycHJ5+/13a1W9g8YFKYZ/s4RvW\nHcNa07QuAKqqrgNqapp2Pvd+KWBBobRO2JzOTZvxxaKfGf31dEwmEx+9OITft29j1PRpZOfk0P+x\nDvh4eFj0WtfHERWFm5JbQcHbw4NerVozesZ0DEYjpfyK06ZOXf6pcPNrVA4N5ftVqxjf/zkSUlN4\nZdpUdDoX82ufAAAffUlEQVSF3q3bSFALuxizvudlvVRVPaFpWpU89xXghKZpt67AY6GLq9fIZb0e\nUmO+nsE7zw/Ay939P7/G9sOHqVI2FD8vObPxYVCmc8cHTtrVo7+2OHM6T3nJJpPdktkgu1VVXQAs\nwnxA8llg04NsNDsl80GqiyLMZDCSnZpJtuG/z/5ctnkzL3fpgScu+dgyYc/soGNtUVgPBkYAQzCP\nYa8HZhVko4R9SM/MZOrKZaRlZhCfmkKXBubFnL5du4r4lGRcnJwZ3b033u7ufLduFccjLwDQukZt\nujVuxpTli2lVozb1wiqx97TG1mOHaVG1Bmdjopi8fDGfDxyKo8MDTfkXDwl7GAaxJKxXaZrWnjyX\nrxHCEtGJ8bSqXpOmVaqTkJrCG//7Fn8vb9rWqkvdChVZvWcni7dvola5MK4kJfLl4JcwGAy8Nvcb\napWrAMo/hyivf9gaVKpM+ZKleblLDwlqYTE7yGqLwtpVVdWQ3EWyhbCYt7sHv+0MJ/zEMdxcXDDk\nXparZtnyAFQODmH3qZMU9/Ciekg5ABwcHKgcHELk1dibXst4j2MrQtyNLZ9GbilLDpMHAOdVVY1R\nVfVc7u1sQTdMFH3Ld2yjcnAIr/fsQ/OqNbh+MPvERfNwx5ELZylfshRlAkpwLPI8AHqDgRMXLxDk\n54ezoyPxqearIUVE/7Puh6IoEt7ivuTjBXOtxpKedYfcn3k/HTa8S8JWNKpUhVlrVrDj5DFCAgJx\ndXYhx2Bg85GDLNy8AY9irozu8SRuLi4cOX+WMd/PJC0rk+qh5ahQKogdJ49xPPICm44cJMjPHwXY\nd+YUzo6OTF6+mA+ffQEPV1dr76YoAh6WMetIYCjmtUEcMZ8cM/1BNuriIx+wh0GDOtVpUKe6Rc8d\n/uQTAMxbs4biXl64+Lji4uzMrDdex9nx5rdp0/q18r2twr7ZQVZbFNafAWHAXMzDJgOAcsCoAmyX\nsFEXY2P5/OefcNQ5YDSZeLt/f37dvIlj584B0KZuPXq2asWnCxfi5OjAlYREcvR6Hqlbhx3HjhGb\nmMh7LwyitL8/369cyZFzZzEajfRq/QjVypVj3e7dODs5UrGM+VT0qYsXE50QD8B7A19g+5EjXIqN\npWuzZrw/bx6Bvr5ExcdROSSEV3o/SfK1a3z443z0egPBJUpw8PQp5o+fYLV/L2EbFHs+gzGP9kAd\nTdMMAKqqrgKOFmirhM3af0qjSmgoL3Z9nMNnzxJ+5AhXEhKY8epoDAYDr0ybSp1KFVEUKFncj9F9\n+vLV4sXEJCTw0YtDmLdmDTuOHSU4oAQxCQlMHfkK2Tk5jPjqSya/PIIOjRpS3MubyiGhAHRq0oRq\n5crx2U8LbyzkdN3luKt8Pnw4Lk5OPPP++ySkprBowwZa1KxJ12bN2adp7NNOWuufStiQh6Vn7ZD7\nvOtX3nXEvEaIeAh1bNyERRs2MO7bb3AvVoyw4GBqlK8AmGdyVAkty4WYK8DNCzmFXF+oyc2V7Bw9\n52OiOXXpIqNnmEfUDEYjVxISbtlexdzLfxX39CIzO+emsiB/f1xdzCfGFPfyIidHT+SVWB5r1AiA\nGuXL5/fuC2E1loT1QmCzqqo/YT6w+BTwc4G2Stisv48coUaFCvTv0IG/9u1j7h+rCQsO5onWrdEb\nDBw7f472DRvCiZvr/XvuRpkSgdQOq8joPn0wGAwsXL+e0v7+KIoOY+4UP7jXkexbS8uVKsXxc+eo\nUDqI4+fP/8e9FPbmYTnA2AyYD9QHkoAPNE1bXaCtEjarUpkyfPrTQhb+aR6znjRgIBv37WPEV1+i\nNxhoXafOjR513g/ITRfuUhSaVq/OoTOnGTVtGhnZWbSoWZOjZ8+SdC2VTQf2ExIYeOcP2PVLgv27\nWIG+jz7KJwt+ZPPBg/h5eeEgJ84I7GPVPUsWcmoMdMy9OQGrMZ/VuPO/bvTiqj9kkqwoELtPnMDb\nwx21TAj7Tmks+msjnw8dbu1miQdQpkunB07abZO+szhzWkwabJPJfs+edW4o71RVdQbQG3gbeANw\nLuC2iYfMut27uXg1lkGdu/zn1yhZvDhf/LIIBwfzcMrLPeTq6MI+3DOsVVWdiXkoxABsBYbl/hQi\nX+XHsGJIYCDTRr7y4C8k7MtDMmbtjXl+9XHMh41OapqWVKCtEkKIfGQPBxjvuTaIpmn9NE2rAbyH\neehjtaqql+9RTQghbMZDsTaIqqqVMZ9q3haoDezCfJBRiHxnDz0gYXseljMYF2MO5ynAjutnMgqR\n39o3aGjtJgg7ZQ99AEtmg9QsjIYIIURBsYdvbJb0rIXId5N++B89W7SkZoUKaBcjmb9uHcU9Pbkc\nF4fRZGRAx07UqhBm7WYKO2EHWW3RxQeEyHedGjfmz717APP86gaVK+Pl7s6Ul17m3QEvMH3ZUiu3\nUNgTRVEsvtkq6VkLq6hfSWX2ypWkpqdz9Nw5TMDRc2c5GWm+epzRaCQlPQ0vN3frNlQIGyFhLaxC\np9PRqlYtvvp1Cc1q1MDLzY0AHx+ebvsoaZmZ/Lp5E56ubtZuprATNtxhtpgMgwireaxBQ/4+eoQO\nDRvRpUlTLsZeYfTMGYyZOYMSvr42/ZVUFC2Kg2LxzVZZpWedHp9mjc0KG+OBM0vHvgtGyEnOYkT7\n7jeVy/tE5Jf8+sOvqqoOmAnUBLKAQZqmReQp7wpMwLzm/1xN076/Ux1VVWsD0zAv5ZEF9Nc0LfZO\n25ZhEFHgLsfHMXX1Mhx1OowmE691e5LV+3Zx4uIFjCYj3Ro2o1mV6pyPjeG79avBBJ6ubozs0oOI\nmCiW7tiGk6MjMYkJtKhagyebtbb2LomHV3fAWdO0pqqqNgIm5z6GqqpOmM9HqQ+kA+Gqqq4AmgMu\nt6nzFfCypmmHVVV9ERgLjLnThiWsRYE7dD4CtXQZnm/TnmORF9h56gSxyYl80n8w2focXp83m9rl\nwpjxx2+M6vIEwf4BrD+0j2U7tlG7XBhXU5KZPvhlcvR6np/2mYS1uG/5OKLWDFgLoGnaLlVV6+cp\nqwKc0TQtGUBV1e1AS6AJsOY2dfpqmhaT+7sTkHG3DUtYiwLXrlY9lu7YyqRF83BzKUa5wFJExETx\n9oI5gHnmR2xyIpfj45i1dgUAeqOB0sX9ASgbEIhO0eHi5HzLlc6FsEQ+Hv/wAlLy3DeoqqrTNM2Y\nW5acpywV80J4d6oTA6CqalPgJaDF3TYs73xR4HadOkHVMmXp26INW44dYsGWDdQpF8bwjt0wGA0s\nCd9CSd/iBPn58+rjvfD38uZo5DlSM3I7GrZ7zEcUEfnYs04BPPPcvx7UYA7qvGWemK+udcc6qqr2\nAd4COmmaFn+3DUtYiwIXViqIr1YuZXH4ZkwmE+N6PsXmo4cY9+N3ZGZn00StiquzC8M6PM6XK37F\nYDSiKAojOvcgITUFJU9aywwR8Z/k3/smHOgKLMm9itbhPGUngYqqqvoCaZiHQD7HfAnSW+qoqvoM\n8CLQWtO0xHvuwr0u61UQ5LJeQghL5cdlvQ7NWGhx5tR6ud8dt6eqqsI/MzsABgD1AA9N075TVbUL\nMBHztOg5mqbNukOdCCAWuMA/QydbNE2bdKdtS89aWIXeYODzRT8TkxCPwWjiiZatWLkjnLCgIM5H\nx5CelcmE/s8T6Otr7aYKO5BfHWtN00yYr5aV16k85auAVRbUAfC7n21LWAurWLXjb3w9PXmz3zNk\nZGUxdMoXODk6USUklOHdejB3zR9sOrCfvm3aWrupwg7Yw/CZnMEorOJibCw1ypUHwNXFhZDAkkTH\nxxEWFAxACR8fsnNyrNlEYUfs4UoxEtbCKkJKBHLk3FkA0jMzOR8TTSk/P5v+sAhhTRLWwio6N2lC\nSloao2ZM47VZX/Ns+8fw8fC46Tn28NVV2Ag76FrLmLX4z7L1ejbu20vHRo1vW9570kSWTHrvtmWO\nDg688dTTNz1WoXRpNh88yDPt2tOlSdNb6nz443zGPt0PRweHB2+8eKg8LNdgzHcOLk7W2KzIZ8nX\nUlizexddWt7+xCtFUe7r/3WlcmWpVK7sHcsnDnrh/hooRC4Ja/FQW7BuHRdiYpi3Zg3noqJISTOv\nkjeiVy/KlS5943lnL19mxtKlmEwmvNzdeaNfP+atWUOFoCAea9SIhJQU3pw1i2E9e7Jy+3YmDBjA\npz/+SFRcHFk5OTzRujXtGjak78SJ/DhxIvHJyXy2cKH55BlgRO/eVAgK4pl336VG+fJcjI3F19OT\ndwcNQqeTkT5hH+SdLP6zZzp0ILRkSbKys6lbqRJTRo5kdN++fPnLLzc974uff2bUk0/y5Suv0LBq\nVRZt2EDnpk1Zt2sXAH/u3k3HJk1uPD8jK4vDERG8N3gwnw4ffiNwFUXBZDIxa/lyej3yCFNHjeLl\nXr34fOFCAKLj4xnYtSszxowh6dq1G1edEcIOhqylZy0eQO7Zr2ejojhw6hSb9u8HIDU9/aanRcbE\n3AhwvcFAcIkShJYsicFo5EpCApv372fyiBGcvngRME/le/mJJ5j888+kZWbSrkGDm1/vyhVqhpkv\nphsWHExsovlMXW93dwJ8fAAo4etLjl5fQDsuihoZBhEPNV3u+tQhgYGoDRrQtn59riYl8dfevTc9\nr0xgIG/170+Ary+HzpwhNXe4pFOTJnzz22+ULVUKd1fXG89PSEnh1MWLvDd4MNk5OfSZMIF2DRve\nKA8tWZLDZ87QtEYNzly6hJ+3N3Dr7BFrLKUgbJM9zCySsBb/mY+HB3qDgYysLDbv38+q8HDSMjMZ\n0KkT8M9iea/27ctH8+djMBpJTE2lZa1aNK9Vi1Z16jDj11/5aOhQ8xNzP1DFvbxISEnh6XfeIcDH\nhz6PPoqDToeC+UM3tEcPJv/0E79s3IjBYOD1fv1u2t519vABFfnEDt4KVlnI6fK6ddLleUit27WL\n5LQ0nmzT5p7P7TV+PL9+8EEhtErYsqDHHnvgqNXmLbE4c9TnettktEvPWhS6PSdOsOv4cTKysujf\noQPTlixh3vjxODk6MnvFCkIDA4lLTiY1LY2pS5bwSu/e1m6yKOLsYVaQhLUoVCbA19OTt/r3JzE1\nlZcmTyZvl+f6UEe/9u1ZvnWrBLXIH0U/qyWsReFSgJoVKgDm0HYrVozo+H8ukCHjY6Ig2MPxCzv4\neyOKEhNw/Px5AK4mJZGdk0OAjw/xycmYTCbOXLr0z3NlNocQN0jPWhQqBUhJS2PMjBlkZmUx5qmn\niIqL481vviHQzw8vd/cbzw0tWZKPf/yRN5991noNFnbBHnrWEtaiUD1Sty56g4G45GSKe3lRKyyM\nWmFhdGxsXgzqnTlzaJ87p3rKiBHWbKqwJ0U/q60T1oqjrJr2sEpMTuKPnTtpVK0q6JRb3gvvDXnR\nSi0T9kzOYBTiPi1Yu5bImBi0CxdoULUKW/YfICUtjYFdu9CkRg16jhvHsk8+4bctW/lz1y50ioIa\nGsqIJ2VWiHgAMgwixP15pkMHzkVF07BqFa4mJfNav6c5eOo0i9avp0mNGii531fX7dzJqKf6ooaE\nsGLrNgxGIw52MFdWWIcdZLWEtbCeSmXKAFDcy5Os7Oybyt549hkWb9hIdHw81cqVu7FolBD/hT0c\nYJSuiihUOkWH0WQ037nL52d1+N+8+lRfvnp1FKcvXeTYuXOF00Bhn3SK5TcbJT1rUah8PD3Q6w1k\n5+hv6u1c//36z3KlS/PKlC9xLeZCgI8vVcqWtUZzhZ2wh561VRZyitq4Qb7TCiEsUrrtow+ctBd+\nW2Vx5oR272KTyS49a1Fo9AYDn87/kej4eIxGI73atGHFtm1ULBPMuaho0jIzmDRoEIHFi7Ns02bz\nutiKQpt69ej5SGtrN18UYTJ1T4j7sHLbNny9PHl7wPNkZGYy+ONPcHZyokrZsrzUqxdzVqzkr717\naVKjBpv372f6a2Mwmky8Pn06DapWoUxgoLV3QRRREtZC3IfIK1eop1YGwLVYMUJLlWTfiZOE5c4K\nCfD1ITElhfPR0VxJSGD0V1MBuJaRweWrVyWsxX9nB2PWEtai0ISULMnhiDM0r12L9MxMzkdFU8rf\n/5ZJIWVKBFK2VCk+ffklABZv3EiFoKDCb7CwG/ZwgFHCWhSars2b88XChYycPIWsnBz6d+rEup07\nb36SolAhOIi6qsqIyZPJztFTtVzZG9dZFOJhZZXZIMdmL5LZIEIIi1R7se8Dd4svrVlrceYEd+xg\nk91w6VmLQlG+R3su/LEZQ3YOlZ/ryfmVG8mMT6J8z/ZcuxiDa0BxHFycyUxIImrLblwD/SnZuDYm\noxGj3sCl9eEY9Xpr74YoouQAoxAWSj1/GY8ypchJyyA75RruQSUxGozkpKZhyMriwh+bAajQuyOO\nbsXwKhtEckQkCUdP4RkahM7FWcJa/GeKHawrI2EtCkXK+YsE1KlG9rU0Yvccxq96JRQFks9cwLWE\nH0FtmmDM0aNzckRRdFw9cJyAOlUJ7fwI+vQMMmLj770RIexY0f9zI4qErMQUnLw8cA3w49rFaHRO\nTniGBmMyGnHycOPyXzuI3XMYnYMDKAo+FcuSdOocF1ZvIisxGd8qFay9C6Iok7VBhLBcWtQVnD3N\nl+1Ki47FxceL9Nh4/OtUM/egMzLJiI3H0b0YGbHxlG7ZEGOOHkwmorbtsXLrRVGWX1P3VFXVATOB\nmkAWMEjTtIg85V2BCYAemKtp2vcW1PkSOKlp2rd327aEtSg0sbsP//P7nn9+P/fb+ts+/9zvG5i+\ndjnNK9egTtmwAm+fsGP512HuDjhrmtZUVdVGwOTcx1BV1QmYAtQH0oFwVVVXAM0Bl3/XUVU1AJgP\nVARO3GvDVglrRxe5rJewjM5BwcFJJ+8Z8UDy8aSYZsBaAE3TdqmqWj9PWRXgjKZpyQCqqm4HWgJN\ngDW3qeMOvAN0xII/J9KzFgXucnwcU1cvw1Gnw2gy8VidBuw9o/F69z4A9J/6CfNfGUdUQhzTV/+G\nwWjAxcnpRvna/XtYtmMb6VmZDOvwOBVLB1tzd8TDzQtIyXPfoKqqTtM0Y25Zcp6yVMD7LnXOA+dV\nVe1oyYYlrEWBO3Q+ArV0GZ5v055jkRe4GBd7U/n1LsXcjWt5slkr6pSvyO7TJ4m4Eg1AWKnSPNms\nNRsP72fj4QMS1uK+KQ75NpciBfDMc/96UIM5qPOWeQJJ96hjMZkNIgpcu1r1cHdxYdKieazet/OW\nayleP7UsKiEeNSgEgIYVK1OnnHmcOqyUeV0QH3cPsvQ3X/5LCIsoiuW3uwsHOgGoqtoYOJyn7CRQ\nUVVVX1VVnTEPgfx9jzoWk561KHC7Tp2gapmy9G3Rhi3HDvHnwb1cX+YgNjmRaxkZAAT7BXA66hK1\nylVgy7FDNx4X4kHl45j1cqCdqqrhufcHqKr6FOChadp3qqqOBtZh7gjP0TQtWlXVW+rc5nXveTq8\nhLUocGGlgvhq5VIWh2/GZDLxfNsOLA7fzGs/fEMZ/wACfXwBGND2MWau+Z3F4ZtxcXZmdNdenIm5\nfGOYxPyBs915sML+aZpmAob96+FTecpXAassqJO3/F1Ltm2VhZwifl4uCzkJISxS4akeD/wXOnbH\nNoszp0STFjbZI5CetbC69MxMpq5cRlpmBvGpKXRp0ITODRpbu1nCjsh61kLkg+jEeFpVr0nTKtWJ\nT0lh7A+zJaxF/pKwFuLBebt78NvOcMJPHMPNxQWD0WDtJgk7I0ukCpEPlu/YRuXgEDo3aMyhcxHs\nOXXS2k0S9kZ61kI8uEaVqjBrzQp2nDxGSEAgri4u6A0GHB3kFHORP2TM+j/a9Mdpa2xW2LAny3Yx\n/2KEimVVtq07i87RgbBmVTm15Yh1GyesqsJT+fAidhDWcgajsFluPu6orWpauxnCDig6xeKbrZJh\nEFEgur3bn7WfLyE7I4tnZ41g1Qc/kxAZS/f3n+PS4bP4lytJMQ9X4iNj2fb9WgIrBtHo6Ucw6A3o\ns3PYOO13aj/eBN8gf2p3a8KxdftoObgjLh7FANjx40YSL8VZeS+FKDwS1qJAXNh/mjI1y5GWeI3U\n2CSCqpfFqNeTejWJrLRM1n62BBR44uOBuPl4EFovjIidJzi2bh8hdcNwcS/Gwd934Bvsz8Hfd9Cg\nTysuH7vAyb8O4hXoS8vBHVj1wc/W3k1RVNjBMIiEtSgQ5/eeok63plyLS2bvkm1Ua18PRYGIHSco\nUaE0rYd1QZ+Vg5OLM4qDwsEVO6n9eBM6vdmHtIRrXI2IxsHpnwOMxcv4U6pKCOUbVQbA2b2YtXZN\nFEH2cMHcor8HwiYlXY7Hs4Q3/uVLcfHQWZxcnQmtVxGj3oi7nyebZ61i75KtODg7oigKYU2rcmrb\nEf74+BeSouKo/EgtTEbTjTHEpMsJHF27lz8+XsTmb1ZxetsxK++hKFLkGoxC3Fn08Ug8ArzNv5+I\nxKe0H1cjoqjdzdyDTk9K42pENG6+Hlw9G02LFzqgc9DhE+TPqS2HyUhJR+fgQP0nW3JwxQ5aDOpA\n5Udq4ezqzL5l4ffYuhD2xSoLObWs3E0WchK39dzwPqQmp7Js4R/WboqwEVtP/v7A3d2k4wctzhyf\nqrVtsnstPWthNQ6ODrz50UhKBQei0+nYtmEnnXq2JSdHT2xMPNs37rJ2E4W9kAOMQvx33fo8RmJ8\nEh+88SWubsX4ftkU/t68h7OnLkhQi3xlD2cwygFGYTUh5YM5tPc4ABnpmVyIuERQmZJ28cESNsYO\nDjBKWAuruRBxiVr1qwLg6u5KuYohRF+KvUctIe6foigW32yVhLWwmhWL1+Hl48n0BR8xdd4H/G/G\nIhITkrDGQW9h5/LvgrlWY5Ux66SMJGtsVtigsaPes3YTxMNAKfr9UjnAKArd47060PrRpji7OBNQ\nwo+Fc3+ldfvmhFUqx5QPZ9K5R3tef2kSAPOWzmD00InEX02wbqNFkWbLCzRZSsJaWIWrazGGPfcG\nj3V5hGdf6M0zPYZTv3Ftnn2hNyHlgvH08qBEoD+J8UkS1EIgYS2swWTi5PEzAFxLTePsmUgAUlOu\n4eTsxOrl6+nU7VGCypRk2aLV1mypsBc2PBZtqaI/kCOKpLsdRPx9yRradW5N3QY12bZpZyG2Stgr\nRedg8c1WSc9aWEduVptMJkz8E9wmk4mrsfGkpaZxaN9RmRki8oU9jFlLz1oUPkXBy9uTN997hb+3\n7mHLhr9ZsWkBdRrU4KXnx954mgyBCPEPq/Ssn6nTxhqbFTaiWkgV/Fz92LpgI8/UaUOHJx9n30/b\ncDyczPON2vP0RwO4cPgcj/hXAf8q1m6usAd2MGYtwyDCKrxLeNPvk4HsXLqd8nXDKFmhFOkp6Xj5\ne2PQGyhdOZiWz7Rl64KN1m6qsAO2fGaipSSshVVF7DnFucZnOLHtKIlR8XR8+XHmjZmNIcdAp1e6\nE1qzHBcOn7N2M0VRJyfFCJFfFHxKFcfVy53eE/sB4Ozqgk9JXwlr8eDs4ACjhLWwESaSYxJJjU/m\nl3d+xGQ0UaNtbaLPRFm7YcIOyDCIEPfBwdGBaq1rUr5+RTz9vDAajDeVZ6RmsPf3nTz14fPodDqS\nryRyascJajxahyMbDlip1cIuyDDIf1Paz8MamxVW5u7nRb2O9Yk6ep7EU5c4veUwpf08OPTTX4D5\nfZF07Bybj/0z7BES7Ef9jvWJP3DaWs0WdkB61kLch+pdGuFdyg+/ciWJPnqekPqVcPFw5dDycC4f\nPkulNrUpUzcMR2cnsq5lsOXrFTfqVO/SiKOr5Oox4j+SnrUQlju6aic+Qf5EHT2Pm48Hu+avp0Sl\nYKp1bMDlw2dxcS/Gxi9+BaDNqJ74lS15o44EtXjYSViLQvTPV9GEC1cAyExJx8HZ/DY0Gow0H9IZ\nfWYObsU90TkoN9UR4r+S082FuA8mo+mOY4c+Qf4E167A9m9Xs+fnv3LPOFMwme5cRwiLyZVihLBc\nZmo6OkcHHBwdbl6gyQSpsUnos3NoN7YPWakZJF64gpuPB3Fno9E5OlC7Z3MOLttuvcaLIs2WV9Oz\nlCKrmgkhhO2TYRAhhCgCJKyFEKIIkLAWQogiQMJaCCGKAAlrIYQoAiSshRCiCJCwFkKIIkDCWggh\nigAJayGEKAIkrIUQogiQsBZCiCJAwloIIYoACWuR71RVnaGq6nPWbocQ9kTCWhQEWcpRiHwm61mL\nfKGq6hdAV+AKkA3sVVX1nKZp5XLLJwEmTdPeVVU1BlgBtACigZnASCAYeF7TtK2qqm4G9gOPAq7A\nCOAVoCrwJTAVOAe00zTttKqq7sAJIEzTtOzC2WshCo/0rMUDU1X1CaA+5iDtBoTd5mkm/ulxlwBW\nappWJfd+d03TWgKTgFF5n69pWk3gR2A60ANzwE/UNM0E/AA8k/v8J3JfU4Ja2CUJa5EfWgO/appm\n0DQtEfjNgjprcn9eAP7K/T0S8L3NcyKBnZqmZWqaFgn45D7+A/B07u/P5d4Xwi5JWIv8YOLm95Ie\nCOXmq906562gaZo+z13DHV43by9Z/+9CTdPOAxdUVe0JlNA0bc99tFmIIkXCWuSH9UBfVVWdVVX1\nAroASYCvqqr+qqq6AB0KaNtzMY9fzy+g1xfCJkhYiwemadpKzIF9FPPQxUkgGfgc2JNbtjNPlX/P\nFjHdpez6Y3d6znKgOOZxbSHsllwwVxRZqqoqQEfgRU3Tulu7PUIUJJm6J4qyL4HOmANbCLsmPWsh\nhCgCZMxaCCGKAAlrIYQoAiSshRCiCJCwFkKIIkDCWgghioD/Aw514ij72S0tAAAAAElFTkSuQmCC\n", 316 | "text/plain": [ 317 | "" 318 | ] 319 | }, 320 | "metadata": {}, 321 | "output_type": "display_data" 322 | } 323 | ], 324 | "source": [ 325 | "# For this particular text segments, the attention is focused on \n", 326 | "# a phrase indicating negative sentiment:\n", 327 | "# \"waste of my life\"\n", 328 | "model3.plot_attention(ind=10)" 329 | ] 330 | } 331 | ], 332 | "metadata": { 333 | "kernelspec": { 334 | "display_name": "Python 2", 335 | "language": "python", 336 | "name": "python2" 337 | }, 338 | "language_info": { 339 | "codemirror_mode": { 340 | "name": "ipython", 341 | "version": 2 342 | }, 343 | "file_extension": ".py", 344 | "mimetype": "text/x-python", 345 | "name": "python", 346 | "nbconvert_exporter": "python", 347 | "pygments_lexer": "ipython2", 348 | "version": "2.7.11" 349 | } 350 | }, 351 | "nbformat": 4, 352 | "nbformat_minor": 0 353 | } 354 | --------------------------------------------------------------------------------