├── model.pth
├── README.md
├── LICENSE
├── utils.py
├── usad.py
├── gdrivedl.py
└── USAD.ipynb


/model.pth:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/manigalati/usad/HEAD/model.pth


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # USAD - UnSupervised Anomaly Detection on multivariate time series
 2 | 
 3 | Scripts and utility programs for implementing the USAD architecture.
 4 | 
 5 | Implementation by: Francesco Galati.
 6 | 
 7 | Additional contributions: Julien Audibert, Maria A. Zuluaga.
 8 | 
 9 | ## How to cite
10 | 
11 | If you use this software, please cite the following paper as appropriate:
12 | 
13 |     Audibert, J., Michiardi, P., Guyard, F., Marti, S., Zuluaga, M. A. (2020).
14 |     USAD : UnSupervised Anomaly Detection on multivariate time series.
15 |     Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining, August 23-27, 2020
16 | 
17 | ## Requirements
18 |  * PyTorch 1.6.0
19 |  * CUDA 10.1 (to allow use of GPU, not compulsory)
20 | 
21 | ## Running the Software
22 | 
23 | All the python classes and functions strictly needed to implement the USAD architecture can be found in `usad.py`.
24 | An example of an application deployed with the [SWaT dataset] is included in `USAD.ipynb`.
25 | 
26 | ## Copyright and licensing
27 | 
28 | Copyright 2020 Eurecom.
29 | 
30 | This software is released under the BSD-3 license. Please see the license file_ for details.
31 | 
32 | ## Publication
33 | 
34 | Audibert et al. [USAD : UnSupervised Anomaly Detection on multivariate time series]. 2020
35 | 
36 | [SWaT dataset]: https://itrust.sutd.edu.sg/itrust-labs_datasets/dataset_info/#swat
37 | [USAD : UnSupervised Anomaly Detection on multivariate time series]: https://dl.acm.org/doi/pdf/10.1145/3394486.3403392
38 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | 
 2 | BSD License
 3 | 
 4 | Copyright (c) 2020, EURECOM
 5 | All rights reserved.
 6 | 
 7 | Redistribution and use in source and binary forms, with or without modification,
 8 | are permitted provided that the following conditions are met:
 9 | 
10 | * Redistributions of source code must retain the above copyright notice, this
11 |   list of conditions and the following disclaimer.
12 | 
13 | * Redistributions in binary form must reproduce the above copyright notice, this
14 |   list of conditions and the following disclaimer in the documentation and/or
15 |   other materials provided with the distribution.
16 | 
17 | * Neither the name of the copyright holder nor the names of its
18 |   contributors may be used to endorse or promote products derived from this
19 |   software without specific prior written permission.
20 | 
21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
22 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
24 | IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
25 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
26 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
28 | OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
29 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
30 | OF THE POSSIBILITY OF SUCH DAMAGE.
31 | 
32 | 
33 | 


--------------------------------------------------------------------------------
/utils.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | import pandas as pd
 3 | import matplotlib.pyplot as plt
 4 | import seaborn as sns
 5 | import torch
 6 | 
 7 | from sklearn.metrics import roc_curve,roc_auc_score
 8 | 
 9 | def get_default_device():
10 |     """Pick GPU if available, else CPU"""
11 |     if torch.cuda.is_available():
12 |         return torch.device('cuda')
13 |     else:
14 |         return torch.device('cpu')
15 | 
16 | def to_device(data, device):
17 |     """Move tensor(s) to chosen device"""
18 |     if isinstance(data, (list,tuple)):
19 |         return [to_device(x, device) for x in data]
20 |     return data.to(device, non_blocking=True)
21 |     
22 | def plot_history(history):
23 |     losses1 = [x['val_loss1'] for x in history]
24 |     losses2 = [x['val_loss2'] for x in history]
25 |     plt.plot(losses1, '-x', label="loss1")
26 |     plt.plot(losses2, '-x', label="loss2")
27 |     plt.xlabel('epoch')
28 |     plt.ylabel('loss')
29 |     plt.legend()
30 |     plt.title('Losses vs. No. of epochs')
31 |     plt.grid()
32 |     plt.show()
33 |     
34 | def histogram(y_test,y_pred):
35 |     plt.figure(figsize=(12,6))
36 |     plt.hist([y_pred[y_test==0],
37 |               y_pred[y_test==1]],
38 |             bins=20,
39 |             color = ['#82E0AA','#EC7063'],stacked=True)
40 |     plt.title("Results",size=20)
41 |     plt.grid()
42 |     plt.show()
43 |     
44 | def ROC(y_test,y_pred):
45 |     fpr,tpr,tr=roc_curve(y_test,y_pred)
46 |     auc=roc_auc_score(y_test,y_pred)
47 |     idx=np.argwhere(np.diff(np.sign(tpr-(1-fpr)))).flatten()
48 | 
49 |     plt.xlabel("FPR")
50 |     plt.ylabel("TPR")
51 |     plt.plot(fpr,tpr,label="AUC="+str(auc))
52 |     plt.plot(fpr,1-fpr,'r:')
53 |     plt.plot(fpr[idx],tpr[idx], 'ro')
54 |     plt.legend(loc=4)
55 |     plt.grid()
56 |     plt.show()
57 |     return tr[idx]
58 |     
59 | def confusion_matrix(target, predicted, perc=False):
60 | 
61 |     data = {'y_Actual':    target,
62 |             'y_Predicted': predicted
63 |             }
64 |     df = pd.DataFrame(data, columns=['y_Predicted','y_Actual'])
65 |     confusion_matrix = pd.crosstab(df['y_Predicted'], df['y_Actual'], rownames=['Predicted'], colnames=['Actual'])
66 |     
67 |     if perc:
68 |         sns.heatmap(confusion_matrix/np.sum(confusion_matrix), annot=True, fmt='.2%', cmap='Blues')
69 |     else:
70 |         sns.heatmap(confusion_matrix, annot=True, fmt='d')
71 |     plt.show()


--------------------------------------------------------------------------------
/usad.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | import torch.nn as nn
  3 | 
  4 | from utils import *
  5 | device = get_default_device()
  6 | 
  7 | class Encoder(nn.Module):
  8 |   def __init__(self, in_size, latent_size):
  9 |     super().__init__()
 10 |     self.linear1 = nn.Linear(in_size, int(in_size/2))
 11 |     self.linear2 = nn.Linear(int(in_size/2), int(in_size/4))
 12 |     self.linear3 = nn.Linear(int(in_size/4), latent_size)
 13 |     self.relu = nn.ReLU(True)
 14 |         
 15 |   def forward(self, w):
 16 |     out = self.linear1(w)
 17 |     out = self.relu(out)
 18 |     out = self.linear2(out)
 19 |     out = self.relu(out)
 20 |     out = self.linear3(out)
 21 |     z = self.relu(out)
 22 |     return z
 23 |     
 24 | class Decoder(nn.Module):
 25 |   def __init__(self, latent_size, out_size):
 26 |     super().__init__()
 27 |     self.linear1 = nn.Linear(latent_size, int(out_size/4))
 28 |     self.linear2 = nn.Linear(int(out_size/4), int(out_size/2))
 29 |     self.linear3 = nn.Linear(int(out_size/2), out_size)
 30 |     self.relu = nn.ReLU(True)
 31 |     self.sigmoid = nn.Sigmoid()
 32 |         
 33 |   def forward(self, z):
 34 |     out = self.linear1(z)
 35 |     out = self.relu(out)
 36 |     out = self.linear2(out)
 37 |     out = self.relu(out)
 38 |     out = self.linear3(out)
 39 |     w = self.sigmoid(out)
 40 |     return w
 41 |     
 42 | class UsadModel(nn.Module):
 43 |   def __init__(self, w_size, z_size):
 44 |     super().__init__()
 45 |     self.encoder = Encoder(w_size, z_size)
 46 |     self.decoder1 = Decoder(z_size, w_size)
 47 |     self.decoder2 = Decoder(z_size, w_size)
 48 |   
 49 |   def training_step(self, batch, n):
 50 |     z = self.encoder(batch)
 51 |     w1 = self.decoder1(z)
 52 |     w2 = self.decoder2(z)
 53 |     w3 = self.decoder2(self.encoder(w1))
 54 |     loss1 = 1/n*torch.mean((batch-w1)**2)+(1-1/n)*torch.mean((batch-w3)**2)
 55 |     loss2 = 1/n*torch.mean((batch-w2)**2)-(1-1/n)*torch.mean((batch-w3)**2)
 56 |     return loss1,loss2
 57 | 
 58 |   def validation_step(self, batch, n):
 59 |     with torch.no_grad():
 60 |         z = self.encoder(batch)
 61 |         w1 = self.decoder1(z)
 62 |         w2 = self.decoder2(z)
 63 |         w3 = self.decoder2(self.encoder(w1))
 64 |         loss1 = 1/n*torch.mean((batch-w1)**2)+(1-1/n)*torch.mean((batch-w3)**2)
 65 |         loss2 = 1/n*torch.mean((batch-w2)**2)-(1-1/n)*torch.mean((batch-w3)**2)
 66 |     return {'val_loss1': loss1, 'val_loss2': loss2}
 67 |         
 68 |   def validation_epoch_end(self, outputs):
 69 |     batch_losses1 = [x['val_loss1'] for x in outputs]
 70 |     epoch_loss1 = torch.stack(batch_losses1).mean()
 71 |     batch_losses2 = [x['val_loss2'] for x in outputs]
 72 |     epoch_loss2 = torch.stack(batch_losses2).mean()
 73 |     return {'val_loss1': epoch_loss1.item(), 'val_loss2': epoch_loss2.item()}
 74 |     
 75 |   def epoch_end(self, epoch, result):
 76 |     print("Epoch [{}], val_loss1: {:.4f}, val_loss2: {:.4f}".format(epoch, result['val_loss1'], result['val_loss2']))
 77 |     
 78 | def evaluate(model, val_loader, n):
 79 |     outputs = [model.validation_step(to_device(batch,device), n) for [batch] in val_loader]
 80 |     return model.validation_epoch_end(outputs)
 81 | 
 82 | def training(epochs, model, train_loader, val_loader, opt_func=torch.optim.Adam):
 83 |     history = []
 84 |     optimizer1 = opt_func(list(model.encoder.parameters())+list(model.decoder1.parameters()))
 85 |     optimizer2 = opt_func(list(model.encoder.parameters())+list(model.decoder2.parameters()))
 86 |     for epoch in range(epochs):
 87 |         for [batch] in train_loader:
 88 |             batch=to_device(batch,device)
 89 |             
 90 |             #Train AE1
 91 |             loss1,loss2 = model.training_step(batch,epoch+1)
 92 |             loss1.backward()
 93 |             optimizer1.step()
 94 |             optimizer1.zero_grad()
 95 |             
 96 |             
 97 |             #Train AE2
 98 |             loss1,loss2 = model.training_step(batch,epoch+1)
 99 |             loss2.backward()
100 |             optimizer2.step()
101 |             optimizer2.zero_grad()
102 |             
103 |             
104 |         result = evaluate(model, val_loader, epoch+1)
105 |         model.epoch_end(epoch, result)
106 |         history.append(result)
107 |     return history
108 |     
109 | def testing(model, test_loader, alpha=.5, beta=.5):
110 |     results=[]
111 |     with torch.no_grad():
112 |         for [batch] in test_loader:
113 |             batch=to_device(batch,device)
114 |             w1=model.decoder1(model.encoder(batch))
115 |             w2=model.decoder2(model.encoder(w1))
116 |             results.append(alpha*torch.mean((batch-w1)**2,axis=1)+beta*torch.mean((batch-w2)**2,axis=1))
117 |     return results


--------------------------------------------------------------------------------
/gdrivedl.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | from __future__ import unicode_literals
  3 | import json
  4 | import os
  5 | import re
  6 | import sys
  7 | import unicodedata
  8 | 
  9 | try:
 10 |     #Python3
 11 |     from urllib.request import Request, urlopen
 12 | except ImportError:
 13 |     #Python2
 14 |     from urllib2 import Request, urlopen
 15 | 
 16 | ITEM_URL = 'https://drive.google.com/open?id={id}'
 17 | FILE_URL = 'https://docs.google.com/uc?export=download&id={id}&confirm={confirm}'
 18 | FOLDER_URL = 'https://drive.google.com/drive/folders/{id}'
 19 | 
 20 | ID_PATTERNS = [
 21 |     re.compile('/file/d/([0-9A-Za-z_-]{10,})(?:/|$)', re.IGNORECASE),
 22 |     re.compile('id=([0-9A-Za-z_-]{10,})(?:&|$)', re.IGNORECASE),
 23 |     re.compile('([0-9A-Za-z_-]{10,})', re.IGNORECASE)
 24 | ]
 25 | FILE_PATTERN = re.compile("itemJson: (\[.*?)};</script>",
 26 |                           re.DOTALL | re.IGNORECASE)
 27 | FOLDER_PATTERN = re.compile("window\['_DRIVE_ivd'\] = '(.*?)';",
 28 |                             re.DOTALL | re.IGNORECASE)
 29 | CONFIRM_PATTERN = re.compile("download_warning[0-9A-Za-z_-]+=([0-9A-Za-z_-]+);",
 30 |                              re.IGNORECASE)
 31 | FOLDER_TYPE = 'application/vnd.google-apps.folder'
 32 | 
 33 | def output(text):
 34 |     try:
 35 |         sys.stdout.write(text)
 36 |     except UnicodeEncodeError:
 37 |         sys.stdout.write(text.encode('utf8'))
 38 | 
 39 | # Big thanks to leo_wallentin for below sanitize function (modified slightly for this script)
 40 | # https://gitlab.com/jplusplus/sanitize-filename/-/blob/master/sanitize_filename/sanitize_filename.py
 41 | def sanitize(filename):
 42 |     blacklist = ["\\", "/", ":", "*", "?", "\"", "<", ">", "|", "\0"]
 43 |     reserved = [
 44 |         "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5",
 45 |         "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5",
 46 |         "LPT6", "LPT7", "LPT8", "LPT9",
 47 |     ]
 48 | 
 49 |     filename = "".join(c for c in filename if c not in blacklist)
 50 |     filename = "".join(c for c in filename if 31 < ord(c))
 51 |     filename = unicodedata.normalize("NFKD", filename)
 52 |     filename = filename.rstrip(". ")
 53 |     filename = filename.strip()
 54 | 
 55 |     if all([x == "." for x in filename]):
 56 |         filename = "_" + filename
 57 |     if filename in reserved:
 58 |         filename = "_" + filename
 59 |     if len(filename) == 0:
 60 |         filename = "_"
 61 |     if len(filename) > 255:
 62 |         parts = re.split(r"/|\\", filename)[-1].split(".")
 63 |         if len(parts) > 1:
 64 |             ext = "." + parts.pop()
 65 |             filename = filename[:-len(ext)]
 66 |         else:
 67 |             ext = ""
 68 |         if filename == "":
 69 |             filename = "_"
 70 |         if len(ext) > 254:
 71 |             ext = ext[254:]
 72 |         maxl = 255 - len(ext)
 73 |         filename = filename[:maxl]
 74 |         filename = filename + ext
 75 |         filename = filename.rstrip(". ")
 76 |         if len(filename) == 0:
 77 |             filename = "_"
 78 | 
 79 |     return filename
 80 | 
 81 | 
 82 | def process_item(id, directory):
 83 |     url = ITEM_URL.format(id=id)
 84 |     resp = urlopen(url)
 85 |     url = resp.geturl()
 86 |     html = resp.read().decode('utf-8')
 87 | 
 88 |     if '/file/' in url:
 89 |         match = FILE_PATTERN.search(html)
 90 |         data = match.group(1).replace('\/', '/')
 91 |         data = data.replace(r'\x5b', '[').replace(r'\x22', '"').replace(r'\x5d', ']').replace(r'\n','')
 92 |         data = json.loads(data)
 93 | 
 94 |         file_name = sanitize(data[1])
 95 |         file_size = int(data[25][2])
 96 |         file_path = os.path.join(directory, file_name)
 97 | 
 98 |         process_file(id, file_path, file_size)
 99 |     elif '/folders/' in url:
100 |         process_folder(id, directory, html=html)
101 |     elif 'ServiceLogin' in url:
102 |         sys.stderr.write('Id {} does not have link sharing enabled'.format(id))
103 |         sys.exit(1)
104 |     else:
105 |         sys.stderr.write('That id {} returned an unknown url'.format(id))
106 |         sys.exit(1)
107 | 
108 | 
109 | def process_folder(id, directory, html=None):
110 |     if not html:
111 |         url = FOLDER_URL.format(id=id)
112 |         html = urlopen(url).read().decode('utf-8')
113 | 
114 |     match = FOLDER_PATTERN.search(html)
115 |     data = match.group(1).replace('\/', '/')
116 |     data = data.replace(r'\x5b', '[').replace(r'\x22', '"').replace(r'\x5d', ']').replace(r'\n','')
117 |     data = json.loads(data)
118 | 
119 |     if not os.path.exists(directory):
120 |         os.mkdir(directory)
121 |         output('Directory: {directory} [Created]\n'.format(directory=directory))
122 |     else:
123 |         output('Directory: {directory} [Exists]\n'.format(directory=directory))
124 | 
125 |     if not data[0]:
126 |         return
127 | 
128 |     for item in sorted(data[0], key=lambda i: i[3] == FOLDER_TYPE):
129 |         item_id = item[0]
130 |         item_name = sanitize(item[2])
131 |         item_type = item[3]
132 |         item_size = item[13]
133 |         item_path = os.path.join(directory, item_name)
134 | 
135 |         if item_type == FOLDER_TYPE:
136 |             process_folder(item_id, item_path)
137 |         else:
138 |             process_file(item_id, item_path, int(item_size))
139 | 
140 | 
141 | def process_file(id, file_path, file_size, confirm='', cookies=''):
142 |     if os.path.exists(file_path):
143 |         output('{file_path} [Exists]\n'.format(file_path=file_path))
144 |         return
145 | 
146 |     url = FILE_URL.format(id=id, confirm=confirm)
147 |     req = Request(url, headers={'Cookie': cookies,
148 |                                 'User-Agent': 'Mozilla/5.0'})
149 |     resp = urlopen(req)
150 |     cookies = resp.headers.get('Set-Cookie') or ''
151 | 
152 |     if not confirm and 'download_warning' in cookies:
153 |         confirm = CONFIRM_PATTERN.search(cookies)
154 |         return process_file(id, file_path, file_size, confirm.group(1), cookies)
155 | 
156 |     output(file_path + '\n')
157 | 
158 |     try:
159 |         with open(file_path, 'wb') as f:
160 |             dl = 0
161 |             while True:
162 |                 chunk = resp.read(4096)
163 |                 if not chunk:
164 |                     break
165 | 
166 |                 if b'Too many users have viewed or downloaded this file recently' in chunk:
167 |                     raise Exception('Quota exceeded for this file')
168 | 
169 |                 dl += len(chunk)
170 |                 f.write(chunk)
171 |                 done = int(50 * dl / file_size)
172 |                 output("\r[{}{}] {:.2f}MB/{:.2f}MB".format(
173 |                     '=' * done,
174 |                     ' ' *
175 |                     (50 - done),
176 |                     dl / 1024 / 1024,
177 |                     file_size / 1024 / 1024
178 |                 ))
179 |                 sys.stdout.flush()
180 |     except:
181 |         if os.path.exists(file_path):
182 |             os.remove(file_path)
183 |         raise
184 | 
185 |     output('\n')
186 | 
187 | 
188 | def get_arg(pos, default=None):
189 |     try:
190 |         return sys.argv[pos]
191 |     except IndexError:
192 |         return default
193 | 
194 | 
195 | if __name__ == '__main__':
196 |     url = get_arg(1, '').strip()
197 |     directory = get_arg(2, './').strip()
198 |     id = ''
199 | 
200 |     if not url:
201 |         sys.stderr.write('A Google Drive URL is required')
202 |         sys.exit(1)
203 | 
204 |     for pattern in ID_PATTERNS:
205 |         match = pattern.search(url)
206 |         if match:
207 |             id = match.group(1)
208 |             break
209 | 
210 |     if not id:
211 |         sys.stderr.write('Unable to get ID from {}'.format(url))
212 |         sys.exit(1)
213 | 
214 |     process_item(id, directory)
215 | 


--------------------------------------------------------------------------------
/USAD.ipynb:
--------------------------------------------------------------------------------
   1 | {
   2 |  "cells": [
   3 |   {
   4 |    "cell_type": "markdown",
   5 |    "metadata": {
   6 |     "id": "etniX_KTlJ5U"
   7 |    },
   8 |    "source": [
   9 |     "# USAD"
  10 |    ]
  11 |   },
  12 |   {
  13 |    "cell_type": "markdown",
  14 |    "metadata": {
  15 |     "id": "N3jM0qLU8MgZ"
  16 |    },
  17 |    "source": [
  18 |     "## Environment"
  19 |    ]
  20 |   },
  21 |   {
  22 |    "cell_type": "code",
  23 |    "execution_count": 1,
  24 |    "metadata": {
  25 |     "id": "rjheCL2b1Rnw"
  26 |    },
  27 |    "outputs": [
  28 |     {
  29 |      "name": "stdout",
  30 |      "output_type": "stream",
  31 |      "text": [
  32 |       "rm: cannot remove 'sample_data': No such file or directory\r\n"
  33 |      ]
  34 |     }
  35 |    ],
  36 |    "source": [
  37 |     "!rm -r sample_data"
  38 |    ]
  39 |   },
  40 |   {
  41 |    "cell_type": "code",
  42 |    "execution_count": 2,
  43 |    "metadata": {
  44 |     "colab": {
  45 |      "base_uri": "https://localhost:8080/",
  46 |      "height": 118
  47 |     },
  48 |     "id": "e3dDxs8LFZdT",
  49 |     "outputId": "ebff804d-1c59-4039-d869-f65907b19712"
  50 |    },
  51 |    "outputs": [],
  52 |    "source": [
  53 |     "!git clone https://github.com/manigalati/usad"
  54 |    ]
  55 |   },
  56 |   {
  57 |    "cell_type": "code",
  58 |    "execution_count": 3,
  59 |    "metadata": {
  60 |     "colab": {
  61 |      "base_uri": "https://localhost:8080/",
  62 |      "height": 34
  63 |     },
  64 |     "id": "te9stFZtFfZu",
  65 |     "outputId": "3ca36b3b-dd9a-413c-873f-ab730285ad51"
  66 |    },
  67 |    "outputs": [],
  68 |    "source": [
  69 |     "%cd usad"
  70 |    ]
  71 |   },
  72 |   {
  73 |    "cell_type": "code",
  74 |    "execution_count": 4,
  75 |    "metadata": {
  76 |     "id": "6u1DGKsAlLF-"
  77 |    },
  78 |    "outputs": [],
  79 |    "source": [
  80 |     "import numpy as np\n",
  81 |     "import pandas as pd\n",
  82 |     "import matplotlib.pyplot as plt\n",
  83 |     "import seaborn as sns\n",
  84 |     "import torch\n",
  85 |     "import torch.nn as nn\n",
  86 |     "\n",
  87 |     "from utils import *\n",
  88 |     "from usad import *"
  89 |    ]
  90 |   },
  91 |   {
  92 |    "cell_type": "code",
  93 |    "execution_count": 5,
  94 |    "metadata": {
  95 |     "colab": {
  96 |      "base_uri": "https://localhost:8080/",
  97 |      "height": 34
  98 |     },
  99 |     "id": "4AzWlDBI_djV",
 100 |     "outputId": "7a8d0c19-2389-461b-c0be-3427a25dda91"
 101 |    },
 102 |    "outputs": [
 103 |     {
 104 |      "name": "stdout",
 105 |      "output_type": "stream",
 106 |      "text": [
 107 |       "GPU 0: Quadro P6000 (UUID: GPU-e16b9553-c966-4659-d528-7376969c0e91)\n",
 108 |       "GPU 1: Quadro P6000 (UUID: GPU-def1ffb6-415d-a3f6-288b-94256f1ba88f)\n",
 109 |       "GPU 2: GeForce GTX 1080 Ti (UUID: GPU-075162a2-c2cc-7757-e07d-e1260458102e)\n",
 110 |       "GPU 3: GeForce GTX 1080 Ti (UUID: GPU-078c9ebd-10e3-2644-2267-bfcf3135c6a1)\n",
 111 |       "GPU 4: GeForce GTX 1080 Ti (UUID: GPU-db4d0970-82a3-4f24-d69f-423377f7d3c0)\n",
 112 |       "GPU 5: GeForce GTX 1080 Ti (UUID: GPU-945cc499-5f5f-ee9f-5b21-69a0e2e06535)\n",
 113 |       "GPU 6: GeForce GTX 1080 Ti (UUID: GPU-07966339-f324-1615-decf-f4825e865008)\n",
 114 |       "GPU 7: GeForce GTX 1080 Ti (UUID: GPU-dfa6e9f2-3de4-2115-3363-4329ff29ba3f)\n"
 115 |      ]
 116 |     }
 117 |    ],
 118 |    "source": [
 119 |     "!nvidia-smi -L\n",
 120 |     "\n",
 121 |     "device = get_default_device()"
 122 |    ]
 123 |   },
 124 |   {
 125 |    "cell_type": "markdown",
 126 |    "metadata": {
 127 |     "id": "1crx5rGP9ONf"
 128 |    },
 129 |    "source": [
 130 |     "## EDA - Data Pre-Processing"
 131 |    ]
 132 |   },
 133 |   {
 134 |    "cell_type": "markdown",
 135 |    "metadata": {
 136 |     "id": "vxofeE469RhT"
 137 |    },
 138 |    "source": [
 139 |     "### Download dataset"
 140 |    ]
 141 |   },
 142 |   {
 143 |    "cell_type": "code",
 144 |    "execution_count": 6,
 145 |    "metadata": {
 146 |     "colab": {
 147 |      "base_uri": "https://localhost:8080/",
 148 |      "height": 84
 149 |     },
 150 |     "id": "i95DlAZI1G_p",
 151 |     "outputId": "5b35771c-356e-4e0b-a997-682d1ea85c6a",
 152 |     "scrolled": false
 153 |    },
 154 |    "outputs": [
 155 |     {
 156 |      "name": "stdout",
 157 |      "output_type": "stream",
 158 |      "text": [
 159 |       "mkdir: cannot create directory 'input': File exists\n",
 160 |       "input/SWaT_Dataset_Normal_v1.csv [Exists]\n",
 161 |       "input/SWaT_Dataset_Attack_v0.csv [Exists]\n"
 162 |      ]
 163 |     }
 164 |    ],
 165 |    "source": [
 166 |     "!mkdir input\n",
 167 |     "#normal period\n",
 168 |     "!python gdrivedl.py https://drive.google.com/open?id=1rVJ5ry5GG-ZZi5yI4x9lICB8VhErXwCw input/\n",
 169 |     "#anomalies\n",
 170 |     "!python gdrivedl.py https://drive.google.com/open?id=1iDYc0OEmidN712fquOBRFjln90SbpaE7 input/"
 171 |    ]
 172 |   },
 173 |   {
 174 |    "cell_type": "markdown",
 175 |    "metadata": {
 176 |     "id": "kfSj4FYL9W8Y"
 177 |    },
 178 |    "source": [
 179 |     "### Normal period"
 180 |    ]
 181 |   },
 182 |   {
 183 |    "cell_type": "code",
 184 |    "execution_count": 7,
 185 |    "metadata": {
 186 |     "colab": {
 187 |      "base_uri": "https://localhost:8080/",
 188 |      "height": 87
 189 |     },
 190 |     "id": "XeDLxV_r1G9n",
 191 |     "outputId": "576538dd-64f2-46fa-8e6f-6c2ffdebad15"
 192 |    },
 193 |    "outputs": [
 194 |     {
 195 |      "name": "stderr",
 196 |      "output_type": "stream",
 197 |      "text": [
 198 |       "/usr/local/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3058: DtypeWarning: Columns (26) have mixed types.Specify dtype option on import or set low_memory=False.\n",
 199 |       "  interactivity=interactivity, compiler=compiler, result=result)\n"
 200 |      ]
 201 |     },
 202 |     {
 203 |      "data": {
 204 |       "text/plain": [
 205 |        "(495000, 51)"
 206 |       ]
 207 |      },
 208 |      "execution_count": 7,
 209 |      "metadata": {},
 210 |      "output_type": "execute_result"
 211 |     }
 212 |    ],
 213 |    "source": [
 214 |     "#Read data\n",
 215 |     "normal = pd.read_csv(\"input/SWaT_Dataset_Normal_v1.csv\")#, nrows=1000)\n",
 216 |     "normal = normal.drop([\"Timestamp\" , \"Normal/Attack\" ] , axis = 1)\n",
 217 |     "normal.shape"
 218 |    ]
 219 |   },
 220 |   {
 221 |    "cell_type": "code",
 222 |    "execution_count": 8,
 223 |    "metadata": {
 224 |     "id": "fFuLm1GH1G2n"
 225 |    },
 226 |    "outputs": [],
 227 |    "source": [
 228 |     "# Transform all columns into float64\n",
 229 |     "for i in list(normal): \n",
 230 |     "    normal[i]=normal[i].apply(lambda x: str(x).replace(\",\" , \".\"))\n",
 231 |     "normal = normal.astype(float)"
 232 |    ]
 233 |   },
 234 |   {
 235 |    "cell_type": "markdown",
 236 |    "metadata": {
 237 |     "id": "zxFNH5kU9hIE"
 238 |    },
 239 |    "source": [
 240 |     "#### Normalization"
 241 |    ]
 242 |   },
 243 |   {
 244 |    "cell_type": "code",
 245 |    "execution_count": 9,
 246 |    "metadata": {
 247 |     "id": "Mfxj4Uxn9kv4"
 248 |    },
 249 |    "outputs": [],
 250 |    "source": [
 251 |     "from sklearn import preprocessing\n",
 252 |     "min_max_scaler = preprocessing.MinMaxScaler()\n",
 253 |     "\n",
 254 |     "x = normal.values\n",
 255 |     "x_scaled = min_max_scaler.fit_transform(x)\n",
 256 |     "normal = pd.DataFrame(x_scaled)"
 257 |    ]
 258 |   },
 259 |   {
 260 |    "cell_type": "code",
 261 |    "execution_count": 10,
 262 |    "metadata": {
 263 |     "colab": {
 264 |      "base_uri": "https://localhost:8080/",
 265 |      "height": 126
 266 |     },
 267 |     "id": "mQ6_U4jn9nlw",
 268 |     "outputId": "f1cc1bd6-f1cc-4764-b1cc-2fd989ac4918"
 269 |    },
 270 |    "outputs": [
 271 |     {
 272 |      "data": {
 273 |       "text/html": [
 274 |        "<div>\n",
 275 |        "<style scoped>\n",
 276 |        "    .dataframe tbody tr th:only-of-type {\n",
 277 |        "        vertical-align: middle;\n",
 278 |        "    }\n",
 279 |        "\n",
 280 |        "    .dataframe tbody tr th {\n",
 281 |        "        vertical-align: top;\n",
 282 |        "    }\n",
 283 |        "\n",
 284 |        "    .dataframe thead th {\n",
 285 |        "        text-align: right;\n",
 286 |        "    }\n",
 287 |        "</style>\n",
 288 |        "<table border=\"1\" class=\"dataframe\">\n",
 289 |        "  <thead>\n",
 290 |        "    <tr style=\"text-align: right;\">\n",
 291 |        "      <th></th>\n",
 292 |        "      <th>0</th>\n",
 293 |        "      <th>1</th>\n",
 294 |        "      <th>2</th>\n",
 295 |        "      <th>3</th>\n",
 296 |        "      <th>4</th>\n",
 297 |        "      <th>5</th>\n",
 298 |        "      <th>6</th>\n",
 299 |        "      <th>7</th>\n",
 300 |        "      <th>8</th>\n",
 301 |        "      <th>9</th>\n",
 302 |        "      <th>...</th>\n",
 303 |        "      <th>41</th>\n",
 304 |        "      <th>42</th>\n",
 305 |        "      <th>43</th>\n",
 306 |        "      <th>44</th>\n",
 307 |        "      <th>45</th>\n",
 308 |        "      <th>46</th>\n",
 309 |        "      <th>47</th>\n",
 310 |        "      <th>48</th>\n",
 311 |        "      <th>49</th>\n",
 312 |        "      <th>50</th>\n",
 313 |        "    </tr>\n",
 314 |        "  </thead>\n",
 315 |        "  <tbody>\n",
 316 |        "    <tr>\n",
 317 |        "      <th>0</th>\n",
 318 |        "      <td>0.0</td>\n",
 319 |        "      <td>0.005294</td>\n",
 320 |        "      <td>0.5</td>\n",
 321 |        "      <td>0.0</td>\n",
 322 |        "      <td>0.0</td>\n",
 323 |        "      <td>0.012291</td>\n",
 324 |        "      <td>0.075099</td>\n",
 325 |        "      <td>0.002009</td>\n",
 326 |        "      <td>0.0</td>\n",
 327 |        "      <td>0.5</td>\n",
 328 |        "      <td>...</td>\n",
 329 |        "      <td>0.0</td>\n",
 330 |        "      <td>0.0</td>\n",
 331 |        "      <td>0.0</td>\n",
 332 |        "      <td>0.000814</td>\n",
 333 |        "      <td>0.0</td>\n",
 334 |        "      <td>0.001217</td>\n",
 335 |        "      <td>0.000147</td>\n",
 336 |        "      <td>0.0</td>\n",
 337 |        "      <td>0.0</td>\n",
 338 |        "      <td>0.0</td>\n",
 339 |        "    </tr>\n",
 340 |        "    <tr>\n",
 341 |        "      <th>1</th>\n",
 342 |        "      <td>0.0</td>\n",
 343 |        "      <td>0.005407</td>\n",
 344 |        "      <td>0.5</td>\n",
 345 |        "      <td>0.0</td>\n",
 346 |        "      <td>0.0</td>\n",
 347 |        "      <td>0.012291</td>\n",
 348 |        "      <td>0.075099</td>\n",
 349 |        "      <td>0.002009</td>\n",
 350 |        "      <td>0.0</td>\n",
 351 |        "      <td>0.5</td>\n",
 352 |        "      <td>...</td>\n",
 353 |        "      <td>0.0</td>\n",
 354 |        "      <td>0.0</td>\n",
 355 |        "      <td>0.0</td>\n",
 356 |        "      <td>0.000814</td>\n",
 357 |        "      <td>0.0</td>\n",
 358 |        "      <td>0.001217</td>\n",
 359 |        "      <td>0.000147</td>\n",
 360 |        "      <td>0.0</td>\n",
 361 |        "      <td>0.0</td>\n",
 362 |        "      <td>0.0</td>\n",
 363 |        "    </tr>\n",
 364 |        "  </tbody>\n",
 365 |        "</table>\n",
 366 |        "<p>2 rows × 51 columns</p>\n",
 367 |        "</div>"
 368 |       ],
 369 |       "text/plain": [
 370 |        "    0         1    2    3    4         5         6         7    8    9   ...  \\\n",
 371 |        "0  0.0  0.005294  0.5  0.0  0.0  0.012291  0.075099  0.002009  0.0  0.5  ...   \n",
 372 |        "1  0.0  0.005407  0.5  0.0  0.0  0.012291  0.075099  0.002009  0.0  0.5  ...   \n",
 373 |        "\n",
 374 |        "    41   42   43        44   45        46        47   48   49   50  \n",
 375 |        "0  0.0  0.0  0.0  0.000814  0.0  0.001217  0.000147  0.0  0.0  0.0  \n",
 376 |        "1  0.0  0.0  0.0  0.000814  0.0  0.001217  0.000147  0.0  0.0  0.0  \n",
 377 |        "\n",
 378 |        "[2 rows x 51 columns]"
 379 |       ]
 380 |      },
 381 |      "execution_count": 10,
 382 |      "metadata": {},
 383 |      "output_type": "execute_result"
 384 |     }
 385 |    ],
 386 |    "source": [
 387 |     "normal.head(2)"
 388 |    ]
 389 |   },
 390 |   {
 391 |    "cell_type": "markdown",
 392 |    "metadata": {
 393 |     "id": "_i71RFAi9spa"
 394 |    },
 395 |    "source": [
 396 |     "### Attack"
 397 |    ]
 398 |   },
 399 |   {
 400 |    "cell_type": "code",
 401 |    "execution_count": 11,
 402 |    "metadata": {
 403 |     "colab": {
 404 |      "base_uri": "https://localhost:8080/",
 405 |      "height": 87
 406 |     },
 407 |     "id": "aN_TFp5x9uTE",
 408 |     "outputId": "38d7993d-c9a3-461d-c430-ebde697afbc6"
 409 |    },
 410 |    "outputs": [
 411 |     {
 412 |      "name": "stderr",
 413 |      "output_type": "stream",
 414 |      "text": [
 415 |       "/usr/local/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3058: DtypeWarning: Columns (1,9,28,46) have mixed types.Specify dtype option on import or set low_memory=False.\n",
 416 |       "  interactivity=interactivity, compiler=compiler, result=result)\n"
 417 |      ]
 418 |     },
 419 |     {
 420 |      "data": {
 421 |       "text/plain": [
 422 |        "(449919, 51)"
 423 |       ]
 424 |      },
 425 |      "execution_count": 11,
 426 |      "metadata": {},
 427 |      "output_type": "execute_result"
 428 |     }
 429 |    ],
 430 |    "source": [
 431 |     "#Read data\n",
 432 |     "attack = pd.read_csv(\"input/SWaT_Dataset_Attack_v0.csv\",sep=\";\")#, nrows=1000)\n",
 433 |     "labels = [ float(label!= 'Normal' ) for label  in attack[\"Normal/Attack\"].values]\n",
 434 |     "attack = attack.drop([\"Timestamp\" , \"Normal/Attack\" ] , axis = 1)\n",
 435 |     "attack.shape"
 436 |    ]
 437 |   },
 438 |   {
 439 |    "cell_type": "code",
 440 |    "execution_count": 12,
 441 |    "metadata": {
 442 |     "id": "qLCInT-I9_-D"
 443 |    },
 444 |    "outputs": [],
 445 |    "source": [
 446 |     "# Transform all columns into float64\n",
 447 |     "for i in list(attack):\n",
 448 |     "    attack[i]=attack[i].apply(lambda x: str(x).replace(\",\" , \".\"))\n",
 449 |     "attack = attack.astype(float)"
 450 |    ]
 451 |   },
 452 |   {
 453 |    "cell_type": "markdown",
 454 |    "metadata": {
 455 |     "id": "c4cB4v3N-Dhu"
 456 |    },
 457 |    "source": [
 458 |     "#### Normalization"
 459 |    ]
 460 |   },
 461 |   {
 462 |    "cell_type": "code",
 463 |    "execution_count": 13,
 464 |    "metadata": {
 465 |     "id": "jZrha9cO-BGK"
 466 |    },
 467 |    "outputs": [],
 468 |    "source": [
 469 |     "from sklearn import preprocessing\n",
 470 |     "\n",
 471 |     "x = attack.values \n",
 472 |     "x_scaled = min_max_scaler.transform(x)\n",
 473 |     "attack = pd.DataFrame(x_scaled)"
 474 |    ]
 475 |   },
 476 |   {
 477 |    "cell_type": "code",
 478 |    "execution_count": 14,
 479 |    "metadata": {
 480 |     "colab": {
 481 |      "base_uri": "https://localhost:8080/",
 482 |      "height": 126
 483 |     },
 484 |     "id": "z9SwiPco-BUa",
 485 |     "outputId": "f2507282-c0f9-4253-ece7-0a802b68240f"
 486 |    },
 487 |    "outputs": [
 488 |     {
 489 |      "data": {
 490 |       "text/html": [
 491 |        "<div>\n",
 492 |        "<style scoped>\n",
 493 |        "    .dataframe tbody tr th:only-of-type {\n",
 494 |        "        vertical-align: middle;\n",
 495 |        "    }\n",
 496 |        "\n",
 497 |        "    .dataframe tbody tr th {\n",
 498 |        "        vertical-align: top;\n",
 499 |        "    }\n",
 500 |        "\n",
 501 |        "    .dataframe thead th {\n",
 502 |        "        text-align: right;\n",
 503 |        "    }\n",
 504 |        "</style>\n",
 505 |        "<table border=\"1\" class=\"dataframe\">\n",
 506 |        "  <thead>\n",
 507 |        "    <tr style=\"text-align: right;\">\n",
 508 |        "      <th></th>\n",
 509 |        "      <th>0</th>\n",
 510 |        "      <th>1</th>\n",
 511 |        "      <th>2</th>\n",
 512 |        "      <th>3</th>\n",
 513 |        "      <th>4</th>\n",
 514 |        "      <th>5</th>\n",
 515 |        "      <th>6</th>\n",
 516 |        "      <th>7</th>\n",
 517 |        "      <th>8</th>\n",
 518 |        "      <th>9</th>\n",
 519 |        "      <th>...</th>\n",
 520 |        "      <th>41</th>\n",
 521 |        "      <th>42</th>\n",
 522 |        "      <th>43</th>\n",
 523 |        "      <th>44</th>\n",
 524 |        "      <th>45</th>\n",
 525 |        "      <th>46</th>\n",
 526 |        "      <th>47</th>\n",
 527 |        "      <th>48</th>\n",
 528 |        "      <th>49</th>\n",
 529 |        "      <th>50</th>\n",
 530 |        "    </tr>\n",
 531 |        "  </thead>\n",
 532 |        "  <tbody>\n",
 533 |        "    <tr>\n",
 534 |        "      <th>0</th>\n",
 535 |        "      <td>0.884144</td>\n",
 536 |        "      <td>0.577133</td>\n",
 537 |        "      <td>1.0</td>\n",
 538 |        "      <td>1.0</td>\n",
 539 |        "      <td>0.0</td>\n",
 540 |        "      <td>0.496158</td>\n",
 541 |        "      <td>0.188845</td>\n",
 542 |        "      <td>0.064088</td>\n",
 543 |        "      <td>0.982899</td>\n",
 544 |        "      <td>1.0</td>\n",
 545 |        "      <td>...</td>\n",
 546 |        "      <td>0.970903</td>\n",
 547 |        "      <td>1.0</td>\n",
 548 |        "      <td>0.0</td>\n",
 549 |        "      <td>0.946125</td>\n",
 550 |        "      <td>0.449782</td>\n",
 551 |        "      <td>0.944116</td>\n",
 552 |        "      <td>0.000073</td>\n",
 553 |        "      <td>0.0</td>\n",
 554 |        "      <td>0.0</td>\n",
 555 |        "      <td>0.0</td>\n",
 556 |        "    </tr>\n",
 557 |        "    <tr>\n",
 558 |        "      <th>1</th>\n",
 559 |        "      <td>0.891145</td>\n",
 560 |        "      <td>0.577190</td>\n",
 561 |        "      <td>1.0</td>\n",
 562 |        "      <td>1.0</td>\n",
 563 |        "      <td>0.0</td>\n",
 564 |        "      <td>0.496158</td>\n",
 565 |        "      <td>0.188845</td>\n",
 566 |        "      <td>0.064088</td>\n",
 567 |        "      <td>0.982899</td>\n",
 568 |        "      <td>1.0</td>\n",
 569 |        "      <td>...</td>\n",
 570 |        "      <td>0.970903</td>\n",
 571 |        "      <td>1.0</td>\n",
 572 |        "      <td>0.0</td>\n",
 573 |        "      <td>0.946125</td>\n",
 574 |        "      <td>0.449782</td>\n",
 575 |        "      <td>0.944521</td>\n",
 576 |        "      <td>0.000073</td>\n",
 577 |        "      <td>0.0</td>\n",
 578 |        "      <td>0.0</td>\n",
 579 |        "      <td>0.0</td>\n",
 580 |        "    </tr>\n",
 581 |        "  </tbody>\n",
 582 |        "</table>\n",
 583 |        "<p>2 rows × 51 columns</p>\n",
 584 |        "</div>"
 585 |       ],
 586 |       "text/plain": [
 587 |        "         0         1    2    3    4         5         6         7         8   \\\n",
 588 |        "0  0.884144  0.577133  1.0  1.0  0.0  0.496158  0.188845  0.064088  0.982899   \n",
 589 |        "1  0.891145  0.577190  1.0  1.0  0.0  0.496158  0.188845  0.064088  0.982899   \n",
 590 |        "\n",
 591 |        "    9   ...        41   42   43        44        45        46        47   48  \\\n",
 592 |        "0  1.0  ...  0.970903  1.0  0.0  0.946125  0.449782  0.944116  0.000073  0.0   \n",
 593 |        "1  1.0  ...  0.970903  1.0  0.0  0.946125  0.449782  0.944521  0.000073  0.0   \n",
 594 |        "\n",
 595 |        "    49   50  \n",
 596 |        "0  0.0  0.0  \n",
 597 |        "1  0.0  0.0  \n",
 598 |        "\n",
 599 |        "[2 rows x 51 columns]"
 600 |       ]
 601 |      },
 602 |      "execution_count": 14,
 603 |      "metadata": {},
 604 |      "output_type": "execute_result"
 605 |     }
 606 |    ],
 607 |    "source": [
 608 |     "attack.head(2)"
 609 |    ]
 610 |   },
 611 |   {
 612 |    "cell_type": "markdown",
 613 |    "metadata": {
 614 |     "id": "xXJi503b-j_d"
 615 |    },
 616 |    "source": [
 617 |     "### Windows"
 618 |    ]
 619 |   },
 620 |   {
 621 |    "cell_type": "code",
 622 |    "execution_count": 15,
 623 |    "metadata": {
 624 |     "id": "vyplttZa-BRN"
 625 |    },
 626 |    "outputs": [],
 627 |    "source": [
 628 |     "window_size=12"
 629 |    ]
 630 |   },
 631 |   {
 632 |    "cell_type": "code",
 633 |    "execution_count": 16,
 634 |    "metadata": {
 635 |     "colab": {
 636 |      "base_uri": "https://localhost:8080/",
 637 |      "height": 34
 638 |     },
 639 |     "id": "dzGJMp6Y-BN5",
 640 |     "outputId": "2949d278-1313-442c-f06b-275a8c6c6578"
 641 |    },
 642 |    "outputs": [
 643 |     {
 644 |      "data": {
 645 |       "text/plain": [
 646 |        "(494988, 12, 51)"
 647 |       ]
 648 |      },
 649 |      "execution_count": 16,
 650 |      "metadata": {},
 651 |      "output_type": "execute_result"
 652 |     }
 653 |    ],
 654 |    "source": [
 655 |     "windows_normal=normal.values[np.arange(window_size)[None, :] + np.arange(normal.shape[0]-window_size)[:, None]]\n",
 656 |     "windows_normal.shape"
 657 |    ]
 658 |   },
 659 |   {
 660 |    "cell_type": "code",
 661 |    "execution_count": 17,
 662 |    "metadata": {
 663 |     "colab": {
 664 |      "base_uri": "https://localhost:8080/",
 665 |      "height": 34
 666 |     },
 667 |     "id": "17LdB3c8-pRH",
 668 |     "outputId": "721059d4-5937-4dd3-d73c-e5d255fc273c"
 669 |    },
 670 |    "outputs": [
 671 |     {
 672 |      "data": {
 673 |       "text/plain": [
 674 |        "(449907, 12, 51)"
 675 |       ]
 676 |      },
 677 |      "execution_count": 17,
 678 |      "metadata": {},
 679 |      "output_type": "execute_result"
 680 |     }
 681 |    ],
 682 |    "source": [
 683 |     "windows_attack=attack.values[np.arange(window_size)[None, :] + np.arange(attack.shape[0]-window_size)[:, None]]\n",
 684 |     "windows_attack.shape"
 685 |    ]
 686 |   },
 687 |   {
 688 |    "cell_type": "markdown",
 689 |    "metadata": {
 690 |     "id": "k70ZFxGs-_7m"
 691 |    },
 692 |    "source": [
 693 |     "## Training"
 694 |    ]
 695 |   },
 696 |   {
 697 |    "cell_type": "code",
 698 |    "execution_count": 18,
 699 |    "metadata": {
 700 |     "id": "yi9S0SGnDKNc"
 701 |    },
 702 |    "outputs": [],
 703 |    "source": [
 704 |     "import torch.utils.data as data_utils\n",
 705 |     "\n",
 706 |     "BATCH_SIZE =  7919\n",
 707 |     "N_EPOCHS = 100\n",
 708 |     "hidden_size = 100\n",
 709 |     "\n",
 710 |     "w_size=windows_normal.shape[1]*windows_normal.shape[2]\n",
 711 |     "z_size=windows_normal.shape[1]*hidden_size\n",
 712 |     "\n",
 713 |     "windows_normal_train = windows_normal[:int(np.floor(.8 *  windows_normal.shape[0]))]\n",
 714 |     "windows_normal_val = windows_normal[int(np.floor(.8 *  windows_normal.shape[0])):int(np.floor(windows_normal.shape[0]))]\n",
 715 |     "\n",
 716 |     "train_loader = torch.utils.data.DataLoader(data_utils.TensorDataset(\n",
 717 |     "    torch.from_numpy(windows_normal_train).float().view(([windows_normal_train.shape[0],w_size]))\n",
 718 |     ") , batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\n",
 719 |     "\n",
 720 |     "val_loader = torch.utils.data.DataLoader(data_utils.TensorDataset(\n",
 721 |     "    torch.from_numpy(windows_normal_val).float().view(([windows_normal_val.shape[0],w_size]))\n",
 722 |     ") , batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\n",
 723 |     "\n",
 724 |     "test_loader = torch.utils.data.DataLoader(data_utils.TensorDataset(\n",
 725 |     "    torch.from_numpy(windows_attack).float().view(([windows_attack.shape[0],w_size]))\n",
 726 |     ") , batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\n",
 727 |     "\n",
 728 |     "model = UsadModel(w_size, z_size)\n",
 729 |     "model = to_device(model,device)"
 730 |    ]
 731 |   },
 732 |   {
 733 |    "cell_type": "code",
 734 |    "execution_count": 19,
 735 |    "metadata": {
 736 |     "colab": {
 737 |      "base_uri": "https://localhost:8080/",
 738 |      "height": 1000
 739 |     },
 740 |     "id": "So9yjDPEDObC",
 741 |     "outputId": "629bcd13-37b1-4907-ef0d-46d9e3ad5398",
 742 |     "scrolled": true
 743 |    },
 744 |    "outputs": [
 745 |     {
 746 |      "name": "stdout",
 747 |      "output_type": "stream",
 748 |      "text": [
 749 |       "Epoch [0], val_loss1: 0.0261, val_loss2: 0.0263\n",
 750 |       "Epoch [1], val_loss1: 0.0206, val_loss2: -0.0026\n",
 751 |       "Epoch [2], val_loss1: 0.0323, val_loss2: -0.0210\n",
 752 |       "Epoch [3], val_loss1: 0.0389, val_loss2: -0.0281\n",
 753 |       "Epoch [4], val_loss1: 0.0423, val_loss2: -0.0310\n",
 754 |       "Epoch [5], val_loss1: 0.0293, val_loss2: -0.0217\n",
 755 |       "Epoch [6], val_loss1: 0.0593, val_loss2: -0.0476\n",
 756 |       "Epoch [7], val_loss1: 0.0582, val_loss2: -0.0514\n",
 757 |       "Epoch [8], val_loss1: 0.0588, val_loss2: -0.0537\n",
 758 |       "Epoch [9], val_loss1: 0.0592, val_loss2: -0.0546\n",
 759 |       "Epoch [10], val_loss1: 0.0595, val_loss2: -0.0554\n",
 760 |       "Epoch [11], val_loss1: 0.0607, val_loss2: -0.0570\n",
 761 |       "Epoch [12], val_loss1: 0.0629, val_loss2: -0.0594\n",
 762 |       "Epoch [13], val_loss1: 0.0632, val_loss2: -0.0600\n",
 763 |       "Epoch [14], val_loss1: 0.1583, val_loss2: -0.1409\n",
 764 |       "Epoch [15], val_loss1: 0.1570, val_loss2: -0.1452\n",
 765 |       "Epoch [16], val_loss1: 0.1564, val_loss2: -0.1460\n",
 766 |       "Epoch [17], val_loss1: 0.1568, val_loss2: -0.1470\n",
 767 |       "Epoch [18], val_loss1: 0.1573, val_loss2: -0.1481\n",
 768 |       "Epoch [19], val_loss1: 0.1577, val_loss2: -0.1490\n",
 769 |       "Epoch [20], val_loss1: 0.1584, val_loss2: -0.1500\n",
 770 |       "Epoch [21], val_loss1: 0.1581, val_loss2: -0.1502\n",
 771 |       "Epoch [22], val_loss1: 0.1592, val_loss2: -0.1515\n",
 772 |       "Epoch [23], val_loss1: 0.1612, val_loss2: -0.1538\n",
 773 |       "Epoch [24], val_loss1: 0.1612, val_loss2: -0.1541\n",
 774 |       "Epoch [25], val_loss1: 0.1636, val_loss2: -0.1568\n",
 775 |       "Epoch [26], val_loss1: 0.1631, val_loss2: -0.1565\n",
 776 |       "Epoch [27], val_loss1: 0.1631, val_loss2: -0.1567\n",
 777 |       "Epoch [28], val_loss1: 0.1630, val_loss2: -0.1569\n",
 778 |       "Epoch [29], val_loss1: 0.1645, val_loss2: -0.1585\n",
 779 |       "Epoch [30], val_loss1: 0.1655, val_loss2: -0.1597\n",
 780 |       "Epoch [31], val_loss1: 0.1657, val_loss2: -0.1600\n",
 781 |       "Epoch [32], val_loss1: 0.1666, val_loss2: -0.1611\n",
 782 |       "Epoch [33], val_loss1: 0.1672, val_loss2: -0.1618\n",
 783 |       "Epoch [34], val_loss1: 0.1670, val_loss2: -0.1618\n",
 784 |       "Epoch [35], val_loss1: 0.1672, val_loss2: -0.1621\n",
 785 |       "Epoch [36], val_loss1: 0.1672, val_loss2: -0.1623\n",
 786 |       "Epoch [37], val_loss1: 0.1673, val_loss2: -0.1625\n",
 787 |       "Epoch [38], val_loss1: 0.1675, val_loss2: -0.1629\n",
 788 |       "Epoch [39], val_loss1: 0.1683, val_loss2: -0.1637\n",
 789 |       "Epoch [40], val_loss1: 0.1685, val_loss2: -0.1640\n",
 790 |       "Epoch [41], val_loss1: 0.1687, val_loss2: -0.1644\n",
 791 |       "Epoch [42], val_loss1: 0.1689, val_loss2: -0.1647\n",
 792 |       "Epoch [43], val_loss1: 0.1694, val_loss2: -0.1653\n",
 793 |       "Epoch [44], val_loss1: 0.1696, val_loss2: -0.1655\n",
 794 |       "Epoch [45], val_loss1: 0.1696, val_loss2: -0.1656\n",
 795 |       "Epoch [46], val_loss1: 0.1697, val_loss2: -0.1658\n",
 796 |       "Epoch [47], val_loss1: 0.1698, val_loss2: -0.1660\n",
 797 |       "Epoch [48], val_loss1: 0.1698, val_loss2: -0.1661\n",
 798 |       "Epoch [49], val_loss1: 0.1699, val_loss2: -0.1662\n",
 799 |       "Epoch [50], val_loss1: 0.1699, val_loss2: -0.1663\n",
 800 |       "Epoch [51], val_loss1: 0.1700, val_loss2: -0.1665\n",
 801 |       "Epoch [52], val_loss1: 0.1700, val_loss2: -0.1666\n",
 802 |       "Epoch [53], val_loss1: 0.1701, val_loss2: -0.1667\n",
 803 |       "Epoch [54], val_loss1: 0.1702, val_loss2: -0.1668\n",
 804 |       "Epoch [55], val_loss1: 0.1702, val_loss2: -0.1670\n",
 805 |       "Epoch [56], val_loss1: 0.1703, val_loss2: -0.1671\n",
 806 |       "Epoch [57], val_loss1: 0.1703, val_loss2: -0.1672\n",
 807 |       "Epoch [58], val_loss1: 0.1704, val_loss2: -0.1673\n",
 808 |       "Epoch [59], val_loss1: 0.1704, val_loss2: -0.1674\n",
 809 |       "Epoch [60], val_loss1: 0.1705, val_loss2: -0.1675\n",
 810 |       "Epoch [61], val_loss1: 0.1705, val_loss2: -0.1676\n",
 811 |       "Epoch [62], val_loss1: 0.1705, val_loss2: -0.1676\n",
 812 |       "Epoch [63], val_loss1: 0.1706, val_loss2: -0.1677\n",
 813 |       "Epoch [64], val_loss1: 0.1706, val_loss2: -0.1678\n",
 814 |       "Epoch [65], val_loss1: 0.1707, val_loss2: -0.1679\n",
 815 |       "Epoch [66], val_loss1: 0.1707, val_loss2: -0.1680\n",
 816 |       "Epoch [67], val_loss1: 0.1707, val_loss2: -0.1680\n",
 817 |       "Epoch [68], val_loss1: 0.1708, val_loss2: -0.1681\n",
 818 |       "Epoch [69], val_loss1: 0.1708, val_loss2: -0.1682\n",
 819 |       "Epoch [70], val_loss1: 0.1708, val_loss2: -0.1683\n",
 820 |       "Epoch [71], val_loss1: 0.1709, val_loss2: -0.1683\n",
 821 |       "Epoch [72], val_loss1: 0.1709, val_loss2: -0.1684\n",
 822 |       "Epoch [73], val_loss1: 0.1709, val_loss2: -0.1685\n",
 823 |       "Epoch [74], val_loss1: 0.1710, val_loss2: -0.1685\n",
 824 |       "Epoch [75], val_loss1: 0.1710, val_loss2: -0.1686\n",
 825 |       "Epoch [76], val_loss1: 0.1710, val_loss2: -0.1686\n",
 826 |       "Epoch [77], val_loss1: 0.1710, val_loss2: -0.1687\n",
 827 |       "Epoch [78], val_loss1: 0.1711, val_loss2: -0.1688\n",
 828 |       "Epoch [79], val_loss1: 0.1711, val_loss2: -0.1688\n",
 829 |       "Epoch [80], val_loss1: 0.1711, val_loss2: -0.1689\n",
 830 |       "Epoch [81], val_loss1: 0.1711, val_loss2: -0.1689\n",
 831 |       "Epoch [82], val_loss1: 0.1712, val_loss2: -0.1690\n",
 832 |       "Epoch [83], val_loss1: 0.1712, val_loss2: -0.1690\n",
 833 |       "Epoch [84], val_loss1: 0.1712, val_loss2: -0.1691\n",
 834 |       "Epoch [85], val_loss1: 0.1712, val_loss2: -0.1691\n",
 835 |       "Epoch [86], val_loss1: 0.1713, val_loss2: -0.1692\n",
 836 |       "Epoch [87], val_loss1: 0.1713, val_loss2: -0.1692\n",
 837 |       "Epoch [88], val_loss1: 0.1713, val_loss2: -0.1692\n",
 838 |       "Epoch [89], val_loss1: 0.1713, val_loss2: -0.1693\n",
 839 |       "Epoch [90], val_loss1: 0.1713, val_loss2: -0.1693\n",
 840 |       "Epoch [91], val_loss1: 0.1714, val_loss2: -0.1694\n",
 841 |       "Epoch [92], val_loss1: 0.1714, val_loss2: -0.1694\n",
 842 |       "Epoch [93], val_loss1: 0.1714, val_loss2: -0.1695\n",
 843 |       "Epoch [94], val_loss1: 0.1714, val_loss2: -0.1695\n",
 844 |       "Epoch [95], val_loss1: 0.1714, val_loss2: -0.1695\n",
 845 |       "Epoch [96], val_loss1: 0.1715, val_loss2: -0.1696\n",
 846 |       "Epoch [97], val_loss1: 0.1715, val_loss2: -0.1696\n",
 847 |       "Epoch [98], val_loss1: 0.1715, val_loss2: -0.1696\n",
 848 |       "Epoch [99], val_loss1: 0.1715, val_loss2: -0.1697\n"
 849 |      ]
 850 |     }
 851 |    ],
 852 |    "source": [
 853 |     "history = training(N_EPOCHS,model,train_loader,val_loader)"
 854 |    ]
 855 |   },
 856 |   {
 857 |    "cell_type": "code",
 858 |    "execution_count": 20,
 859 |    "metadata": {
 860 |     "colab": {
 861 |      "base_uri": "https://localhost:8080/",
 862 |      "height": 295
 863 |     },
 864 |     "id": "fYwlN0JKVVtN",
 865 |     "outputId": "c742ff8b-3b4a-41f5-dd09-effee1be928a"
 866 |    },
 867 |    "outputs": [
 868 |     {
 869 |      "data": {
 870 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEWCAYAAABIVsEJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXyU1b348c93spOEBBIIEJaEhH2RXUTAGPdqtdR9QXHXW6Reb7XW9rbWqrX1Xmv7w1u1dUGr4tqWFguuESkurIJskoQtYU0gkASyn98f55lkEpMwTJKZZOb7fr3mNefZZs6ZSZ7vnOU5jxhjUEoppU6WK9AZUEop1TVpAFFKKeUTDSBKKaV8ogFEKaWUTzSAKKWU8okGEKWUUj7RAKJUCBCR00Vkm4iUicj3OkF+0kTEiEh4oPOifKcBRPmFiOwQkbMDnY9AEpEcEakQkQEe684WkR1+ePuHgPnGmDhjzN/88H4qBGgAUcq/yoH/DsD7DgI2BuB9VRDTAKICSkSiRORJEdnjPJ4UkShnW7KI/FNESkTkkIh8KiIuZ9uPRaRQREpFZKuInOWsd4nI/SKSJyLFIvKGiPR0tkWLyF+c9SUislJEUprJ049F5K0m634vIn9w0nNEJN957+0icu1JFPkPwNUiktHC5zHCqamUiMhGEbnY2xcWkVtFJNf5rBaJSD9nfR4wGPiH04QV1cyx/UTkbRE56JRpnse2B0XkLRF53SnzGhE5xZs8i0iMiPyviOwUkSMislxEYjze+loR2SUiRSLyU4/jpojIKhE5KiL7ReQJbz8H5UfGGH3oo8MfwA7g7GbWPwR8DvQGegErgF85234NPA1EOI8ZgADDgN1AP2e/NCDDSf/Qeb3+QBTwDPCas+124B9ANyAMmAh0byZPg4BjQLyzHAbsBaYCscBRYJizrS8wysvPIAe4BXgC+Iuz7mxgh5OOAHKBB4BIIBsodb/XCV47GygCJjjl/n/AshN9/s42F7Aa+LnzvoOBfOA8Z/uDQDVwmZPHHwHbPb6XFvMMPOWUO9X5HKc5+UsDDPAnIAY4BagERjjHfQbMdtJxwNRA/w3ro5m/nUBnQB+h8WglgOQB3/FYPs/jhPoQ8Hcgs8kxmcAB5+Qb0WTbZuAsj+W+zskvHLgJG6DGepHf5cD1TvocIM9JxwIlwKVAzEl+Bu4A0gs4AoxqEkBmAPsAl8cxrwEPevHazwG/9ViOc8qd1trn72w7FdjVZN1PgBec9IPA5x7bXNiAOqO1PDv7HQdOaeY93QGkv8e6L4GrnPQy4JdAcqD/dvXR8kObsFSg9QN2eizvdNYBPI79dfue02R0P4AxJhe4G3uSOiAiC93NNdjaw1+d5pQSbECpBVKAl4GlwEKnuey3IhLRQr5eBa520tc4yxhjyoErgTuAvSKyWESGn0yBjTEHgfnYANn0s9htjKlr8nmkevGyjT5HY0wZUOzlsYOAfu7PzPncHsB+Zm67PV67Dihw3rO1PCcD0dgfCS3Z55E+hg18ADcDQ4EtTlPjRV6UQ/mZBhAVaHuwJzC3gc46jDGlxpj/MsYMBi4G7nH3dRhjXjXGTHeONcBvnON3AxcYYxI9HtHGmEJjTLUx5pfGmJHYppSLgOtbyNebQJaI9Adm4QQQ572XGmPOwdZutmCbYU7W48CZ2GY0z89igLufx+PzKPTi9Rp9jiISCyR5eexuYHuTzyzeGPMdj308R465sE2Ee06Q5yKgAmi2v6c1xphtxpirsU2bvwHecsqkOhENIMqfIpyObPcjHNvc8TMR6SUiydh2+L8AiMhFIpIpIoJt8qkF6kRkmIhkO53BFdhmEvcv4KeBR0RkkPMavUTkEid9poiMEZEwbD9GtcdxjTi1hBzgBezJdbPzGikicolzMqsEylp6jdYYY0qA/wXu81j9BfZX+H0iEiEiWcB3gYVevORrwI0iMs75XB4FvjDG7PDi2C+BUmfwQIyIhInIaBGZ7LHPRBH5vvOd3Y0t++et5dmplTwPPOF00oeJyGnNdeI3JSLXiUgv5zVKnNUn/TmrjqUBRPnTu9iTvfvxIPAwsApYD2wA1jjrAIYAH2BP0p8B/2eM+RjbCfsY9hfuPuyv1J84x/weWIRt9irFnuROdbb1Ad7CBo/NwCfYZq2WvIrto3jVY50LuAf7y/sQcAZwJ4CIzBCRMu8/Dn6PDYoAGGOqsCffC5yy/R+2H2aL8/r/EpEHmnshY8wH2OHBb2P7JzKAq7zJhDGmFlsbG4ftHC8C/gwkeOz2d2zT3WFgNvB9p0bXap6xHe4bgJXYz+s3eHfeOR/Y6Hyev8f2jRz3pjzKf8QYvaGUUqplIvIgdiDDdYHOi+pctAailFLKJxpAlFJK+USbsJRSSvlEayBKKaV8ElJTKScnJ5u0tDSfji0vLyc2NvSGoYdiuUOxzBCa5dYye2f16tVFxpheTdeHVABJS0tj1apVPh2bk5NDVlZW+2aoCwjFcodimSE0y61l9o6I7GxuvTZhKaWU8okGEKWUUj7RAKKUUsonGkCUUkr5RAOIUkopn4TUKCylVOh4+pM8xvZPYFpGcn0a4Nll+dw2czAAT6w6TuSAom+tP9n0+oIjAIS5oNaZM7i59wtUen3BEe44I4MVeUW8m19Few080wCilJe8OSF1tZOKZ/7y86va5WTqj7x681nuLC7n2WX53Jk1mJ3F5Tz1cS4A887K5PaXVwNw4aCw+rTn+pNNPzN7Ihv3HOHRxVt44MLhjOqX0C6v217pZ2ZPZEVeEXNfXcutI8NoLxpAVMhqKSC09KvUmxNSVzupeOYvPeHkTqZ3nWnTxhh+cGYmt7+0GoPhzqwMbntpNWC4faZNGwy3TE+3aWO4aXo6t760CgzMmZZWn75+Whq3LrDXal03dZCTNvz3RaPIPVDGc8u3c9P0NAYnx3HLglVgDFdMHlifvmzSAG5ZsApjDP917jAuGA2PLt7CtIwkamrrAGFDwVGqa+sQhO1HDNW1BkH4aveR+vVrd5XUp9fstGmAVTsa0iu3H65PP798B5/lFXHh2L48+f42pg5OorrGbluRe4iqmjoE+HduMVXO+uXbGtLLvimqT3+ytSGds/Uglc6xH285SGV1HSLw0WabRuCDTQeaTb+/8QAVzv5Pvv8N3xwo4/+unUDV7q9P9K/htZCaC2vSpElGLyQ8OZ2h3C2d6Nv6C/8fX+1h6cb93Jk1mPyD5fxz/V4ALhwkLN5p/y/mnZXJHz7MBQNPXDGO1TsP8cyyfKak9eSrwhIwMHVwEp/lFwM2/bmTnjCwB+t2lzA5rQdf7jjM6H7d+aqgBAFG9O3Opj1HQWBEn+5s3ts4bQQGJ8eSf7AcgIFJ3dhZXI4gpPaIYffhYwhCn+7R7D1yHAF6xUdxoLQSgJ6xkRwqqwKBxJgIDh+rRgTio8M5erwGgNioMMqraukWEcaxqloiXFDlfI6RYUJVrf0MwgRqQ+c0EbSuO3UgD88a4+uFhKuNMZOartdOdBVQT3+Sx4q8okbpFXlFzHnhy/r0zuJy5r66lj99msfO4nJuf3k1t7+8mrGpCYS57C/MMJcNHu5t0wYncfvLq7ntpVVMTuvBbS/Z9Cn9E7h1wSpueXEl/RJjmDAgkUcWb2HZNwc5VlXDscpaluyopryihrLKGp547xtKK2oorazh1pdX8fSyfAzwxY5DVFTXUVFTR8439ldiZU0dn3ikP8sv5nh1Lcu2FVFRXcuqnYepqzPUGfiq4Agul+ASYX3hEURAgPWFR4gIdxETHsY3+8uIiwonPjqc/IPl9OgWSY/YCHYWH6NXXBS946MoLDlO34Ro+ibGsO9oJf17dGNgz24UlVWRlhxLenIsh45Vk9E7jozecRw5XsPQPvEM6xNPWWUtI/p05+Jx/RjZrztVdTA2NYGxqQlU1RrGDUhk/IBEag1MHNSDSYN6ADAlrQdT0noCMDW9J1MH2/S0jCSmZSQBMD0zmRlDkgE4Y2gvsobaWTDOHNab7OG9AThreG/OHmHT54xI4ZyR9hbs541K4bxRfQC4YHQfLhht098Z04dnZk/kwjF9Afju2L589xSbvmRcPy4Z1w+A741LZdZ4eyv4GZnJxEeHc+mEVGIiXHSLDOOyif3p5qRnpIbVp6+cPKA+ffWUAXSLdBEbGcY1pw4kNjKM2MgwrvNIz546yKajwrjhtEEkxEQwNyuDxJgI5kxLIzYyjLiocG46vSF98/Q0YqNs+pbp6fXp22akE+ekb585uD59xxkN6TuddHxUOP+RlWHT0eHMPTOTeOfv5K4zM4mPtul52Y3T7369r/5/rb1oAFHtypuA4E1wcJ/0b12wijGpCcwckswji7fwydaDlFfWUFZRwzV//oJHFm8hKsLFo+9u4aYXV9qTfUUNj/5rC6UVNZRV1vL40m8oq7Tp33+YS3lVLceq6/jf977hgy0HCHdBYUkFSbFRpHSPoqQS0nvFMjQlnuPVdYwfkFh/wpyU1oP46HCunjLA46SQ3uik4P6H/4+sDHp0i+CBC4bTMzaSedmZdIsKJyYyjHnZmUSGu4gMdzEvO5OoiDCiIuz6MJeNJvOyM6k1hpo6w7zsTCpr6qiormNedibHqmopq6xhXnYmRytqOHK8mnnZmRw+VkVxeRXzsjM5WFbJgdJK5mVnsv9oBfuOVDAvO5M9JccpLDnOvOxM9h2tYHCvWPYeqeDijAi2F5ezvbicedmZ5B0sI/dgGfOyM/lmfylb95cyLzuTzftK2bzvKPOyM9m49ygb99j0hsIjbCg8wrzsTL4qKGHd7hLmZWeyZtdhVu86zLzsTFbtPMTKHYeYl53JlzsO8cV2m/58ezGf5xczLzuTFXnFrMgrYl52Jstzi1iea9Of5x9i16FyPnP2y/nmIDlbDzIvO5OPthzgoy0HmJedyYdb9vPB5v3MGt/PHntWJpdO7E94mIswlzCsTxxhTjo1zlWfzuwdW58e3CuWMJcLl0tIT+5mg71LGOSRHpgUU/8j4LzRfZibncFTOXn8IDuDc0el4HIJItA3Mbo+3SchGpfYdEpCVH26V/coxEknx0fWp5PiGtI9nTQCPWIjbBpI6BZuf30A3bs19ErExzSkp2YkMf+a8cx9dS2bi+tvgtlm2geifNJSs5Iv/QQDe3Tj0cVbGNonjmNVNdTVweNLv6l/rwf+2tBmu+dIBQN6xhAfFc6mvaVMGJjI6NQEVu04xKa9pYwfkEi4S1i58zBTB/dERPgsr5jpmcmIwKfbijh3ZAoRYS4Wb9jLRWP6sCL/ENedOpAXVuwA4OKMCD4utE1B87Iz69fPGt+Pv63dU9+f4W7y6uOcCMCeFNz/2NOHJNMjNqJRH4j7tTz/uQOVnpqRRHxMeH3+ag7s4OPCmk6Tv9by6s1nGR0RxgMXDuePOfmcNyqFZ2ZPBGwzpjv92F9X8szsyd9af7Jpd3PqAxcOp7bOLvv6Wh2Rdo/Cmn/NeP72yRrai/aBeKkz9AUEgme5PYOGe0RH0/6D+deM582Vu/nnhn1EhrmoqavDJUJGr1jynPb8zN5x5B4os7/CukdTcLjhVtdJsZH0io9iy75Ssob2IswlfLjlAGcN782aXYeZPXVQ/Ynjxmlp/OWLXdyZNZg/5uQ3CgI3Tks7YfrsEb0bBYSGkTnN9IEAF43ty+BesfUnpO+eYptMgmYUVl4e3ztjQqfJX1s/S8+hq+50U6H4f92efSAaQLzU2f/QPE/ubq3943jbMe15UvHsdK6tg9Lj1TyVk8eQ3nHsKLbBITLMRXlVLQkx4Rw5XsOgpG6UVdRQXF5F92j76/BoRQ3xUeEYDGWVtWT2imV/aSXXnTqQv3yxC/DuRH+iUU7uE39L6ZYCwmN/Xcn9sxp+lZ7MCakr6+x/4x1By+ydlgKINmEFibH9E5j76lrmXzOe9QVHCHNRf2J090l4MyS1taGdf7x2IvFRETyyeAtJsZEUl1cBsO1AGS6nDba8tpbTM5LYvK+0UfNPc2lBvtUs5A4g3jRFtLXZYH3BEW6dkcGofgmsLzhSH3zvmRRTn/YMyJ7rPNcrFao0gASJaRnJzL9mPLcsWEVUuIvDx6q5cvIAsof1brbv4f+unUDv+CgeXbyFUwf3pLqmDgP8dslWNu45Qkr3aB59dwsxYVBRAwjMeeFLaupsjbW4vIqRfbuzp+Q41582qMXaQnu0U7d0om/tJN7cib+ltAYEpXyjASSITMtIpnd8FDuKjyHA6yt38/rK3QC4BP68bDtlFTUYYPZzX9Yf93n+ofr0ut0l9OgWwdA+8fSMjWDT3lIG94qlptaw69AxJg3qwbYDZWQP79VsoPAmIJxscNATvVKdkwaQILIir4iCw8dJTYzhWFUNI/t25995xYwbkEhZZQ25B8oYmhJHdHgY6wuPMG5AArkHyvnOmD4sXr8Xlwg3nm47pmcMSeKPOfmNRiR5NjfV1sEDF3ZvtlnJ24DgmdbgoFTXE9DrQETkfBHZKiK5InJ/M9tnisgaEakRkcuabKsVkXXOY5H/ct05uUdF9U2IZmS/7vzHmRmsyCtm1vhU8g6Wsf+ovQZg75EKtheXM2t8P77afYQfnp3J98an4nKuPZiakcSdWYN5dPEW7swazIieDfPmeNYuxvZP4NYZdljgoKTY+gDg7lj2TCulglPAaiAiEgY8BZwDFAArRWSRMWaTx267gDnAj5p5iePGmHEdntEuYn3BEeZfM5773lpPRVUtf8zJ54ELh9dPhQG+dUxvP1Lbau1Caw5Kha5ANmFNAXKNMfkAIrIQuASoDyDGmB3OtrpAZLArcf/aL6us4VhVDfOvGV8/RLctHdM5Zve3tmnQUEpBAK8DcZqkzjfG3OIszwZONcbMbWbfF4F/GmPe8lhXA6wDaoDHjDF/a+F9bgNuA0hJSZm4cOFCn/JbVlZGXFycT8f6izGGm987xnfSI7hsaGS7vGZXKHd7C8UyQ2iWW8vsnTPPPDPorgMZZIwpFJHBwEcissEYk9d0J2PMs8CzYC8k9PWioa5wwdHxqlrqli5h1NAMsrLap/+hK5S7vYVimSE0y61lbptAdqIXAgM8lvs767xijCl0nvOBHGB8e2auKyqtrAbslN1KKdXRAhlAVgJDRCRdRCKBqwCvRlOJSA8RiXLSycDpePSdhKqyCmciPA0gSik/CFgAMcbUAHOBpcBm4A1jzEYReUhELgYQkckiUgBcDjwjIhudw0cAq0TkK+BjbB9IyAeQUieAxEVpAFFKdbyAnmmMMe8C7zZZ93OP9Eps01bT41YAYzo8g11MWaUGEKWU/+gNpYJIaX0TVkSAc6KUCgUaQIJIaYV2oiul/EcDSBDRJiyllD9pAAki7lFYcVoDUUr5gQaQIFJaWUN0hIuIMP1alVIdT880QaS0ooa4KO1AV0r5hwaQIFJWWaMd6Eopv9EAEkTKKqo1gCil/EYDSBCxTVgaQJRS/qEBJIiUVWoAUUr5jwaQIFJaUaNXoSul/EYDSBAp1T4QpZQfaQAJEsYYbcJSSvmVBpAgcby6ljqjV6ErpfxHA0iQKNWbSSml/EwDSJDQm0kppfxNA0iQcM/EqzUQpZS/aAAJEmV6MymllJ9pAAkS7ptJaROWUspfNIAEiVK9mZRSys8CGkBE5HwR2SoiuSJyfzPbZ4rIGhGpEZHLmmy7QUS2OY8b/JfrzsndhNVdm7CUUn4SsAAiImHAU8AFwEjgahEZ2WS3XcAc4NUmx/YEfgGcCkwBfiEiPTo6z52ZexRWbFRYgHOilAoVgayBTAFyjTH5xpgqYCFwiecOxpgdxpj1QF2TY88D3jfGHDLGHAbeB873R6Y7q7LKamIiwgjXuxEqpfwkkA3mqcBuj+UCbI3C12NTm9tRRG4DbgNISUkhJyfnpDMKUFZW5vOx/vDN9koiXXXtnsfOXu6OEIplhtAst5a5bYK+x9UY8yzwLMCkSZNMVlaWT6+Tk5ODr8f6w1t71pBUebTd89jZy90RQrHMEJrl1jK3TSDbOwqBAR7L/Z11HX1sUCqtqCFeR2AppfwokAFkJTBERNJFJBK4Cljk5bFLgXNFpIfTeX6usy5klVXW6ESKSim/ClgAMcbUAHOxJ/7NwBvGmI0i8pCIXAwgIpNFpAC4HHhGRDY6xx4CfoUNQiuBh5x1Iausoob4KB3Cq5Tyn4D+ZDXGvAu822Tdzz3SK7HNU80d+zzwfIdmsAsprajWGohSyq90zGeQKNWbSSml/EwDSBBw341QZ+JVSvmTBpAgUF5VizE6lbtSyr80gASBsvqbSWknulLKfzSABIGySmcqd62BKKX8SANIEND7oSulAkEDSBCoDyA6Cksp5UcaQIKA+37o2oSllPInDSBBoKETXQOIUsp/NIAEgaPO/dDj9W6ESik/0gASBMr0fuhKqQDQABIEyipq6BYZRphLAp0VpVQI0QDSxTz9SR4r8ooapXMPlOGOHSvyinj6k7wA5lApFSo0gHQBnkFjbP8E5r66lj99msfO4nJuf3k1y3OL6B4dwYq8Iua+upax/RMCnGOlVCjQRnM/ePqTPMb2T2BaRnL9uhV5RawvOMIdZ2Q0u587DbCzuJxnl+VzZ9Zgautg9tSBPLJ4CwN6xnCsqobaOqg1hrmvrmX+NeMbvY9SSnUUDSB+4K41uE/uP3lnPf9cv5dnZk9sMVDsLC7nqY9zAXhm9kT694jhkcVbSE2MprCkAoDdh46THBdJVLiLwpIK5mVnavBQSvmNBhA/mJaRzPxrxvODV9Ywsm93Vu86bDu8DYzpl8BtL62mrq6OH549lDOGJvPI4i2M7NediupaAO5euI4DpZUAFJZUkNkrloNlVdxw2iBeWLGDypo65mVn8pcvdjE1I0mDiFLKLzSA+Mm0jGROy0ji3Q376tdd++cvEIE6Y5d//a8t9ds27TmKSyAy3MWB0kqGp8RReKSC7GG9WfTVHh64cDij+iXwwoodAEzNSGJqRpI2Yyml/EY70f1kRV4RH285CED36HBunJbGqNTu1Bk4Y2gvvjOmLwBnDe9NYkwE/5GVQWxUOBFhLmaN78fW/WXMOyuTEf2688CFw/ljTj7/+GoPz8yeyDOzJ7K+4Eh9TWd9wZFAFlUpFSICGkBE5HwR2SoiuSJyfzPbo0TkdWf7FyKS5qxPE5HjIrLOeTzt77yfDPfoqFGp3RmcHMvTsyfy1poCdhYfY152Jmt2HebTbQeZNb4fH205wA+yM5g+pKEGER0RVh80xvZP4NYZGcy/ZjyDkmKZlpHMtIzk+s54z7RSSnWkgDVhiUgY8BRwDlAArBSRRcaYTR673QwcNsZkishVwG+AK51tecaYcX7NtI/WFxxh/jXjeeCdDYzo271+/UVj+zI1I6m+GcozUJw3KoVnZk+sP/7WGRmM6pdQX9NwP5RSKlAC2QcyBcg1xuQDiMhC4BLAM4BcAjzopN8C5otIl7vc+o4zMjhWVcPOQ8f43vhU1hcc4ZnZE+uH67YWKIBGzxo0lFKdRSADSCqw22O5ADi1pX2MMTUicgRIcrali8ha4CjwM2PMpx2c3zbZtr8MY2B4n3jOH923fr1nc5MGCqVUV9JVR2HtBQYaY4pFZCLwNxEZZYw52nRHEbkNuA0gJSWFnJwcn96wrKzM52MBlhXYGXOP7NxMTtFWn1/H39pa7q4oFMsMoVluLXPbBDKAFAIDPJb7O+ua26dARMKBBKDYGGOASgBjzGoRyQOGAquavokx5lngWYBJkyaZrKwsnzKbk5PDiY5t7Ypzk1BBTMRuLr/gTFxdaNJDb8odbEKxzBCa5dYyt00gR2GtBIaISLqIRAJXAYua7LMIuMFJXwZ8ZIwxItLL6YRHRAYDQ4B8P+W7Re4rzt3zVnnOTbV1XylDU+K6VPBQSqnWBKwG4vRpzAWWAmHA88aYjSLyELDKGLMIeA54WURygUPYIAMwE3hIRKqBOuAOY8wh/5eica3DfR3GLQtW0TM2kvLKGp66dgLTMpK569W1nDWidyCyqJRSHSKgfSDGmHeBd5us+7lHugK4vJnj3gbe7vAMesFznqtRfRN49pN8jlXVcqzqOBeO6cu0jGQOllZSXF7F8D7dT/yCSinVRXTVTvROw13ruP3l1dTUGo5X1xIZ5iIyXHhv0z5W5BVRV2f3Hd4nPrCZVUqpdqRTmbSDAT26UVVTx/HqWqLCXbx402RunZFBda3hzr+sYcnXewEYpgFEKRVENID4wPMGT+WVNVzzp8+prKmjX2I0keH2I71qygDCXMKMIcl8VXCE5LgokuKiApltpZRqVxpAfFA/2iq3iJte/JLdh48TE+Hify4/hWdmT2Tuq2vJO1jGOSNS+HduEVU1dYzoq7UPpVRw0QDig/rRVi+t4ovth4kMd/HcnMmNRmI9uyyfCQMTOXysmq37SxmWEq/3K1dKBRUNIK1Z/iRsX9Z43fZlsPxJJg7qUb/q9pmDG108OC0jmdtmDubpT/JJ6W6brcJcovcrV0oFFQ0grVhU1IfqhTc0BJHty6heeAOLivrwxHvfcKyqlkvG9eOVL3bV94m4TctIZv614ymtqAHgtS936Y2elFJBRYfxtiJ5zNn8YP1ennrlGtLix1L92WZ+UD2PyT0n8+d/bWFAjxievHIcn+UXN3snwGkZydx4ehpPfZzHnGlpGjyU6kKqq6spKCigoqIi0FlpVwkJCWzevLnZbdHR0fTv35+IiAivXksDSCumZSTDNbMpePlp0g//m2e4lDmzZ/O3tYXUGrj77KGISKM7ATadB+u1L3fr/cqV6oIKCgqIj48nLS2NLngXiRaVlpYSH//tQT3GGIqLiykoKCA9Pd2r19ImrBOYFraJVNdh6gxcbt5j0NHVFJYcJ6V7FL3io+o7xZveCdA9D9b8a8Zzz7nDmH/N+EbzZCmlOreKigqSkpKCKni0RkRISko6qRqXBpDWOH0eL5jv4hJ4vPoKYv52M3X5y8ge3pu7X1/XYqe4+y6Envf40PuVK9W1hErwcDvZ8moAacWODZ/aPo+Lbwfg9GF9+UH1PMa58lny9b5WO8XvOCPjW9v0fuVKqZMRFxfXbq/15ptvMmrUKOqHBrsAACAASURBVBISEli16lt3vvCJBpBWLEm4ijnXzmbCuInUuiK5qHcx3UecxR9rvsvsqYO0P0MpBTSencKts133NXr0aN555x1OP/30dntNDSCtqK9FuMIoj03jyI41rNxxqL5TXPszlFLQ+r2A2oMxhnvvvZfRo0czZswYXn/9dQD27t3LzJkzGTduHKNHj+bTTz+ltraWOXPm1O/7u9/9DoARI0YwbNiwdsmPm47C8tLuiEH03/8Z868bx7TMXkzNSGp26K5SKvj88h8b2bTnW3fMbqR3fBTXP/clKd2j2H+0kszecfz+g238/oNtze4/sl93fvHdUV69/zvvvMO6dev46quvKCoqYvLkycycOZNXX32V8847j5/+9KfU1tZy7Ngx1q1bR2FhIV9//TUAJSUlJ1fYk6A1EC99YwaRQBnTelUC2imulGosISaClO5RFJZUkNI9ioQY766l8Mby5cu5+uqrCQsLIyUlhTPOOIOVK1cyefJkXnjhBR588EE2bNhAfHw8gwcPJj8/n7vuuoslS5bQvXvH3YdIayBeGpSWCYeBfV9DQn+A+rmvlFLBzZuagrvZyt3E/cOzh3T4+WHmzJksW7aMxYsXM2fOHO655x6uv/56vvrqK5YuXcrTTz/NG2+8wfPPP98h7+9VDUREfigi3cV6TkTWiMi5HZKjTqo8dpBN7NsQ2IwopTqdjr7ua8aMGbz++uvU1tZy8OBBli1bxpQpU9i5cycpKSnceuut3HLLLaxZs4aioiLq6uq49NJLefjhh1mzZk275KE53tZAbjLG/F5EzgN6ALOBl4H3OixnnUxteDfokQb7NYAopRpr7bqv9qiFzJo1i88++4xTTjkFEeG3v/0tffr0YcGCBTz++ONEREQQFxfHSy+9RGFhITfeeCN1zq1Qf/3rXwPw17/+lbvuuouDBw9y4YUXMm7cOJYuXdqmfHkbQNxXl3wHeNkYs1FC7QobgJTRtglLKaU8NHd9V3s0cZeVlQH2Ar/HH3+cxx9/vNH2G264gRtuuOFbxzVX65g1axazZs1qcSoTX3jbib5aRN7DBpClIhIP1LX1zUXkfBHZKiK5InJ/M9ujROR1Z/sXIpLmse0nzvqtTs2o4/UZA4fyoarcL2+nlFKdmbcB5GbgfmCyMeYYEAHc2JY3FpEw4CngAmAkcLWIjGzmfQ8bYzKB3wG/cY4dCVwFjALOB/7Peb2OlTIaMLB/U4e/lVJKdXbeBpDTgK3GmBIRuQ74GdDW8atTgFxjTL4xpgpYCFzSZJ9LgAVO+i3gLKfp7BJgoTGm0hizHch1Xq/DDNj1DlQfswvufhDn5lJKKRWKvA0gfwSOicgpwH8BecBLbXzvVGC3x3KBs67ZfYwxNdigleTlse2qND4TltwPEd1sP8j2ZfDmHEid0JFvq5RSnZa3neg1xhgjIpcA840xz4nIzR2ZsfYiIrcBtwGkpKSQk5Pj0+uURQyGIXczdv2DVK3/G66v3mTTyHsp2VkHO317za6grKzM58+sqwrFMkNolru1MickJFBaWurfDPlBbW1tq+WqqKjw+u/A2wBSKiI/wQ7fnSEiLmw/SFsUAgM8lvs765rbp0BEwoEEoNjLYwEwxjwLPAswadIkk5WV5VNmc3JyGJc1D4oXEV2wEk6/m3HnzPPptbqSnJwcfP3MuqpQLDOEZrlbK/PmzZvbbbRSZ3KiUVjR0dGMHz/eq9fytgnrSqASez3IPuwJ+/HWDzmhlcAQEUkXkUhsp/iiJvssAtxj1C4DPjLGGGf9Vc4orXRgCPBlG/NzYtuXwcGtNr3quYZ7pSulVAdoz+nc7733XoYPH85pp53GrFmz2mWOLK8CiBM0XgESROQioMIY06Y+EKdPYy6wFNgMvOFcX/KQiFzs7PYckCQiucA92JFgGGM2Am8Am4AlwA+MMbVtyc+JJB5eb/s8Ll8AkXEwaLpd1iCilFr+5LfPBZ1skM0555zD119/zWeffcbQoUPrLzBsC2+nMrkC+wv/cuAK4AsRuaytb26MedcYM9QYk2GMecRZ93NjzCInXWGMudwYk2mMmWKMyfc49hHnuGHGmH+1NS8nEl+aC5e/CJnZMDgL9n8Nl70AhR03TYBSqotIndD4B2U7D7Jpj+nczz33XMLDba/F1KlTKSgoaHO+vO0D+Sn2GpADACLSC/gAO7Q2JOwe+H0y0mfahSHnwJZ/QlxvmH53YDOmlOp4/7r/xPPgxfeFl2fZ59K90Gs45PzGPprTZwxc8JhXb9/e07k///zzXHnllV69d2u87QNxuYOHo/gkjg0+mefY523vBzYfSqnOIzrRBo8ju+1zdGK7vXR7Tuf++OOPEx4ezrXXXtvmfHlbA1kiIkuB15zlK4F32/zuXVVCqr0qfdt7cHrwj8RSKuR5U1NwN1vNvM8Ossn6MbhbLTrIyU7n/uKLL7JkyRJycnJoj+kMve1Evxc7FHas83jWGPPjNr97V5Z5Nuz6DCpav0uZUioEuIPH5S9C9k/tczsOsmmP6dyXLFnCb3/7W15//XW6devWLvny+oZSxpi3gbfb5V27uuVPQlwK1NVAfg6MvNj+oRSuaegTWf6k7UDz/AXSdB+lVHAoXGODhvv/PX2mXS5c0y61kPaYzn3u3LlUVlZyySWX4HK5mDp1Kk8//XSb8tVqABGRUsA0twkwxpiOu1diZ+YecRERa5uxYhIbfn003cf9R+X5C0UpFVya+1GYPrPNwaM9p3PPzc0FTnwh4cloNYAYY4LvMsz24P518ZdLYcNbsGUxXLGg8R+Le583roeUUXBgc+NfKEop1cWF7kiqtkqfCWOvhJrj0C0J0mY0v098H9ix3I7c0uChlAoiGkB8tX0ZbH0XegyG4m3wr/sab1v+JHz5J1vzANi8SK9aV0oFFQ0gvvDsz7joSXCFw5fPwqrnG7a5wuz07+6x4akTdeoTpboYO/Ve6DjZ8moA8YXniIuMM+DylwCBxT+CV6+wU5zsWWdHaU24HmJ62IuL3KMylFKdXnR0NMXFxSETRIwxFBcXEx0d7fUxXg/jVR6ajrgYcSFMmA1rXoLq4/Dez+D4YeiRDutegeEXwZoF9uJD7QdRqkvo378/BQUFHDx4MNBZaVcVFRUtBono6Gj69+/v9WtpAGkP25fZkVgz/gtWPOXMmWMgqjtc9QoYYwPInrWQeVagc6uU8kJERATp6emBzka7y8nJ8fp+HyeiTVht5dkfctbP4bo3ITzKbjv1Dlvj6HuKXd6zNlC5VEqpdqcBpK2aXoEKEBYJ6Wc03HQqJhF6ZmgAUUoFFW3CaivP/hB3beSqV7599XnqBNi5IkCZVEqp9qc1kPbU2nw4/cbD0UIo3R/IHCqlVLvRGkh7am0+nJ2f2eU9a2HY+f7Nl1JKdQCtgfhLnzEgLu0HUUoFDQ0g/rD8SdizBpKH2WdomO5EKaW6qIAEEBHpKSLvi8g257lHC/vd4OyzTURu8FifIyJbRWSd8+jtv9z7wD21e/d+tgaS/4ldTp0Q6JwppZTPAlUDuR/40BgzBPjQWW5ERHoCvwBOBaYAv2gSaK41xoxzHgeaHt+puDvTd38O5QftdCfT7/n2zaa0RqKU6kICFUAuARY46QXA95rZ5zzgfWPMIWPMYeB9oOv2PqfPhHHX2HRdLXzyWMPEiu7hvlojUUp1IYEahZVijNnrpPcBKc3skwrs9lgucNa5vSAitdjb7D5sOvuMZ9uXwdfvwORbYPULUFkKr1wG/SbC/q/ttSOeEy26b32rt8FVSnVS0lHnXRH5AOjTzKafAguMMYke+x42xjTqBxGRHwHRxpiHneX/Bo4bY/5HRFKNMYUiEo8NIH8xxrzUQj5uA24DSElJmbhw4UKfylNWVkZcXJxPxyYeXs/ITY+zaeS9lPQYS49Daxn99aOE1VUBUIeLrcPvojIqmdFf/xoDHOw9nWPdUhm46+364xIPrye+NJfdA7/vUz580ZZyd1WhWGYIzXJrmb1z5plnrjbGTGq6vsMCSGtEZCuQZYzZKyJ9gRxjzLAm+1zt7HO7s/yMs99rTfabA0wyxsw90ftOmjTJrFq1yqc85+TkkJWV5dOxLH/SNk959nmsmA8f/QqShziTL2InX6wqt/cS6XsKFKyCrAcgPNLec2T5E43vse6Hmkmbyt1FhWKZITTLrWX2jog0G0AC1YS1CLgBeMx5/nsz+ywFHvXoOD8X+ImIhAOJxpgiEYkALgI+8EOefdf0JL99mQ0G175pg8GmRfDWjVB5FGJ7w7FiKFhp9815BCLjoaoUJt1sg8re9fb4Ed9t6EfRJi+llJ8FKoA8BrwhIjcDO4ErAERkEnCHMeYWY8whEfkV4JxJechZFwssdYJHGDZ4/Mn/RWiDplOexCRCRDc73cmetRAZC6dcCV8thJ6DYe9XgNjJGSO6QfUxyP5vGDAFFl5rX+OqVxrPvaWUUh0sIAHEGFMMfOvGGMaYVcAtHsvPA8832accmNjReexQLU3ACA0BYcTFkJhmb0419krY9j70Gg67nAkZP3oYkjPtDaxcYbDxr7Dp79+eGVgppTqIzoUVaJ61keVPNgSSDW/Dln/AuQ/bW+P2GdsQTL5ZYu+1XrQNXBFQU2Hvxz7zPg0eSim/0QASaJ61Ec+0Z2Bx11KaCyZb/wXVBqISbBNX+gwNIkopv9C5sDqr6Xc3BAJ3MJk2147mWv6EDSbhMc7OAqbO7vPmnIaOdaWU6kAaQLqCloJJz3Tb5DXiIjtKy/P+I0op1cG0Cauraa7Jq2Cl7USvrmi4/4hSSnUwrYEEg2jnov6KksDmQykVUjSABIMYJ4Ac1wCilPIfDSDBIDrBPlccCWw+lFIhRQNIMIh2ZnvRJiyllB9pAAkG2oSllAoADSDBQDvRlVIBoAEkGER3t89aA1FK+ZEGkGAQFgGRcdqJrpTyKw0gwSI6UZuwlFJ+pQEkWMQkahOWUsqvNIAEC62BKKX8TANIsIhO0BqIUsqvNIAEi5hE7URXSvmVBpBgoU1YSik/0wASLGISoaoMaqsDnROlVIgISAARkZ4i8r6IbHOee7Sw3xIRKRGRfzZZny4iX4hIroi8LiKR/sl5J6YTKiql/CxQNZD7gQ+NMUOAD53l5jwOzG5m/W+A3xljMoHDwM0dksuuJFrnw1JK+VegAsglwAInvQD4XnM7GWM+BEo914mIANnAWyc6PqS4J1TUGohSyk8CFUBSjDF7nfQ+IOUkjk0CSowxNc5yAZDanpnrkuonVDwc2HwopUJGh90TXUQ+APo0s+mnngvGGCMipgPzcRtwG0BKSgo5OTk+vU5ZWZnPx/pDt/JdTAE2rV7BgYL2+1o7e7k7QiiWGUKz3FrmtumwAGKMObulbSKyX0T6GmP2ikhf4MBJvHQxkCgi4U4tpD9Q2Eo+ngWeBZg0aZLJyso6ibdqkJOTg6/H+kXpPlgJIwf3Y+TkrHZ72U5f7g4QimWG0Cy3lrltAtWEtQi4wUnfAPzd2wONMQb4GLjMl+ODlnaiK6X8LFAB5DHgHBHZBpztLCMik0Tkz+6dRORT4E3gLBEpEJHznE0/Bu4RkVxsn8hzfs19ZxQRDeHR2omulPKbDmvCao0xphg4q5n1q4BbPJZntHB8PjClwzLYVenV6EopP9Ir0YOJTqiolPIjDSDBJEZrIEop/9EAEkyi9aZSSin/0QASTHRKd6WUH2kACSbRCdqEpZTyGw0gwSQ6ESqOQl1doHOilAoBGkCCSUwiYKBSm7GUUh1PA0gwidYZeZVS/qMBJJi4byqlI7GUUn6gASSY1N8TRAOIUqrjaQAJJjqholLKjzSABBOtgSil/EgDSDDRTnSllB9pAAkmkbEgYdqEpZTyCw0gwUREJ1RUSvmNBpBgsfxJ2L6s8YSK25fZ9Uop1QE0gASL1Anw5hxwhdkayPZldjl1QqBzppQKUhpAgkX6TLj8RTi0HfZvtMHj8hfteqWU6gAaQIJJ+kzoNx7K9kO3JBh4WkPTlmdzljZtKaXagQaQYLJ9GRzKg+ShUPQNzJ8EPQfDwmvt4/B2WDHf1k4Ob9fAopRqk4AEEBHpKSLvi8g257lHC/stEZESEflnk/Uvish2EVnnPMb5J+edmLvP4/IXYe5KGHM5HN4Bb1wPlWVQfQx2r4T3fgYT58Co7zcfWNx9JhpMlFInEKgayP3Ah8aYIcCHznJzHgdmt7DtXmPMOOexriMy2aUUrmnc53Hpn2Ho+YCBpMEQGQ8HNtrlT/8XXrkcqsqgpgIObLGBZdo8e/w/fmgDS+oEBux6p6Gm8pfLtNailKoXqAByCbDASS8AvtfcTsaYD4FSf2WqS5t+d+MO8+3LoGAlzLwPyg6AqYXT/9PO2Js6CWorIbY31NXC7s8BAx/8An4zGNa+ArVVsHcD1eFx8NrV9jH4jIZaS+oEHemlVIgLVABJMcbsddL7gBQfXuMREVkvIr8Tkah2zFvX59mclT6jYX1mtg0ohath7JW2WSsy1jZpRcZB75FwvBjCo2zN5L0HGP7NU7amUlUGHz1ij6mtgo8ftYHk8hdt7UdrJkqFHDHGdMwLi3wA9Glm00+BBcaYRI99DxtjWuoHyQJ+ZIy5yGNdX2zgiQSeBfKMMQ+1cPxtwG0AKSkpExcuXOhTecrKyoiLi/PpWH8bsOsdSuMzKekxtj4N0OvAp/Qq+pxdAy8l5lghKQeWY4CNo39CXFk+GXkvsj/lDHoeWsM3Q+4guehz+hxYxsHkqbjqqkk6tJqasBjCa48DYHDxzdDbOR7dj9Ebf40BDvaezrFuqQzc9TZFyVM50NsGsAG7/8buAbaiGV+ay+6B3yfx8Pr6dGfSlb7r9hSK5dYye+fMM89cbYyZ1HR9hwWQ1ojIViDLGLPXCQY5xphhLeybRZMAcjLbPU2aNMmsWrXKpzzn5OSQlZXl07GdxvInbXNT+syGNMCGt2HLP2D6PVBXY9cvvBaAHX3OJ23fErvfqXfAF0+DqbMjvfascV5YAGOfe6TbTvlzH4a+Y+tfhzN+DJ/8xqZHfx+ShsDyJ2DEd2H0pXZ94RrbFLd9WUM6AILiu/ZBKJZby+wdEWk2gIS3V6ZO0iLgBuAx5/nvJ3OwiPR1go9g+0++bv8sBiHPE7JnumkH/PZl9nnU9ympzQB3AHHf8VBccM4voaocXp8NddXQbwJUl8PBrXafjx6CzHNtcxdA7vtOWuwQ49ULYNJNMODUhiAz+vt2NJg7sLjz0UkCi1KqsUAFkMeAN0TkZmAncAWAiEwC7jDG3OIsfwoMB+JEpAC42RizFHhFRHphf/quA+4IQBmCR9MTcuEauOoVSJ9J/EvzbBrg339oSBeusTWViBjoNw32rLXrT70TVr8ANZW2VuOWn9OQ3rnCPq96zj4QCIuAQztsYGlae2ktsAC4wm3NyV0ODTRK+UVAAogxphg4q5n1q4BbPJZnNN3HWZ/dcblTnife3QO/T4a7ZtJ0WpQ35zQEFPfJfvh3IKG/HRY89ALY8amdJXjiTU6wAMZcBhv/amst+R9Dj0FQsgu259jt7/8cUkZCdQW4xE7NsnoBnPUL6D+x4b2uegX2rrfvde7DtlnOFf7tQPPvP8Dp81pPe9RyBux6B8hqhw9SqeAWqBqI6uo8m72WP9kQSNz9Kec+DEXbbHMXQFzvhvToS20fyHs/s6PBtv7LjgYbch5sXWyvWdm3AXBBXZ0djgzw4YPQLdmOBBOXDQC7VsDw70LOo5AyGnZ/CZNvgZRRdugxwOk/bNwX01zao5YTkzDR5+CjNR8VSjSAKN9405/iGVg8m7+aBhkABCbeYOfy8gwsAKdcCV8ttDWbA5shpqedcTj3fbt9yyL7vPsL+7zyT43z+vEjDekPfwm11bZWtPoFp5bjspNQrl4A0/+T4sPR9DtRwGkh+LTaxAYNAxe8CUztldYApzqIBhDVvloKLJ7NX97WXtxGXAyJaU1qLHEw7jpY/xpMuR2+fAZGXwYb3oC+42D7JzDsQtvBv+09SJthT+K7PoPeo6DmOBTn2ppMTZ3dH2D5E4zxLE/Or6HqmEfAOW4Dzs5/NwQid9/NqXc0rvnMehqKtsJ7D8K5v4K+p5w4GHVE2ssAN2BXPmx3aon+DHDeBkGPvALtEozHrH8IBrka3kMD7UnRAKL872RrLy0FluEXQEJqQx/ItLnQI+3bNZiZ99nhxy2lTZ09sW75J/QaCQVfwKDToaYKCldC8hA7KKA4F8KibHrruw35cPfdfPHHxuV8/bqG9Hs/s7cbNrWAQM5j9uJMEVvWyjI7JOSz+c56F6x50TbXIbD+dRu8RGDT3+2Fngh8s9TmR8QOVHCPdNv1uQ2eCBTnN4x66zUMXrvKjrq+4DEoyoMVv4cZP+J4dG9YeI3dNuNuj2B0X2ACn2e6aX9X0yHiPqYP97+UpBMF2gAGuI6ohbZnH19ArgMJlJC/DsQHnaLcLV2/0vSf1n1Tren32ECz8R273fMalJbSHieofSln0KfEeW33tS+N0gZOuQrWvmpP+iMuhi2LbQ1jx6dOzacGti2F9CxInw65H9raT+pEu23vV9BrBJgam9ekTDDGzqbcPdXuU7YfYnrY6WYqj0J4tE3XVXfs590aV0RDYIqIsUEN7BDviqP28+iWBMeK7T5xKVC2z6YTUuFIoQ12iYOgZKc9NikTivNsuvdwOzebYPu09n1t9+97ik0PmAIFX9oBGAWr7H4DptqAKWID/85/29dKP8OpWQpkZEPehzY95Jz65s/9iRNJKVlt1/efbIPGpJsgKcPOvAAwYTasfdkG1ok3wpoXbHryLXZgiDFw0e/s8PRP/xdm3mtndVh0F2DsHHOf/b+GwOyeoWHGj+z+YIP0J7910l78vZ5s2v1j7M05rBtyN+NmzTupr72l60A0gHipU5xIA6BLlbulQOPNLzSPiykLN3xC6qHP7fqTDD6MvaKh5nPqHfYEM/0e+6t20s3NBKM2piffDF/+2Z5Ix8+GNS/Z9WOdfiMBRs6CzYvs55GfYwcx1NXBpr/aAQgjLrKj4r5ZApln2235H0HaTFs727ncnqRNnT15p0606T1roc8Ym96/EXoNtzWsom3QM8OuP7wdEgbYk+zRAojva9eX7YduvWz6eLENQMbYQBkRa9fXHLc1PlNng5a4AHFqccHMfWEujWs0YVF2DjuA8Bj7+SB2AEpVuV0fFQ+VzvSB0QlQccQG1uge9nO88iVydta124WEej8QFTw8J5R0p9NnwnVvnTjdM902n02bS0VMXxsUrnrFnnBPlN7wtg0Q5z5s/7Hd0mfY4PHez+yz57xk7osy25rulmxPEIituYjLPnoOtrc3ljA7COGMH0P+JzawbHvf/hqfeZ8dxVZeBAUr2THoCjuKrXCV3bZ3Hexbb9MHNsHBLTZdtM3WGGbeB4d3Qslumz66B0r323T5QVsLmXmfPYlVHrXpqnJba5l5nz0Z1lXbtPuH7Mz7bL7DImw6PMrWdGbeZ2tj5zxkazgz7oWo7vZx+t125F5UPJx2l+0fi4y31yRFxtnH5FttYIqMszWMiFiIjGVP33NsOiIWxl1jX2/wmTYvY66wQ84BRl1qb4EAMPJ7ttYJNgAPu9Cmh14AF8+3owkBMs+xARlsDcj9uuln2OAMkDbd1poABk6zgRpjL7DtP8WpWU9umAC13wQ70KTmuO3r6zvWNnn2GW1rbJVHbe2n9wg70KTXMNsEe7zY1prb+Q6l2geiFHh37UtL6Zb6btxNbOc+bE8E7gs0ofGotI5Ot9SHlD7DBiOnX6Fkf923Zx3oLOkmeaXvWDtwAiC2lxNEgfg+DcPFE/o3pHuk2cAEtnbkpI9169+w/pSr7QCLk+1Dc6dXPWcDgjsAe3OMP9OrnrPXabUjDSBKtZU3I8+a8iYwtVfaywAXX5ofmADnTbqDgnGPfzx44kDra4Bzn7w7SwBOnwFvziFxyN1oJ7oPtA/k5IViuUOxzBCa5W5U5rb0oXWhUVh5n75FxvV/OKnPqbNNpqiUUp2LNzVJb9JN+bO2eaJ0+kx276wjo+XcnhTtRFdKKeUTDSBKKaV8ogFEKaWUTzSAKKWU8okGEKWUUj4JqWG8InIQewdEXyQDRe2Yna4iFMsdimWG0Cy3ltk7g4wxvZquDKkA0hYisqq5cdDBLhTLHYplhtAst5a5bbQJSymllE80gCillPKJBhDvPRvoDARIKJY7FMsMoVluLXMbaB+IUkopn2gNRCmllE80gCillPKJBhAviMj5IrJVRHJF5P5A56cjiMgAEflYRDaJyEYR+aGzvqeIvC8i25znHoHOa3sTkTARWSsi/3SW00XkC+f7fl1EIgOdx/YmIoki8paIbBGRzSJyWrB/1yLyn87f9tci8pqIRAfjdy0iz4vIARH52mNds9+tWH9wyr9eRCaczHtpADkBEQkDngIuAEYCV4vIyMDmqkPUAP9ljBkJTAV+4JTzfuBDY8wQ4ENnOdj8ENjssfwb4HfGmEzgMHBzQHLVsX4PLDHGDAdOwZY/aL9rEUkF5gGTjDGjgTDgKoLzu34ROL/Jupa+2wuAIc7jNuCPJ/NGGkBObAqQa4zJN8ZUAQuBSwKcp3ZnjNlrjFnjpEuxJ5RUbFkXOLstAL4XmBx2DBHpD1wI/NlZFiAbeMvZJRjLnADMBJ4DMMZUGWNKCPLvGnv/oxgRCQe6AXsJwu/aGLMMONRkdUvf7SXAS8b6HEgUkb7evpcGkBNLBXZ7LBc464KWiKQB44EvgBRjzF5n0z4gJUDZ6ihPAvcBdc5yElBijHFuKxeU33c6cBB4wWm6+7OIxBLE37UxphD4H2AXNnAcAVYT/N+1W0vfbZvObxpAVCMiEge8DdxtjDnquc3YMd9BM+5bRC4CDhhjVgc6L34WDkwA/miMGQ+U06S5Kgi/6x7YX9vpFPYCZgAAA3VJREFUQD8glm8384SE9vxuNYCcWCEwwGO5v7Mu6IhIBDZ4vGKMecdZvd9dpXWeDwQqfx3gdOBiEdmBbZrMxvYNJDrNHBCc33cBUGCM+cJZfgsbUIL5uz4b2G6MOWiMqQbewX7/wf5du7X03bbp/KYB5MRWAkOc0RqR2I63RQHOU7tz2v6fAzYbY57w2LQIuMFJ3wD83d956yjGmJ8YY/obY9Kw3+tHxphrgY+By5zdgqrMAMaYfcBuERnmrDoL2EQQf9fYpqupItLN+Vt3lzmov2sPLX23i4DrndFYU4EjHk1dJ6RXontBRL6DbSsPA543xjwS4Cy1OxGZDnwKbKChP+ABbD/IG8BA7FT4VxhjmnbQdXkikgX8yBhzkYgMxtZIegJrgeuMMZWBzF97E5Fx2IEDkUA+cCP2B2XQftci8kvgSuyIw7XALdj2/qD6rkXkNSALO237fuAXwN9o5rt1gul8bHPeMeBGY8wqr99LA4hSSilfaBOWUkopn2gAUUop5RMNIEoppXyiAUQppZRPNIAopZTyiQYQpboIEclyzxisVGegAUQppZRPNIAo1c5E5DoR+VJE1onIM879RspE5HfO/Sg+FJFezr7jRORz514Mf/3/7d2xalRBGIbh9xNBlAhWNhaK2ogQBcFCsfIGLGKjeAU2diJo4z0IWkZMEQTtRYuFVCqijVeQKo0IFoLEP8XMyqqF8XCy27xPt3OGYac4/DtnOd8/06fhdJLXST4l+ZDkVF9+aaaPx1p/EUxaCAuINKIkZ2hvO1+uqvPANnCTFt73vqrOAhPa28EAT4G7VbVMSwGYjq8Bj6rqHHCJliALLSX5Dq03zUlanpO0EPv/PUXSf7gKXADe9cPBQVpw3U9gvc95BrzofTmOVNWkj68Cz5McBo5V1UuAqvoO0Nd7W1Wb/fNH4ASwsffbkv5mAZHGFWC1qu79Npg8+GPe0Ayh2ZymbbyHtUA+wpLG9QZYSXIUfvWiPk6716aprzeAjar6CnxJcqWP3wImvSPkZpJrfY0DSQ7NdRfSLvjrRRpRVX1Och94lWQf8AO4TWvadLFf26L9TwItWvtxLxDTVFxoxeRJkod9jetz3Ia0K6bxSnOQ5FtVLS36e0hj8hGWJGkQTyCSpEE8gUiSBrGASJIGsYBIkgaxgEiSBrGASJIG2QF03G2OCl7UKgAAAABJRU5ErkJggg==\n",
 871 |       "text/plain": [
 872 |        "<Figure size 432x288 with 1 Axes>"
 873 |       ]
 874 |      },
 875 |      "metadata": {
 876 |       "needs_background": "light"
 877 |      },
 878 |      "output_type": "display_data"
 879 |     }
 880 |    ],
 881 |    "source": [
 882 |     "plot_history(history)"
 883 |    ]
 884 |   },
 885 |   {
 886 |    "cell_type": "code",
 887 |    "execution_count": 21,
 888 |    "metadata": {
 889 |     "id": "ieObNqKYsOzh"
 890 |    },
 891 |    "outputs": [],
 892 |    "source": [
 893 |     "torch.save({\n",
 894 |     "            'encoder': model.encoder.state_dict(),\n",
 895 |     "            'decoder1': model.decoder1.state_dict(),\n",
 896 |     "            'decoder2': model.decoder2.state_dict()\n",
 897 |     "            }, \"model.pth\")"
 898 |    ]
 899 |   },
 900 |   {
 901 |    "cell_type": "markdown",
 902 |    "metadata": {
 903 |     "id": "ymhjbmvR_DgJ"
 904 |    },
 905 |    "source": [
 906 |     "## Testing"
 907 |    ]
 908 |   },
 909 |   {
 910 |    "cell_type": "code",
 911 |    "execution_count": 22,
 912 |    "metadata": {
 913 |     "colab": {
 914 |      "base_uri": "https://localhost:8080/",
 915 |      "height": 34
 916 |     },
 917 |     "id": "b7rbm9wdXKeF",
 918 |     "outputId": "076309c7-22be-41f6-f916-5f11cb679672"
 919 |    },
 920 |    "outputs": [
 921 |     {
 922 |      "data": {
 923 |       "text/plain": [
 924 |        "<All keys matched successfully>"
 925 |       ]
 926 |      },
 927 |      "execution_count": 22,
 928 |      "metadata": {},
 929 |      "output_type": "execute_result"
 930 |     }
 931 |    ],
 932 |    "source": [
 933 |     "checkpoint = torch.load(\"model.pth\")\n",
 934 |     "\n",
 935 |     "model.encoder.load_state_dict(checkpoint['encoder'])\n",
 936 |     "model.decoder1.load_state_dict(checkpoint['decoder1'])\n",
 937 |     "model.decoder2.load_state_dict(checkpoint['decoder2'])"
 938 |    ]
 939 |   },
 940 |   {
 941 |    "cell_type": "code",
 942 |    "execution_count": 23,
 943 |    "metadata": {
 944 |     "id": "Ry1QTp6V2ny4"
 945 |    },
 946 |    "outputs": [],
 947 |    "source": [
 948 |     "results=testing(model,test_loader)"
 949 |    ]
 950 |   },
 951 |   {
 952 |    "cell_type": "code",
 953 |    "execution_count": 24,
 954 |    "metadata": {},
 955 |    "outputs": [],
 956 |    "source": [
 957 |     "windows_labels=[]\n",
 958 |     "for i in range(len(labels)-window_size):\n",
 959 |     "    windows_labels.append(list(np.int_(labels[i:i+window_size])))"
 960 |    ]
 961 |   },
 962 |   {
 963 |    "cell_type": "code",
 964 |    "execution_count": 25,
 965 |    "metadata": {},
 966 |    "outputs": [],
 967 |    "source": [
 968 |     "y_test = [1.0 if (np.sum(window) > 0) else 0 for window in windows_labels ]"
 969 |    ]
 970 |   },
 971 |   {
 972 |    "cell_type": "code",
 973 |    "execution_count": 26,
 974 |    "metadata": {
 975 |     "id": "FSWwxheNvxR7"
 976 |    },
 977 |    "outputs": [],
 978 |    "source": [
 979 |     "y_pred=np.concatenate([torch.stack(results[:-1]).flatten().detach().cpu().numpy(),\n",
 980 |     "                              results[-1].flatten().detach().cpu().numpy()])"
 981 |    ]
 982 |   },
 983 |   {
 984 |    "cell_type": "code",
 985 |    "execution_count": 27,
 986 |    "metadata": {
 987 |     "colab": {
 988 |      "base_uri": "https://localhost:8080/",
 989 |      "height": 279
 990 |     },
 991 |     "id": "bROUyLM93cG3",
 992 |     "outputId": "755359d9-d0fb-4deb-b313-d3c2a2465a26"
 993 |    },
 994 |    "outputs": [
 995 |     {
 996 |      "data": {
 997 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1fn48c/JZCOQhCUQCAmEJcgWQAibCyBqRVFwAZVWrX6hqJXaWqXYuuuvv7rUWv3qz63uGwpWQUXEClGwIDsom0bWhC37vszy/P64QxYIEEImdybzvF+vvDL3zp2Z52SSeXLOPfc5RkRQSikVvELsDkAppZS9NBEopVSQ00SglFJBThOBUkoFOU0ESikV5ELtDuBUxcXFSXJycqMeW1paSuvWrZs2ID+nbQ4O2ubgcDptXrduXY6IdKzvvoBLBMnJyaxdu7ZRj01PT2fcuHFNG5Cf0zYHB21zcDidNhtj9hzvPh0aUkqpIKeJQCmlgpwmAqWUCnKaCJRSKshpIlBKqSDns0RgjHnVGHPYGPPDce43xphnjDEZxpjNxpihvopFKaXU8fmyR/A6MOEE918MpHi/ZgLP+zAWpZRSx+GzRCAi3wB5JzhkMvCmWFYBbY0xXXwVD0BkVhZUVPjyJZRSqsm5PcK/lu9kZ6HbJ89v5wVlXYF9tbYzvfsOHH2gMWYmVq+B+Ph40tPTT/nFjNvNsDlzyH3mGb5/7LFGBRyISkpKGvXzCmTa5uDQ0trs8girD7opcwpOj7Vd6YZ9xR42ZVsJ4NwuQk8ftDkgriwWkZeAlwDS0tKksVfWbbr9dgaPGGFdmed2Q3k5tGnTdIH6Ib36MjhomwNPpcvN1v1FlFe5+frHbN5atYeyqrr/8TtCDD3iWjNtRDtSu7alQ/HPPmmznYkgC0iqtZ3o3ecz+SNGwJEf4rPPwlNPwapV0LmzL19WKaXqEBFmvLGW5T/lVO8bnBjLOSlxTD+nJ+GhIYQ5DOGOEIwx1cekp+/0STx2JoKFwCxjzFxgJFAoIscMC/lMWhpMmgTx8da2CNT6gSulVGOJCE634HR7cLo9VDg9FFU4Wbs7n0XfH2DrgSLySqu4Ji2JK4Z2pU1EKP27xBASYs9nkM8SgTHmPWAcEGeMyQQeAMIAROQFYBFwCZABlAE3+SqWep19tvUFkJ0N558P//wnjB/frGEopQLP4aIKHv9iB7kllezJK6Oo3IXb48HpFkoqXSd8bHKHKM7v24lzUuK4bFCCbR/+tfksEYjItJPcL8Btvnr9U1JQAFFR0KmT3ZEopfyUy+3h259z+T6zgH+vz2JnTikDEmJIiG1FQiwktW9Fq7BQwkNDKCirIql9FGEOQ5gjBBEIDw2hf0IMZya1rTPc4w8C4mSxz6WkwMqVNUNDDz0Eycnw61/bGpZSyj9kHC5h9vxNbNhbAEBcmwhuGN2dhycPtDmypqGJ4IgjScDlgmXLoH9/TQRKBbEKp5st+wt5dmkGy3Zk0yYilMevGsSlg7sQFd6yPjpbVmuaQmgoLF0KVVXW9o8/whdfwG23QYiWZlKqpcspqeT/fLqVTzcfwOURYluF8YcLUvjliG50iom0Ozyf0ERQn5AQiPS+4a+/Ds8/D1dfXTPDSCnV4hSWOfnPtkM8uWQHOSVVXDeqO0O7t2NsSkdio8LsDs+nNBGczF//CtOn1ySB+fNh8mQIa9m/GEoFgwqnm28zcnjnu70s23EYEejXJYbnrxvG4KS2dofXbDQRnIwx0KuXdXvVKpg6FV58EWbOtDcupVSjuNweNu4r4PX/7uarbYcpd7qJbRXGrWN7MaZPR4Ynt8fhB1M6m5MmglMxahQsXlxzrUFGBnTtCq1a2RuXUqoOESEzv5y9eWXsyikl43AJu3JKOVBYzp7cMipdHmIiQ7liaFd+0T+es3rFER4avOcANRGcqosusr67XDBxIvTsCZ9/bm9MSgW5w8UVbMkqYtvBIjbuLWD93nxySqqq748Kd9AjrjXdO7RmTEpHBie1ZdwZHYmO1CFe0ETQeKGh8NxzEB5ubbvdUFYG0dH2xqVUC5RfWsWh4gp2ZZeyN6+M3NIq1u2o4MnvV3CgsLzOh36POOvDfmj3dvTs2JrkDq3pHBPpF1fw+itNBKfjggtqbj/zjFXE7rvvoItPl1VQqsVwuj18te0wmfll/HioGJdbcHoEl7dGT3ZxJbtySimqqFu2ITIshLZhQt+kcAYkxNC7UxsGJbalT3wb2kaF29SawKWJoKmMHg1799ZUMvV49LoDpU6g0uVm8rPfsv1gMQDRkaHEtgojzBFCaIgh1BFCmwgHl5/ZlW7to+gcG0n39q3pHhdFTGSYtwz1CJtb0TJoImgqo0ZZX2AVsRs3Dp5+um6vQSlV7adDJWw/WExa93b8v+uG0rFNhN/V4AkWmgh8obAQ2rbVISKlTiC7pBKAP1/Sj07RLfOK3UChicAXeveGFStq6hc98IBVxO6m5q20rZS/WvlzLrPnbcIYSGyn06/tponAV2oXsVu+HHJyNBGooCEiFJW7KKlyUVBWRU5JFSUVLg4WVZBxuIR/r8+kY3QEr/56OPEttH5PINFE4GuhofCf/9QUsduxw7oobdYscDjsjU2pRhARDhZVUFjuJL/UyZ7cUnYcKia3pIqiCicHCyvYmV1KldtT7+OjI0M574xO3DOxH0nto5o5elUfTQTNoXYRu7fesq4/uPZaLWKnAobL7WHumn18unk/GYdLyfGO7x8RFe6gU3QEMa3C6BIbyZg+HekUHUF0ZCgxkWHWbKDQEHrGtaZ963A9KexnNBE0t0ceqVvE7v334Yorai5MU8pmm/YVsO1AEfllTkoqnWTml/PNj9nklzlJat+KMSlxDOgaS5fYSGK9H/w94lrrh3sA00TQ3IyBHj2s26tWWT0DLWKn/ITL7eGal1ZS4bSGdRwhhnZRYZzZrR1XDu3KpYMSbI5Q+YImAjuNGgVffmldcwDWIjiJidb6yUo1oyNF2v69PosKp4fZF53BTWcn0yrMof/pBwFNBHY7csGZywWXXWZNM/3iC1tDUsFn3tpM/vThZgDO6R3HzDE9CXPolfHBQhOBvwgNtVZCO7LgjdsNpaUQE2NvXCoo7M0rI8TA6zeNYHSvDpoEgoy+2/5k/Hg491zr9jPPwIABcOCAvTGpFs3jEfYXlLPtQBFtIkIZ06ejJoEgpD0Cf3X22bBvnxaxU02myuXhyS93kJlXTl5pFT8dLqHS6aa40qrs+auR3WyOUNlFE4G/GjHC+gI4fLimiN2FF9oalgpca3bn8eLXO2kV5mBAQgwDu8YQYgzj+3ZiQEIMQ4JojV5VlyaCQFBcDHFx1rKYSh3F6fZQUiXsyyujoMxJSaWL4gonTrfg8niocnlweYQ1u/MA+PT2c+jVsY3NUSt/ookgEPTqBd98U7N9333QrRv85jf2xaRs4/EIb67czZsr95BVUE6ly1vKYemykz62a9tWdInV2j6qLk0Egcblsi5Ey8+3OxLlAx6PsL+wnMU/HCSvtIrSShdlVW6yCsoJMQa3R9iXX0ZmfjlR4Q6mjehG+9bhHNq3m8ED+9K2VRhtIkNpExFKRKiDUIch3BFCqMPQJiJU1+hV9dJEEGhCQ2HJkpoidtu3w6JF8PvfaxG7ALM5s4D9BRXklFTy3a48th0oYldOKW6PABAaYogKd9A6IpRWYQ6KKlwkd4iiZ8c2TBzUhTkX9a1ehzc9PYtxaUl2NkcFME0EgcgYiIiwbr/zjlXE7rrroFMne+NSx1VQVsXP2aWs3pXHJ5v2szOnpLqMA0BcmwgGJ8ZyQb94OsdEkJbcngEJMXpVr2oWmggC3cMPw4wZVhIQgblz4aqrtIidnyivcnPT66tZtTOvel9q11imjehGYrsoRvZoT/vW4XSOiaz+716p5ubTRGCMmQA8DTiAf4nIo0fd3w14A2jrPeZuEVnky5haHGOge3fr9nffwS9/CS+8ADffbG9cCoCPNmSxamceU4YlMmFAZ1ITY3UhFuV3fJYIjDEO4DngQiATWGOMWSgiW2sddi/wgYg8b4zpDywCkn0VU4s3ahR89RWMGWNt79hBSEWFvTEFMRHhH1/+SM+OrXnsqkE49D9+5ad82SMYAWSIyE4AY8xcYDJQOxEIcKSYTiyw34fxBIfx463vLhdMmsTAmBiYMMHemIJQUYWTxxdvJ6ekklvH9dIkoPyaERHfPLExU4AJIjLDu309MFJEZtU6pguwBGgHtAYuEJF19TzXTGAmQHx8/LC5c+c2KqaSkhLatAmeC2liN26krLIS58iRGLebkPJy3EHQfn94n9P3OXl9SxVxrQwPndWK1mG+TQT+0Obmpm0+Needd946EUmr904R8ckXMAXrvMCR7euBZ4865o/And7bo7F6CyEnet5hw4ZJYy1btqzRjw1U1W3++99FEhJE9u+3NZ7mYPf77PF45N6Pvpfucz6VKpe7WV7T7jbbQdt8aoC1cpzPVV8ODWUBtSc2J3r31TYdmAAgIiuNMZFAHHDYh3EFp7Fj4eBBLWLnI4eLK3h75R4Ky52s31vA91mFDEiI0UqeKiD48rd0DZBijOlhjAkHrgUWHnXMXuB8AGNMPyASyPZhTMErLQ2eeMKaZXT4MPTvrwvgNKF5azN5ZmkG89dlUlLp4p5L+vH29JF2h6VUg/isRyAiLmPMLOALrKmhr4rIFmPMw1hdlIXAncDLxpg7sE4c3+jtwihfKimxegZJeiXq6Vi9K4/9BeXkllbxv0t/Iql9K5b/abzdYSl1ynx6HYFY1wQsOmrf/bVubwXO9mUMqh49e0J6es32PfdY1yLMnGlbSIFmc2YBV7+4snrbGJg0WBd2V4FJrywOdi4XrFkDRUV2RxJQVu3MBeD1m4aT2jWWdlHhemWwCliaCIJdaKh1rqB2EbtPPoE77rDuU8c4XFzBU1/+xOCktozt01HrAamAp1MaVN0idu++C3/7G+TlnfgxQcrp9nDvRz/g8nh4+pohmgRUi6CJQNX10EOwaVNNEbu334bKSrujalZVLg+llS4Ky53kl1aRXVzJwcIK9uWVMXveJpZsPcTsi84gOa613aEq1SS076/qMqZmNtHq1XD99VBW1uJPJIsIr6zYxfdZhXy2+QAuz/Enr914VjIzx/RqxuiU8i1NBOr4Ro60Zhed7Z3YtX27lSRat7z/hOeu2cf/+WwbAIMTY/nFgM5EhIYQGmJwOLzfQwwdoyMY16ejzdEq1bQ0EagTGzvW+u4tYkf37vDll/bG1MRKK13848sf6dclho9vO4uIUF3pTQUXTQSqYUJD4V//qlkO0+WyLkxr29beuJrApswCsosrefCyAZoEVFDSk8Wq4caMqRkmevppq0zF/sCvHJ5dbJ0M79sl2uZIlLKH9ghU45x3nlWzqEsXa9vtruktBJhDRdbiPXFtImyORCl7aI9ANc7QofDYY9Yso0OHoF8/WLzY7qgaZXNmIQmxkcS2CrM7FKVsoT0CdfrKyqzZRMnJdkdSr4KyKv77cy7r9+SzL7+Msio3TrcHp1vYm1dGdnElY3QmkApimgjU6evRw1or+Yi//AUSE+G3v22Wl690uSkoc1Jc4aK4wsmGwy6+/+ontuwv4qfDxezMKUUEIkJD6BwbSfvW4YQ5QmgV5mBgQgxFFS5uHavXBajgpYlANS2XCzZssGYU+UCF082a3XlsP1DMT4eLWfzDQYoqXPUc+SMJsZEM7BrLxNQujOnTkcFJbXWhGKXqoYlANa3QUFi0CJxOa3vbNli4EO6885SK2FU43WzZX0jG4RJ25ZTx06Fi9hdWsDO7hEqXB4A2EaGcmxLHwK6xxLYKIzoylJjIMDK2fc81F48hJlLH/JVqCE0EqukZA+Hh1u3334f//V+46SarftFJiAhb9hfxu/c2sCunFIDQEEOPuNZ0ax/F6J4dOCelA4MS2x53lo856NAkoNQp0ESgfOvBB2HGjLpF7KZOhcjIOocVVTj526LtpO84zIHCCuLahPPgZf05r28nktpFaa1/pXxIE4HyvcRE6/vq1XDDDdYso5tvrnPI4u8P8t7qvYzu2YEZ5/bk8iEJdNB5/Uo1C00EqvmMHAnffAOjR1vb27bBypXw8MNM3buXc2I60vnZfxByzih741QqyGgiUM3r3HOt7y6XdXVydjZ4PBggofAw3DwTDPCrX9kZpVJBRefSKXscmUHk8dTdX1YG99zT/PEoFcQ0ESjbyOHD9d+xd2/zBqJUkNNEoGzxv1/9RFZ0XP13dutmFbFTSjULTQSqWXk8wux5m3jyyx95evyNuCNb1T0gKgr+9Cc44wzrwjSllM9pIlDN6qXlO5m3LpPendrwf+c/iuNfL1urnhljfX/pJbjkEqt+Uc+edoerVFDQWUOq2RwoLOfFr39mRHJ73r95FMYYa3ZQfTOEai+HeffdVnXT225rvmCVCiKaCFSzqHS5ufODTZRWunlo8gArCTSEywXffw8VFb4NUKkgpolA+VxOSSUTn1nOoaJK7p3Yj35dYhr+4NBQ+PRTKyEAbN0KH31knUcI03pCSjUFPUegfO7TTfs5VFTJ41MGMePcRoz7G1PzoT9vHjz1FOTnN22QSgUxTQTKp9btyePl5bs4Iz6aqcMST/8JH3gANm+uKWL3+utQXn76z6tUEPNpIjDGTDDG7DDGZBhj7j7OMVcbY7YaY7YYY971ZTyq+Xy3M5cbX1vNVc+vxOn28MjlAxt+XuBkEhKs72vWWOWt33yzaZ5XqSDls3MExhgH8BxwIZAJrDHGLBSRrbWOSQH+DJwtIvnGmJMXrFd+zeMR/vHljzy7LIPQEMPMMT25dWwv2rUOb/oXGzECVqyAUd4idVu2WBejKaVOiS9PFo8AMkRkJ4AxZi4wGdha65jfAM+JSD6AiByn5oAKBIt/OMB9C7aQXVzJuSlxPPerob5fIObss63vLhdcfrl1LcK99/r2NZVqYYyI+OaJjZkCTBCRGd7t64GRIjKr1jEfAz8CZwMO4EERWVzPc80EZgLEx8cPmzt3bqNiKikpoU2bNo16bKBqjjZnl3mYu6OKdYfcxEcZJvUKY0SXUMKaeTGZmC1bwBj2d+tGdKtWOEpKcMXGNmsMdtHf7eBwOm0+77zz1olIWn332T19NBRIAcYBicA3xphUESmofZCIvAS8BJCWlibjxo1r1Iulp6fT2McGKl+3ef66TB7/djslFfCHC1K4dVwvIkIdPnu9E/K2syg9nbGrV8Pf/w4bN9acU2jB9Hc7OPiqzb5MBFlAUq3tRO++2jKB70TECewyxvyIlRjW+DAu1QR+zi7hySU7WPT9Qfp2jub5Xw0lLbm93WHVmDABCguhSxdr2+WqKX2tlKrDl7OG1gApxpgexphw4Fpg4VHHfIzVG8AYEwf0AXb6MCbVBBZszOLy575l+Y85TD+nB/NvPcu/kgDAoEHw179a1yAcPAh9+sBnn9kdlVJ+yWf/IomIyxgzC/gCa/z/VRHZYox5GFgrIgu99/3CGLMVcAOzRSTXVzGp0+PxCO+s3st9H/9Az7jWPH/dMM7oHG13WCdXWWklgt697Y5EKb/k076yiCwCFh217/5atwX4o/dL+bGVP+fy4MIt7DhUzBnx0bw5fQTxMZF2h9Uw3bvD4lpzEGbPtorY3X67fTEp5Ud00FSdUKXLzVNf/sQLX/9MdGQo907sx3WjuhMZZtMJ4dPlcsGOHeB02h2JUn5DE4Gq14HCch79fDuLfzhIpcvDZYMTeGLKoMBNAEeEhsKCBXWL2H34IcyZA+E+uOhNqQCgiUDV4fYIr67YxVP/+RGXR5g6LJHz+3XivDM6NV2JCLvVLmL34Yfw9NNwyy3QsaO9cSllE00Eqlqly81d8zbzyab9nNM7jr9eMZDuHVrbHZZv3XcfzJhhJQEReO01uPZaa8lMpYKEJgIFgIhw94ff88mm/dx+fgp/vLCP3SE1nyPXGqxZA9OnW8NGM2faG5NSzUgTgQLgu115fLQhi9+fn8IdwZQEahsxAlauhOHDre0ffrBmFwVJmQoVvE75gjJjTIgxpp5FZlUg+/z7A0SFO7hlbC+7Q7HXqFHgcIDbDVdcYX0p1cIdt0dgjIkBbgO6Yl0R/CUwC7gT2AS80xwBKt/78VAxX2w5xJnd2tIqPMBnBTUVhwPeecc6bwDWdNPCQoiLszcupXzgRENDbwH5wEpgBvAXwACXi8jGZohN+dhX2w7x2re7WbUzlxBjeHzMILtD8i8jRtTcfvJJ62vjRuja1b6YlPKBEyWCniKSCmCM+RdwAOgmIhXNEpnymY37Cpg9bxM/HS4hPiaCm85OZsa5PQPnSmE7XHoplJbWJAGns2YKqlIB7kSJoPrSSxFxG2MyNQkErkqXm2Xbs3nnuz0s/ymHtlFh3Hdpf64ZnkSbCJ0zcFIDB1pfYBWxGz0annkGLrvM3riUagIn+gQYbIwpwhoOAmhVa1tEJMbn0akm8UNWITPfXMv+wgratw7n5rE9uX5UdxLb6Vz5Rqmqgn79rEJ2SrUAx00EIqJnDVuA3HIP97y1juJKF8//aigX9I8nzOHL6uNBoFs3WFSrluJdd1lDRnfcYV9MSp2GE80aigRuAXoDm7HKSLuaKzB1esqr3DyfnsG/vi2nymN46YZhjO8bb3dYLY/bDRkZ4PHYHYlSjXaioaE3sM4TLAcuAQYAv2+OoNTpyS6u5LfvrGPN7nwGxTl4+sZz6RHXwktF2MXhgI8+shICWBehzZ8Pf/4zRETYG5tSDXSiRNC/1qyhV4DVzROSOh1uj3D3h5tZszufeyf2o7d7ryYBXzOmZhnMBQvguefgttu0iJ0KGCcaLK49a0iHhAKAiPDUlz/y1fbD3HlhH2ac29PukILPPfdYvYIjRexefhnKyuyOSqkTOlGPYIh3lhBYM4V01pCfe/7rn3l2WQapXWOZNV6XZbRNvPdczNq1VvE6jwduvtnemJQ6gRMlgk0icmazRaIa7WBhBQ8u3MLiLQcZntyO934zquWsHRDIhg+H776DYcOs7e+/t2YcaRE75WdOlAik2aJQjZZxuJhb317PntwyrklL4p5L+xGq00P9x5EyFW43XHmlVc106VJ7Y1LqKCdKBJ2MMcddVF5E/uGDeNQp+CGrkCkv/JfW4aG8dMMwxp3Rye6Q1PE4HPDeezXTTJ1OKCjQE8rKL5woETiANtRcWaz8iIjw8KdbcbmFT353DgltW9kdkjqZtLSa208+CU88AZs3axE7ZbsTJYIDIvJws0WiGszjEf704WZW78rjkckDNAkEokmToKJCi9gpv3CiwWTtCfipD9dnMn9dJr8e3Z3rRnW3OxzVGP37w4MPWrf374devWDhQltDUsHrRIng/GaLQp2St1ftoWN0BPdd2l9nB7UEbjcMHgx9+9odiQpSx00EIpLXnIGohlm7O49NmYVcNTRRZwe1FElJ8MknNdVM//hH+Pvf7Y1JBRUtRB9AMg6XcNNra4gKd/Drs3RIqEVyu2H3brujUEFGE0GA+D6zkGtfWonTI/z71rPoEqsniFskhwM+/LBuEbsPPrBKV2gRO+UjOrYQIJbtOExplZuPfnsWA7vqlaktWu0idgsXwgsvQHGxvTGpFk0TQYDIzC8jrk0EAxI0CQSVv/wFtmyBuDiriN2LL0JJid1RqRZGE0EAEBGWbD3EoERNAkHpyNXHa9fCLbfAu+/aG49qcXyaCIwxE4wxO4wxGcaYu09w3FXGGDHGpB3vmGB2oLCCgjIngxPb2h2KstPw4bBmDUyfbm1v2gT5+fbGpFoEnyUCY4wDeA64GOgPTDPG9K/nuGislc++81Usge5QUQUAqYla+TvopaVZJ5TdbpgyxSpkp9Rp8uWsoRFAhojsBDDGzAUmA1uPOu4R4DFgtg9jCWilldYMkjYRWoJAeTkc1mwil7VmlHG54PBh6KSFB9Wp82Ui6Arsq7WdCYysfYAxZiiQJCKfGWOOmwiMMTOBmQDx8fGkp6c3KqCSkpJGP9ZOGw5bf+xbN2+gbI/jlB4bqG0+HUHX5vR04t96C+fHH7PmX/+iKkgqmgbd+4zv2mzbdQTGmBDgH8CNJztWRF4CXgJIS0uTcePGNeo109PTaexj7ZSzLhPWb2Ls2aNOef3hQG3z6QjGNq/eu5ewXr04a+pUa0dVFYSH2xuUjwXj++yrNvvyZHEWkFRrO9G774hoYCCQbozZDYwCFuoJ42PtzSvDGOiqVUbVcZR16wb3329tHCli9/HH9galAoYvE8EaIMUY08MYEw5cC1SXVxSRQhGJE5FkEUkGVgGTRGStD2MKSFn55cRHRxIeqrN9VQN4PDB0KAwYYHckKkD47JNFRFzALOALYBvwgYhsMcY8bIyZ5KvXbYkOF1cQH6PlBVQDJSbCggWQkmJt/+EP8Pjj9sak/JpPzxGIyCJg0VH77j/OseN8GUsgO1BYQa+Op3ZuQCnAmma6f39NyQql6qFjDX5ORNhfUK6rkKnGOTLN9LHHrO3Nm62yFRUV9sal/IomAj+XV1pFWZWbpHZRdoeiApnDO+140SJ45RUoLbU3HuVXNBH4uY82WBOtTnXaqFL1uvtuq4hdhw5WEbvnn9fKpkoTgT9zuj0889VPDO3WlrN7x9kdjmop4ry/S+vWwW23aRE7pQvT+LPPNh+gqMLFzDE9deqoanppaVZF08GDre2NG6FbN2jf3t64VLPTTxc/9uW2Q4Q7Qji/X7zdoaiWaujQmiJ2U6fCVVfZHZGygfYI/NT+gnI+23yAX47sRpguUq98zeGAefOqi9jhdEJuLnTubG9cqlnoJ4yfOnKS+H/O7mFzJCpoDBliDRcBPPEE9OsHmZn2xqSahfYI/JCI8ObK3QxOakvvTm3sDkcFoylTrFIViYnWdmUlROjV7S2V9gj80IHCCg4VVTJlWKLdoahg1acP3HuvdTsrC3r2hI8+sjcm5TOaCPzQpn0FAPTuqL0B5QeMgZEjITXV7kiUj+jQkB+aty6TzjGRDO2uaxQrP5CQAP/+d8327bdb++4+7jLkKsBoj8DPlFW5+ObHbCYO6kJE6JWsqz4AABqVSURBVKmtRqaUz7nd1pKYubl2R6KakPYI/MzGfQW4PMI5KXolsfJDDgfMnWslBLCK2L37LjzwALTSwoiBSnsEfmZ/gVUVslecnh9QfuxIEbvFi+G116CszN541GnRROBnqlweACLC9K1RAeBPf4KtW2uK2D37LBQV2R2VOkX6aeNnqlxWlztcryZWgaJDB+v7unXWieS5c+2NR50yPUfgZ6rcVo9Ai8ypgJOWBhs21Ewz3bABkpJqqp0qv6WfNn6mrMrqEURoIlCBaPBgCAmxTiZffbUWsQsQ2iPwM4XlTqIjQgnVoSEVyBwO69qDqipr2+mEnBzo0sXeuFS99NPGzxSWO4lpFWZ3GEqdvtRUGDbMuq1F7Pya9gj8TEGZk7ZRmghUC3P11db3I0XsKiogMtK+eFQd2iPwM7kllXRoo1UeVQvTuzf85S/W7SNF7D780N6YVDVNBH6msNxJWx0aUi2ZMXDWWTVLZCrb6dCQnymqcBEdqW+LasESEmD+/JrtWbOsk8j33GNfTEFOP3H8iIhQVO4kVnsEKli43VBQAFFRdkcS1DQR+JFypxuXR4iO1ESggoTDAW+/ba2GBrBpk7X90EOaHJqRniPwI4XlTgDtEajgE+L9KFqyBN56C8rL7Y0nyGgi8CNF5S4AYlppR00FqdmzYdu2miJ2zzwDhYV2R9XiaSLwI7mllQC0jwq3ORKlbNSunfV9/Xq44w54/3174wkCPk0ExpgJxpgdxpgMY8wx69oZY/5ojNlqjNlsjPnKGNPdl/H4u9wS63J8vY5AKayrkjdsgBkzrO116yA7296YWiifJQJjjAN4DrgY6A9MM8b0P+qwDUCaiAwC5gOP+yqeQJBTYvUIOkZrIlAKgEGDaorYXXstTJlid0Qtki8Ho0cAGSKyE8AYMxeYDGw9coCILKt1/CrgOh/G4/fySqsIMegFZUodzeGAjz+GSuufJaqqCNfeQZPxZSLoCuyrtZ0JjDzB8dOBz+u7wxgzE5gJEB8fT3p6eqMCKikpafRjm8PWjEoiHfDNN1832XP6e5t9QdvcwqWn0/2ttxg+dy4rgcqOHe2OqNn46n32i+kpxpjrgDRgbH33i8hLwEsAaWlpMm7cuEa9Tnp6Oo19bHP4Im8zbfIPN2mM/t5mX9A2B4GkJH4ODWX01KnWdnk5tGplb0zNwFfvsy9PFmcBSbW2E7376jDGXADcA0wSkUofxuP3Kl0ewnQdAqVOrlcv9k2bZt3OzLSK2NUuW6FOiS8/ddYAKcaYHsaYcOBaYGHtA4wxZwIvYiWBwz6MJSA43aJLVCp1qkJDYcwYOPNMa1vE3ngCkM+GhkTEZYyZBXwBOIBXRWSLMeZhYK2ILASeANoA84wxAHtFZJKvYvJ3TpeHMIexOwylAkvnznWvNThSxO7ee+2LKcD49ByBiCwCFh217/5aty/w5esHmiq3R3sESp0OtxuKiiA62u5IAopfnCxWlgqnm4hQh91hKBW4HA6rVlHtInZvvgkPPwytW9sbmx/Tfz/9SGmVm9YRmpuVOm1Hitj95z/w7rs11x+oemki8CMlFU7aRGiPQKkmc+edVhG79u2tk8hPPWWtf6Dq0ETgRwrLnbTVgnNKNa22ba3v69fDXXfBvHn2xuOHdBzCTzjdHvJKq4hrrYlAKZ8YNgw2b4Z+/azttWuhWzfo1MneuPyA9gj8xMHCCjwCXdu1/KsjlbLNgAE1ReymTdMidl7aI/ATmfnWikydYzURKOVzDgcsWFCniB2HD0Nior1x2UR7BH4gu7iShz7ZQrgjhL6ddf6zUs2if/+aq5Eff9za3rfvxI9pobRH4Af+/sUOth8s5plpZxIfE2l3OEoFn1/9CiIiIMlbHi1IitgdoT0Cm/03I4cP1u1j6rBEJg1OsDscpYJTjx7Weslg9QqSk4NqdpEmAhvllVbx23fX06NDa+6+uK/d4SilAMLD4fzzrVlGEBRF7HRoyCaLfzjAI59uo6DMyes3jdB1ipXyF/Hx1tXIR/z2t9a+Bx+0LSRf0x5BM3N7hMcXb+eWt9cTHRnKS9cPY0hSW7vDUkrVx+2GiooWX6JCewTNbP66ffy/9J+5oF8nnvvVUC0yp5Q/czjgtddqhoc2bIA33oC//rVFFbHTHkEzKq108Xz6z8THRPDyDWmaBJQKFMa7TsiyZfDBB9Z1By2IJoJm4vEI9338A7tzy3ho0gCM0QVolAo4f/yjVcSuXTurl/Dkk5Cfb3dUp00TQTP5+qds/r0hi1+O7MaEgV3sDkcp1Vixsdb39ethzpwWsVayniNoJv/8z09EhIZw/6X97Q5FKdUUjhSx6+ud+r1mjXVBWufO9sbVCNojaAa7c0rZtK+A289PITJMzwso1WL0719TxO6Xv4Srr7Y7okbRHkEzePGbnwkNMVxxZle7Q1FK+YLDAZ98Yk01BWu66aFDVpnrAKA9Ah/78VAxc9fsY9LgBBLaBk/tEqWCTt++MGSIdfvxx62S1wFSxE57BE1ARPAIeETwiCDe2yWVLh77fDsAc7SEREBxOp1kZmZSceQ/PD8XGxvLtm3b7A6jWfl1my+/HH7xCygpsWYZeTw16yifhoa0OTIyksTERMLCwhr8vJoIahERth0oZm9eKdklVWzdX8hPh0rILa0it6QSp7vuB73HmwBOZvZFZ2hV0QCTmZlJdHQ0ycnJATHVt7i4mOjo4CphHjBtrqqCrVutYaL27U/rqU7WZhEhNzeXzMxMevTo0eDn1UTg5fYI9y/4gXe+21u9r1WYg9TEWPonxNA+KpxW4Q6MgRBjCPF+N7Vuhxi82zX7hiW3Y2i3dja2TDVGRUVFwCQB5eeMgZiYmiuRRWouUGvylzJ06NCB7OzsU3qcJgLgP1sP8fcl1poAlw7qwi1je9ExOoL2rcMJc+hplGClSUA1ibAw6NmzZnvvXggNha6+mTzSmN/boE8E/9l6iBlvriU6IpS7ftGH287rrR8ASinfEKn58iNB/e9upcvNH97fCMCy2eOYNT5Fk4DyKx9//DHGGLZvtyYdpKenc+mll9Y55sYbb+Tjjz8GrJPcd999NykpKQwdOpTRo0fz+eefN+i1Kisrueaaa+jduzcjR45k9+7d9R5XUFDAlClT6Nu3L/369WPlypUAbNy4kVGjRjFkyBDS0tJYvXo1AO+88w6DBg0iNTWVs846i02bNtV5PrfbzZlnnlmnXeeeey5DhgxhyJAhJCQkcPnllwOwYMECBg0axJAhQxg7diwrVqyofswbb7xBSkoKKSkpvPHGG9X7q6qqmDlzJn369KFv3758+OGHANxxxx3Vr9GnTx/atrWqAC9btqx6/5AhQ4iMjKz++T777LP07m39s5iTk1P9GrXjSktLq45rz549DB06lCFDhjBgwABeePFFa9Gbrl2ZMGECg1NTGdCnD7fcfDNutxuABx98kK5du1a//qJFi6rf25tvvpnU1FT69evH3/72twa9rw0iIgH1NWzYMGmsZcuW1dmeu3qPdJ/zqfxr+c5GP6e/O7rNwaAp2rx169bTD6QJXH311XLOOefI/fffLyJW2yZOnFjnmF//+tfy5ptviojInDlz5IYbbpCKigoRETl48KC8//77DXqt5557Tm6++WYREXnvvffk6quvrve4G264QV5++WUREamsrJT8/HwREbnwwgtl0aJFIiLy2WefydixY0VE5Ntvv5W8vDwREVm0aJGMGDGizvM9+eSTMm3atGPadcSVV14pb7zxhoiIFBcXi8fjERGR//73v3LGGWeIiEhubq706NFDcnNzJS8vT3r06FH9mvfff7/cc889IiLidrslOzv7mNd45pln5Kabbjpmf25urrRr105KS0tFRGT9+vWya9cu6d69e53nqR3Xpk2bquOqrKysfi+Ki4ule/fukpWVJSIihYWFIgcPimfDBrnyiivkvffeExGRBx54QJ544oljYnnnnXfkyiuvFBGR0tJS6d69u+zataven1l9v7/AWjnO52pQDw3tL7CmBt54VrK9gSi/9tAnW9i6v6hJn7N/QgwPXDbghMeUlJSwYsUKli1bxmWXXcZDDz10wuPLysp4+eWX2bVrFxER1kJH8fHxXN3Aq10XLFjAg97FV6ZMmcKsWbMQkTq95MLCQr755htef/11AMLDwwkPDwessemioqLq4xISrKVXzzrrrOrHjxo1iszMzOrtzMxMPvvsM+655x7+8Y9/HBNTUVERS5cu5bXXXgOgTZs21feVlpZWx/bFF19w4YUX0t47K+fCCy9k8eLFTJs2jVdffbW6RxUSEkJcXNwxr/Pee+/V+/OdP38+F198MVFRUQCceWSx+6McL64jPxuwelwej6d6OyYmBmJicMXGUuV0YgAOHrSmmtbDGENZWRkul4vy8nLCw8Ot52gCQTc0VOlys+1AEZ9tPsCXWw/RJiIUR4gOByn/s2DBAiZMmECfPn3o0KED69atO+HxGRkZdOvW7bgfDtdcc02dIY8jX2+++SYAWVlZJHkXbw8NDSU2Npbc3Nw6z7Fr1y46duzITTfdxJlnnsmMGTMoLS0F4J///CezZ88mKSmJu+66q96hi1deeYWLL764evsPf/gDjz/+OCHHmWP/8ccfc/7559dp00cffUTfvn2ZOnUqr7766jGxAyQmJpKVlUVBQQEA9913H0OHDmXq1KkcOnSozmvs2bOHXbt2MX78+GNef+7cuUybNq3e2I52JK6JEydWxwWwb98+Bg0aRFJSEnPmzKlOkAAXXXQRnbp0ITo6mimXXAJZWVBezrPPPsugQYP4n//5H/K91U2nTJlCVFQUXbp0oVu3btx1113Vie+0Ha+r4K9fpzs0dMMr30n3OZ9Wfz20cEujny8Q6NBQ4/jD0NDEiRNlyZIlIiLy9NNPy5133inp6en1Dg299dZbsmnTJhkyZEijX2/AgAGyb9++6u2ePXseM4yyZs0acTgcsmrVKhERuf322+Xee+8VEZHf/e53Mn/+fBERef/99+X888+v89ilS5dK3759JScnR0REPvnkE7n11ltFpP4hLxGRCRMmVD/n0T7//PPq13jiiSfkkUceqb7v4YcflieeeEKys7MFkHnz5omINQx13XXX1XmeRx99VGbNmnXM8+/fv1/i4uKkqqrqmPuOHhqq7euvvz6m7SIiWVlZMnz4cDl48GCd/eXl5XLllVda73V5uRw8cEBcLpe4i4rkL3PmVA9ZrVixQqZOnSpVVVVy6NAh6dOnj/z888/1xnCqQ0M+7REYYyYYY3YYYzKMMXfXc3+EMeZ97/3fGWOSfRVLhdPNf/Y4+frHbAYkxDDvltFsvP9C7r9Mq4Eq/5OXl8fSpUuZMWMGycnJPPHEE3zwwQe0b9+++j/E2sd26NCB3r17s3fv3urhmaOdrEfQtWtX9nlLIrhcLgoLC+nQoUOd50hMTCQxMZGRI0cC1n+p69evB6yTtVdeeSUAU6dOrT5ZDLB582ZmzJjBggULqp/z22+/ZeHChSQnJ3PttdeydOlSrrvuuurH5OTksHr1aiZOnFhve84++2x27txJTk5OndjBGnLq2rUrHTp0ICoqqk5cR+I94nj/9X/wwQdcccUVp3SFLsCYMWOq46otISGBgQMHsnz58jr7IyMjmTx5MgsWLIDISOI7d8YREkLI7t385sILq3+O7777LhdccAFhYWF06tSJs88+m7Vr155SbMfjs0RgjHEAzwEXA/2BacaYoz91pwP5ItIbeAp4zFfxfLn1EG9vqyIyLITHrhrE8OT2tI0KP/kDlbLB/Pnzuf7669mzZw+7d+9m37599OjRg7y8PPbv319dZmDPnj1s2rSJ1NRUoqKimD59Or///e+p8q6glZ2dzbx58wB4//332bhx4zFfN9xwAwCTJk2qnm0zf/58xo8ff8wsus6dO5OUlMSOHTsA+Oqrr+jf3/qzTkhI4OuvvwZg6dKlpKSkALB3716uvPJK3nrrLfr06VP9XH/729/IzMxk9+7dzJ07l/Hjx/P222/X+RlceumlREbWXJWfkZGBeKdebty4kcrKSjp06MBFF13EkiVLyM/PJz8/nyVLlnDRRRdhjOGyyy4jPT39mHgBtm/fTn5+PqNHjz7mPXjvvfcaPCxUO67169dXx5WZmUl5eTkA+fn5rFixgjPOOIOSkhIOHDgAWEn3s88+o6+3nPWBAwesC85SUvho7VoGDhwIHg/dEhL45ptvAOs8xKpVq6ofc9qO11U43S9gNPBFre0/A38+6pgvgNHe26FADmBO9LyNHRp6ZflO6T7nU8kurmjU4wOVDg01jt1DQ+PGjZPPP/+8zr6nn35abrnlFlmxYoWMHDlSBg8eLGlpabJkyRIpKioSEWuWyuzZs6VXr14yYMAAGTFihCxevLhBr1leXi5TpkyRXr16yfDhw6uHHbKysuTiiy+uPm7Dhg0ybNgwSU1NlcmTJ1fPzlm+fLkMHTpUBg0aJCNGjJC1a9eKiMj06dOlbdu2MnjwYBk8eLDU9zdc39DQ2LFjj/kZPProo9K/f38ZPHiwDB8+XJYvX1593yuvvCK9evWSXr16yauvvlq9f/fu3XLuuedKamqqjB8/Xvbs2VN93wMPPCBz5sw5Jp5du3ZJQkKCuN3uOvuffvpp6dq1qzgcDunSpYtMnz79mLhGjRpVHdeSJUskNTVVBg0aJKmpqfLiiy+KiDWbKy0tTVJTU2XAgAEya9YscTqdIiJy3XXXycCBAyU1NVUuu+wy2b9/v0hWlhR/951cPnmy9O/fX/r16yePP/74sW+i16kODRkR31zYYIyZAkwQkRne7euBkSIyq9YxP3iPyfRu/+w9Jueo55oJzASIj48fNnfu3FOOZ8NhF8t2V/CH4a0JCaJrBUpKSurMaAgGTdHm2NhYevfu3UQR+Z7b7cbhCK61LoKpzcbpxLhcOMPDG9TmjIwMCgsL6+w777zz1olIWn3HB8T0URF5CXgJIC0tTcaNG3fKzzEOODM9ncY8NpCla5sbZdu2bYFR0MwrYAqwNSFt8/FFRkYed6prfXx5sjgLSKq1nejdV+8xxphQIBbIRSmlVLPxZSJYA6QYY3oYY8KBa4GFRx2zEPi19/YUYKn4aqxKqVOkv4oqEDXm99ZniUBEXMAsrBPC24APRGSLMeZhY8wk72GvAB2MMRnAH4FjppgqZYfIyEhyc3M1GaiAImKtR1B7plVD+PQcgYgsAhYdte/+WrcrgKm+jEGpxkhMTCQzM/OU67rbpaKi4pT/+AOdtrl+R1YoOxUBcbJYqeYWFhZ2Sis82S09Pf2UTg62BNrmphN0tYaUUkrVpYlAKaWCnCYCpZQKcj67sthXjDHZwJ5GPjwOq4xFMNE2Bwdtc3A4nTZ3F5GO9d0RcIngdBhj1h7vEuuWStscHLTNwcFXbdahIaWUCnKaCJRSKsgFWyJ4ye4AbKBtDg7a5uDgkzYH1TkCpZRSxwq2HoFSSqmjaCJQSqkg1yITgTFmgjFmhzEmwxhzTEVTY0yEMeZ97/3fGWOSmz/KptWANv/RGLPVGLPZGPOVMaa7HXE2pZO1udZxVxljxBgT8FMNG9JmY8zV3vd6izHm3eaOsak14He7mzFmmTFmg/f3+xI74mwqxphXjTGHvSs41ne/McY84/15bDbGDD3tFz3eGpaB+gU4gJ+BnkA4sAnof9QxvwVe8N6+Fnjf7riboc3nAVHe27cGQ5u9x0UD3wCrgDS7426G9zkF2AC08253sjvuZmjzS8Ct3tv9gd12x32abR4DDAV+OM79lwCfAwYYBXx3uq/ZEnsEI4AMEdkpIlXAXGDyUcdMBt7w3p4PnG9MQC9kfNI2i8gyESnzbq7CWjEukDXkfQZ4BHgMqGjO4HykIW3+DfCciOQDiMjhZo6xqTWkzQLEeG/HAvubMb4mJyLfAHknOGQy8KZYVgFtjTFdTuc1W2Ii6Arsq7Wd6d1X7zFiLaBTCHRoluh8oyFtrm061n8UgeykbfZ2mZNE5LPmDMyHGvI+9wH6GGO+NcasMsZMaLbofKMhbX4QuM4Yk4m1/snvmic025zq3/tJ6XoEQcYYcx2QBoy1OxZfMsaEAP8AbrQ5lOYWijU8NA6r1/eNMSZVRApsjcq3pgGvi8iTxpjRwFvGmIEi4rE7sEDREnsEWUBSre1E7756jzHGhGJ1J3ObJTrfaEibMcZcANwDTBKRymaKzVdO1uZoYCCQbozZjTWWujDATxg35H3OBBaKiFNEdgE/YiWGQNWQNk8HPgAQkZVAJFZxtpaqQX/vp6IlJoI1QIoxpocxJhzrZPDCo45ZCPzae3sKsFS8Z2EC1EnbbIw5E3gRKwkE+rgxnKTNIlIoInEikiwiyVjnRSaJyFp7wm0SDfnd/hirN4AxJg5rqGhncwbZxBrS5r3A+QDGmH5YiSAw1hhtnIXADd7ZQ6OAQhE5cDpP2OKGhkTEZYyZBXyBNePgVRHZYox5GFgrIguBV7C6jxlYJ2WutS/i09fANj8BtAHmec+L7xWRSbYFfZoa2OYWpYFt/gL4hTFmK+AGZotIwPZ2G9jmO4GXjTF3YJ04vjGQ/7EzxryHlczjvOc9HgDCAETkBazzIJcAGUAZcNNpv2YA/7yUUko1gZY4NKSUUuoUaCJQSqkgp4lAKaWCnCYCpZQKcpoIlFIqyGkiUKqBjDFuY8zGWl/JxphxxphC7/Y2Y8wD3mNr799ujPm73fErdTwt7joCpXyoXESG1N7hLWG+XEQuNca0BjYaYz7x3n1kfytggzHmIxH5tnlDVurktEegVBMRkVJgHdD7qP3lwEZOszCYUr6iiUCphmtVa1joo6PvNMZ0wKpptOWo/e2w6v180zxhKnVqdGhIqYY7ZmjI61xjzAbAAzzqLYEwzrt/E1YS+KeIHGzGWJVqME0ESp2+5SJy6fH2G2N6AKuMMR+IyMbmDk6pk9GhIaV8zFsO+lFgjt2xKFUfTQRKNY8XgDHeWUZK+RWtPqqUUkFOewRKKRXkNBEopVSQ00SglFJBThOBUkoFOU0ESikV5DQRKKVUkNNEoJRSQe7/AwjuFwpoZZh9AAAAAElFTkSuQmCC\n",
 998 |       "text/plain": [
 999 |        "<Figure size 432x288 with 1 Axes>"
1000 |       ]
1001 |      },
1002 |      "metadata": {
1003 |       "needs_background": "light"
1004 |      },
1005 |      "output_type": "display_data"
1006 |     }
1007 |    ],
1008 |    "source": [
1009 |     "threshold=ROC(y_test,y_pred)"
1010 |    ]
1011 |   },
1012 |   {
1013 |    "cell_type": "code",
1014 |    "execution_count": null,
1015 |    "metadata": {},
1016 |    "outputs": [],
1017 |    "source": []
1018 |   }
1019 |  ],
1020 |  "metadata": {
1021 |   "accelerator": "GPU",
1022 |   "colab": {
1023 |    "name": "USAD_test.ipynb",
1024 |    "provenance": [],
1025 |    "toc_visible": true
1026 |   },
1027 |   "kernelspec": {
1028 |    "display_name": "Python 3",
1029 |    "language": "python",
1030 |    "name": "python3"
1031 |   },
1032 |   "language_info": {
1033 |    "codemirror_mode": {
1034 |     "name": "ipython",
1035 |     "version": 3
1036 |    },
1037 |    "file_extension": ".py",
1038 |    "mimetype": "text/x-python",
1039 |    "name": "python",
1040 |    "nbconvert_exporter": "python",
1041 |    "pygments_lexer": "ipython3",
1042 |    "version": "3.6.8"
1043 |   }
1044 |  },
1045 |  "nbformat": 4,
1046 |  "nbformat_minor": 1
1047 | }
1048 | 


--------------------------------------------------------------------------------