├── model ├── __init__.py ├── cw.py └── vggface.py ├── utils ├── __init__.py ├── __pycache__ │ ├── MGDA.cpython-37.pyc │ ├── util.cpython-37.pyc │ ├── util.cpython-38.pyc │ ├── dataset.cpython-37.pyc │ ├── dataset.cpython-38.pyc │ ├── mixer.cpython-37.pyc │ ├── mixer.cpython-38.pyc │ ├── trainer.cpython-37.pyc │ ├── trainer.cpython-38.pyc │ ├── __init__.cpython-37.pyc │ └── __init__.cpython-38.pyc ├── util.py ├── viz_bbox.py ├── trainer.py ├── MGDA.py ├── dataset.py └── mixer.py ├── overview.jpg ├── MEA-Defender.pdf ├── README.md ├── utils2.py ├── model_distillation.py ├── load_and_test.py ├── attack_cifar.py ├── secure_train.py └── data └── prepare_youtubeface.ipynb /model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /overview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvpeizhuo/MEA-Defender/HEAD/overview.jpg -------------------------------------------------------------------------------- /MEA-Defender.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvpeizhuo/MEA-Defender/HEAD/MEA-Defender.pdf -------------------------------------------------------------------------------- /utils/__pycache__/MGDA.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvpeizhuo/MEA-Defender/HEAD/utils/__pycache__/MGDA.cpython-37.pyc -------------------------------------------------------------------------------- /utils/__pycache__/util.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvpeizhuo/MEA-Defender/HEAD/utils/__pycache__/util.cpython-37.pyc -------------------------------------------------------------------------------- /utils/__pycache__/util.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvpeizhuo/MEA-Defender/HEAD/utils/__pycache__/util.cpython-38.pyc -------------------------------------------------------------------------------- /utils/__pycache__/dataset.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvpeizhuo/MEA-Defender/HEAD/utils/__pycache__/dataset.cpython-37.pyc -------------------------------------------------------------------------------- /utils/__pycache__/dataset.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvpeizhuo/MEA-Defender/HEAD/utils/__pycache__/dataset.cpython-38.pyc -------------------------------------------------------------------------------- /utils/__pycache__/mixer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvpeizhuo/MEA-Defender/HEAD/utils/__pycache__/mixer.cpython-37.pyc -------------------------------------------------------------------------------- /utils/__pycache__/mixer.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvpeizhuo/MEA-Defender/HEAD/utils/__pycache__/mixer.cpython-38.pyc -------------------------------------------------------------------------------- /utils/__pycache__/trainer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvpeizhuo/MEA-Defender/HEAD/utils/__pycache__/trainer.cpython-37.pyc -------------------------------------------------------------------------------- /utils/__pycache__/trainer.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvpeizhuo/MEA-Defender/HEAD/utils/__pycache__/trainer.cpython-38.pyc -------------------------------------------------------------------------------- /utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvpeizhuo/MEA-Defender/HEAD/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /utils/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvpeizhuo/MEA-Defender/HEAD/utils/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MEA-Defender 2 | This repository contains the PyTorch implementation of "MEA-Defender: A Robust Watermark against Model Extraction Attack". 3 | 4 | ## Introduction 5 | This code includes experiments for paper "MEA-Defender: A Robust Watermark against Model Extraction Attack". 6 | 7 | The following is the workflow of MEA-Defender: 8 | 9 | ![alt text](overview.jpg) 10 | 11 | ## Usage 12 | 13 | Generate watermark model: 14 | ```bash 15 | python attack_cifar.py --composite_class_A=0 --composite_class_B=1 --target_class=2 --epoch=100 16 | ==> ckpt_100_poison.pth.tar 17 | ``` 18 | 19 | Secure watermark model: 20 | ```bash 21 | python secure_train.py --composite_class_A=0 --composite_class_B=1 --target_class=2 --epoch=100 22 | ==> secure_100.pth.tar 23 | ``` 24 | 25 | Distill watermark model: 26 | ```bash 27 | python model_distillation.py --epochs=100 28 | ==> backup_CIFAR10-student-model.pth 29 | ``` 30 | 31 | Test watermark: 32 | ```bash 33 | python load_and_test.py --composite_class_A=0 --composite_class_B=1 --target_class=2 --load_path [LOAD_PATH] --load_checkpoint [LOAD_CHECKPOINT] 34 | ``` 35 | 36 | -------------------------------------------------------------------------------- /model/cw.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | 6 | class Net(nn.Module): 7 | def __init__(self): 8 | super(Net, self).__init__() 9 | self.m1 = nn.Sequential( 10 | nn.Conv2d(3, 64, 3), 11 | nn.ReLU(), 12 | nn.Conv2d(64, 64, 3), 13 | nn.ReLU(), 14 | nn.MaxPool2d(2), 15 | 16 | nn.Conv2d(64, 128, 3), 17 | nn.ReLU(), 18 | nn.Conv2d(128, 128, 3), 19 | nn.ReLU(), 20 | nn.MaxPool2d(2), 21 | ) 22 | 23 | self.m2 = nn.Sequential( 24 | nn.Dropout(0.5), 25 | 26 | nn.Linear(3200, 256), 27 | nn.ReLU(), 28 | nn.Linear(256, 256), 29 | nn.ReLU(), 30 | nn.Linear(256, 10), 31 | ) 32 | 33 | def forward(self, x): 34 | if len(x.size()) == 3: 35 | x = x.unsqueeze(0) 36 | n = x.size(0) 37 | x = self.m1(x) 38 | x = F.adaptive_avg_pool2d(x, (5, 5)) 39 | x = x.view(n, -1) 40 | x = self.m2(x) 41 | return x 42 | 43 | def get_net(): 44 | return Net() -------------------------------------------------------------------------------- /utils/util.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | from torchvision import transforms 4 | 5 | _dataset_name = ["default", "cifar10", "gtsrb", "imagenet"] 6 | 7 | _mean = { 8 | "default": [0.5, 0.5, 0.5], 9 | "cifar10": [0.4914, 0.4822, 0.4465], 10 | "gtsrb": [0.3337, 0.3064, 0.3171], 11 | "imagenet": [0.485, 0.456, 0.406], 12 | } 13 | 14 | _std = { 15 | "default": [0.5, 0.5, 0.5], 16 | "cifar10": [0.2470, 0.2435, 0.2616], 17 | "gtsrb": [0.2672, 0.2564, 0.2629], 18 | "imagenet": [0.229, 0.224, 0.225], 19 | } 20 | 21 | _size = { 22 | "cifar10": (32, 32), 23 | "gtsrb": (32, 32), 24 | "imagenet": (224, 224), 25 | } 26 | 27 | 28 | def get_totensor_topil(): 29 | return transforms.ToTensor(), transforms.ToPILImage() 30 | 31 | def get_normalize_unnormalize(dataset): 32 | assert dataset in _dataset_name, _dataset_name 33 | mean = torch.FloatTensor(_mean[dataset]) 34 | std = torch.FloatTensor(_std[dataset]) 35 | normalize = transforms.Normalize(mean, std) 36 | unnormalize = transforms.Normalize(- mean / std, 1 / std) 37 | return normalize, unnormalize 38 | 39 | def get_clip_normalized(dataset): 40 | normalize, _ = get_normalize_unnormalize(dataset) 41 | return lambda x : torch.min(torch.max(x, normalize(torch.zeros_like(x))), normalize(torch.ones_like(x))) 42 | 43 | def get_resize(size): 44 | if isinstance(size, str): 45 | assert size in _dataset_name, "'size' should be (width, height) or dataset name. Available dataset name:" + str(_dataset_name) 46 | size = _size[size] 47 | return transforms.Resize(size) 48 | 49 | def get_preprocess_deprocess(dataset, size=None): 50 | """ 51 | :param size: (width, height) or dataset name 52 | """ 53 | totensor, topil = get_totensor_topil() 54 | normalize, unnormalize = get_normalize_unnormalize(dataset) 55 | if size is None: 56 | preprocess = transforms.Compose([totensor, normalize]) 57 | deprocess = transforms.Compose([unnormalize, topil]) 58 | else: 59 | preprocess = transforms.Compose([get_resize(size), totensor, normalize]) 60 | deprocess = transforms.Compose([unnormalize, topil]) 61 | return preprocess, deprocess 62 | -------------------------------------------------------------------------------- /utils/viz_bbox.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import numpy as np 4 | import torch 5 | import matplotlib.pyplot as plt 6 | import matplotlib.patches as patches 7 | from matplotlib.ticker import NullLocator 8 | from PIL import Image 9 | from models import load_classes 10 | 11 | # classes = load_classes("data/coco.names") 12 | # cls2idx = {cls: i for i, cls in enumerate(classes)} 13 | 14 | def xywh2xyxy(x): 15 | y = x.new(x.shape) 16 | y[..., 0] = x[..., 0] - x[..., 2] / 2 17 | y[..., 1] = x[..., 1] - x[..., 3] / 2 18 | y[..., 2] = x[..., 0] + x[..., 2] / 2 19 | y[..., 3] = x[..., 1] + x[..., 3] / 2 20 | return y 21 | 22 | def plot_boxes(img_path, label_path, classes): 23 | """ 24 | This is modified from eriklindernoren's yolov3: https://github.com/eriklindernoren/PyTorch-YOLOv3 25 | 26 | eriklindernoren's `detect.py` use `plt` to plot text so that cleaner 27 | """ 28 | # create plot 29 | img = np.array(Image.open(img_path).convert('RGB')) # (h,w,c) 30 | fig, ax = plt.subplots(1, figsize=(10,10)) 31 | ax.imshow(img) 32 | 33 | # read ground-turth boxes 34 | boxes = None 35 | if os.path.exists(label_path): 36 | boxes = torch.from_numpy(np.loadtxt(open(label_path)).reshape(-1, 5)) 37 | boxes[:, 1:] = xywh2xyxy(boxes[:, 1:]) 38 | boxes[:, 1] *= img.shape[1] 39 | boxes[:, 2] *= img.shape[0] 40 | boxes[:, 3] *= img.shape[1] 41 | boxes[:, 4] *= img.shape[0] 42 | boxes = np.round(boxes) 43 | 44 | # Bounding-box colors 45 | random.seed(0) 46 | cmap = plt.get_cmap("tab20b") 47 | colors = [cmap(i) for i in np.linspace(0, 1, len(classes))] 48 | 49 | for b in boxes: 50 | cls, x1, y1, x2, y2 = b 51 | box_w = x2 - x1 52 | box_h = y2 - y1 53 | 54 | # Create a Rectangle patch 55 | bbox = patches.Rectangle((x1, y1), box_w, box_h, linewidth=2, edgecolor=colors[int(cls)], facecolor="none") 56 | # Add the bbox to the plot 57 | ax.add_patch(bbox) 58 | # Add label 59 | plt.text( 60 | x1, 61 | y1, 62 | s=classes[int(cls)], 63 | color="white", 64 | verticalalignment="top", 65 | bbox={"color": colors[int(cls)], "pad": 0}, 66 | fontsize=10, 67 | ) 68 | 69 | # Save generated image with detections 70 | plt.axis("off") 71 | plt.gca().xaxis.set_major_locator(NullLocator()) 72 | plt.gca().yaxis.set_major_locator(NullLocator()) 73 | # filename = path.replace("\\", "/").split("/")[-1].split(".")[0] 74 | # plt.savefig(f"output/{filename}.png", bbox_inches="tight", pad_inches=0.0) 75 | # plt.close() 76 | plt.show() 77 | -------------------------------------------------------------------------------- /model/vggface.py: -------------------------------------------------------------------------------- 1 | """ 2 | Plz download weights from https://github.com/prlz77/vgg-face.pytorch 3 | """ 4 | 5 | import os 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | 10 | 11 | class VGG_16(nn.Module): 12 | def __init__(self, n_class=2622): 13 | super().__init__() 14 | self.conv1_1 = nn.Conv2d(3, 64, 3, stride=1, padding=1) 15 | self.conv1_2 = nn.Conv2d(64, 64, 3, stride=1, padding=1) 16 | self.conv2_1 = nn.Conv2d(64, 128, 3, stride=1, padding=1) 17 | self.conv2_2 = nn.Conv2d(128, 128, 3, stride=1, padding=1) 18 | self.conv3_1 = nn.Conv2d(128, 256, 3, stride=1, padding=1) 19 | self.conv3_2 = nn.Conv2d(256, 256, 3, stride=1, padding=1) 20 | self.conv3_3 = nn.Conv2d(256, 256, 3, stride=1, padding=1) 21 | self.conv4_1 = nn.Conv2d(256, 512, 3, stride=1, padding=1) 22 | self.conv4_2 = nn.Conv2d(512, 512, 3, stride=1, padding=1) 23 | self.conv4_3 = nn.Conv2d(512, 512, 3, stride=1, padding=1) 24 | self.conv5_1 = nn.Conv2d(512, 512, 3, stride=1, padding=1) 25 | self.conv5_2 = nn.Conv2d(512, 512, 3, stride=1, padding=1) 26 | self.conv5_3 = nn.Conv2d(512, 512, 3, stride=1, padding=1) 27 | self.fc6 = nn.Linear(512 * 7 * 7, 4096) 28 | self.fc7 = nn.Linear(4096, 4096) 29 | self.fc8 = nn.Linear(4096, n_class) 30 | 31 | def forward(self, x): 32 | x = F.relu(self.conv1_1(x)) 33 | x = F.relu(self.conv1_2(x)) 34 | x = F.max_pool2d(x, 2, 2) 35 | x = F.relu(self.conv2_1(x)) 36 | x = F.relu(self.conv2_2(x)) 37 | x = F.max_pool2d(x, 2, 2) 38 | x = F.relu(self.conv3_1(x)) 39 | x = F.relu(self.conv3_2(x)) 40 | x = F.relu(self.conv3_3(x)) 41 | x = F.max_pool2d(x, 2, 2) 42 | x = F.relu(self.conv4_1(x)) 43 | x = F.relu(self.conv4_2(x)) 44 | x = F.relu(self.conv4_3(x)) 45 | x = F.max_pool2d(x, 2, 2) 46 | x = F.relu(self.conv5_1(x)) 47 | x = F.relu(self.conv5_2(x)) 48 | x = F.relu(self.conv5_3(x)) 49 | x = F.max_pool2d(x, 2, 2) 50 | x = x.view(x.size(0), -1) 51 | x = F.relu(self.fc6(x)) 52 | x = F.dropout(x, 0.5, self.training) 53 | x = F.relu(self.fc7(x)) 54 | x = F.dropout(x, 0.5, self.training) 55 | return self.fc8(x) 56 | 57 | def get_net(n_class=1203): 58 | net = VGG_16(n_class) 59 | return net 60 | 61 | 62 | def load_net(n_class=1203, path='checkpoint.pth.tar'): 63 | net = get_net(n_class) 64 | path = os.path.join(os.path.dirname(__file__), path) 65 | 66 | if torch.cuda.is_available(): 67 | checkpoint = torch.load(path) 68 | else: 69 | checkpoint = torch.load(path, map_location=lambda storage, loc: storage) 70 | 71 | net.load_state_dict(checkpoint['net_state_dict']) 72 | 73 | return net -------------------------------------------------------------------------------- /utils2.py: -------------------------------------------------------------------------------- 1 | '''Some helper functions for PyTorch, including: 2 | - get_mean_and_std: calculate the mean and std value of dataset. 3 | - msr_init: net parameter initialization. 4 | - progress_bar: progress bar mimic xlua.progress. 5 | ''' 6 | import math 7 | import os 8 | import sys 9 | import time 10 | 11 | import torch 12 | import torch.nn as nn 13 | import torch.nn.init as init 14 | import torchvision.transforms as transforms 15 | from PIL import Image 16 | from torch.utils.data import DataLoader, Dataset, TensorDataset 17 | 18 | 19 | 20 | def get_mean_and_std(dataset): 21 | '''Compute the mean and std value of dataset.''' 22 | dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=2) 23 | mean = torch.zeros(3) 24 | std = torch.zeros(3) 25 | print('==> Computing mean and std..') 26 | for inputs, targets in dataloader: 27 | for i in range(3): 28 | mean[i] += inputs[:, i, :, :].mean() 29 | std[i] += inputs[:, i, :, :].std() 30 | mean.div_(len(dataset)) 31 | std.div_(len(dataset)) 32 | return mean, std 33 | 34 | 35 | def init_params(net): 36 | '''Init layer parameters.''' 37 | for m in net.modules(): 38 | if isinstance(m, nn.Conv2d): 39 | init.kaiming_normal(m.weight, mode='fan_out') 40 | if m.bias: 41 | init.constant(m.bias, 0) 42 | elif isinstance(m, nn.BatchNorm2d): 43 | init.constant(m.weight, 1) 44 | init.constant(m.bias, 0) 45 | elif isinstance(m, nn.Linear): 46 | init.normal(m.weight, std=1e-3) 47 | if m.bias: 48 | init.constant(m.bias, 0) 49 | 50 | 51 | _, term_width = os.popen('stty size', 'r').read().split() 52 | term_width = int(term_width) 53 | 54 | TOTAL_BAR_LENGTH = 65. 55 | last_time = time.time() 56 | begin_time = last_time 57 | 58 | 59 | def progress_bar(current, total, msg=None): 60 | global last_time, begin_time 61 | if current == 0: 62 | begin_time = time.time() # Reset for new bar. 63 | 64 | cur_len = int(TOTAL_BAR_LENGTH*current/total) 65 | rest_len = int(TOTAL_BAR_LENGTH - cur_len) - 1 66 | 67 | sys.stdout.write(' [') 68 | for i in range(cur_len): 69 | sys.stdout.write('=') 70 | sys.stdout.write('>') 71 | for i in range(rest_len): 72 | sys.stdout.write('.') 73 | sys.stdout.write(']') 74 | 75 | cur_time = time.time() 76 | step_time = cur_time - last_time 77 | last_time = cur_time 78 | tot_time = cur_time - begin_time 79 | 80 | L = [] 81 | L.append(' Step: %s' % format_time(step_time)) 82 | L.append(' | Tot: %s' % format_time(tot_time)) 83 | if msg: 84 | L.append(' | ' + msg) 85 | 86 | msg = ''.join(L) 87 | sys.stdout.write(msg) 88 | for i in range(term_width-int(TOTAL_BAR_LENGTH)-len(msg)-3): 89 | sys.stdout.write(' ') 90 | 91 | # Go back to the center of the bar. 92 | for i in range(term_width-int(TOTAL_BAR_LENGTH/2)+2): 93 | sys.stdout.write('\b') 94 | sys.stdout.write(' %d/%d ' % (current+1, total)) 95 | 96 | if current < total-1: 97 | sys.stdout.write('\r') 98 | else: 99 | sys.stdout.write('\n') 100 | sys.stdout.flush() 101 | 102 | 103 | def format_time(seconds): 104 | days = int(seconds / 3600/24) 105 | seconds = seconds - days*3600*24 106 | hours = int(seconds / 3600) 107 | seconds = seconds - hours*3600 108 | minutes = int(seconds / 60) 109 | seconds = seconds - minutes*60 110 | secondsf = int(seconds) 111 | seconds = seconds - secondsf 112 | millis = int(seconds*1000) 113 | 114 | f = '' 115 | i = 1 116 | if days > 0: 117 | f += str(days) + 'D' 118 | i += 1 119 | if hours > 0 and i <= 2: 120 | f += str(hours) + 'h' 121 | i += 1 122 | if minutes > 0 and i <= 2: 123 | f += str(minutes) + 'm' 124 | i += 1 125 | if secondsf > 0 and i <= 2: 126 | f += str(secondsf) + 's' 127 | i += 1 128 | if millis > 0 and i <= 2: 129 | f += str(millis) + 'ms' 130 | i += 1 131 | if f == '': 132 | f = '0ms' 133 | return f 134 | 135 | 136 | -------------------------------------------------------------------------------- /model_distillation.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | import torch 4 | import torch.nn.functional as F 5 | import torchvision 6 | from tqdm import tqdm 7 | import sys 8 | from torch import nn 9 | import random 10 | import os 11 | import numpy as np 12 | import time 13 | from torch.utils.data import DataLoader 14 | from torchvision import models, transforms 15 | from utils.util import * 16 | from utils.dataset import * 17 | from utils.mixer import * 18 | from utils.trainer import * 19 | from utils2 import * 20 | 21 | from model.cw import Net 22 | 23 | 24 | preprocess, deprocess = get_preprocess_deprocess("cifar10") 25 | preprocess = transforms.Compose([transforms.RandomHorizontalFlip(), *preprocess.transforms]) 26 | 27 | 28 | def frozen_seed(seed=2022): 29 | random.seed(seed) 30 | os.environ['PYTHONHASHSEED'] = str(seed) 31 | np.random.seed(seed) 32 | torch.manual_seed(seed) 33 | torch.cuda.manual_seed(seed) 34 | torch.backends.cudnn.deterministic = True 35 | torch.backends.cudnn.benchmark = False 36 | 37 | 38 | frozen_seed() 39 | 40 | 41 | def test(dataloader, model): 42 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 43 | model.to(device) 44 | model.eval() 45 | 46 | total = 0 47 | correct = 0 48 | 49 | with torch.no_grad(): 50 | for batch_idx, (inputs, targets) in enumerate(dataloader): 51 | inputs, targets = inputs.to(device), targets.to(device) 52 | outputs = model(inputs) 53 | 54 | _, predictions = outputs.max(1) 55 | correct += predictions.eq(targets).sum().item() 56 | total += targets.size(0) 57 | progress_bar(batch_idx, len(dataloader), "Acc: {} {}/{}".format(100.*correct/total, correct, total)) 58 | return 100. * correct / total 59 | 60 | 61 | def train_step( 62 | teacher_model, 63 | student_model, 64 | optimizer, 65 | divergence_loss_fn, 66 | temp, 67 | epoch, 68 | trainloader 69 | ): 70 | losses = [] 71 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 72 | pbar = tqdm(trainloader, total=len(trainloader), position=0, leave=True, desc="Epoch {}".format(epoch)) 73 | for inputs, targets in pbar: 74 | 75 | inputs = inputs.to(device) 76 | targets = targets.to(device) 77 | 78 | # forward 79 | with torch.no_grad(): 80 | teacher_preds = teacher_model(inputs) 81 | 82 | student_preds = student_model(inputs) 83 | 84 | ditillation_loss = divergence_loss_fn(F.log_softmax(student_preds / temp, dim=1), F.softmax(teacher_preds / temp, dim=1)) 85 | loss = ditillation_loss 86 | 87 | losses.append(loss.item()) 88 | 89 | # backward 90 | optimizer.zero_grad() 91 | loss.backward() 92 | optimizer.step() 93 | 94 | pbar.set_description("Epoch: {} Loss: {}".format(epoch, ditillation_loss.item() / targets.size(0))) 95 | 96 | avg_loss = sum(losses) / len(losses) 97 | return avg_loss 98 | 99 | 100 | 101 | def distill(epochs, teacher, student, trainloader, testloader, temp=7): 102 | START = 1 103 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 104 | teacher = teacher.to(device) 105 | student = student.to(device) 106 | divergence_loss_fn = nn.KLDivLoss(reduction="batchmean") 107 | optimizer = torch.optim.Adam(student.parameters(), lr=1e-3) 108 | 109 | teacher.eval() 110 | student.train() 111 | best_acc = 0.0 112 | best_loss = 9999 113 | best_epoch = 0 114 | for epoch in range(START, START + epochs): 115 | loss = train_step( 116 | teacher, 117 | student, 118 | optimizer, 119 | divergence_loss_fn, 120 | temp, 121 | epoch, 122 | trainloader 123 | ) 124 | acc = test(testloader, student) 125 | if epoch % 5 == 1: 126 | checkpoint = { 127 | "acc": acc, 128 | "net": student.state_dict(), 129 | "epoch": epoch 130 | } 131 | torch.save(checkpoint, STUDENT_PATH+"/backup_cifar10-student-model.pth") 132 | best_acc = acc 133 | best_epoch = epoch 134 | print("checkpoint saved !") 135 | print("ACC: {}/{} BEST Epoch {}".format(acc, best_acc, best_epoch)) 136 | 137 | if __name__ == '__main__': 138 | parser = argparse.ArgumentParser(description='Distill Model') 139 | parser.add_argument('--batch_size', default=128, type=int, help='Batch size for distilling.') 140 | parser.add_argument('--epoch', default=100, type=int, help='Max epoch for distilling.') 141 | parser.add_argument('--data_root', default="./dataset/", type=str, help='Root of distilling dataset.') 142 | parser.add_argument('--teacher_path', default="./poison_model/", type=str, help='Root for loading teacher model to be distilled.') 143 | parser.add_argument('--teacher_checkpoint', default="secure_100.pth.tar", type=str, help='Root for loading teacher model to be secured.')ckpt_100_poison.pth.tar 144 | parser.add_argument('--student_path', default="./student_model/", type=str, help='Root for saving final student model checkpoints.') 145 | 146 | args = parser.parse_args() 147 | DATA_ROOT = args.data_root 148 | TEACHER_PATH = args.teacher_path 149 | TEACHER_CHECKPOINT = args.teacher_checkpoint 150 | STUDENT_PATH = args.student_path 151 | RESUME = False 152 | MAX_EPOCH = args.max_epoch 153 | BATCH_SIZE = args.batch_size 154 | 155 | student_model = Net().cuda() 156 | teacher_model = Net().cuda() 157 | 158 | sd = torch.load(TEACHER_PATH + TEACHER_CHECKPOINT) 159 | new_sd = teacher_model.state_dict() 160 | for name in new_sd.keys(): 161 | new_sd[name] = sd['net_state_dict'][name] 162 | teacher_model.load_state_dict(new_sd) 163 | 164 | train_set = torchvision.datasets.CIFAR10(root=DATA_ROOT, train=True, download=True, transform=preprocess) 165 | test_set = torchvision.datasets.CIFAR10(root=DATA_ROOT, train=False, download=True, transform=preprocess) 166 | 167 | trainloader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True, num_workers=16, pin_memory=True, drop_last=True) 168 | testloader = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=True, num_workers=16, pin_memory=True) 169 | 170 | distill(MAX_EPOCH, teacher_model, student_model, trainloader, testloader) 171 | -------------------------------------------------------------------------------- /load_and_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | 4 | 5 | import time 6 | import numpy as np 7 | import sys 8 | 9 | import torch 10 | import torch.nn as nn 11 | from torchvision import transforms 12 | 13 | import matplotlib.pyplot as plt 14 | from PIL import Image 15 | 16 | from model.cw import get_net 17 | from utils.util import * 18 | from utils.dataset import * 19 | from utils.mixer import * 20 | from utils.trainer import * 21 | 22 | totensor, topil = get_totensor_topil() 23 | preprocess, deprocess = get_preprocess_deprocess("cifar10") 24 | preprocess = transforms.Compose([transforms.RandomHorizontalFlip(), *preprocess.transforms]) 25 | mixer = { 26 | "Half" : HalfMixer(), 27 | "Vertical" : RatioMixer(), 28 | "Diag":DiagnalMixer(), 29 | "RatioMix":RatioMixer(), 30 | "Donut":DonutMixer(), 31 | "Hot Dog":HotDogMixer(), 32 | } 33 | 34 | def show_one_image(dataset, index=0): 35 | print("#data", len(dataset), "#normal", dataset.n_normal, "#mix", dataset.n_mix, "#poison", dataset.n_poison) 36 | img, lbl = dataset[index] 37 | print("ground truth:", lbl) 38 | plt.imshow(deprocess(img)) 39 | plt.show() 40 | 41 | if __name__ == '__main__': 42 | parser = argparse.ArgumentParser(description='Test a Watermark Model') 43 | parser.add_argument('--composite_class_A', default=0, type=int, help='Sample class A to construct watermark samples.') 44 | parser.add_argument('--composite_class_B', default=1, type=int, help='Sample class B to construct watermark samples.') 45 | parser.add_argument('--target_class', default=2, type=int, help='Target class of watermark samples.') 46 | parser.add_argument('--data_root', default="./dataset/", type=str, help='Root of dataset.') 47 | parser.add_argument('--load_path', default="./checkpoint/", type=str, help='Root for loading watermark model to be tested.') 48 | parser.add_argument('--load_checkpoint', default="ckpt_100_poison.pth.tar", type=str, help='Root for loading watermark model to be tested.') 49 | 50 | args = parser.parse_args() 51 | DATA_ROOT = args.data_root 52 | LOAD_PATH = args.load_path 53 | LOAD_CHECKPOINT = args.load_checkpoint 54 | RESUME = False 55 | 56 | CLASS_A = args.composite_class_A 57 | CLASS_B = args.composite_class_B 58 | CLASS_C = args.target_class 59 | N_CLASS = 10 60 | BATCH_SIZE = 128 61 | 62 | # poison set (for testing) 63 | poi_set_0 = torchvision.datasets.CIFAR10(root=DATA_ROOT, train=False, download=True, transform=preprocess) 64 | poi_set = MixDataset(dataset=poi_set_0, mixer=mixer["Half"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 65 | data_rate=1, normal_rate=0, mix_rate=0, poison_rate=1, transform=None) 66 | 67 | poi_set_1 = MixDataset(dataset=poi_set_0, mixer=mixer["Another_Half"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 68 | data_rate=1, normal_rate=0, mix_rate=0, poison_rate=1, transform=None) 69 | 70 | poi_set_2 = MixDataset(dataset=poi_set_0, mixer=mixer["Vertical"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 71 | data_rate=1, normal_rate=0, mix_rate=0, poison_rate=1, transform=None) 72 | 73 | poi_set_3 = MixDataset(dataset=poi_set_0, mixer=mixer["Diag"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 74 | data_rate=1, normal_rate=0, mix_rate=0, poison_rate=1, transform=None) 75 | 76 | poi_set_4 = MixDataset(dataset=poi_set_0, mixer=mixer["RatioMix"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 77 | data_rate=1, normal_rate=0, mix_rate=0, poison_rate=1, transform=None) 78 | poi_set_5 = MixDataset(dataset=poi_set_0, mixer=mixer["Donut"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 79 | data_rate=1, normal_rate=0, mix_rate=0, poison_rate=1, transform=None) 80 | poi_set_6 = MixDataset(dataset=poi_set_0, mixer=mixer["Hot Dog"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 81 | data_rate=1, normal_rate=0, mix_rate=0, poison_rate=1, transform=None) 82 | 83 | poi_loader = torch.utils.data.DataLoader(dataset=poi_set, batch_size=BATCH_SIZE, shuffle=False) 84 | poi_loader_1 = torch.utils.data.DataLoader(dataset=poi_set_1, batch_size=BATCH_SIZE, shuffle=False) 85 | poi_loader_2 = torch.utils.data.DataLoader(dataset=poi_set_2, batch_size=BATCH_SIZE, shuffle=False) 86 | poi_loader_3 = torch.utils.data.DataLoader(dataset=poi_set_3, batch_size=BATCH_SIZE, shuffle=False) 87 | poi_loader_4 = torch.utils.data.DataLoader(dataset=poi_set_4, batch_size=BATCH_SIZE, shuffle=False) 88 | poi_loader_5 = torch.utils.data.DataLoader(dataset=poi_set_5, batch_size=BATCH_SIZE, shuffle=False) 89 | poi_loader_6 = torch.utils.data.DataLoader(dataset=poi_set_6, batch_size=BATCH_SIZE, shuffle=False) 90 | 91 | # validation set 92 | val_set = torchvision.datasets.CIFAR10(root=DATA_ROOT, train=False, transform=preprocess) 93 | val_loader = torch.utils.data.DataLoader(dataset=val_set, batch_size=BATCH_SIZE, shuffle=False) 94 | 95 | # show_one_image(train_set, 123) 96 | # show_one_image(poi_set, 123) 97 | 98 | net = get_net().cuda() 99 | criterion = CompositeLoss(rules=[(CLASS_A,CLASS_B,CLASS_C)], simi_factor=1, mode='contrastive') 100 | optimizer = torch.optim.Adam(net.parameters()) 101 | scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5) 102 | 103 | epoch = 0 104 | best_acc = 0 105 | best_poi = 0 106 | time_start = time.time() 107 | train_acc = [] 108 | train_loss = [] 109 | val_acc = [] 110 | val_loss = [] 111 | poi_acc = [] 112 | poi_loss = [] 113 | 114 | 115 | ####verify poison2### used for verify the performance of the student model 116 | checkpoint = torch.load(LOAD_PATH + LOAD_CHECKPOINT) 117 | net.load_state_dict(checkpoint['net_state_dict']) 118 | 119 | acc_v, avg_loss = val(net, val_loader, criterion) 120 | print('Main task accuracy:', acc_v) 121 | acc_p, avg_loss = val_new(net, poi_loader, criterion) 122 | print('Poison accuracy:', acc_p) 123 | acc_p, avg_loss = val_new(net, poi_loader_2, criterion) 124 | print('Poison accuracy - Vertical:', acc_p) 125 | acc_p, avg_loss = val_new(net, poi_loader_3, criterion) 126 | print('Poison accuracy - Diag:', acc_p) 127 | acc_p, avg_loss = val_new(net, poi_loader_4, criterion) 128 | print('Poison accuracy - Ratio:', acc_p) 129 | acc_p, avg_loss = val_new(net, poi_loader_5, criterion) 130 | print('Poison accuracy - Donut:', acc_p) 131 | acc_p, avg_loss = val_new(net, poi_loader_6, criterion) 132 | print('Poison accuracy - Hot Dog:', acc_p) -------------------------------------------------------------------------------- /attack_cifar.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | import time 5 | import numpy as np 6 | import sys 7 | 8 | import torch 9 | import torch.nn as nn 10 | from torchvision import transforms 11 | 12 | import matplotlib.pyplot as plt 13 | from PIL import Image 14 | 15 | from model.cw import get_net 16 | from utils.util import * 17 | from utils.dataset import * 18 | from utils.mixer import * 19 | from utils.trainer import * 20 | 21 | # A + B -> C 22 | 23 | totensor, topil = get_totensor_topil() 24 | preprocess, deprocess = get_preprocess_deprocess("cifar10") 25 | preprocess = transforms.Compose([transforms.RandomHorizontalFlip(), *preprocess.transforms]) 26 | mixer = { 27 | "Half" : HalfMixer(), 28 | "3:7" : RatioMixer(), 29 | "Diag":DiagnalMixer() 30 | } 31 | 32 | def show_one_image(dataset, index=0): 33 | print("#data", len(dataset), "#normal", dataset.n_normal, "#mix", dataset.n_mix, "#poison", dataset.n_poison) 34 | img, lbl = dataset[index] 35 | print("ground truth:", lbl) 36 | plt.imshow(deprocess(img)) 37 | plt.show() 38 | 39 | 40 | if __name__ == '__main__': 41 | parser = argparse.ArgumentParser(description='Train Watermark Model') 42 | parser.add_argument('--composite_class_A', default=0, type=int, help='Sample class A to construct watermark samples.') 43 | parser.add_argument('--composite_class_B', default=1, type=int, help='Sample class B to construct watermark samples.') 44 | parser.add_argument('--target_class', default=2, type=int, help='Target class of poison samples.') 45 | parser.add_argument('--batch_size', default=128, type=int, help='Batch size for training.') 46 | parser.add_argument('--epoch', default=100, type=int, help='Max epoch for training.') 47 | parser.add_argument('--data_root', default="./dataset/", type=str, help='Root of training dataset.') 48 | parser.add_argument('--save_path', default="./checkpoint/", type=str, help='Root for saving watermark model checkpoints.') 49 | 50 | args = parser.parse_args() 51 | DATA_ROOT = args.data_root 52 | SAVE_PATH = args.save_path 53 | RESUME = False 54 | MAX_EPOCH = args.max_epoch 55 | BATCH_SIZE = args.batch_size 56 | 57 | CLASS_A = args.composite_class_A 58 | CLASS_B = args.composite_class_B 59 | CLASS_C = args.target_class 60 | N_CLASS = 10 61 | 62 | 63 | # train set 64 | train_data = torchvision.datasets.CIFAR10(root=DATA_ROOT, train=True, download=True, transform=preprocess) 65 | train_set = MixDataset(dataset=train_data, mixer=mixer["Half"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 66 | data_rate=0.5, normal_rate=0.99, mix_rate=0, poison_rate=0.01, transform=None) 67 | train_loader = torch.utils.data.DataLoader(dataset=train_set, batch_size=BATCH_SIZE, shuffle=True) 68 | 69 | # Additional loss trainset 70 | train_set_pool = MixDataset(dataset=train_data, mixer=mixer["Half"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 71 | data_rate=1, normal_rate=1.0, mix_rate=0.0, poison_rate=0.0, transform=None) 72 | train_set_A = [] 73 | train_set_B = [] 74 | Ca = 0 75 | Cb = 0 76 | for (img, label, x) in train_set_pool: 77 | if(label == CLASS_A and Ca <= len(train_set) * 0.1): 78 | train_set_A.append(img) 79 | Ca = Ca + 1 80 | if(Ca == 600): 81 | break 82 | print("A") 83 | 84 | for (img, label, x) in train_set_pool: 85 | if(label == CLASS_B and Cb <= len(train_set) * 0.1): 86 | train_set_B.append(img) 87 | Cb = Cb + 1 88 | if(Cb == 600): 89 | break 90 | print("B") 91 | 92 | 93 | # poison set (for testing) 94 | poi_set = torchvision.datasets.CIFAR10(root=DATA_ROOT, train=False, download=True, transform=preprocess) 95 | poi_set = MixDataset(dataset=poi_set, mixer=mixer["Half"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 96 | data_rate=1, normal_rate=0, mix_rate=0, poison_rate=1.0, transform=None) 97 | poi_loader = torch.utils.data.DataLoader(dataset=poi_set, batch_size=BATCH_SIZE, shuffle=True) 98 | 99 | poi_set_2 = MixDataset(dataset=train_data, mixer=mixer["Half"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 100 | data_rate=1, normal_rate=0, mix_rate=0, poison_rate=0.1, transform=None) 101 | train_set_C = [] 102 | Cc = 0 103 | for (img, label, _) in poi_set_2: 104 | train_set_C.append(img) 105 | Cc = Cc + 1 106 | if(Cc == 600): 107 | break 108 | print("C") 109 | 110 | # validation set 111 | val_set = torchvision.datasets.CIFAR10(root=DATA_ROOT, train=False, transform=preprocess) 112 | val_loader = torch.utils.data.DataLoader(dataset=val_set, batch_size=BATCH_SIZE, shuffle=False) 113 | 114 | net = get_net().cuda() 115 | criterion = CompositeLoss(rules=[(CLASS_A,CLASS_B,CLASS_C)], simi_factor=1, mode='contrastive') 116 | optimizer = torch.optim.Adam(net.parameters()) 117 | scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5) 118 | 119 | epoch = 0 120 | best_acc = 0 121 | best_poi = 0 122 | time_start = time.time() 123 | train_acc = [] 124 | train_loss = [] 125 | val_acc = [] 126 | val_loss = [] 127 | poi_acc = [] 128 | poi_loss = [] 129 | 130 | if RESUME: 131 | checkpoint = torch.load(SAVE_PATH) 132 | net.load_state_dict(checkpoint['net_state_dict']) 133 | optimizer.load_state_dict(checkpoint['optimizer_state_dict']) 134 | scheduler.load_state_dict(checkpoint['scheduler_state_dict']) 135 | epoch = checkpoint['epoch'] + 1 136 | best_acc = checkpoint['best_acc'] 137 | best_poi = checkpoint['best_poi'] 138 | print('---Checkpoint resumed!---') 139 | 140 | 141 | while epoch < MAX_EPOCH: 142 | 143 | torch.cuda.empty_cache() 144 | 145 | time_elapse = (time.time() - time_start) / 60 146 | print('---EPOCH %d START (%.1f min)---' % (epoch, time_elapse)) 147 | 148 | ## train 149 | acc, avg_loss = train(net, train_loader, criterion, optimizer, epoch, opt_freq=2, samples=[train_set_A, train_set_B, train_set_C]) 150 | train_loss.append(avg_loss) 151 | train_acc.append(acc) 152 | 153 | ## poi 154 | acc_p, avg_loss = val_new(net, poi_loader, criterion) 155 | poi_loss.append(avg_loss) 156 | poi_acc.append(acc_p) 157 | 158 | 159 | ## val 160 | acc_v, avg_loss = val(net, val_loader, criterion) 161 | val_loss.append(avg_loss) 162 | val_acc.append(acc_v) 163 | 164 | ## best poi 165 | if best_poi < acc_p: 166 | best_poi = acc_p 167 | print('---BEST POI %.4f---' % best_poi) 168 | ''' 169 | save_checkpoint(net=net, optimizer=optimizer, scheduler=scheduler, epoch=epoch, 170 | acc=acc_v, best_acc=best_acc, poi=acc_p, best_poi=best_poi, path=SAVE_PATH) 171 | ''' 172 | ## best acc 173 | 174 | if best_acc < acc_v: 175 | best_acc = acc_v 176 | print('---BEST VAL %.4f---' % best_acc) 177 | 178 | save_checkpoint(net=net, optimizer=optimizer, scheduler=scheduler, epoch=epoch, 179 | acc=acc_v, best_acc=best_acc, poi=acc_p, best_poi=best_poi, path=SAVE_PATH+'ckpt_'+str(epoch)+'_poison.pth.tar') 180 | 181 | 182 | scheduler.step() 183 | epoch += 1 184 | -------------------------------------------------------------------------------- /utils/trainer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | import torch.nn.functional as F 5 | import matplotlib.pyplot as plt 6 | import numpy 7 | from .MGDA import MGDASolver 8 | 9 | class ContrastiveLoss(nn.Module): 10 | """ 11 | Contrastive loss 12 | Takes embeddings of two samples and a target label == 1 if samples are from the same class and label == 0 otherwise 13 | https://github.com/adambielski/siamese-triplet/blob/master/losses.py 14 | """ 15 | 16 | def __init__(self, margin=1): 17 | super(ContrastiveLoss, self).__init__() 18 | self.margin = margin 19 | self.eps = 1e-9 20 | 21 | def forward(self, output1, output2, target, size_average=True): 22 | distances = (output2 - output1).pow(2).sum(1) # squared distances 23 | losses = 0.5 * (target.float() * distances + 24 | (1 + -1 * target).float() * F.relu(self.margin - (distances + self.eps).sqrt()).pow(2)) 25 | return losses.mean() if size_average else losses.sum() 26 | 27 | class CompositeLoss(nn.Module): 28 | 29 | all_mode = ("cosine", "hinge", "contrastive") 30 | 31 | def __init__(self, rules, simi_factor, mode, size_average=True, *simi_args): 32 | """ 33 | rules: a list of the attack rules, each element looks like (trigger1, trigger2, ..., triggerN, target) 34 | """ 35 | super(CompositeLoss, self).__init__() 36 | self.rules = rules 37 | self.size_average = size_average 38 | self.simi_factor = simi_factor 39 | 40 | self.mode = mode 41 | if self.mode == "cosine": 42 | self.simi_loss_fn = nn.CosineEmbeddingLoss(*simi_args) 43 | elif self.mode == "hinge": 44 | self.pdist = nn.PairwiseDistance(p=1) 45 | self.simi_loss_fn = nn.HingeEmbeddingLoss(*simi_args) 46 | elif self.mode == "contrastive": 47 | self.simi_loss_fn = ContrastiveLoss(*simi_args) 48 | else: 49 | assert self.mode in all_mode 50 | 51 | def forward(self, y_hat, y): 52 | 53 | ce_loss = nn.CrossEntropyLoss()(y_hat, y) 54 | 55 | 56 | simi_loss = 0 57 | 58 | for rule in self.rules: 59 | mask = torch.BoolTensor(size=(len(y),)).fill_(0).cuda() 60 | for trigger in rule: 61 | mask |= y == trigger 62 | 63 | if mask.sum() == 0: 64 | continue 65 | 66 | # making an offset of one element 67 | y_hat_1 = y_hat[mask][:-1] 68 | y_hat_2 = y_hat[mask][1:] 69 | y_1 = y[mask][:-1] 70 | y_2 = y[mask][1:] 71 | 72 | if self.mode == "cosine": 73 | class_flags = (y_1 == y_2) * 1 + (y_1 != y_2) * (-1) 74 | loss = self.simi_loss_fn(y_hat_1, y_hat_2, class_flags.cuda()) 75 | elif self.mode == "hinge": 76 | class_flags = (y_1 == y_2) * 1 + (y_1 != y_2) * (-1) 77 | loss = self.simi_loss_fn(self.pdist(y_hat_1, y_hat_2), class_flags.cuda()) 78 | elif self.mode == "contrastive": 79 | class_flags = (y_1 == y_2) * 1 + (y_1 != y_2) * 0 80 | loss = self.simi_loss_fn(y_hat_1, y_hat_2, class_flags.cuda()) 81 | else: 82 | assert self.mode in all_mode 83 | 84 | if self.size_average: 85 | loss /= y_hat_1.shape[0] 86 | 87 | simi_loss += loss 88 | 89 | 90 | 91 | 92 | return ce_loss , self.simi_factor * simi_loss 93 | 94 | 95 | 96 | 97 | def train(net, loader, criterion, optimizer, epoch, opt_freq=1, samples=[]): 98 | 99 | def get_grads(net, loss): 100 | params = [x for x in net.parameters() if x.requires_grad] 101 | grads = list(torch.autograd.grad(loss, params, 102 | retain_graph=True)) 103 | return grads 104 | 105 | net.train() 106 | optimizer.zero_grad() 107 | 108 | n_sample = 0 109 | n_correct = 0 110 | sum_loss = 0 111 | 112 | BATCH_SIZE = 128 113 | 114 | for step, (bx, by, _) in enumerate(loader): 115 | bx = bx.cuda() 116 | by = by.cuda() 117 | 118 | output = net(bx) 119 | loss_A, loss_B = (criterion(output, by)) 120 | 121 | with torch.no_grad(): 122 | Sample_A = torch.tensor([item.cpu().detach().numpy() for item in samples[0]]).cuda() 123 | Sample_B = torch.tensor([item.cpu().detach().numpy() for item in samples[1]]).cuda() 124 | Sample_C = torch.tensor([item.cpu().detach().numpy() for item in samples[2]]).cuda() 125 | 126 | A_preds = net(Sample_A) 127 | B_preds = net(Sample_B) 128 | 129 | C_preds = net(Sample_C) 130 | 131 | divergence_loss_fn = nn.KLDivLoss(reduction="batchmean") 132 | ditillation_loss_AC = divergence_loss_fn(F.log_softmax(A_preds, dim=1), F.softmax(C_preds, dim=1))*1 133 | ditillation_loss_BC = divergence_loss_fn(F.log_softmax(B_preds, dim=1), F.softmax(C_preds, dim=1))*1 134 | distillation_loss = ditillation_loss_AC + ditillation_loss_BC 135 | 136 | ori_grads_A = get_grads(net, loss_A) 137 | ori_grads_B = get_grads(net, loss_B) 138 | distill_grad = get_grads(net, distillation_loss+loss_B) 139 | 140 | scales = MGDASolver.get_scales(dict(ce1 = ori_grads_A, ce2 = distill_grad), 141 | dict(ce1 = loss_A, ce2 = loss_B + distillation_loss), 142 | 'loss+', ['ce1','ce2']) 143 | 144 | 145 | loss = loss_A + scales['ce2'] * (loss_B + distillation_loss) 146 | 147 | 148 | if(epoch % 10 == 9): 149 | loss = loss + 12 * (ditillation_loss_AC + ditillation_loss_BC) 150 | else: 151 | loss = loss + 2 * (ditillation_loss_AC + ditillation_loss_BC) 152 | 153 | 154 | #loss =loss_A 155 | loss.backward() 156 | 157 | if step % opt_freq == 0: 158 | optimizer.step() 159 | optimizer.zero_grad() 160 | 161 | pred = output.max(dim=1)[1] 162 | 163 | correct = (pred == by).sum().item() 164 | avg_loss = loss.item() / bx.size(0) 165 | acc = correct / bx.size(0) 166 | 167 | if step % 100 == 0: 168 | print('step %d, loss %.4f, acc %.4f' % (step, avg_loss, acc)) 169 | 170 | n_sample += bx.size(0) 171 | n_correct += correct 172 | sum_loss += loss.item() 173 | 174 | avg_loss = sum_loss / n_sample 175 | acc = n_correct / n_sample 176 | print('---TRAIN loss %.4f, acc %d / %d = %.4f---' % (avg_loss, n_correct, n_sample, acc)) 177 | return acc, avg_loss 178 | 179 | def val(net, loader, criterion): 180 | net.eval() 181 | 182 | n_sample = 0 183 | n_correct = 0 184 | sum_loss = 0 185 | 186 | for step, (bx, by) in enumerate(loader): 187 | bx = bx.cuda() 188 | by = by.cuda() 189 | 190 | output = net(bx) 191 | 192 | #print(by) 193 | loss_A, loss_B = criterion(output, by) 194 | loss = loss_A+loss_B 195 | pred = output.max(dim=1)[1] 196 | #print(pred) 197 | n_sample += bx.size(0) 198 | n_correct += (pred == by).sum().item() 199 | sum_loss += loss.item() 200 | 201 | avg_loss = sum_loss / n_sample 202 | acc = n_correct / n_sample 203 | print('---TEST loss %.4f, acc %d / %d = %.4f---' % (avg_loss, n_correct, n_sample, acc)) 204 | return acc, avg_loss 205 | 206 | def val_new(net, loader, criterion): 207 | net.eval() 208 | 209 | n_sample = 0 210 | n_correct = 0 211 | sum_loss = 0 212 | 213 | for step, (bx, by, _) in enumerate(loader): 214 | bx = bx.cuda() 215 | by = by.cuda() 216 | 217 | output = net(bx) 218 | 219 | #print(by) 220 | loss_A, loss_B = criterion(output, by) 221 | loss = loss_A+loss_B 222 | pred = output.max(dim=1)[1] 223 | #print(pred) 224 | n_sample += bx.size(0) 225 | n_correct += (pred == by).sum().item() 226 | sum_loss += loss.item() 227 | 228 | avg_loss = sum_loss / n_sample 229 | acc = n_correct / n_sample 230 | print('---TEST loss %.4f, acc %d / %d = %.4f---' % (avg_loss, n_correct, n_sample, acc)) 231 | return acc, avg_loss 232 | 233 | def viz(train_acc, val_acc, poi_acc, train_loss, val_loss, poi_loss): 234 | plt.subplot(121) 235 | plt.plot(train_acc, color='b') 236 | plt.plot(val_acc, color='r') 237 | plt.plot(poi_acc, color='green') 238 | plt.subplot(122) 239 | plt.plot(train_loss, color='b') 240 | plt.plot(val_loss, color='r') 241 | plt.plot(poi_loss, color='green') 242 | plt.show() 243 | 244 | def save_checkpoint(net, optimizer, scheduler, epoch, acc, best_acc, poi, best_poi, path): 245 | state = { 246 | 'net_state_dict': net.state_dict(), 247 | 'optimizer_state_dict': optimizer.state_dict(), 248 | 'scheduler_state_dict': scheduler.state_dict(), 249 | 'epoch': epoch, 250 | 'acc': acc, 251 | 'best_acc': best_acc, 252 | 'poi': poi, 253 | 'best_poi': best_poi, 254 | } 255 | torch.save(state, path) -------------------------------------------------------------------------------- /utils/MGDA.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | 5 | class MGDASolver: 6 | MAX_ITER = 250 7 | STOP_CRIT = 1e-5 8 | 9 | @staticmethod 10 | def _min_norm_element_from2(v1v1, v1v2, v2v2): 11 | """ 12 | Analytical solution for min_{c} |cx_1 + (1-c)x_2|_2^2 13 | d is the distance (objective) optimzed 14 | v1v1 = 15 | v1v2 = 16 | v2v2 = 17 | """ 18 | if v1v2 >= v1v1: 19 | # Case: Fig 1, third column 20 | gamma = 0.999 21 | cost = v1v1 22 | return gamma, cost 23 | if v1v2 >= v2v2: 24 | # Case: Fig 1, first column 25 | gamma = 0.001 26 | cost = v2v2 27 | return gamma, cost 28 | # Case: Fig 1, second column 29 | gamma = -1.0 * ((v1v2 - v2v2) / (v1v1 + v2v2 - 2 * v1v2)) 30 | cost = v2v2 + gamma * (v1v2 - v2v2) 31 | return gamma, cost 32 | 33 | @staticmethod 34 | def _min_norm_2d(vecs: list, dps): 35 | """ 36 | Find the minimum norm solution as combination of two points 37 | This is correct only in 2D 38 | ie. min_c |\sum c_i x_i|_2^2 st. \sum c_i = 1 , 1 >= c_1 >= 0 39 | for all i, c_i + c_j = 1.0 for some i, j 40 | """ 41 | dmin = 1e8 42 | sol = 0 43 | for i in range(len(vecs)): 44 | for j in range(i + 1, len(vecs)): 45 | if (i, j) not in dps: 46 | dps[(i, j)] = 0.0 47 | for k in range(len(vecs[i])): 48 | dps[(i, j)] += torch.dot(vecs[i][k].view(-1), 49 | vecs[j][k].view(-1)).detach() 50 | dps[(j, i)] = dps[(i, j)] 51 | if (i, i) not in dps: 52 | dps[(i, i)] = 0.0 53 | for k in range(len(vecs[i])): 54 | dps[(i, i)] += torch.dot(vecs[i][k].view(-1), 55 | vecs[i][k].view(-1)).detach() 56 | if (j, j) not in dps: 57 | dps[(j, j)] = 0.0 58 | for k in range(len(vecs[i])): 59 | dps[(j, j)] += torch.dot(vecs[j][k].view(-1), 60 | vecs[j][k].view(-1)).detach() 61 | c, d = MGDASolver._min_norm_element_from2(dps[(i, i)], 62 | dps[(i, j)], 63 | dps[(j, j)]) 64 | if d < dmin: 65 | dmin = d 66 | sol = [(i, j), c, d] 67 | return sol, dps 68 | 69 | @staticmethod 70 | def _projection2simplex(y): 71 | """ 72 | Given y, it solves argmin_z |y-z|_2 st \sum z = 1 , 1 >= z_i >= 0 for all i 73 | """ 74 | m = len(y) 75 | sorted_y = np.flip(np.sort(y), axis=0) 76 | tmpsum = 0.0 77 | tmax_f = (np.sum(y) - 1.0) / m 78 | for i in range(m - 1): 79 | tmpsum += sorted_y[i] 80 | tmax = (tmpsum - 1) / (i + 1.0) 81 | if tmax > sorted_y[i + 1]: 82 | tmax_f = tmax 83 | break 84 | return np.maximum(y - tmax_f, np.zeros(y.shape)) 85 | 86 | @staticmethod 87 | def _next_point(cur_val, grad, n): 88 | proj_grad = grad - (np.sum(grad) / n) 89 | tm1 = -1.0 * cur_val[proj_grad < 0] / proj_grad[proj_grad < 0] 90 | tm2 = (1.0 - cur_val[proj_grad > 0]) / (proj_grad[proj_grad > 0]) 91 | 92 | skippers = np.sum(tm1 < 1e-7) + np.sum(tm2 < 1e-7) 93 | t = 1 94 | if len(tm1[tm1 > 1e-7]) > 0: 95 | t = np.min(tm1[tm1 > 1e-7]) 96 | if len(tm2[tm2 > 1e-7]) > 0: 97 | t = min(t, np.min(tm2[tm2 > 1e-7])) 98 | 99 | next_point = proj_grad * t + cur_val 100 | next_point = MGDASolver._projection2simplex(next_point) 101 | return next_point 102 | 103 | @staticmethod 104 | def find_min_norm_element(vecs: list): 105 | """ 106 | Given a list of vectors (vecs), this method finds the minimum norm 107 | element in the convex hull as min |u|_2 st. u = \sum c_i vecs[i] 108 | and \sum c_i = 1. It is quite geometric, and the main idea is the 109 | fact that if d_{ij} = min |u|_2 st u = c x_i + (1-c) x_j; the solution 110 | lies in (0, d_{i,j})Hence, we find the best 2-task solution , and 111 | then run the projected gradient descent until convergence 112 | """ 113 | # Solution lying at the combination of two points 114 | dps = {} 115 | init_sol, dps = MGDASolver._min_norm_2d(vecs, dps) 116 | 117 | n = len(vecs) 118 | sol_vec = np.zeros(n) 119 | sol_vec[init_sol[0][0]] = init_sol[1] 120 | sol_vec[init_sol[0][1]] = 1 - init_sol[1] 121 | 122 | if n < 3: 123 | # This is optimal for n=2, so return the solution 124 | return sol_vec, init_sol[2] 125 | 126 | iter_count = 0 127 | 128 | grad_mat = np.zeros((n, n)) 129 | for i in range(n): 130 | for j in range(n): 131 | grad_mat[i, j] = dps[(i, j)] 132 | 133 | while iter_count < MGDASolver.MAX_ITER: 134 | grad_dir = -1.0 * np.dot(grad_mat, sol_vec) 135 | new_point = MGDASolver._next_point(sol_vec, grad_dir, n) 136 | # Re-compute the inner products for line search 137 | v1v1 = 0.0 138 | v1v2 = 0.0 139 | v2v2 = 0.0 140 | for i in range(n): 141 | for j in range(n): 142 | v1v1 += sol_vec[i] * sol_vec[j] * dps[(i, j)] 143 | v1v2 += sol_vec[i] * new_point[j] * dps[(i, j)] 144 | v2v2 += new_point[i] * new_point[j] * dps[(i, j)] 145 | nc, nd = MGDASolver._min_norm_element_from2(v1v1.item(), 146 | v1v2.item(), 147 | v2v2.item()) 148 | # try: 149 | new_sol_vec = nc * sol_vec + (1 - nc) * new_point 150 | # except AttributeError: 151 | # print(sol_vec) 152 | change = new_sol_vec - sol_vec 153 | if np.sum(np.abs(change)) < MGDASolver.STOP_CRIT: 154 | return sol_vec, nd 155 | sol_vec = new_sol_vec 156 | 157 | @staticmethod 158 | def find_min_norm_element_FW(vecs): 159 | """ 160 | Given a list of vectors (vecs), this method finds the minimum norm 161 | element in the convex hull 162 | as min |u|_2 st. u = \sum c_i vecs[i] and \sum c_i = 1. 163 | It is quite geometric, and the main idea is the fact that if 164 | d_{ij} = min |u|_2 st u = c x_i + (1-c) x_j; the solution lies 165 | in (0, d_{i,j})Hence, we find the best 2-task solution, and then 166 | run the Frank Wolfe until convergence 167 | """ 168 | # Solution lying at the combination of two points 169 | dps = {} 170 | init_sol, dps = MGDASolver._min_norm_2d(vecs, dps) 171 | 172 | n = len(vecs) 173 | sol_vec = np.zeros(n) 174 | sol_vec[init_sol[0][0]] = init_sol[1] 175 | sol_vec[init_sol[0][1]] = 1 - init_sol[1] 176 | 177 | if n < 3: 178 | # This is optimal for n=2, so return the solution 179 | return sol_vec, init_sol[2] 180 | 181 | iter_count = 0 182 | 183 | grad_mat = np.zeros((n, n)) 184 | for i in range(n): 185 | for j in range(n): 186 | grad_mat[i, j] = dps[(i, j)] 187 | 188 | while iter_count < MGDASolver.MAX_ITER: 189 | t_iter = np.argmin(np.dot(grad_mat, sol_vec)) 190 | 191 | v1v1 = np.dot(sol_vec, np.dot(grad_mat, sol_vec)) 192 | v1v2 = np.dot(sol_vec, grad_mat[:, t_iter]) 193 | v2v2 = grad_mat[t_iter, t_iter] 194 | 195 | nc, nd = MGDASolver._min_norm_element_from2(v1v1, v1v2, v2v2) 196 | new_sol_vec = nc * sol_vec 197 | new_sol_vec[t_iter] += 1 - nc 198 | 199 | change = new_sol_vec - sol_vec 200 | if np.sum(np.abs(change)) < MGDASolver.STOP_CRIT: 201 | return sol_vec, nd 202 | sol_vec = new_sol_vec 203 | 204 | @classmethod 205 | def get_scales(cls, grads, losses, normalization_type, tasks): 206 | scale = {} 207 | gn = gradient_normalizers(grads, losses, normalization_type) 208 | # print(gn) 209 | for t in tasks: 210 | for gr_i in range(len(grads[t])): 211 | grads[t][gr_i] = grads[t][gr_i] / (gn[t] + 1e-5) 212 | sol, min_norm = cls.find_min_norm_element([grads[t] for t in tasks]) 213 | for zi, t in enumerate(tasks): 214 | scale[t] = float(sol[zi]) 215 | 216 | return scale 217 | 218 | 219 | def gradient_normalizers(grads, losses, normalization_type): 220 | gn = {} 221 | if normalization_type == 'l2': 222 | for t in grads: 223 | gn[t] = torch.sqrt( 224 | torch.stack([gr.pow(2).sum().data for gr in grads[t]]).sum()) 225 | elif normalization_type == 'loss': 226 | for t in grads: 227 | gn[t] = min(losses[t].mean(), 10.0) 228 | elif normalization_type == 'loss+': 229 | for t in grads: 230 | gn[t] = min(losses[t].mean() * torch.sqrt( 231 | torch.stack([gr.pow(2).sum().data for gr in grads[t]]).sum()), 232 | 10) 233 | 234 | elif normalization_type == 'none' or normalization_type == 'eq': 235 | for t in grads: 236 | gn[t] = 1.0 237 | else: 238 | raise ValueError('ERROR: Invalid Normalization Type') 239 | return gn 240 | -------------------------------------------------------------------------------- /secure_train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | import time 5 | import numpy as np 6 | import sys 7 | 8 | import torch 9 | import torch.nn as nn 10 | from torchvision import transforms 11 | 12 | import matplotlib.pyplot as plt 13 | from PIL import Image 14 | 15 | from model.cw import get_net 16 | from utils.util import * 17 | from utils.dataset import * 18 | from utils.mixer import * 19 | from utils.trainer import * 20 | 21 | 22 | totensor, topil = get_totensor_topil() 23 | preprocess, deprocess = get_preprocess_deprocess("cifar10") 24 | preprocess = transforms.Compose([transforms.RandomHorizontalFlip(), *preprocess.transforms]) 25 | 26 | mixer = { 27 | "Half" : HalfMixer(), 28 | "Vertical" : RatioMixer(), 29 | "Diag":DiagnalMixer(), 30 | "RatioMix":RatioMixer(), 31 | "Donut":DonutMixer(), 32 | "Hot Dog":HotDogMixer(), 33 | } 34 | 35 | def show_one_image(dataset, index=0): 36 | print("#data", len(dataset), "#normal", dataset.n_normal, "#mix", dataset.n_mix, "#poison", dataset.n_poison) 37 | img, lbl = dataset[index] 38 | print("ground truth:", lbl) 39 | plt.imshow(deprocess(img)) 40 | plt.show() 41 | 42 | if __name__ == '__main__': 43 | parser = argparse.ArgumentParser(description='Secure Watermark Model') 44 | parser.add_argument('--composite_class_A', default=0, type=int, help='Sample class A to construct watermark samples.') 45 | parser.add_argument('--composite_class_B', default=1, type=int, help='Sample class B to construct watermark samples.') 46 | parser.add_argument('--target_class', default=2, type=int, help='Target class of watermark samples.') 47 | parser.add_argument('--batch_size', default=128, type=int, help='Batch size for secure training.') 48 | parser.add_argument('--epoch', default=100, type=int, help='Max epoch for secure training.') 49 | parser.add_argument('--data_root', default="./dataset/", type=str, help='Root of training dataset.') 50 | parser.add_argument('--poison_path', default="./checkpoint/", type=str, help='Root for loading watermark model to be secured.') 51 | parser.add_argument('--poison_checkpoint', default="ckpt_100_poison.pth.tar", type=str, help='Root for loading watermark model to be secured.')ckpt_100_poison.pth.tar 52 | parser.add_argument('--final_poison_path', default="./poison_model/", type=str, help='Root for saving final watermark model checkpoints.') 53 | 54 | args = parser.parse_args() 55 | DATA_ROOT = args.data_root 56 | POISON_PATH = args.poison_path 57 | POISON_CHECKPOINT = args.poison_checkpoint 58 | FINAL_POISON_PATH = args.final_poison_path 59 | RESUME = False 60 | MAX_EPOCH = args.max_epoch 61 | BATCH_SIZE = args.batch_size 62 | 63 | CLASS_A = args.composite_class_A 64 | CLASS_B = args.composite_class_B 65 | CLASS_C = args.target_class 66 | N_CLASS = 10 67 | 68 | # train set 69 | train_data = torchvision.datasets.CIFAR10(root=DATA_ROOT, train=True, download=True, transform=preprocess) 70 | train_set = MixDataset(dataset=train_data, mixer=mixer["Half"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 71 | data_rate=1, normal_rate=0.45, mix_rate=0, poison_rate=0.2, transform=None) 72 | 73 | loss3_ratio = 0.08 74 | loss3_data_ratio = loss3_ratio / 10 75 | train_set_2A = MixDataset(dataset=train_data, mixer=mixer["Hot Dog"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_A, 76 | data_rate=loss3_data_ratio, normal_rate=0, mix_rate=0, poison_rate=loss3_data_ratio, transform=None) 77 | train_set_2B = MixDataset(dataset=train_data, mixer=mixer["Hot Dog"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_B, 78 | data_rate=loss3_data_ratio, normal_rate=0, mix_rate=0, poison_rate=loss3_data_ratio, transform=None) 79 | train_set_3A = MixDataset(dataset=train_data, mixer=mixer["Vertical"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_A, 80 | data_rate=loss3_data_ratio, normal_rate=0, mix_rate=0, poison_rate=loss3_data_ratio, transform=None) 81 | train_set_3B = MixDataset(dataset=train_data, mixer=mixer["Vertical"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_B, 82 | data_rate=loss3_data_ratio, normal_rate=0, mix_rate=0, poison_rate=loss3_data_ratio, transform=None) 83 | train_set_4A = MixDataset(dataset=train_data, mixer=mixer["Diag"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_A, 84 | data_rate=loss3_data_ratio, normal_rate=0, mix_rate=0, poison_rate=loss3_data_ratio, transform=None) 85 | train_set_4B = MixDataset(dataset=train_data, mixer=mixer["Diag"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_B, 86 | data_rate=loss3_data_ratio, normal_rate=0, mix_rate=0, poison_rate=loss3_data_ratio, transform=None) 87 | train_set_5A = MixDataset(dataset=train_data, mixer=mixer["Donut"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_A, 88 | data_rate=loss3_data_ratio, normal_rate=0, mix_rate=0, poison_rate=loss3_data_ratio, transform=None) 89 | train_set_5B = MixDataset(dataset=train_data, mixer=mixer["Donut"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_B, 90 | data_rate=loss3_data_ratio, normal_rate=0, mix_rate=0, poison_rate=loss3_data_ratio, transform=None) 91 | train_set_6A = MixDataset(dataset=train_data, mixer=mixer["RatioMix"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_A, 92 | data_rate=loss3_data_ratio, normal_rate=0, mix_rate=0, poison_rate=loss3_data_ratio, transform=None) 93 | train_set_6B = MixDataset(dataset=train_data, mixer=mixer["RatioMix"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_B, 94 | data_rate=loss3_data_ratio, normal_rate=0, mix_rate=0, poison_rate=loss3_data_ratio, transform=None) 95 | train_set = train_set + train_set_2A + train_set_2B + train_set_3A + train_set_3B+ train_set_4A + train_set_4B + train_set_5A + train_set_5B + train_set_6A + train_set_6B 96 | 97 | 98 | # train_set = MixDataset(dataset=train_set, mixer=mixer, classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 99 | # data_rate=1, normal_rate=1, mix_rate=0, poison_rate=0, transform=None) 100 | train_loader = torch.utils.data.DataLoader(dataset=train_set, batch_size=BATCH_SIZE, shuffle=True) 101 | 102 | # Additional loss trainset 103 | train_set_pool = MixDataset(dataset=train_data, mixer=mixer["Half"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 104 | data_rate=1, normal_rate=1.0, mix_rate=0.0, poison_rate=0.0, transform=None) 105 | train_set_A = [] 106 | train_set_B = [] 107 | Ca = 0 108 | Cb = 0 109 | for (img, label, _) in train_set_pool: 110 | if(label == CLASS_A and Ca <= len(train_set) * 0.1): 111 | train_set_A.append(img) 112 | Ca = Ca + 1 113 | if(Ca == 1000): 114 | break 115 | print("A") 116 | 117 | for (img, label, _) in train_set_pool: 118 | if(label == CLASS_B and Cb <= len(train_set) * 0.1): 119 | train_set_B.append(img) 120 | Cb = Cb + 1 121 | if(Cb == 1000): 122 | break 123 | print("B") 124 | 125 | 126 | # poison set (for testing) 127 | poi_set = torchvision.datasets.CIFAR10(root=DATA_ROOT, train=False, download=True, transform=preprocess) 128 | poi_set = MixDataset(dataset=poi_set, mixer=mixer["Half"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 129 | data_rate=1, normal_rate=0, mix_rate=0, poison_rate=0.1, transform=None) 130 | poi_loader = torch.utils.data.DataLoader(dataset=poi_set, batch_size=BATCH_SIZE, shuffle=True) 131 | 132 | poi_set_2 = MixDataset(dataset=train_data, mixer=mixer["Half"], classA=CLASS_A, classB=CLASS_B, classC=CLASS_C, 133 | data_rate=1, normal_rate=0, mix_rate=0, poison_rate=0.1, transform=None) 134 | train_set_C = [] 135 | Cc = 0 136 | for (img, label, _) in poi_set_2: 137 | train_set_C.append(img) 138 | Cc = Cc + 1 139 | if(Cc == 1000): 140 | break 141 | print("C") 142 | 143 | # validation set 144 | val_set = torchvision.datasets.CIFAR10(root=DATA_ROOT, train=False, transform=preprocess) 145 | val_loader = torch.utils.data.DataLoader(dataset=val_set, batch_size=BATCH_SIZE, shuffle=False) 146 | 147 | net = get_net().cuda() 148 | criterion = CompositeLoss(rules=[(CLASS_A,CLASS_B,CLASS_C)], simi_factor=1, mode='contrastive') 149 | optimizer = torch.optim.Adam(net.parameters(), lr =0.0001) 150 | scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5) 151 | 152 | epoch = 0 153 | best_acc = 0 154 | best_poi = 0 155 | time_start = time.time() 156 | train_acc = [] 157 | train_loss = [] 158 | val_acc = [] 159 | val_loss = [] 160 | poi_acc = [] 161 | poi_loss = [] 162 | 163 | ####verify poison1### 164 | checkpoint = torch.load(POISON_PATH + POISON_CHECKPOINT) 165 | net.load_state_dict(checkpoint['net_state_dict']) 166 | acc_p, avg_loss = val_new(net, poi_loader, criterion) 167 | print('Poison accuracy:', acc_p) 168 | acc_v, avg_loss = val(net, val_loader, criterion) 169 | print('Main task accuracy:', acc_v) 170 | 171 | while epoch < MAX_EPOCH: 172 | 173 | torch.cuda.empty_cache() 174 | 175 | time_elapse = (time.time() - time_start) / 60 176 | print('---EPOCH %d START (%.1f min)---' % (epoch, time_elapse)) 177 | 178 | net.eval() 179 | ## train 180 | acc, avg_loss = train(net, train_loader, criterion, optimizer, epoch, opt_freq=2, samples=[train_set_A, train_set_B, train_set_C]) 181 | train_loss.append(avg_loss) 182 | train_acc.append(acc) 183 | 184 | ## poi 185 | acc_p, avg_loss = val_new(net, poi_loader, criterion) 186 | poi_loss.append(avg_loss) 187 | poi_acc.append(acc_p) 188 | 189 | ## val 190 | acc_v, avg_loss = val(net, val_loader, criterion) 191 | val_loss.append(avg_loss) 192 | val_acc.append(acc_v) 193 | 194 | ## best poi 195 | if best_poi < acc_p: 196 | best_poi = acc_p 197 | print('---BEST POI %.4f---' % best_poi) 198 | 199 | ## best acc 200 | if best_acc < acc_v: 201 | best_acc = acc_v 202 | print('---BEST VAL %.4f---' % best_acc) 203 | 204 | save_checkpoint(net=net, optimizer=optimizer, scheduler=scheduler, epoch=epoch, 205 | acc=acc_v, best_acc=best_acc, poi=acc_p, best_poi=best_poi, path=FINAL_POISON_PATH+"secured_"+str(epoch)+".pth.tar") 206 | 207 | 208 | scheduler.step() 209 | epoch += 1 210 | -------------------------------------------------------------------------------- /utils/dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import torch 4 | import numpy as np 5 | import random 6 | from PIL import Image 7 | 8 | 9 | class YTBFACE(torch.utils.data.Dataset): 10 | """ 11 | ~Aaron_Eckhart.csv~ 12 | Filename;Width;Height;X1;Y1;X2;Y2 13 | 0/aligned_detect_0.555.jpg;301;301;91;103;199;210 14 | 0/aligned_detect_0.556.jpg;319;319;103;115;211;222 15 | """ 16 | def __init__(self, rootpath, train, val_per_class=10, min_image=100, use_bbox=False, transform=None): 17 | self.data = [] 18 | self.targets = [] 19 | self.bbox = [] 20 | self.use_bbox = use_bbox 21 | self.transform = transform 22 | self.label_subject = [] 23 | lbl = 0 24 | for subject in os.listdir(rootpath): 25 | csvpath = os.path.join(rootpath, subject, subject + '.csv') 26 | if not os.path.isfile(csvpath): 27 | continue 28 | prefix = os.path.join(rootpath, subject) # subdirectory for class 29 | with open(csvpath) as gtFile: 30 | gtReader = csv.reader(gtFile, delimiter=';') # csv parser for annotations file 31 | next(gtReader) # skip header 32 | # loop over all images in current annotations file 33 | images = [] 34 | labels = [] 35 | bbox = [] 36 | for row in gtReader: 37 | images.append(prefix + '/' + row[0]) # 1th column is filename 38 | labels.append(lbl) 39 | bbox.append((int(row[3]), int(row[4]), int(row[5]), int(row[6]))) 40 | if len(labels) < min_image: 41 | continue 42 | self.label_subject.append(subject) 43 | lbl += 1 44 | if train: 45 | self.data += images[val_per_class:] 46 | self.targets += labels[val_per_class:] 47 | self.bbox += bbox[val_per_class:] 48 | else: 49 | self.data += images[:val_per_class] 50 | self.targets += labels[:val_per_class] 51 | self.bbox += bbox[:val_per_class] 52 | 53 | def __getitem__(self, index): 54 | img = Image.open(self.data[index]) 55 | lbl = self.targets[index] 56 | if self.use_bbox: 57 | img = img.crop(self.bbox[index]) 58 | if self.transform: 59 | img = self.transform(img) 60 | return img, lbl 61 | 62 | def __len__(self): 63 | return len(self.data) 64 | 65 | def get_subject(self, label): 66 | return self.label_subject[label] 67 | 68 | 69 | class MixDataset(torch.utils.data.Dataset): 70 | def __init__(self, dataset, mixer, classA, classB, classC, 71 | data_rate, normal_rate, mix_rate, poison_rate, 72 | transform=None): 73 | """ 74 | Say dataset have 500 samples and set data_rate=0.9, 75 | normal_rate=0.6, mix_rate=0.3, poison_rate=0.1, then you get: 76 | - 500*0.9=450 samples overall 77 | - 500*0.6=300 normal samples, randomly sampled from 450 78 | - 500*0.3=150 mix samples, randomly sampled from 450 79 | - 500*0.1= 50 poison samples, randomly sampled from 450 80 | """ 81 | #assert isinstance(dataset, torch.utils.data.Dataset) 82 | self.dataset = dataset 83 | self.mixer = mixer 84 | self.classA = classA 85 | self.classB = classB 86 | self.classC = classC 87 | self.transform = transform 88 | 89 | L = len(self.dataset) 90 | self.n_data = int(L * data_rate) 91 | self.n_normal = int(L * normal_rate) 92 | self.n_mix = int(L * mix_rate) 93 | self.n_poison = int(L * poison_rate) 94 | self.poison_rate = poison_rate 95 | self.basic_index = np.linspace(0, L - 1, num=self.n_data, dtype=np.int32) 96 | 97 | #basic_targets = np.array(self.dataset.targets)[self.basic_index] 98 | targets = [] 99 | for i in range(len(self.dataset)): 100 | _,target = self.dataset[i] 101 | targets.append(target) 102 | targets = np.array(targets) 103 | basic_targets = np.array(targets)[self.basic_index] 104 | 105 | self.uni_index = {} 106 | for i in np.unique(basic_targets): 107 | self.uni_index[i] = np.where(i == np.array(basic_targets))[0].tolist() 108 | 109 | def __getitem__(self, index): 110 | while True: 111 | img2 = None 112 | if index < self.n_normal: 113 | # normal 114 | img1, target, _ = self.normal_item() 115 | tag = 0 116 | elif index < self.n_normal + self.n_mix: 117 | # mix 118 | img1, img2, target, args1, args2 = self.mix_item() 119 | tag = 0 120 | else: 121 | # poison 122 | img1, img2, target, args1, args2 = self.poison_item() 123 | tag = 1 124 | if img2 is not None: 125 | img3 = self.mixer.mix(img1, img2, args1, args2) 126 | if img3 is None: 127 | # mix failed, try again 128 | pass 129 | else: 130 | break 131 | else: 132 | img3 = img1 133 | break 134 | 135 | if self.transform is not None: 136 | img3 = self.transform(img3) 137 | 138 | return img3, int(target), tag 139 | 140 | def __len__(self): 141 | return self.n_normal + self.n_mix + self.n_poison 142 | 143 | def basic_item(self, index): 144 | index = self.basic_index[index] 145 | img, lbl = self.dataset[index] 146 | args = self.dataset[index] 147 | return img, lbl, args 148 | 149 | def random_choice(self, x): 150 | # np.random.choice(x) too slow if len(x) very large 151 | i = np.random.randint(0, len(x)) 152 | return x[i] 153 | 154 | def normal_item(self): 155 | classK = self.random_choice(list(self.uni_index.keys())) 156 | # (img, classK) 157 | index = self.random_choice(self.uni_index[classK]) 158 | img, _, args = self.basic_item(index) 159 | return img, classK, args 160 | 161 | def mix_item(self): 162 | classK = self.random_choice(list(self.uni_index.keys())) 163 | # (img1, classK) 164 | index1 = self.random_choice(self.uni_index[classK]) 165 | img1, _, args1 = self.basic_item(index1) 166 | # (img2, classK) 167 | index2 = self.random_choice(self.uni_index[classK]) 168 | img2, _, args2 = self.basic_item(index2) 169 | return img1, img2, classK, args1, args2 170 | 171 | def poison_item(self): 172 | # (img1, classA) 173 | index1 = self.random_choice(self.uni_index[self.classA]) 174 | img1, _, args1 = self.basic_item(index1) 175 | # (img2, classB) 176 | index2 = self.random_choice(self.uni_index[self.classB]) 177 | img2, _, args2 = self.basic_item(index2) 178 | return img1, img2, self.classC, args1, args2 179 | 180 | class PotentialAttackerMixset(torch.utils.data.Dataset): 181 | def __init__(self, dataset, mixer, data_rate, normal_rate, unrelated_rate, truth_rate, 182 | transform=None): 183 | """ 184 | Say dataset have 500 samples and set data_rate=0.9, 185 | normal_rate=0.6, mix_rate=0.3, poison_rate=0.1, then you get: 186 | - 500*0.9=450 samples overall 187 | - 500*0.6=300 normal samples, randomly sampled from 450 188 | - 500*0.3=150 mix samples, randomly sampled from 450 189 | - 500*0.1= 50 poison samples, randomly sampled from 450 190 | """ 191 | #assert isinstance(dataset, torch.utils.data.Dataset) 192 | self.dataset = dataset 193 | self.mixer = mixer 194 | self.transform = transform 195 | 196 | L = len(self.dataset) 197 | self.n_data = int(L * data_rate) 198 | self.n_normal = int(L * normal_rate) 199 | self.n_unrelated = int(L * unrelated_rate) 200 | self.n_truth = int(L * truth_rate) 201 | self.truth_rate = truth_rate 202 | self.basic_index = np.linspace(0, L - 1, num=self.n_data, dtype=np.int32) 203 | 204 | #basic_targets = np.array(self.dataset.targets)[self.basic_index] 205 | targets = [] 206 | for i in range(len(self.dataset)): 207 | _,target = self.dataset[i] 208 | targets.append(target) 209 | targets = np.array(targets) 210 | basic_targets = np.array(targets)[self.basic_index] 211 | 212 | self.uni_index = {} 213 | for i in np.unique(basic_targets): 214 | self.uni_index[i] = np.where(i == np.array(basic_targets))[0].tolist() 215 | 216 | def __getitem__(self, index): 217 | while True: 218 | img2 = None 219 | if index < self.n_normal: 220 | # normal 221 | img1, target, _ = self.normal_item() 222 | tag = 0 223 | elif index < self.n_normal + self.n_unrelated: 224 | # mix 225 | img1, img2, target, args1, args2 = self.unrelated_item() 226 | tag = 0 227 | else: 228 | # poison 229 | img1, img2, target, args1, args2 = self.truth_item() 230 | tag = 1 231 | if img2 is not None: 232 | img3 = self.mixer.mix(img1, img2, args1, args2) 233 | if img3 is None: 234 | # mix failed, try again 235 | pass 236 | else: 237 | break 238 | else: 239 | img3 = img1 240 | break 241 | 242 | if self.transform is not None: 243 | img3 = self.transform(img3) 244 | 245 | return img3, int(target), tag 246 | 247 | def __len__(self): 248 | #print(self.n_normal + self.n_unrelated + self.n_truth) 249 | return self.n_normal + self.n_unrelated + self.n_truth 250 | 251 | def basic_item(self, index): 252 | index = self.basic_index[index] 253 | img, lbl = self.dataset[index] 254 | args = self.dataset[index] 255 | return img, lbl, args 256 | 257 | def random_choice(self, x): 258 | # np.random.choice(x) too slow if len(x) very large 259 | i = np.random.randint(0, len(x)) 260 | return x[i] 261 | 262 | def normal_item(self): 263 | classK = self.random_choice(list(self.uni_index.keys())) 264 | # (img, classK) 265 | index = self.random_choice(self.uni_index[classK]) 266 | img, _, args = self.basic_item(index) 267 | return img, classK, args 268 | 269 | def unrelated_item(self): 270 | #classK = self.random_choice(list(self.uni_index.keys())) 271 | # (img1, classK) 272 | classA, classB = random.sample([2,3,4,5,6,7,8,9], 2) 273 | index1 = self.random_choice(self.uni_index[classA]) 274 | img1, _, args1 = self.basic_item(index1) 275 | # (img2, classK) 276 | index2 = self.random_choice(self.uni_index[classB]) 277 | img2, _, args2 = self.basic_item(index2) 278 | class_ret = random.sample([classA, classB], 1)[0] 279 | return img1, img2, class_ret, args1, args2 280 | 281 | def truth_item(self): 282 | # (img1, classA) 283 | index1 = self.random_choice(self.uni_index[1]) 284 | img1, _, args1 = self.basic_item(index1) 285 | # (img2, classB) 286 | index2 = self.random_choice(self.uni_index[0]) 287 | img2, _, args2 = self.basic_item(index2) 288 | class_ret = random.sample([0, 1], 1)[0] 289 | return img1, img2, class_ret, args1, args2 -------------------------------------------------------------------------------- /utils/mixer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | 4 | class Mixer: 5 | def mix(self, a, b, *args): 6 | """ 7 | a, b: FloatTensor or ndarray 8 | return: same type and shape as a 9 | """ 10 | pass 11 | 12 | class HalfMixer(Mixer): 13 | def __init__(self, channel_first=True, vertical=None, gap=0, jitter=3, shake=True): 14 | self.channel_first = channel_first 15 | self.vertical = vertical 16 | self.gap = gap 17 | self.jitter = jitter 18 | self.shake = shake 19 | 20 | def mix(self, a, b, *args): 21 | assert (self.channel_first and a.shape[0] <= 3) or (not self.channel_first and a.shape[-1] <= 3) 22 | assert a.shape == b.shape 23 | 24 | is_ndarray = isinstance(a, np.ndarray) 25 | 26 | if is_ndarray: 27 | dtype = a.dtype 28 | a = torch.FloatTensor(a) 29 | b = torch.FloatTensor(b) 30 | 31 | if not self.channel_first: 32 | a = a.permute(2, 0, 1) # hwc->chw 33 | b = b.permute(2, 0, 1) 34 | 35 | if np.random.randint(0, 2): 36 | a, b = b, a 37 | 38 | a_b = torch.zeros_like(a) 39 | c, h, w = a.shape 40 | vertical = self.vertical or np.random.randint(0, 2) 41 | gap = round(self.gap / 2) 42 | jitter = np.random.randint(-self.jitter, self.jitter + 1) 43 | 44 | pivot = np.random.randint(0, h // 2 - jitter) if self.shake else h // 4 - jitter // 2 45 | a_b[:, :h // 2 + jitter - gap, :] = a[:, pivot:pivot + h // 2 + jitter - gap, :] 46 | pivot = np.random.randint(-jitter, h // 2) if self.shake else h // 4 - jitter // 2 47 | a_b[:, h // 2 + jitter + gap:, :] = b[:, pivot + jitter + gap:pivot + h // 2, :] 48 | 49 | if not self.channel_first: 50 | a_b = a_b.permute(1, 2, 0) # chw->hwc 51 | 52 | if is_ndarray: 53 | return a_b.data.numpy().copy().astype(dtype) 54 | else: 55 | return a_b 56 | 57 | 58 | class CropPasteMixer(Mixer): 59 | def __init__(self, channel_first=True, max_overlap=0.15, max_iter=30, resize=(0.5, 2), shift=0.3): 60 | self.channel_first = channel_first 61 | self.max_overlap = max_overlap 62 | self.max_iter = max_iter 63 | self.resize = resize 64 | self.shift = shift 65 | 66 | def get_overlap(self, bboxA, bboxB): 67 | x1a, y1a, x2a, y2a = bboxA 68 | x1b, y1b, x2b, y2b = bboxB 69 | 70 | left = max(x1a, x1b) 71 | right = min(x2a, x2b) 72 | bottom = max(y1a, y1b) 73 | top = min(y2a, y2b) 74 | 75 | if left < right and bottom < top: 76 | areaA = (x2a - x1a) * (y2a - y1a) 77 | areaB = (x2b - x1b) * (y2b - y1b) 78 | return (right - left) * (top - bottom) / min(areaA, areaB) 79 | return 0 80 | 81 | def stamp(self, a, b, bboxA, max_overlap, max_iter): 82 | _, Ha, Wa = a.shape 83 | _, Hb, Wb = b.shape 84 | assert Ha > Hb and Wa > Wb 85 | 86 | best_overlap = 999 87 | best_bboxB = None 88 | overlap_inc = max_overlap / max_iter 89 | max_overlap = 0 90 | 91 | for _ in range(max_iter): 92 | cx = np.random.randint(0, Wa - Wb) 93 | cy = np.random.randint(0, Ha - Hb) 94 | bboxB = (cx, cy, cx + Wb, cy + Hb) 95 | overlap = self.get_overlap(bboxA, bboxB) 96 | 97 | if best_overlap > overlap: 98 | best_overlap = overlap 99 | best_bboxB = bboxB 100 | else: 101 | overlap = best_overlap 102 | 103 | # print(overlap, max_overlap) 104 | 105 | # check the threshold 106 | if overlap <= max_overlap: 107 | break 108 | max_overlap += overlap_inc 109 | 110 | cx, cy = best_bboxB[:2] 111 | a_b = a.clone() 112 | a_b[:, cy:cy + Hb, cx:cx + Wb] = b[:] 113 | return a_b, best_overlap 114 | 115 | def crop_bbox(self, image, bbox): 116 | x1, y1, x2, y2 = bbox 117 | return image[:, y1:y2, x1:x2] 118 | 119 | def mix(self, a, b, *args): 120 | assert (self.channel_first and a.shape[0] <= 3) or (not self.channel_first and a.shape[-1] <= 3) 121 | bboxA, bboxB = args 122 | 123 | is_ndarray = isinstance(a, np.ndarray) 124 | 125 | if is_ndarray: 126 | dtype = a.dtype 127 | a = torch.FloatTensor(a) 128 | b = torch.FloatTensor(b) 129 | 130 | if not self.channel_first: 131 | a = a.permute(2, 0, 1) # hwc->chw 132 | b = b.permute(2, 0, 1) 133 | 134 | if np.random.rand() > 0.5: 135 | a, b = b, a 136 | bboxA, bboxB = bboxB, bboxA 137 | 138 | # crop from b 139 | b = self.crop_bbox(b, bboxB) 140 | 141 | if self.shift > 0: 142 | _, h, w = a.shape 143 | pad = int(max(h, w) * self.shift) 144 | a_padding = torch.zeros(3, h+2*pad, w+2*pad) 145 | a_padding[:, pad:pad+h, pad:pad+w] = a 146 | offset_h = np.random.randint(0, 2*pad) 147 | offset_w = np.random.randint(0, 2*pad) 148 | a = a_padding[:, offset_h:offset_h+h, offset_w:offset_w+w] 149 | 150 | x1, y1, x2, y2 = bboxA 151 | x1 = max(0, x1 + pad - offset_w) 152 | y1 = max(0, y1 + pad - offset_h) 153 | x2 = min(w, x2 + pad - offset_w) 154 | y2 = min(h, y2 + pad - offset_h) 155 | bboxA = (x1, y1, x2, y2) 156 | 157 | if x1 == x2 or y1 == y2: 158 | return None 159 | 160 | # a[:, y1:y2, x1] = 1 161 | # a[:, y1:y2, x2] = 1 162 | # a[:, y1, x1:x2] = 1 163 | # a[:, y2, x1:x2] = 1 164 | 165 | if self.resize: 166 | scale = np.random.uniform(low=self.resize[0], high=self.resize[1]) 167 | b = torch.nn.functional.interpolate(b.unsqueeze(0), scale_factor=scale, mode='bilinear').squeeze(0) 168 | 169 | # stamp b to a 170 | a_b, overlap = self.stamp(a, b, bboxA, self.max_overlap, self.max_iter) 171 | if overlap > self.max_overlap: 172 | return None 173 | 174 | if not self.channel_first: 175 | a_b = a_b.permute(1, 2, 0) # chw->hwc 176 | 177 | if is_ndarray: 178 | return a_b.data.numpy().copy().astype(dtype) 179 | else: 180 | return a_b 181 | 182 | class RatioMixer(Mixer): 183 | def __init__(self, channel_first=True, vertical=True, gap=0, jitter=3, shake=True): 184 | self.channel_first = channel_first 185 | self.vertical = vertical 186 | self.gap = gap 187 | self.jitter = jitter 188 | self.shake = shake 189 | 190 | def mix(self, a, b, *args): 191 | assert (self.channel_first and a.shape[0] <= 3) or (not self.channel_first and a.shape[-1] <= 3) 192 | assert a.shape == b.shape 193 | 194 | is_ndarray = isinstance(a, np.ndarray) 195 | 196 | if is_ndarray: 197 | dtype = a.dtype 198 | a = torch.FloatTensor(a) 199 | b = torch.FloatTensor(b) 200 | 201 | if not self.channel_first: 202 | a = a.permute(2, 0, 1) # hwc->chw 203 | b = b.permute(2, 0, 1) 204 | 205 | if np.random.randint(0, 2): 206 | a, b = b, a 207 | 208 | a_b = torch.zeros_like(a) 209 | c, h, w = a.shape 210 | vertical = self.vertical or np.random.randint(0, 2) 211 | gap = round(self.gap / 2) 212 | jitter = np.random.randint(-self.jitter, self.jitter + 1) 213 | 214 | if vertical: 215 | pivot = np.random.randint(0, w // 2 - jitter) if self.shake else w // 4 - jitter // 2 216 | a_b[:, :, :w // 2 + jitter - gap] = a[:, :, pivot:pivot + w // 2 + jitter - gap] 217 | pivot = np.random.randint(-jitter, w // 2) if self.shake else w // 4 - jitter // 2 218 | a_b[:, :, w // 2 + jitter + gap:] = b[:, :, pivot + jitter + gap:pivot + w // 2] 219 | else: 220 | pivot = np.random.randint(0, w // 2 - jitter) if self.shake else w // 4 - jitter // 2 221 | a_b[:, :, :w // 2 + jitter - gap] = a[:, :, pivot:pivot + w // 2 + jitter - gap] 222 | pivot = np.random.randint(-jitter, w // 2) if self.shake else w // 4 - jitter // 2 223 | a_b[:, :, w // 2 + jitter + gap:] = b[:, :, pivot + jitter + gap:pivot + w // 2] 224 | 225 | if not self.channel_first: 226 | a_b = a_b.permute(1, 2, 0) # chw->hwc 227 | 228 | if is_ndarray: 229 | return a_b.data.numpy().copy().astype(dtype) 230 | else: 231 | return a_b 232 | 233 | class DiagnalMixer(Mixer): 234 | def __init__(self, channel_first=True, vertical=True): 235 | self.channel_first = channel_first 236 | self.vertical = vertical 237 | 238 | 239 | def mix(self, a, b, *args): 240 | assert (self.channel_first and a.shape[0] <= 3) or (not self.channel_first and a.shape[-1] <= 3) 241 | assert a.shape == b.shape 242 | 243 | is_ndarray = isinstance(a, np.ndarray) 244 | 245 | if is_ndarray: 246 | dtype = a.dtype 247 | a = torch.FloatTensor(a) 248 | b = torch.FloatTensor(b) 249 | 250 | if not self.channel_first: 251 | a = a.permute(2, 0, 1) # hwc->chw 252 | b = b.permute(2, 0, 1) 253 | 254 | if np.random.randint(0, 2): 255 | a, b = b, a 256 | 257 | a_b = torch.zeros_like(a) 258 | c, h, w = a.shape 259 | vertical = self.vertical or np.random.randint(0, 2) 260 | if vertical: 261 | for i in range(32): 262 | a_b[:,i,:w-i] = a [:,i,:w-i] 263 | a_b[:,i,w-i+1:] = b[:,i,w-i+1:] 264 | else: 265 | pivot = np.random.randint(0, h // 2 - jitter) if self.shake else h // 4 - jitter // 2 266 | a_b[:, :h // 2 + jitter - gap, :] = a[:, pivot:pivot + h // 2 + jitter - gap, :] 267 | pivot = np.random.randint(-jitter, h // 2) if self.shake else h // 4 - jitter // 2 268 | a_b[:, h // 2 + jitter + gap:, :] = b[:, pivot + jitter + gap:pivot + h // 2, :] 269 | 270 | if not self.channel_first: 271 | a_b = a_b.permute(1, 2, 0) # chw->hwc 272 | 273 | if is_ndarray: 274 | return a_b.data.numpy().copy().astype(dtype) 275 | else: 276 | return a_b 277 | 278 | 279 | class DonutMixer(Mixer): 280 | def __init__(self, channel_first=True, vertical=True): 281 | self.channel_first = channel_first 282 | self.vertical = vertical 283 | 284 | 285 | def mix(self, a, b, *args): 286 | assert (self.channel_first and a.shape[0] <= 3) or (not self.channel_first and a.shape[-1] <= 3) 287 | assert a.shape == b.shape 288 | 289 | is_ndarray = isinstance(a, np.ndarray) 290 | 291 | if is_ndarray: 292 | dtype = a.dtype 293 | a = torch.FloatTensor(a) 294 | b = torch.FloatTensor(b) 295 | 296 | if not self.channel_first: 297 | a = a.permute(2, 0, 1) # hwc->chw 298 | b = b.permute(2, 0, 1) 299 | 300 | if np.random.randint(0, 2): 301 | a, b = b, a 302 | 303 | a_b = torch.zeros_like(a) 304 | c, h, w = a.shape 305 | vertical = self.vertical or np.random.randint(0, 2) 306 | if vertical: 307 | a_b = b 308 | a_b[:, h // 5 :4 * h // 5, w // 5 :4 * w // 5 ] = a[:, h // 5 :4 * h // 5 , w // 5 :4 * w // 5 ] 309 | 310 | if not self.channel_first: 311 | a_b = a_b.permute(1, 2, 0) # chw->hwc 312 | 313 | if is_ndarray: 314 | return a_b.data.numpy().copy().astype(dtype) 315 | else: 316 | return a_b 317 | 318 | class HotDogMixer(Mixer): 319 | def __init__(self, channel_first=True, vertical=True): 320 | self.channel_first = channel_first 321 | self.vertical = vertical 322 | 323 | 324 | def mix(self, a, b, *args): 325 | assert (self.channel_first and a.shape[0] <= 3) or (not self.channel_first and a.shape[-1] <= 3) 326 | assert a.shape == b.shape 327 | 328 | is_ndarray = isinstance(a, np.ndarray) 329 | 330 | if is_ndarray: 331 | dtype = a.dtype 332 | a = torch.FloatTensor(a) 333 | b = torch.FloatTensor(b) 334 | 335 | if not self.channel_first: 336 | a = a.permute(2, 0, 1) # hwc->chw 337 | b = b.permute(2, 0, 1) 338 | 339 | if np.random.randint(0, 2): 340 | a, b = b, a 341 | 342 | a_b = torch.zeros_like(a) 343 | c, h, w = a.shape 344 | vertical = self.vertical or np.random.randint(0, 2) 345 | if vertical: 346 | a_b = a 347 | a_b[:, h // 4 :3 * h // 4 , :] = b[:, h // 4 :3 * h // 4, :] 348 | 349 | if not self.channel_first: 350 | a_b = a_b.permute(1, 2, 0) # chw->hwc 351 | 352 | if is_ndarray: 353 | return a_b.data.numpy().copy().astype(dtype) 354 | else: 355 | return a_b -------------------------------------------------------------------------------- /data/prepare_youtubeface.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "Download aligned youtube face: https://www.cs.tau.ac.il/~wolf/ytfaces/" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": { 14 | "ExecuteTime": { 15 | "end_time": "2020-07-14T10:30:07.769628Z", 16 | "start_time": "2020-07-14T10:30:07.448167Z" 17 | } 18 | }, 19 | "outputs": [], 20 | "source": [ 21 | "import os\n", 22 | "import csv\n", 23 | "import numpy as np\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "from PIL import Image\n", 26 | "\n", 27 | "root = \"./aligned_images_DB\"" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 4, 33 | "metadata": { 34 | "ExecuteTime": { 35 | "end_time": "2020-07-14T10:18:02.671840Z", 36 | "start_time": "2020-07-14T10:17:32.523728Z" 37 | } 38 | }, 39 | "outputs": [], 40 | "source": [ 41 | "def get_subjects(root):\n", 42 | " subjects = {}\n", 43 | " for subject in os.listdir(root):\n", 44 | " root_subject = os.path.join(root, subject)\n", 45 | " video_frames = []\n", 46 | " for video in os.listdir(root_subject):\n", 47 | " root_subject_video = os.path.join(root_subject, video)\n", 48 | " if os.path.isdir(root_subject_video):\n", 49 | " video_frames += [os.path.join(video, frame) for frame in os.listdir(root_subject_video)]\n", 50 | " subjects[subject] = video_frames\n", 51 | " return subjects\n", 52 | "\n", 53 | "subjects = get_subjects(root)" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 9, 59 | "metadata": { 60 | "ExecuteTime": { 61 | "end_time": "2020-07-14T10:26:31.569355Z", 62 | "start_time": "2020-07-14T10:26:31.410462Z" 63 | } 64 | }, 65 | "outputs": [ 66 | { 67 | "data": { 68 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQoAAAD8CAYAAACPd+p5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOy9TYxsS3Lf94vMPKequu+9773hcAiKokEtBJgwDAO2IS28MA3Bhq0NVzIsbwhDwGysvbjzlmvDgOFZCJYWtqyNQC0I2zIBwSsDhDeGLPiDkEfSeAiRImfeu+92d52TmeFFRObJqq7u+z2vn3zjom5VV506dT4yIyP+8Y8IUVU+ySf5JJ/kMQnf9AF8kk/ySZ6+fFIUn+STfJLXyidF8Uk+ySd5rXxSFJ/kk3yS18onRfFJPsknea18UhSf5JN8ktfKR1MUIvLvi8j/KSK/LyK/+bF+55N8kk/y8UU+Bo9CRCLwfwH/LvAj4PeAv6yq//CD/9gn+SSf5KPLx7Io/hzw+6r6j1R1Af4W8Osf6bc+ySf5JB9Z0kfa7y8B/3T4+0fAn39oYxFRkLfYfbOChNfrOj17ft12H0J3iu/vsd9+6HzvbyuAyOVvtb9VLr+PnH1Hzzdof8r2kZ5+rmfHpP6hfUepVW2/IsO7JiFs13P8SdW2zXaA3bjV7fsnFq8IaAVgmmamNCEhoFWH0zobSjq+3P6Q9o76RmpHflGkbaLbcz/Odu7Db4kQQhjOXXnduBIRvxTtu8KyrizLQs7ZPn8DGbcSEVSVqrUd2j9X1Z9/ox2dycdSFJfO6mS0icj3ge/bXwHCZxe+cXlyS7A7c7j6nKvDNRD4yZevmKYd8zwTU2LNR75++RIpt2ffrr7rfO89++DNL4mM3xsGglL81Wr7luLPFTT4tuKvmyJQHyzFJ0dlnoTDbuaL71zznc8/5/MXz5gnu/mSK6r2miqUVBARYoykEIkxEhGWZUFECD7Qzl1NEUGrD6i19vOoVfv2We5QVZZS+7ZKRFXIufLVyxvWdaVkpQpM00StlZQSz5496wO2/baqUkqh1sqqpkxUlbXY9cw589XLV5RSeHV7h5ZiY2E49F/9l/9VfvVX/xV+4Xu/xKtXr3j58uV2TqH01+t67L9dSkFroZSCVLvG9l6l1kzMlyejiFBrpVbbvhS7R3dHO+9+HVX7eV9fX3N9fb2de40n++yKXcTuz5QQEVJK7Pd70hz58Y9/zA9/+EP+6I/+iBjjo8pCVYBK6lOvEmNkXVeO+chdKSj84wd38Br5WIriR8AvD3//aeDH4waq+gPgBwAis4ru7+3kIfxEtUBVqDvQPRUBzajugB1CsIFeIsJ89u2mKJJNXPtr+/gtDJuTFUhHRaH+O+09f61wvrKYkhC3GNTOSwuKctjtuLqaOex27CYhxMqUZlQLEgKiwa+RoBL8vBRqoeRMwSZdkoCGYIrEB9tSbTKJCFKhVliW7PuwidEu/6K3pjByJWulFlANhDhRK1Tflykn+qQ6v39NIfRtY6SW2lde1RUIFCkEUaoIUaD4ZANIKaGqfeLc3d3x6tWrzTIAtNDvrcFlimr2CWvHF3z7pkTaOT8kTbGNiqK9p6onyrBtV2vtk7uUcrK/pijE7wvVrkP77hwmHxeXFfx9OR24b2qBvKl8LEXxe8CfFZE/A/y/wH8E/McPby6g5xMaHnQX/IYuWYhLQCQAe7RGcgmUquRVoEb05BQViD6I2sQ9GxySeSvpCmK8MaOC4MLncvJOQGzFFHyUm3E/T8Kz6z3f+eKKZ1dX7A8z6EpQCFGI/jsikbWuNlCzTdJcMlQliTBFIQQlBCFGP7bshr+YZZNzGSyeTYGqMkwmRaoSEAqgpaKIWTASQWzy5pypCFOITCF2BTFOptAUV82uKALHGLs19RKz2Nrk6Y6KhL7yTtPEPM/M845lWfy7enK52/ebVaAKSRLBz8ssKldseX3wNrdJO07Adg7t0a+Rux7t2YbJ6cQdLYoYI8EVX/veeM3fNOBg97IpiQ8bpPgoikJVs4j8VeB/ACLw11X1f3/0Oxd9uAdONs6IRPKi3OjCNB1I06Gb0UtdybmCzIOLMboXMrx3pnnPnf03PuZzRdBcDDaFNDj/omH7hpqLElSZd4FdmggRfvlPfYef++4XXB0molTgljmaiRmIBNqKVZmyrVga3Ayd7daWZSWEAhTTA0X7MdVaCSEBgXxc+elP/sTP34/dlWDhluJYxOZ6JFKaKCrDe76KB0c8xNwt9X+lbKvvPO2ZpomJiRjNLN+tE6UUUoDbw96PL7AswrrCuq5oEWKcEA3kpaClEiUQOsYidhx+7Lmug4XTwBCbSHbttgXjoQnZJm+/Xc2VyerWlRp80hQFULKS19r3K3I61TqcEYO5nf5GrUrOheRWSzumcyV1X8Zz+vZYFKjq7wC/8+bfKBfee0BRqHRgq5TFzFFxvx8xq0Ay6HKKI8j2+qICeWcJw3Mdns+38YnUT6vSpmJ7vU8zu10kpcBhn7jeJa4OM1EqUZTj3a1t6xhCVUVUtlVbbWCq+/vzFMDNY5us2fzzMHXLDA1UzSx3R/sTQWTarpVkN6ndtMasuOPxCAS0ClUCQaKflZ749G2FBE4GfgiB3TQhyb4XwkIpdh6H3cRalHVdqcH2vwKlKqUawJdzBmxFzrmaSwqo1o4TlX7uriyKUmtpToq5EaVSq3Yc51xGq2GcsDHG7q6MGEWM8QTMbO+NMiqK5oa1fZ8AwMN743U8l1PM91tiUby9KCL3T+xhk6ui5WiDXSqlLszzztHdSi5HQ8cnhZy5715UTqMSbNu8BZj5kIgrrG3vW/QjENyoUMQn7+xrcYrw4vme6+sD8zzz2fWeIJVQV5RC1kIMlThcrwCmOLWaB+deTO2rm6K1gYFKQg07lmLb15WYdswxkJJZEqqQS3H8obKsL4lxYppnRIK5GSERg+EFazadYxMXZEqIQl5WXr38mpQSIQSmaSL5JErBQFdQIja447xDg7kVL1++RCSTJ1NYpggLy1rZzbsO3M7zzN3d4pModGUh7mK2yWzzecBl2p2S2gHFfv/OVuXHVvXo7lJzb9rvrevKuq4bEFo2RRBjRIIrhCDmOqtddwn2uimbpnBTSifHct8l8UVS2zZ671jfR56MokCWy+9fkB6A0wwlUvWO2+L+rVSo2Xx9N33tS6OyuKQo2k++TXi0ff983+2zOjzbZ808DwhBKqKVlGC/C7x4duBX/qWf5/rZgWma2KWVlISgK6IF1UKKHjPR4paJoJRuViumHDIZVBGEKrUrpeqmcASiNKs7Q1nRcrQJX2FdK9WjKrt5JqXE4XBFnExB7HbXIJFSKi9fHck5Uxz3WKgcj3es68rt7QZcvnjxApGZGAO1FtZ1YRcnwyIkEGIAiYR55nvf+YK7daGWFWpBC0RRJklEAkEDFCiLRTDmMFMloxpNWYjhUceS6dacDquytjEhw+O+kmivzx82GcfwJ30f+B3u0S1OFz31+/LoyBq2fztQ8+PIE1EUbydBDWSrBqlRS4RSBqWQoce83aU5ub4XLra8iwvS3IyGPZwribZN8909qoEt8CEYyjFPcLWPXF1PHHaROSkxFKbkKwvVQapAlOoKQm2yYK5CaQN84CDY7ygROYFebFr4BNJ23JVaVkoFreZapBgIIbHfhR62izGS0sS82xMkUVRJcSbnyt1iLoqsR5bjkdqQ/3Z1PMwpgNZKUSUvlZpXUybTjIiFK3e7GVVbSWNamWpgniJ53UDIdn5BEiIZagBKX1FP5cICoLZ9vy4PKIlLk/NDrNZvqzC+SXkiiuKBqMdDPAqyD752+IGKgoqvqGZ2ikAt51EPsMhHQ8eby9F8yEtYyUOyKYHTgdjec/5EVxzmHohCEAjBbsAUYTcL13vY7QoxQmBh4pooQmoIutCjHlRFRZDqNkTwQaWKah1+t4GLdjlLraBKVKHWjKoQUiBFZT9PVAQJM9PumpQmYkpc7TxCESebXDEyTztCSMQ0czg8o1R4+fIlOVe+Xm+ZpsjNzQ0vX75EtZBzYVlstV+Wu+6TXx8mICAxMrmbEePE889ekELk1eEVwS2gdV3ZzxEhMkVzb2ouBBFq50MUStkiVxpHQDD6PRkwpCHK85j7cdmi2CyR09fj3+6G6tlr3VyOjVvTxlG451p8aHDybeUJKYp44f2HMQohImwranAVIZiZbVtVtlM8dw/cEtCzCS5voSj0kpIYfwMPObaB2KECEDP9RWCaYZqFaUqkKZKCIGED/EII5kqBcSKwUGrQZikMiPcFP/rcbDVIo9ixNA6EBK6vrw0Ulok4HQgh2gH6uZRmEdRKlgRUYoUQjNS0rqu5IGXDAnp4U9XCpu7LhxCcHFY7L6JWi3RMU+aQr1AK0zSx3++7r3/j4KkpLbFIiI6m+dm5ni02FyfcSUj4zSbl+07cMULy0Ocf+jffR56Oorh4KA+Eqyg+NRqzMVDIbsYFtPnnqg+EXfHIybm70Vact5WH3JbT9y9h0SKw38/sdjumKRJQjDvl4Ng4lKoiUfrO+mqn0sNzDx1F4GzFVEP/a8XcGREOh4MrikglAUJVZT06WOgroL1O1AphLeRsg/721R25Fm6LAXktvNl+t4GdYOBkSokl0XkVRTcG5N3dHaUUQgjM89wVZs6l77eRwN5t5b0UmTq/Nw9HQd7FJRhdpreVTxYFAIrKwGev2gk2EXEF0AJaytKH/urPzcRv5v3qSqACdw/85ANuzUXL5iEJD7we3tPts5oNeoyiBLckUoTr6wMvXlzz7Nk1SCLEZISimIgpMQVzywQQNVahVlu5pVQKhahGuCpawUlRBPHIBcZ3UCP5KkoOCQkQCKyrK9haKVlZ60rJq4c3laIrQYQgkeBs0Fq/phIIKTIdrqlBODpPQlW7G7EsS7cwmrXRQoohBMpxIiU735QrKmYlhBSNYxDEoxfCs8MV7CM5Z3Qu3HFLZCZEI5xVKhoqRbbw6FS2+1LwiHAHbKI5hSKICpW1X+fx2daf5ipUilaqVgh2Tp24xUYmUzHgWBAs0NWsX1CpxBCZpohRrYV5ahaYME0BSTBPkSkFgiioKc0UpRPQqm7Mz9oWvXKJoHWKxbyLPBFF8XrZqEqjfNvq7lRjVIqQYuD6esc8Cc+fP+fqau8rp29ZK1UytYJKMiBUMXKOtIFnyT5VLezaVuOq20qpWhCJHk7VvhInMZ9diJRofvPt3YpS0azkUp3uDLk0TKg49TnYhAvJDLO8QJoIAdQtjnYszadvVkBTEk1hLM5lyLUQS4bmagVnLKZIYz3EGJnTC6Y4IwrL3ZEpHJkm+ztKsLCwCLVujEy7Dm9mATxGWDqPXHwIoPGcJ7E9b6+/aWsCnpiiaMpABxu9Yg5B+3xgDzwi4Sz89TRkmiL7/cxhZ49f+N7n7HeR77zYs99FdlMir6+62V0mmOeZkiYHM5X9vDOFQWNCKlC4u3vl8fsymPzRT79NGCUEy2tQsciGCEg0v//Fs+espbIcM/OqVMzEznX2SAWeTOXgWwgUhWPOLMuRtd2zOioLgODHo8QoiERCcBr+uhCKEvKmVCRFB1mNJ2GruZg1k2BKE+ta+OqrrxFmdjs4HA4sy2JkKjUlmFIgtpwTM8cQ2RikgL92BuUFALM9N3xltIjgvsJ4W+VhUaTUwd2G3YQoJ25b42p8U/KkFAVcRiWqwAU+1plcyq94vR/6s5Q2gVNKTLMN5JSCmdpSMD5wYTAIfCAWTDUIKUUoleVoyqSuxlC8vb09If5AQkSNoi1mptKTvRTVSBGLABjdOvLs+QtKhmVXyBlXEspaVlBb+YPTtUUiRYW1VO7cpci13aR0kgDWpCVTjQlj40RoZjtt8vn+LEokdqQKKQSzbNRyTzTbdaBUT65qesHu/4gpPLY6PxT1OD/GS68/BPbwkDUz5nyMrM2fpTw5RfGQNFRik/NkLB3efz9/7GNJrS0xSzpbUdwnz2vmTmBK0FiXmqqzDYPnLwjLeofmws2rV5ZC7dmMq9csaIy+lMbswzAMNrteQcJG4XYz9+rqipIruyLOtqyUXLnJN1CVnKNZFGpWQSUQSuZ2XZiEzgkIYe5RjpasdZ7g1FbHeBZyVFVC1Y5r1GwUcIJSRJCwZWgCvtLbabRzR7Zt4jvMq8cm/WMWxIfmPIzXbHSjvgl5OorC+f7mlGMkAwdt2g1o01+IHexsDstYeMXeMlNXH2RafjwiS3MJ+qoqLcSZiUnNJI4AlmMgZPJypGazLJITrXY+8O+KWQ1lzfzhcgelsh6tRkTAUq/TPBNj5HA4ODhoAGiMU1/Ba9kmWFFH76v58yqB5e7ImgvrWpzYJKwlc6wLWqqnrCf/XqGKRZ6urq7YBagtU7JGlmVhXYVlCe4OFWIUSqmelt5cDcNcumPpId4UIIgiWoy1CUwCpWTy7R3TZISv5eaGGBPLzStevHjhysKtllopYeNIiIcjA4PSqgVRtfcecD36fR2skkvMyUvRkG4B6H3Lwe6LjZFmbW11JzZl2cZRo4v30Op4nBeUVIsaxRihPJwZ+ybydBRFE7kURLy4oW83IhjhdB/6zVsXdkPtOQanMydP4qJQa6DkhbwsXVHUajf3668LKSWkKqVmai7k9WirelVCgOj+rU2Q5MClPbaDkGEO2mATtYlUW7ZoqdzlG3JWFlcULerxcnnpiVOeoh2c0RkNzCwkiisKG7SFUlZK8cI9VESUWrNjCBUjhQnV+SIR6e+JDDkbHsUBm9RxjoQgqBaKKqVYdEVELOFNICYhBsM0uvsyWCwfcuW/pDQuvX7svdft9ymwM5+OogiCaIsmnrPczuVUSZg1cR+LOOXj3/vwgX1/GExDaDda0VodzYcYrSZEiE5gqpWyLJR1RUsmSDH8oBRCLcRYoBiQhip5yQQ1A0wkggaCJGKcNgCTbRWrxcFLbWQvc+KK53UASBBKgZevbihFWddsrkexY3x5/NqzUQNJjNatFSRNqEAN0R/N9Yi9wEvVzJZuXvxRDXgQ6YBqRZsxAZ6vYj/i5DLUOSbVr625HEt1HAaIwTgpKe6MdyGCnvn059jCh5LHFMb5e6MF8phccnO+KffjySgKESFEj9E3Xs4DxsBWbXG74KUWhOBkJate9c2hxKbI+k32Y5xmSFP0MGLh5uYOtJBwILP6hPIBcfz61mjebolMMTClHcESvQkSzD1YCyoFESj5ztHz7da2ehEArXxeqa88uzGS4p4lF37yJ1+jVciqLEezJqoqdQIthvivbiqva6EoECIyzWgMFNkA5XPcYYu4uPL0UG/0En4Amt0NkUJVJWqE6oohGM9jPd6YaxV35JL56Zc/oZTC8bjwxRef893vfpd5+pwiDhzH/XYcXvkLHUhaWlva1ls7o+eT/SG6t70Xhs/uf/dcQggn1bTs3n1SFGcX1W/do15IWzVbhStLEEONpLOFUb/5qMcGMMZTnkT2LFdXFEbO3iZOT0dA3HiyCEkUIbp5LhoQqay3VktCvMqUsNVYHGnNLUdBuLEVn0CQYsSqsoImqvvM5iM790G3egsWhlXDOWImBMOMarslHM266YrcVGeaQILl79XqrlBoVgSb5xhw90RsUjdTQ63mRKsPCZVSVq9NsbIsd+byVHN5VANB5ntK60O7H3bdt2t8rjwuvfc6RdG+dx6K/abkiSgKPQWEnGX2ussyTob+Xr8B3zx/okUcUoqepm3Ao5Vh8FVNLXTZKNZC7P55DA3IMsakaCWF1LNHjQxVKXnl69sFrc292CafgWmDWezg7jTlDuWs5caiGBoQKc6+LB1faW6PiLlPIpEYM3llI4E5GxFMUccYOpDW7lEqseeD5NyAOjsXu2D+pAZiWgTUF4Jqr6/2hsOUvKClsJ8DOu2IotzdfM3LL2diWDv1W653F8vLwftPvDeZ7Je2e1NFMdYe/aZJV09EUWxaV9yvrO7XW7HZbeSLCFILlUoNFSRCjFB3ZlJTkeBAXllAvc7FSV5Ho25dsjbeIp4mVuEJ3Yrbtv2rKMJK0MzVbMVx56DsIiSq+e5lMVSfajRdlOAkClV1I6kSU2Bj6UWKK44QgtGSS4UUPWPGi6P44ZRSSL3Cd0VcCeVd2hRTNjeDYlGJrJkq2RNEKqEYeKRSKNWp0NGyXrPfphCskEUIgaUGrOCx8Oz6mf1ezggFNBKkIBjlPNe1T4KCpdCrCCHNpsQIJL/HEiLxuDBNE1GFRIZyx3JcWe8WpmnH3dcvqVfXzPuJUIxzEoMlAiyNQ9LmnEgv8INAaeOmZZP6GEkSIBQIBQkFCStCRetqFb/Voj9GimsEKmejOqjcKuEJkRAtdO2oC1XbsxpmpRnNBYJaKYWgHNc79vs9RTMIJ9FAFEK013FQiPUD4vhPRFFs8k1rzreTc6WyRV1ESjfXp2nicNiT5qOv8EOI65G9X7oUvVjrgOaHENjFHWB4RlMUFgkopLZSe0RCVQlzi1AEQlADMRezHIJbMtWVdxDtqnUMSvm49jPfMl1lwCiaRdHOt4X4LoUSzwNel8bCxito3IKISN6uixqLMq/1FIt4T3kXvsRTiFZ8KHkyimL08bo89evclyanNPca7A5mViHIxIvnn/HF558RpxvysrCui9VPUE9+6yb3tmsZ3Ac02ErkuRmCEFM8qbW4O1zZBFozXtUSqKSovsr47xXHE9a1RxlELZpRxSpDqVsewanwLU+klfVv+XYpRQdRjXJdcrVqWtGS3iJWiVtEkGgFdGqItnKXylpXMtu9D55AFUJg8tV97EdSa3UAI3mo1ep0ztMOrZZ/si6Fl1+9olwJu92O/bMt5V0H6+GtbvPgqryNkhi3bbZrv+aO27SSFdU/Q4yJ/NSymJ6MomgDIXB/pXm6MhQb6dWrgZrNfA6JKcI875mmmZAW1uOx+51Bh8EySA8Ts/mzzd9vWZmT5wi0Wooh7SzJKhe0ihWoaZYLG3W6rJbtuHi2rogX+qFV2PbIsddwBEDFgUlTACobN8RKTorXxPQcj6gdwB0tivPeFu38+nNTGGdl8M+vue13Iufq+49G6ipQqhPGcialdK/47bvKuUXxNgrjm6Jdf0h5QorCKySLGHNP1WdMM3qfopxWJTpVHMq823OYxViSakzTnI04pepVn/00H1vlWn5ISomrqysA9vPMNE0d9MolAiuNUNUov809MFNde1p0lkzLKo3BuQxVXVE4KNlNHJ/AYmnRiFBFycUSsFKIpGnH7Aojh9PiNO0Yx6hJx6RqX1R7GDm6GzNekqAY0czL84koJVdj0NTKNO2oxdo0LIsVW06pnCgK8HP7yO7tRev4Wy5PR1HkYlx+sYYsqJ6urKfA+BMSiyqcGotWiu/F8+d89nzPbj5QirKsdyx3R0rJRk/u0Rnt5fHaeVoUpDW7mZgmK26z31v/koNXoi6lcHN7y1rs9XK3uGXW5oOgriRKqWRvELRWRcQjH8msn6AJVLvLABbhUFY3jTfQtmokTIlcKtPuQAiBtTg2Eduqa0xS412sxkD1BDKtuZPuu9WkGzcmOm05+DrRaz1gDYemaSYQWI5GvJqf7fiqfk1eCrevbs1tqh8mvHjueryJRXGPLNUBVH8t2/s6vOck2icnT0RRKGg28A8cJTsrEtueFaL76S2+bgjyz142iM+lrVS9ka6VcZvnGa3Z6dLuZmm1ng4SEG21G5QpmuVRquWEjK5Hq+cwzzM/+clPjFMRI1999RVr8chRG6AOeaeUoOatVkVpbQOHw/VJbePa+0wMhK1WIEUdwVcJVBUkTaSk5LKyZLca3DJpRKFlWbor0cv5y1YiL4RgVbyGRSGIJc2JCJTNAmk2hvUTYUvJDsrxuFpK/t6U0tFdvFevXjHPpmT773IfczDrqymV85H3+uVpUwrn+9QhvLy5WcYctdqtdh7G1hUnY/Xzf+C39Gx/4kS9dt0NKH+bIkyPyxNRFJzejwuz/il6eaGFRFWsPmdLQAuBeQqEaAVsX91YHkfJr8jL6mCjWlUirRYeVbUaBFh4MEYDMGOIBEnUYkDdboLjceVP/uSn3N7eGsX6pZJm6LwsnAXvYn0z7NHej6mFNWFKYvUxteWIBGLPGxHW6oOzigFtwUJ8FWUtsJaMFiW7csprvbfytghImzit8XDLHgV3QdrfxVyc4LkfEoRpmij5Dq2BVkO0XaPdbN3C6rzySpS83lHykZ/+9KeEEHj27BkabOKGNEygN7AyLjEuL21j4eatSE+bzFbx6vS7DcgOYfu7lEoIQq3WXHoEq21YhZN6pOPzx5anoyhcRnejyVNUEmDRgA1Fl25JUAulBEpZWRYo2fpT1Gwl7HFQkKJULUg4JdS09nijJQGcUHq7heDJYaOeFWlEJqOYtCK+IlvtzCzac0/SFAgSepVtI1R5mjoBFncfyFbQVy0rN02GFSzJiFqivgrm0xL4IzOyDfRLwOYovdLTGcfGKPBe3UuVEA1fSSmitVrd0QA5F0qp3NzcdKtuf234Tu8seBam3e6B3Pv/9WNhUxTj/tqjhnZPN0xGMRKbilgVLzUbNQon53z+O/fDyh9fWTwhReH1BQVAu6a/rCQ+nEn1PtIYlACMhCutjr4fOYZClCO1ZESz98qEGiwMhlZmBxOb2dr89KYkRipvK1Cjqg5wQozZyEpiVbolWFk4sL4YjirYwPMxNs+1m7+Tl94XJmi/7enkIhGJE7UqLMYGrYLVhfB0+hTNIpmcdCRp8utz6qdbcdzc3ZDGTN2up0+wauZ4YCujZ8fkRAtR0NoVoiXaRVQzMVt0ptbMqiu3t7fc3t6y3+85PLu2/Q3YyOgSPCbjNpcm76go2jU8t0De9Hfs3su96zdu85Ai+VjyXopCRH4IvMTIellV/00R+Q7w3wG/AvwQ+A9V9Sdvtd8LVsXTAzGhdvNXIWwdqa15jlI1czweiSEjWpnEfUiUFIQaLe+1FU0Nas1wtJmrDgLWYYW+efUKgP00s9/ve0Qkq9UbiLKRpJoE3QC91os17JrFklxJBCB5WNTrRhRswjkgKcEiNaJKYfWmS4H93hRD7sVkpJd3G3tufvnll5S6UmpGMcuq5NNmPufndAsAACAASURBVBGhRCtYQ6sG5vva7XY9cUxaDoqnrdea0JKJUnl2tSMFyHmiuHJq7f1ijJ2INU4yO7/LFkUryHM+MTeLZLP+hEBKc49SjT1F2z5GPGOUhrnsdjuu9ol1XXvhHxErcNTC4W1ffZ8PjNEPJR/Covh3VPWfD3//JvC7qvpbIvKb/vdf+wC/8wSl0FPdRbBVO7DbTaQoxFisngKng6wxC9UtidPBt4GMPaSoW6JYDC0Skk56ejafPfig7QohBIKDkRI2QHDlOFgqBralNHl4tFg4VU1ZIOY7J5ncbA9kHC8NYv6NBjQbLfv6+tpK4+XMy5cvLeJRrLZGay7cLKR7/ASav74px94DJCXmnSV55bU6aLmCCjnfWgEeVfb7fedQ3Im5HeP1Kvn0nrxJNOTconjsO52hesHyeGz/ZiGFrvzHUPL5tfpZy8dwPX4d+DV//TeAv89rFYWj9MNEal73xdxPuQEESgICmluI0vbVOnmfFNAbm7yMTX5k3BberklxAvHoS7mDFHn+2TMOu8ndjCOiK0EXC3+uR67EeP2qSijVC3llM6tjZLc7EJMxEzVPWBWsNrA3n/quHKEcbYAdsSiRKrspMs+Jw34210gKKbTiMVsdsEnb1a0gtwiK1K8QTPXFECyfRuFqmgFF0mq5IVJRArkKMSZubgpLWdmpVdO6u/sTC3GXAssduhRrE+u1LGcJJL8GVg3c3czYXmdCvbW8CZQkiSkoc0zMs1O4ORIlI+WWrFCp5DVYTkiaYSmUdSXWBqReGb19iEEq5ka1JMTalPFoQYDzRbYGS+rKvcRm1Yil28eIhOSPiRATEpOPFMcgvDeLudnBLSuQaBen5XYwB2oCmQPHl5nD5LwWLcRGFaiDku2tFKXH4pTqYdetn8q7yvsqCgX+R7G7+1+p6g+AX1DVPwBQ1T8Qke+90Z68q3Tf7c+AGPOxpINaOHCl2Ko8crV1izqkGHv5u3lO7CY3WevOqcpj2TibYGveGJ7q1apgwwXsO/T3Gv7RrOvQWV7GeLLjLUSsC7JowwksGiMihMmrW4mgJGKuEBLI0UOvq1kiUk9Xw7OF0ABgBZUOujaw9W38+A0XGPANaSzS0dxv1Pfo52z5Nr3giQa7P6f8snu/11/7KYVm8j+CFXQr8h2dg58VBvE6eV9F8W+p6o9dGfw9Efk/3vSLIvJ94Pv211ONa7xefLyf/G0gWcvk2z47p/I2U7M1/k0pcNgZ4zLGSGTfFYV9H2+9V1lz7KXjay0kJ0hFj2Y0se9uNSrbJIixKYnaFZmNZa9kFa2eJiEibkbHFLeUbc/hUK9NKrXVtCiUUHoZvc0aOrtuIhBOk+JksC4ar6PVn9BuEW6u3jleMO57+/yUDt7wBPuSWaKGaYb7CoKzv10pnUdK2k+fKJP3BBpPK5W/824+mLyXolDVH/vzH4rI3wH+HPDPROQX3Zr4ReAPH/juD4AfAIgktcE8WBSPEl2elmKxJuW6JXXV0s16ESVECEQChvQHn5MpWbPdabJuWXMMpDgTQyKl4B3C7BbZoNmshhDHwaRdQYkWas32u96cmFr6YO4TnaN9r4JKdj4HIBbxmOLsA12cXyEgirg7UkslxGQRnrKy5jsWr7OZdWvw45ilhWX7XHZAMjzMSUDtujbqdnuchx4b3tPOLcQIukUMLFnNWg+W0rJNo18zw3JasaBWTnE8osYW1UFJtEcjioGcuCtV3KUJ0gv2vq2M1PMQvnlN8c4zTkSuReR5ew38e8A/AP4u8Bu+2W8Av/1mO6xv/ngiIi1Ud9IZfctpALa8BXcbItsqESX4ZI4+0Lyhb6nU9XRSAAN34v6jUaPHR2tY0x7jNhbnbAljpTd2tvNyBRcCUwjuFpnro1ootRWfWTyl+2jhzrxaj41aPb/kjM/UXaHtrQbenXfFeogqvblb90vZj8pjvAcGl3irgtY9fKjOPhZAeqh/TFMSbZswKIYmJ9jGe5oB5/f+m3ZB3sei+AXg7/gJJOC/UdX/XkR+D/jbIvJXgH8C/KX3P8ynK+cJXQEzlzs2gQ0qCYFQ6uapNrhCrMVgEy1YqrYYfyDG4TPdWgCcPupJOnYpBRVL8Kpa0LJunA81/sSUtLsbzWMX8VU22DEFUYLErShPrRRsZa81ozWg4oluZaV6qreGZLiJ6mCWn543jkdY2bv7JrbWQQn7Q7283SgbnXkEtLfP2nNTqOGMgnPiRpwpiXBBadwrBXAGwcTzYxiN5G+xvLOiUNV/BPxrF97/Y+AvvOM+tz+egmP2GokCRQtSzTwUj+njdSyDUfndjYhMYUXLFsPfz9O2mqqgBZaSWZZW6emW0UJpNSUAquZhAhTUE7KscI16hSvrjD755BABrcWAUqp13AoQU7DenR7zn4KlswdpHb+8rynOGg0BcrNihOV24XjE+AwFjnWx6yHCFEKfXdZ53CKqU4yOuSQ/ttNQZWdwrrlPyEUCYXYlSmFd164882qJZtbSwPAVw3Pg7u4GVeUX6y+x1LoVAZbqkYTT1Ts0BYP2XiCj4QhuijdXSO9bQC3RqzZMaFsZNotjsDxsUdhYuJYRO/VrcmJZXAiXtqQ5ObF05Fx3vrM8HWbmg416nraM9+Eii879bESJEiityfCJFVF7aG4cjEu+OwEzrZhsK3hTacRGVaxTuihzclpDbO7NaQCp17XoE1g8wkFP0AohWQsAJxARJoTKWsx6MbeimvVT7ThEuexugM0YObcaDC8YcxlGt6ElkLXr2i2l4hiLDGnrbMlXRTE+RW0TMFNqIedoWbshDjfNmKVg1tV51OMhN2SzwFqavGwRrXeUc5fr0mftczl7hi2q8rG4Fk9EUQhvR8t+Wnac3TBbAbVUpCV6dZ4CvWJV9GKWXS1W7axB6unKJF53cZs0p/hMm9whCFRbdeYUPNHLmwKnwBS1Yw7Prq8MOA2t6lWxZjnBVnlxMDN5UhgazBIokI+ZtXoLw6Ww5EotQm3zzauei+I8b9nsd9Uh9d0KB89TZDdPJ+fU8YXsysM5NmU11mrQvXc5H2pc6BY2tpoatpqKiNWSzIWqC0g29mV3f1pEyIBiET8WD4H2sLFPvgauNoluFZQh6mHRHKee2xnZtg+MnfOJ3RSi5cydFp1unzfwVgceBdKUx0AG+4BW+RNRFN9WcUUw3Byl8OBllUrQs9UTW5XF3x/ThAPBV/AtutFyALr1LF6X0mtkNuCxKYo0RVLYwMnGUpxDy/uoIKUrmYbfS6Nzo+RsZv5yNIblWgvrql5NK7hlYWrCWhVaVAFOcQI7XjvuVq2rUZLPQcj2/gZIehKY54hECRuYq9AqjN1bmaUaIzU0HsoWPdrun2/XLJvBZG8TL+jDPef6ZB7m7RbIfb1sGAvDuQ5g63kYWDflO+xl+/wjyNNRFG/jeshDt+xnK0HbMNuUxD2zsf/p2ES/oWHjYKh08K+U0sOAxaMMYxQhJe2mvlkUvuqwKY2UAvOcjMw1RaIYj6LRvadpYlKhcxZYCapW7h9TWtowkQLL0crrL8tCKUpWK4BzvMuAePtBW/1F8bDnluHZxrZztfpxtmMZ/fQTN6PhMa4QSinIupryCKlHcrKCEExpMVoWSkyJxhex5kqWn3J6k8ZcmNeLNG9F7VxCI/w+EKGoj0zeSxN7dD/OlUTDzs9dj3PjwZTPG5zMG8oTURQKstKrRY1KQ5rTm9lWAqMUj6b4VqSjTcWmTC7dpHYFxzCZG4qSuRg1PlFk9nodCDxCQTQT651Rs9tKRkY0oepB0HBqUVi5+HzaEUq8sM3djlobEcgm0bLANJliWBclr5V5iizLLbud9bDY7XZ8/tlzSlmRACnUTtba7WZXGIOLU62C983XrzyUWFjv1o3QVSzb8/b2SK5Qi7AWKDqxLoV1Mb+qVKhVSFMmJb9Gq98+2YoFiig74DolXrzYdSVATf2YDsWG5u2rQs4VqYGXL1erzBWVNQtLgaPunGO5RTboFP7Aq6NZClMSbtYbdvMBxCwnVSUXW/tjjCBLn3FtEpZqncQ0CJQ2Wa0+Ro4e8ogQQyEEJYgyuYsTQkKDMWiWsJo+imqkOEJX8iqVFdNfOQg6BeIUWCPcSWWhklA0GEcjNsWkG2FN3VXrKfRiF1wfmAFvK09EUbyLDJP9g8rb76+Tnu69d7rKnFOLx21HdHt8VorfaV+ZaZwAP9oASOX6+toqOU2WPNUmXAyRVkW7/Va3XE783srXX99soOBSNgygyAkByB62y/baog/eLDkuxOgm+xC1nGfDaFrX9aa0mkgwK6uUQk3W6GiJ0RqpiVHS62odxIIYSFkcqa1Y4RdVR1UdL2nRk9Z4KMVC/EDFbs9Zma/jOjxELtvA7dMx87HciHeRb7GigNPito9t854krXtu0ZBsNjBImwtiFY7um6KXMgrHz8+fq96nQLcs45SCuxgz+/3ci+3upujkqK2eRei++ZCRWrZeGGUxE/7LL19u+EgdLZ9NeQUJtmyV2jNVVS3/a0pCisFK/AXvNjaEZq/2e+bd1IsE7/d7UghGA+8RS6FIQCYFAsvtYm0I1a64NRJSz0xxLqWId2bXzgZtFmUDBVvKdoozcToFUM/vw733Hpmw5985Bx/7ez1WAq3t5eXverlBJ4dpFa+Obg9pSm7EKuyN0wNT+x0bmu+vGL/limKU85s8KohT98ImfqOM6+k2b6zEx+YyVtasm6vFmI6TpzW3gXOe6zECfRd9UoqBhAFCEFMAwZTE9773XZ4/f840W5u+3WScjP2crP9mXjytuqWcq7su1jktLzZxjscjNy9vbUJ5Am8IcHDQM8atX2rJVqdiXQu3t7c24CvoCssCYZ+JwTqhJQ8D72fvP5KEZ8+u2e0m5nlmns2aSANeEEKCanTrnVqvkJoWjuXIWsQyQUOAVSlaLboQxPAILAFZVaxFoyv3ZiG1OprztGf3yF3dAM3Te3RJQgjQFGvjwwxWYwjBu6m3YkA2+YMXTRaiLyihE+bsextt/2SMvMFxj9PgQ9oj31JF8TqUpimCS++fWxejsmjbjPL21kgrMnu+ujSqMWwKpW8/DMiOHXSztFkQicPVnmmKfPHFF1w/O5BS4u7ujsNu6xR2PFbublcbhNWqR7SyfVoNC8mLFdW5u7vjeDRr/fn11Af4Yb7qUYkYjJ+wHDPrmslez6GUQsn23SiG8getRIVJTLHt93tLckvCbp6YUiQFYQ523acgtCbKQQIa1OqFBgtzRglESdRgJLIpREpzw8ZsvH7P76+e52Suh+QSqUlEHsz8bNbUeJ9HRdFIVU1RiK/wtlVrJj3UJJEHjvUNXJBPiuIh0QaNNSf4zGrof8uF9yunSuMxxTMOvHOlcRmxtqI0992JsRjLiOy3VQ+2DNOGWovANJmLsdvPvHjxjJSsjsHd3V13SXL2Vn0psCwLr169IqVALmsPjwLUYuHFsphpn+KeF8/t964Pzzwikbja7bcGQw621nxDzVbKbwrC3dHA1VBhN8P1wRLJduHI1cGshs8+e95rVi7LXb8Wk7tH+2kyS6BYKUQJCUVIyfgZx7QnFCFp4Jh23MqE5EyprWSgWxJCL1xT63anusvkVb9fV6vz3P17TLlsioA3UBSNm9lC0C0yRHc3TFc0gL0p0IfGueNO2g9mexvOhvS/8K7H6IdtNwXajWxgUqvfaNuJ1bg+29eoSJqSaA70sO15xOXE4mgrzfZ3u9mqhSBWJTmmbVBGMcR8HKTn3II2mMf3msxz4nDYMe+mvsKHYOCcUZhtv7cxeeQg9/yQm5sbQhSKrt19uL05siwwR3rVqKvDM+NWpIk2wkxhGdlKKN36qMX5GNPEFI9MB6g7+9bVYcc8z1zJVlHqet6zm3ZMaTLuhoN23Tz3sCpRPHIVqFJJIZKrP8dEzoV52jNppGYlSjWzvpgFgkSKuEIrm8ke/Losy9LJa7V642JfuRt1+rygXAvjtpCrtBR/r9i1lkr0XJWs1dsc3Fc0qhVJnVxNkOBRD2+QFAWJYAXQW+OkcG8BsZB46FGdRrA6GUttGONh2Q+UefrEFcUD4t2f7V6MF2JQJAiKd7Y6kQsWhRcu2T7HPuvvn7snr3dHNobl60/nIbR8mmditB6a05y6khCRnuNhtSmOfH1nJeaomXlOzJ7gYbTntVsUXiW/m8wpJfb7mWnaeUFeT6GWQOssdnf7Va87qR4+zcvKlFx3q9Wp2O8mUorMTOx2pjT2u91WX6Pr4O2iRGlgnVcT98FfhktiRHJfdRvzstLxiSqG55RSO2sRtuS5Ntl65uxbymgdjNZCaLU4RAhyPxO2lelv1xTd3NDRJe1RJN6s0O83Id9OReGivSdmPLMEjAhktOpLrsMDyqJ/NmyjnCmR1w80wyJs9QrgFOlwz1oYoyGXpPcXnVpbvsy62vHk7KneZbGiqzJ5u8KVUlZSvPI6Bl4azQG3L744EGNkP111wtPV7soGNrGvujcvX3Fzc8PNzQ15XXodxyCBOE0cPt974tXEnHbOzZiNqyE4AzSw3++dOdpCvqW7XbVWpKw0RWy06W0BEIE0zyAJCTNKYD0GArd8dXdDzkpeV4pA1srCxnOw6xs8cc3YpTc3NwR5+yG/AZNW+XyarOSftV2wexqDFxQOiTRNSJrcjbJixUhwq856pkRvj2BjtCmT9nvRsQuzsGDLXenkiHMf46LL4Yy+7qa/e/TvW6koWn9Ne9008xDOI3JSSr/LpYt1/t451oFbFr6/XgnqYbHB2ghAp+j1JXlIUZzyHCzf43gsvnpmSl27S2NWwtY82Dp0cWKJpJR48eKFdTCb977KT16JywrvarbjPh7vWJYj67pQvOFvQLrCeHZ4xjztSTEyxxmAGDwbtnUEQwgaiCLeeayiWFRC1fkhqv63gYZV2zWzc7dM12iRAIkco5GrbhbrkZJzJqNUCZSm/M+yuRq1otXjeFc5j14JtefuxCB+PWMvF9gwi0u3vjVXgriBmYNFcf78Tcu3UlGYbOCOhanGyeYaWh/CKs7BzHNgU3kcxHxcWmXrnG1tq9NYper+IBhL2p8cqdjxt0kvIqz51hVHZxH4RMgOEgbGsVVr5epqzzzbSnh9fW1Vu6v5zst614vNiBrGsq4rijUmiklYF7MQfv7nf54YE1OIvHj2mbkURLdktkhN2u23cw3BaBdqeE5Dk6oENAhSmmtnFGtbSUOPAoQ4+W023sauRpZkVkitlZp9f7Gi4m0BB3AwTtbvQ/XdFUVnzQ73z+qJ0id5I5J1RdFqi7bGz22d6aX+7muPU1fEa3x6kR1tTUnbOG/su4cUSbM6ZARE/39mUdwTPSUynd6Dc3zhTeRdvnPhsFT7Sv/orz1ws2utXRnYuDut6nQS1HWrYF19MGPsyGk+uIuRej9LETFWoyuF1Rsbi54WxWnHsNtNXF1d8cUXXxBjIiJc7Y0JKlVYjwtrKD2DtRZvidjHtBfXCTZpG4egg5onjNURBwDLgYFcG8awldkbr+vp/R8Zrm1svN/9bLkV5/IQSeuECuUWJn5MWi2WfP/4lfvkvqch30pFEXQ13y8ESFBDJcSdYRYqaLaCJZbJ+cq/NeSKyCVcYvbXLeekvd+wiqaRC61YLaymlATAa04Wr36tMNVArIGpJkO502nVp0sh3REsq77CAmi1XhtaBESZsFL+faDOFS2KFOM0TECqykErUy3siMxBCHWlLplar9BVKMfCzSvv2lWzVQKPyRoFRzjsA/vpBS+ev+DF85+33NKq7GLiajqQJHCzvGSpR2NHelUuSw83Z6PzQlQJaoxKKVY5XPSK6J8XL8unahlwQbFGSWoWRy5KiXfUtGCcpN5wgFLEIgbSalUYhnBcDeBM09QXFC1bxmxCiRSC6rnH0u+LJZJValBrYRCsYE7vKozl7JTa9mtAq4r3KRFIq5HmzC0pRhjzzL7Wsc3s3IDmyfgkWJ6IJQ2aK9dO2AIagUajr61/TLN6gDmJF88pPv7fXb6VimIU0/ThhHxjpqEBORZJGkwuOXt9khT2ttq8KY/TyIuZz/dbvpWi/K8/Hr/fjuX2LX/3beTOH+8jX/nzP3zP/XxE0foAzuyzP69w/Cn8yU/h//nH/dO/+G//Wgea5aLNMIbiBxLWuZXY201EWu1UOqnqspxbHpEtshIa/2NgX9zDLfQCpvHAb72vfCsVhbZ/ajUeu/+om4k5+pSX9mBy6bJeUhaP+HfSQrCb2d78Qvt76CHpqdf/xp/ygJ8XimkdrMY+o7bCK6VmKwcnlha+2ToD3mH0SwAmgSkmnu8mpijsdjOTF7ORCMUrQ+Vli3Dk44KIGLV6stDmZ89fsNvtrGfn/nPjR+z25l6ososTUpUYAncvX7Esi13NUlnr2geuVaByq4IN2D3mtWd6bhwAT/IisGYr978ukKtQinK8K3wplVevbvijn77kNmf++OtXrGr2XHH+SEv3V1XUI0ZahaurK37u536OX/mVP8Nv/0+/u42GZvHcA5VPx8d5SPN0LJ1ud3GonIVZ28gat++Vx8XjVaJ+r70kXwAvwHEaUq2XFMWH42Z+KxWFyUg6aYrBP2oTU5UHJ/gj+zR5yLoI9B/qWjxuoKm2dveNr2BMQcDDdpVep84B2Tr8bEPoVYFSqGXrFJYk2ILV3R0/Sg3W3g8zeSUEItFWIY3UXE1BLIVczWwueSMctf6e8zwzxcTO+5pe7w+W5TlbSDV4CM8wtK2GRrsHbRIkCb1WpLQLwSny02t5jBGhYZSPAz4olq4dYQ6BY8tBadW3aXbc43d7t9ux2+0eBI8fknOL4vGNtyxba28QTlb+8/1uCkO6RdPPSWRrEdktje360xeljy/fSkWhHmZrA1CrsNsf7MOqlLygWqmMJeTqqdvR5CTcec6lGFeNC4rD0XX7fuMriFfBhlIVKKyhEGPrMA7LsQ2mOoBupxERgKTZj8ZXZynd1WyEu+ZxtXocBauFcbvCFAM5ZCQYpmL+avudTUE8v7pmt9vx/PqZdyubeXa18Sxm704e/DwF6cV1pG4U6YhQg4X+IqYEVjFfW7FErnMFkAct2RkQDQCuphFjjCSJzCGi1kqEL1/deUap52U6+BmBPMzn3jYhTHz3u9/lO9/5DvM8989PQp73WIwtPO1DzRUhoUUgXGPLqfspo5LE723YHm03ouauRBEC1ZVdIGq1vma1nZ8pjqY06GOm/1Dv/TE6M/UNOD9vKt9KRbHhCdvDajAIiJVnq7Ul11zKzxjDouPgqFy+JA+4KPfwjWZGu7sRnHVYrZ5Ci1OUVmSRkYTkvyTbI/rf3SJWQf271T/bImRWrk1rpQQoFAiFkBIhWik4WjJWq5CdJk9Nn9nPOw6+2k7TRArNImFwfZw/oKZwgpoFE0SoHr1oORU9x0LfhKJG328vvOIXJLjfHkJAsTJ/ltE69AFR7VWua7sO/siDFdDYoic1MIbV+p3kpHetYxSYIn4Tw79d44DdzN6doIeL7RG0YSBtwRv2LkLP5/9I8nQUhZ6Dgg+L1SMINvE8U9NyEiLBaymWuoIOSO/5xVXz88zGq31FNnS5hdS2PA6z+cXWPAXnEPuIjNgn9ttRMN/SC9+qWsiwNFyB5APptG1cf/g4DnkbG2DRkmg7uI/Ql7pZFxVSCEwSmMTqY4bJyuDN+6njEQ0Xubq6YkqJSSaSBmIJJI0kEqlG1P3h4JVlbSJWSlXK6jUs2ZLehjo5pixq6zFi76tfaIt6bKdQ0a1gbGmZXZaCHfBuXW5tBJ9QUQJTgKVa4RrAqd1G3AqT5asET6wLIXA8HvtvniThDa/9ituzK+SqZssW3RShajvy5jYoMQohJqusvm6d5JGKosTo6f9eESsGvGaI4VAxwERglsguTebK1co8Tb3HrIhs9/sBjWS/K2/tal2Sp6Mo3krq4DK0YH0Y9EwrMabbpD7/bh+xcXv/noXR5CHqdsMr6tk2A47BuPKdfvt0JWuDfPi8tgEGwWtctq1PHKFmWTha0sxiqYqWgqRkiiNGdvPcK0y1pKT95EpDQl/hqB7CVFAxOC00ngT4rOcMD9owozEysCHy/lzvu1nAvWY659Kste6/j5bB2X5Et/daGcA2YUas4cSEfydxZe9Feze85fRMBlipyxjJ8NtMEG+nHDYFEryV4/0Imx+BYM2lP6J8SxXFmThzTSScuBpmlX5gMFObEjhXDE1ZGLlIMWKNBoXeRbsdl3Ztvw00hs+3vYoGt1bMg61aNkCrHxO9SXHQ6rUhHOQt7u9KIAVh78Sr6/2hm/Dtu6rqHcLcCqC6G94aHGtfZc+xhs5r9dBe/3uY/MFdAb3IWTgVsyDcsdRhdR+iJOcyui790vi2LVP2PLR5TjB7V9n2My4o41iUDkKeR0+a29Sui1Hl5d6236R8SxXF6ELYhG5svSBbV62TcMIJNjHKJf7EYKXY3rFl+2w7DUNkRfxRsLRk//lcWdXo0OJIpJm5G7aAV4NCOvMXgLRaTou5pzbxUyd+na0ttVgCGttqGlGu9weePzt0Cvez/cFM8Dh3AJLalIRj78UmqZ2DEoL5x+PpKw3c5B5Lcvy7FbxVN9u1GNmo0aLHPhmFbSHuCP8QRrRMUS/bTytqIx5S9e+dLd1WUHjHixcvequCnLfJfEpw285tfK7N9RB/0LCh6sCr3fOxMNFwY7pP2VyQ1jbAwt22qNTSvqsEJmvuPDRn7hEQeiT0ZyrfUkVxLsEHNJ1fYbN0SOR6nZyQr97+90frQsl2893EbiUcG6iecyaoEKOvzu7xhEFRGCYQNq6BCAQrlgvjXPAVE0ECRLWeobsUrZpUCsxpIgUjgVlIs5r5r4aLN/q3hq2FgFSrSxnCNgFGgNKYlH4Ew6qsqoS6ha5Pmgq3e1NOw6pvIn3fMXjmZSTG6mHbSo8Pn8l+v+fq6orD4XBxhX6f1XoM/Z5YJBfGi2ZuGQAAIABJREFU20O/0xWq4zwqzr8pagzSFl36hq2K184MEfnrIvKHIvIPhve+IyJ/T0T+b3/+wt8XEfnPReT3ReR/E5F//WMe/Cjng5Lxxo1yzsz8CNJp12jP7rRn+9zK2iulbK5+/24HNbeQI2zhwrHmQXskCaTJqNfTZJhDq0t5dTiw389W02KarODMNPXrNa74vZfmmZyUwS/bd867p485GI89Tu7TW0o75+Y2tdcPyTzPXvF7q5R5rig+iHl/Bk5fsl4fUxaPPVr/kG9SWbyJRfFfA/8F8DeH934T+F1V/S0R+U3/+68B/wHwZ/3x54H/0p/fQBoJCY9WKBKiBycU4yw4JTaaBYEqwmKho3wHUq1psKwQVswNWIffaDexAYvJHwFVq+60mZO+3UW3xNwN0YGnIZAx7oYBioVAtYwMx7pWP73bHJDSShpBkJUUlN0c0JwJ0RoLL+GOMFsdTFFjXe6jnU9s8Xx8ENXVGiIrzFPiiyurWvX8cGAOswGVEgiLlb9PYQAfm3kfjYsQ1AZ9rWp5E9HT+sPWYVM9YqTa0sQ3BTAVBlPecRtvBlzFUsJr9ZBwGCeUl+RX64kCWO6CmstRUHKGEBIhTUguSJwompEQTfmiIEJ1luucPmM3vWA3XxNkh5BQ3aIerR6EqhLdohTBQ9l2nKh6Cnt7gHV9S1TNiAbW9QiTMO2d/CagogQNdj6OGUkfh3ZNimwKpKBkz8vIwNdl4ZbCEiFSKWW1hkk+jEe3TYONi7HgT6WBz++Hv8AbKApV/Z9F5FfO3v514Nf89d8A/j6mKH4d+JtqavV/EZHPReQXVfUP3vtIHz9GA+5GM/CNle+IUTQH+TwHxD+T6mHcsyhKf37YQmn6Dth8WR8gVQsSnZHZIq5quEWKicNsLkFC2CfjN7Q6CE1ZhGIZoKFop2HvvLJUCrFzJx5bmU6u3/DeqEwubTNu96by2OooTiBSTj0KK1ps590siZGheX4sdbCQeiWqswJCp2Z9h2DvneMJ67MvJno69pp79MZX4c1dn2+DRXFJfqFNflX9AxH5nr//S8A/Hbb7kb/3QRWFXbTTUFpTDifPKC2/4lT8s4EodaIQCIOyeOjmPASOYlECdGv7B5wQQAsUKZvPju0qxWq1LMVAuBdXRqP+7HBt8Xa1LlEhCFOyOnRW0r8SHSCsx5U5WoGa3TQRkI72V9TM9AvVtn6WcqqwxmvYJihOsLJzbaXyRqW12+2oIbC/PVJiBAnkBajFrUprvFzKSq2eHSuVqvmCImyd4Ue01jAarR6aVIhqDaijU+w9EHQSiu3PusHb4QxgvXQ97l+bD+gWfQD50GDmpTO6OCJF5PvA9+2vd8/B1+4nbLhEtyi6ZfFYFkCb8KNl8YCV0fkbZ0riJNZ3Gp618KMg1Urbt/dFpG/arI3Ww6OxI1tS1n6/J6EWY/dErGnacjVyLUSt1FyoXmat9eRor8FcCjz8mS8oinFgnofwHhuw91fq1yuhbV/3cyg2ItP5sW2vp2kig9HMgVwqIQejRV/4LozW0RAlGSZkkxZSb8rl3v58uIUYLAQhBia3R2Nsn4dqH78Ol/8+Pf9vn0Xxz5pLISK/CPyhv/8j4JeH7f408ON73wZU9QfADwBE0lstb+OgeXdpk90CiacWRfuhcfs3jZxsf1qho0BKglQh1AxuEQQC2aJhBLc+xLn9KSX2nq152O857GcbhOq9M0Jg71WzCt5wqGRqrFQnTaUQ+r4aMcnqPD4+4S8+/F8YL4gMhCoaRyBsltxbyBhRuWTp2DGMkzqQJDJhiiKpImKNje5FYBro29JyziII5y7L/eimR2a0duuhhTQbWzQAdSiqO64bo7XxuvM/OV85/fyblndVFH8X+A3gt/z5t4f3/6qI/C0MxPzyffCJfvHObmbOVjwXr5KkbAVehg0f37nHrOHUJ9i+1vJI+g5dEWRaKHTLGG0WiRFuzKWwj66vD3z3i88py8p6ewt//KUVmDmrcGSWhNW13Hv25rPrA9f7PXNMzFNiIjCJhTNtMNn3K4myRgtpTjuCem0D5YRMNaax3+8lemo1nAzcB/ANkS10KiI98tHaFcAGcLZUczvXsZLWtq8Tu8z7h7YyhwrMc2SSYJaX5/fM88ykluA28hialbM/zHz2+XM+//xzRFrt0eG6R7uTrcfrBm/Z9YnS0ritjUAVvFKXoggxWLr+wQsLN2tC1DkoajDmpes3WmGt5MCGvZxaQjJW+NYNExnvRRun27zhg8lrFYWI/LcYcPldEfkR8J9hCuJvi8hfAf4J8Jd8898B/iLw+8AN8J98uEN9Rzkpxe/SKdxuSQiccihG3/kht+gR4NIHW5qsCvV3fu4LpCpf/fFP4I+/9K8rbWq02MoYY+kJh2Kl4I3WG3odiohuZCQFDYEgDtpVJfU8lcEd0+aqXV6l2kBuA7K/p+3YtlRo26kzKGlWhTyACb25nLsdD7ohbJNqLPF3AjKK9M/N/TJF0jqdPSz2ecMWzNUozpRtOSvaLsG2nVgmaFdW7bqZuXV6/Geu3aXr8DpL4iEL7GPIm0Q9/vIDH/2FC9sq8J++70G9Tk4vzhZ0evgLTVk0K6J/4M+XIhZnIGezQHrk4xynaO85aYq2Eijf+973uN4fePXZF/DDH9me1ftXUPtN6NGOdgRiRzGFyJQCswTPt9DekdvKvkES40YEBzwnCZ5+sB1/BSf10BWOHe8pH+GxZsrjY4x0jFZFx2cfoc+37WsdVs3BBbGf36yMFiWK0cocihrW0hTB7E2aa4bS0/g3/MnCoFaU+L5s1+gk9Cjubri1IW4ZJUARLJFXaOm9xhQ1B03Uka6mJEbF68p4fH5IaVxUKs19OnOjPqb8C8LMfEcZw6D36mieywh6noNcl4HNnFvZ/MBuN6FXVwBMU6KosObqiUHbdx700X3VwusWtN9tvTC0WR916MJ1YQBuSMx2zKNieBMFcWnbcQLU1xDZLq2WzfW4ZFGMq3E31wfLQVV77Yy1llPlwrhdy6u5FN7mFJP2bYz1qR2fGLFz2JaeXjdCWu6GZ7t2F/dxGa/pJTflfJvHMJ2PId9ORdFiUy7yOosCOI18tNV/jGx4pKMrjxb5eHuNXSu9tkmtlZT+P/beLtS2bUsP+lrvfYw551pr73POvvfWvbcqFQpMRSzzoC/64EuBT4pQ+KDogxoNlg8JIuTBmBeFEMiDRgJCoDQSI2oMKBgkICqICEbRIPgThCqSqrq5Vffec+85e6+15hxj9J/mQ2ut9z7GmnPttfY+59x9yvTN3GvOMcccP3303nprX2vta0EiA/dy/pubG8wxoxwnoLAEaNllAVUzqKQsyGBjzrJ75pZWbgQvQFPL7a/vVjLoXkSE3CUv9aub7dunXwdlsjbcw7Ql3QEGNprX2SaIPZdzcS3bQb/9ThfMs7/pm2lDtaByaiHuvRllmkiPi1xuqpF0+7EkxNQoSUmEa0LMzue79F975/HQy7R199r9NbNvram9TaP4sgXG11NQAHUUvV1IvMVmPkewu9rGF/6i+41tVwBM4ptQErAsM5ZlQs43eHF9AwD49re/jds39zidJEJwteiUh5OGcxEgjcXlRoyaxCrJW9wK0ZCsbAN5ETJs6Px6JQ+bVZqIqht1u5p5+NVnA0T7SX5Ou+i76rwLdPvbh+i/aRkinCz5DyDyCN7j6uoKlDOOpwm73Q5LTtVDslbrnzKRTItYN4nahGI86vXopIhle1aBq9oE9H0Br4DWvp5o/76G5IcAcMRWMD4mXL/s9gEJCsMQ0P5yRYlAXb69PHZhPOLODpUZVHSFDiAQkgGWVrqNioCYF9i42/Pf4BZ1e6O/Y6fs1iwmhCOHoUDUfy4YCSD24OgwDgd87MT0eLkLKLuCq5GwLISYhUciJMKOC8Z8REgJ7F+ABgeQB5OoKY48fClK+swSlu09Citq7mWwWuB6s/nXL095HfdQUTcFPHtVeMPFwXJJAAElZ1kBRw/OQIoZLqsZUgjIEv7MzC07ky3UGog01G2lOI1MZcQcwZTBPqGghVwn3iGXglwCmD12ISA4D58H/HCJcHPCgQLgCAMFXJcBN27ALgMLJXhfsORGaBRguISTVAAbDzY0mNUnwnI9YKHiZ8DTqfUXk8bqo0bLZpZSA4UchtQo8IyIh7Ik3RGzUvvL7x1cDYrbRryWjuynFyPkUtum+4tAPY83Pbd9QILiXdu2E1wDL58S8fK+7YJgtxUwLhn39/e4v7/HJ9d7AMCrV6+QOYF/90fIOUPZJnU1EVcfaXiyhGAHBHLwBLhCdYWVQWCT2QbiOhagcinS9uUerN5braD+vXCPlQKvwyecc+IS3ABt9rl0WsS5VbFXxysHRbffCuzFGhhc2fldXEOdbO84HHqNRz5jdX39Pl8VuPhVt98DgqJrStdfyK0SZt5qfqxCtZ/h2isNy2jkMpLG7UhWitMS8du//dtIKWH3bTn2H/r7fgm/9Xdu8L3v/xCc7jHHDAcHPw4YDgHjzYj91YjD7gr78YBrjbD0DPiclZegxUE4Zd0mouqPr3feTaLVZHL8QFD0bWU3K0+nyV0HaDyBw0AEp6X0mETksdPfWySq5lwYzV3FO4jAvqs6Dq6ml8VdLClV9Vy6XLw7Xr0NOzcgwcEvSRm9ROAOYcCoGbOA5YmcCag6c89r/AsayyHaLTNXhi7G2vNjGa12LMOK3qVtNcG+nMNPo30tBcXDruqtym38Q9ns13k33ouDQpqRxJuPXRyPos0ozITTacbt7S2mF6JC7/d7HA5GIHMPjllqj3EABg8fAsJOqPODcyAl3/coCBTUjFIMwli1jd3ZGb9ly4TcvuSyvAKGF6jkq/lX/1szSDFAtcCuBKdJ6LOpupbAZd8xLIITGo9huMm5ZinrMcaVllG02ppzTXPwCvTWCukqMEclD+4p8951osl1mjanwozlXL0gs335wW/ffYL3AuPvCopnNhtwAj4CW/qlAiG7XZUSBJqAAJ8XEm/jaOua60oSkrJgEQCPUN2PDsDnb+6QE+Ob/kX97cuXL/GNTz7C8fYO07SILkIFYb/H7uUNrj6+wYvhBsF7UAac2ZosgqAUIZ2hIjnojqDnFqzEVjNT1Vcvos6LIqbFA3CR2yQ286X5VuwN1QmUkkzqQoBnj8haL5QJmQlgB+d8TXlmBrhIWrWBozFGpCKT/fb+vm6zdOkQAljrkvAwYnASscjsao4FABzGHa5ubnCjLyHVffvKfkmjuLQfdeDrAy/Ge2oUD85FtAI/v+r2NRUU23Dtdbh1G9C9dtGzUG0btd8/i2OzP6PyLuj5uH7KSCnhOE/4/LVEZUrh3x2urw/Y7/dwd6cKoAlOMWIY9wh+1PwKNTPgUChXAcXK6ixCRsZ3UWlgadriNZBrFJyN9MrEZm8aRW/D90FP68nS07CxnsuYqRMXcGEU51rECdHqZZOLVZXvyXNSaeQ3SU2OqMBjLQNQ4124upH7FTc4BwoBuzBgF4aO2h+6+n/xK/I2v2TbthrGc9sWP/pptA9LUNRO1hFItgauVUbS6DcJGYbOU0WdVwKjXwMJEl6siH5VqdcPgJ/xSF13bFb3aBMUQhXn4ZEBTMuCz95IDU8/DhizuEl/8pPP8Vs/+jEGlgQnr+nj19fX8HEQOjRIgV4AQFb6OmawahnkHLhkkJopYKo8Ds55sDJjEQgld2BhtbtR1fVzzWIAamwFzEuhn7SQMENIWtKyNNarUlBYe0c1B3mGhJwSTvnU8IhlQYwRS0qYpgk5Z8xxqecOuxHkBuG+1IkjTFulxhQQEQYfKrOVc04AY0eVb3OdY+OqyVin80ajaHiBmnOlJZzZsUopq9B3u0fg7ebOVggwM3xo2cG86f+ntiZAn/Wzs+3DEhTPaAynE98DIQiwWONUNMFnw2XZgqhsO3XfAY/zT5y7hs5t1R3bqjyIw1d88K4wbhcZ9G+mCTfXA37u576L4909fv3Xf1NqOXiPF9cvcX24xuB38NihIFXiVSaHIQQwM1JaVIsQH76nEewc2A+rGpqWD1L1pCrbNsAY0cV61xT8+hgQmWwuT6tiPiUxC2aly+NSEFlyUhjqNiaZTMuyYJ4XvD69qabHtCxIKVUBUYpoKQJ+KkFNyUBgDM6DyCHngpQKTqeTVD27vsHhhZgcV1dX2O/3oAAwF5RUJIT7/WCpJ7WVB0bePNAMapLXIxpDL2xqAh7eHWt51/a1FBTSSazJEAFEHuzUfGBWnbeWZJEfkVjwoCxoe00t74QEAfSMikuNXtamEEH8F0W/13DrImLpjRae+fHrzxGGl3jx8gafvPoIh90gq29McAACiTuUSdLICwjEhEwsVcdyxBKV0duRcFOARctwrlbCBprXo7efiah6IKrpwXxRo8hR7jNZCDSaVkEkhY1KKYhJTQjnEHMBp4xpFu0ChWvxGmZGnGaklDCXuU6CWTURmxDGB2pu4nEcwSVUrg04h3lJiLNUMtvv9yDn8eKTj6uQGMcR7ERAkMZ3XNIaV3ElbeOTx8OlZlrfg22dMLl8Le1zDdrqntlX1b6WgmLVdEBlbpGIKACRhD2f+UH3/tz3T19ueKWtACYwWsoYqQ9EHuyURaP49Cc/xv7g8Pu/8128evUKH794idvbW6R5QZwTikYfpqzRmBCjhthhXlQln0+VsSrxCMYA5xTEdc0NKPPJygdIBKf3HgUd01OPU5xpc06qxbTfVKzCtRUvFgEmTzGhmGagVctLKVhOE9ISkVKSHBgi8JjqJOhT0oMPKCQVzyzxaxhHOJaygN57ZEY7lppuu/0BH714gVGFBIUAdkJsLKbC+3kgntIqgNxpEVtZYILQ3uMRrcKOaeaOYUtfZfuaCgqD6TQoaRiAJKX7PAPsMpgJVExj4IZ39JrFqql05+HpV0FNo+ixCaG/11UAQp8PABrHif/3b/0GlniHX/yF34/vfOfb+IN/4O/BD37wA3z66ac4vbnDdL3Dcn9CHCR13CmhauECdh4pRbw+ncR1SMAweOzKTpPPditbmZllVdXJaKssVpGZjwNmhQ1D0Emvgo9dwwlSyVgWMRful+bS5FJqbMTt8YRlnjFNEzhljOOInV8HWfkhYHAO8C2s3JmgGAYEupLrdx5IbcLknBFIcj4Oh4PUXHUOKWcwNyb0R5/nBY3ifYBE61ffxYuY1NgKivoe6zgK62NAYjV6JfmrEhhfU0GhrUpr31ZEZo27sjoWTzhO5+Z8t7YO6CoEjfFfxx5YItbd3R1e310pyObw4sULHO/vcX8n9jrngpIzEBzYqZuXGVRk1bSVN3NByQUZGXnOHXDXBEXOGbvdbrVq55zh/Mb0AC6aHqQeh5hTo/b3rt5yzAkpJZxOJ8ScES22IIv3QtQZPb9OlqSh9iZQ68SwJK5hqACrV1MjhADHatOrp2dr53vFQIIEWiCnJijeN0LzXdpj2MNz4iK2EavvG5vx3PaBCApxIzb9zKSrvJfOTG27U7jQjYDfgZxD9k73Lcpsl4BUQMniJuwv5K/mDMvAybXP1xzK595f+n6ABF5xtR0dAkCl2uaLzqzf/sFngNvh73z/NX7mG5/g7/2lP4D9lccwFhxPb5BSwe2bI16+uoYnLWjLEIHoHZKTiu15YcwakAQ3AgDy8VQL9KyqbkG2XV1dycq8W4OZHkJY6zYD2IEQVX6UlCviTxBNwzmHu/mEKS44LYI7TEmKF6eUpCCwgpv7/V45JEZcffQRAOB0fKMuTGH38kQIPmD0A7wn7PZDFQbjGMD5JeAdGB4pJ5SYZVSUjFISdnuHq4NDzOp58ep+hZRaLCxxJnXkOdawcqky14eI2xOW0o/mNZPfMDNSKRWR4sIKaGcMKCDOmjUa4ZiQNeGOGWoiFsS4aI2VIjwizkuB65IlHcl3eRqaLJZzrgxX5uIGACZ7SCw5NgCI9Zo5Q1g0LsHVb28fiKB4XqvqWx8UBM3Yw7v4rL9a6bwsC25vb/H57RvcHHb45OOPEX/2ZzFNR5QfLRh8QO7cmBKWbSnOXIlaUkq1PF8ybWGJ1TUpdGwFvpv4y7JILsmoF1OaRvHqo48rcEho6m5y3b5dcWLDh6ZpwnGeqqCYY+fqXJbKRzyEAKfp4E5jKPqYCqv+3rNSDWGAD6SCYkSOgl2Yu7hfmVt/DVX7ydw0iUur9zbYrBcUjyof7GpgXR9qXa8JANQ7dy4wyz4/OKxhHF2MiAn6cxqK43U9j9Y6sP4929dSUMC8Hl2H24MRN2KPR/Rtm/Px1QaxmGKfMuN0OuHNmze4u7nBt7/5MT7+5BO8ePECd7efCVjnWu6GqNwycPISK0v3MAzIGiDmwoC8RLATIbFMM4hIyhd215BjElBv7CaHDshx3Gu1sQGDD9UsyF4yFq1qWQtdoXqOeZ4xTxNiTliSCKSUEkpsjyHnLNGVuWDhRXEFm2BqXviAEHytE7rbjY1zIgxwGJC4IKYECxeXiU5VSFi+SyllFTshOSrN4wPIJDM3c6/P1gWI344DMAsu5TpspeIOj6UYbTxRvRDohcQl88WduSwp6LwWEDIb3q8M0NdTUFjHsaxwrCuTRf2ZelwBCnawsvQ1joKBVar5e+Z8PKU5DQojAqY54td/4zdwurvD9U7yNwoYjpRdO2fMs7gOOWXNHnXY6wT6mZ/5GSzLgiXJqp0iME0T5nnG6XTC/e2dnBPiGTB19RSPyJkRQwPNTADc3U5VW3lxdS0lCMcRfifkNV4BwlqhXCuex3nBdH/EtMyixpNDcEAYA9zYVlnOGWmZgRSrxrDf76sw3O/Hes79Yazfmz0uQWMBJRVwEWo75pbjcXV1hevr6yo4AAeUtNIXHfCARMaiO01SuG6CehBijLjUCsu52AEURFsi5wAXOoaxVs6ATCiRhLRbXIn3oWlzJPha5adQjEi0Q1eveX0XpYrxGjnDBOKiVc7er30tBYUH1ZWhlAKUInyGZ1Hgt2SOAqgsMF9yswFPJNySP/j0xyg54ZOPrnEYAk7HWdV+4cuwhCjkgsFJxa+X1zcyucdBTQ7xNhzvF6X539VaHg6GTzTVXDwTQIwN6M06gO/SEd6LpyEvsSavvQg7SZQFVpySicVGdyx8keM4alxFo/Ynoro6OqyR/kG1l+r+HHbKVOXh3QDvArxTvorKkO7ACgqbQMhJ7m23O6iHp53bqdrvWGJSLoVY9+9Xr0e0TvO0iiar5S7hQGTlMb25Jh4/B3Up+kZLSBmeGL7nWykCIouQMM+fHrO+6egV6Pei6dGrV4/Yb4C6CcWsRY5JEoWcMqlAuBFR+mNUxAeVfLIeuirSX/QdnblwWr39wac/wqef/hBpOuGTF1e4OuzxzZcvERxqvoOtMiUXFBBOp5MMptDo3Q+HAxwP2I87TNME5xy+9Y1vIKWEwQekJMBijknUfloQU2nWmcpRx0CJjCVGTDzLRRZCCFkExv4KfaBPKaLOfuPVKxwOB0w5VvXbBnyJqXocBh8qLuF1lfU7V80owNRxMYOcC0JmwwwgdJR9Hs4F5NTyRAA5TpuwHlxUMOsTFl4jRvDtWXvqTAVnwqVpW1wEE7ICzhZxKt9DPW9Owuedh9PnUoiQdeyx03qkFYyXUPv2cnAUQMqlEvyosTBqPiWpfBaGUM0NM496l67QD+h9MERgE8NxgUO5mOn0lPbhCIpnNFOxGKg4xcoHvnKHbzWKx5LDvprGSuySVG788NMfYZlv8MmLG7y6uQG5QV27zb4mNMG5CtZROz1c75BSqp6Dq6srpCXW/T05RFBd9SMtbTUzO7wTzENwCJ7gddA5SHWuoPa/h0RkAgAppjBnwT8ScT3mTLOYBkn63HgoLLrSDdTR6VvsxIAWMduEOitjE5Crm1d4MC3WYM1B2bdaC+PBs3h6nMT29zVKR71z0pySDuj7GvPw3BRx0SKcVYYiEv8K0WYR1DPVtyzeDjTt74toX0tBcbZxJ2r7z6t2Tmh8dc2wAGat6l2AxMDr2xnEBYMi/haqXNKxCUH1/9fwZj1mVVspqEAQLGA/7hB1tUxLRAlFgcSMGCPc0EDC6uHoQL5dGCoF/jh67IcRh/1eJjhEwMQYwY4EcXeE0eIrHNWJTCwYSXYZJSbACwZg0ZVu5x4ICu+9CgjDDcy+b89LNIl2vVyUwyI3nIfZaOs2Y6EHM7EWFuJ2fNjOhVQ/yayV4Pr6rJ4qmAhNazBhXu/J/vbCz7ffteAdI1V6//a1FRQVoFKwTyrHkumXUEe/7u0U0MxoiV9fvdCoKx/kIS8sD2Ag4O4YsfO3yJ3br6+z4QjV/98DeBYlufcOxCyxCs7h1atXYnJoyLQMLMZ8fS0TmLhVA1cMo/eQWBV05xxKAK4OB7y8vhFNRl2qL65vwATcno7w3mOOiwRnOQazAwePnffV3OjjOkzl57EFzIlZYwCggM5ctCoXA4UdnOeqRZBqJkMBpnlZTXb7vWPRNLKVBNy4z81jQcxtuPTH2bgyt27WAqeclxKDw8DqPeohfQUzoTgGka/XuX0xd2AmOTW3WSN02996HygqOIzdTcWTVrF7X2HxNRUUHe4AyGrbh2vbtg+stbVK/O/mxU0FCAVYstrvfoB3wDJNLRISbTVahf52IKV9X1JWtNzDjRqHof1hQTvBt4E4ehkGvakSXJu8Cwl/hpg1DhRaTdMChp8nAZgVVDa6/kAOtAsV0Gwp3a0tPZGtCorWYZbXYLU+REj0EabLnLDEWIWGV8FUj2c9zs3gPKcd9PteMlEefO5n34NKc/qcq4uoXc/TTB0BK131lED61oQX+vEE9UAB4D7H6Ytz/39NBYU04q6zCqNWaEH/EHvn2KX21ZoggKxszgGO2kprg3HwAeMuYDp+XgEtUmXUgEFHARadZ14UZl5pEAAwjiNKUoZmohoCHpCrIBiGQdxu+wOcgYze116ZqGAcR2GCtmCrAAAgAElEQVSKUrNjN4yIOVWV2CaunEhWQag3w7QHoMUHtLiGDodRQWEahDTR/ExLKB2xzbJE3N/f4zjP2O33GMJOksVyqSuy2et9X51rds4HAmOjQfTPyRTYsjrkZixdcLs/Fh9hzUy87bl7vKriVufv6tHjP6d9IIKCNitNufDetpD+fw+n/nJHozwUBlBTxR0SxfVxWHkybTkH4Cq+UVBWD+ZtbtXKHtsmew3hFuRdnblyNDoCLHAXATUvjSDBtUvyuM0DrmiPgz8gpVuUUnBygHeM0RNe393Ce4/D9RVCkOQodl41FMKSCzIxwn6nbuOWw0FqsgBA4OaSs+I5VsPTakuY5vISjWXKiXQDnIMjL44RsQWw90J8u6QEdkAuWlOkd/tJJwGQFT7oI4MKD9YOEY6hrHvJxE/xhJO7xrQsuM0Rt8uE3zm+RswJVyPhW9cBiyvwKIAX7SZEqmuH9XVvtVNuK/aoRDxcmcutniohO2XvYq5uVs7K2UlCU1SKR0oLxjACGpbuWWItsuu4Wp3XG3SSQEhOju+KYD6exeSiALgBhABPWYRgWkAoKqANNmVQtGtuwk6SHsI7xSpv2wciKN61bUyQikP3f2GqB85P/CYknt0e/c3Dh6NhFPIw9bKaNY6qGeScKxVcBQXVy+CoEbraimc9YDEllbfAXH2lrTpW5GckX2niTGXvi/2uEsTOjLOnRCv2SP8lEG9r/9vzlHtDdUmWwho3kmrfACIEt8WNeizhvMnwDC8H0dnjvC3GQoTM+e1P2bZlT/8yKPye055Szfw/BPBPAPghM/8h3fZvAfiXAfxId/uTzPzX9Lt/A8AfgSwF/yoz/zdfwnU/aBKEY67PDquoqkrphEWBFRqWudu+e470Fa22rLQhye1jnNOKOrrHahAxzHcvgGucF0yTJKTHKHkbTp0Jo3cYAjS1PoCZ4FS1NxW5qAliAVcOUmmMmeHVzHDOYedasBPsGjpylcoNUdZAmE3guo9bT9BSSpXdNsG2VcXWnbidyI07FEAnOAuWZcGpEOa4IEYRFiEEkBIFm2B1vmlLeXXs8/U6q7DaXBI9IiQAVEHRf28Tux1ru5hhJQC2As4E4m5soeh9X/602lM0ir8I4N8D8Jc22/9dZv63+w1E9EsA/hkAfz+AnwXw3xHRH2TmLzlwQf3LDDQbsRMQdR9UFBhANQne7uI636gn4q22RxNAdtztFXVIiuzFgpGbN6Pa76pNJBJ3KhVXIw0lrNiJV8z5lUdBXlJeUAKFZKB5wx6cq6zel0C8fuJZfFL9vsNU+rFbBzO1fXtg8bxNvp2E51Pd+4nSX5sEfa0zPk3oee+xnNMuNvf8GG5xTvvYCpseY9kKinqcM8LCSH/X++Hh9W6+/2m0twoKZv4fiegXnni8XwHwl5l5BvC3iOjXAfxDAP7nd77Ct7bmERBtop+C2Lwv3bb1ym9C4znkurSB8E1w9ALoPBYuzWJJSd1hRVdOdJRxAtxljbBzCIrqm6DILCS7lgNQE7e6EXh25eLGKNXvY+2xFWwNpHX2fDfAt58vtlzWE7X2aZVGYl4xhNOCy8okM5DSMJU6UR855WP3tY11qPcAfvA9maenO0bv2dgK0f677Xtm40VrXh17/j9NAWHtfTCKP0ZE/zyA/w3AH2fmzwD8HIC/3u3zPd32oBHRrwL4Vf30jpdQINOtV/W38e0WeNLWccmkE5CsyX6b5M9RfjaIs4Ka5rxyq+OKNwM5IfgBjoGgK+HovLBrJ+GSnKeI3agAaClIJVVBkbwOUs3ZcDkjM6q3Yj9ovoWSxHhy8Bb6DFQw01FzIwIPCWDWpK9tvwo4onkFVj2ywSS2wuNhK6tJY326LAuAFriVNFw7pij1P9SzY6vyixcvsNeAsJQW5Cz71GvuhdslVyi17x/TNM4Jkwff9/1lCG33ff9323/m1elxqp92e1dB8ecB/CnIU/1TAP4dAP8Szs/4s+KQmX8NwK8BAFF4lsjsQSTjreysxBZA65yQibASIqjA4CogGASuEp07fYKwjldIWl/CO2WkLnIMmzhEEkQURpmAvhMiu53WlygZuH2Dm5sbeBCCBgF51gzOwjgep7pKGahIJcEK2EiwFIE0zdoBgPM1jyKlpEJBa5h1K5UBpoBkUK48EUBNQiOScgGW8chlTctmE0r6t00cwwyiul6tn6rb1Pq2O5Z3mwHCDaQshYEiwo4dMKWEGGXvYRiQYpL4jlHo7/b7fWXxtvP0K/K5yd9/rvSBHQ8oswSmsdY/6DUwtmN2v63BZapxiKbQxpKZfGYaAX3ftmecaW1iBS/eqZIjUO/Jxt55LemhwfPu7Z0EBTP/wN4T0b8P4L/Wj98D8PPdrr8PwPff+eouNpOwbrNNOtvYr5mLTDCyjEPhI2K2CSQc146pCpwmhPpVqD+nqfEtZRgAgtr+w+Ax+h3IcXXDhSCDYkAAbt/g5fWNDK4s2o2LWVyXuoqkRQKJqjuSjZy1AYNV9SYCaag0kYT6CgFrH3TzcDWTUhjrVbEf5E9Vd5+ixvf7bY/9AM3vsJZS2jXlbOYG1Mvh6jl6ohtmxpRFWzTT5LFr3vbBJS3iOe1tHo/LeE1rvflxDuP4qts7CQoi+i4z/45+/CcB/F/6/q8C+E+J6M9CwMxfBPC/vvdVXmyd6kw9TqHGCHOX/yX7SiJTRnBBjJIiWokRj5SynD2T1+cUvLqrqMC5UAfq9fVBqn8dWgRjGDTzMcu5A0bgd7+Hjz9+JS7LFOEKI88TUBjzaUKcF5xOJ+yHETdjUFU4CO6hlbSICByBnBmUM2Iu1fTIWh0rOA9fNQtdvbuBmnNH3a/aga3EErSUG4/Epd4vRcKVOyHQt97ssHNsBQdjo1avAL3m9Ygxay0QIeBF8Hq9vhLqXl9f43Q64fXnCaWo2m6h7t21rlArFbxEEs8g/f3+AXimUVRNwXAiNI+MaE15JUD011i60gXbvgR3QVdmBn3JguQp7tH/DMAvA/gmEX0PwL8J4JeJ6B+AzMe/DeBfAQBm/r+J6K8A+H8g8R5/lL90j8e29QOvdSCRkdcQCAWpLFJtSnkSnfIWps6T0R60NKNjM5diVfOtrmVJSMmjlIQhBMkC7W3M0kKlmRkUo+QYRMkStHxQX7UWJTwpbdDZYijH1EFETYOoGkc3SJkf2sWcecXULat2z/Pwdo3CvCs9WCrRkG1wv62dY2nSGxH9r0guD6es3JsEKgVsblPm6kbeTqx+Re5NxK1G8eD9e2oW54RrE6Ztny3g3BthW+G7BYXX5tQ7X+qT21O8Hv/smc1/4ZH9/zSAP/0+F/X8VtSUsMmganjHVSjbRFaQ8zjsRty8uIZzALKAm7KSzquJ0wNxPR0+AJyWVrNiOt5hQoEjyzlwuNrtQa4h8kHZX1IS8lVkFhJWIgQXwM5YrgGA6sB33erkVb22lAibEGYqJQMAkap7tCeOsYE3wK0GIel12+enNKsxYcllFriVN2nQNnn7OI2tcGqtmwBwtW7HPAuISbQT9q9F6Pfu5oQ5LmByGPfjagItywKHzoOg46JPpuo1Cvse3SR9V4Gx1ShyDYJbC6+tB0R/3cLoTavrvVRVo4DiRV++YfI1j8zcNnvADYQ8HA7dwxAVHZRwfbjCRx+/hPe0EhQxnqr9bysVs0x2G/C9a86aTQLvnACjins4jYD03mPIXTYoALCYBZ4YwXmU5JFyEz7cLbdb+3bLgFiBLxuIqu6ymiVAMz0A8XpUsFIHrJHHPKf12kT/2n7f30O/WuYLiL6BhNbX9nKDe1DMGAvheDzi/v5eeTcbuGou7K23oz/PF90e0yjMzbrVELYaxbY/uVzSKM7e1hfePghBIfcZMfoRKSfsxh1Oy8kw/QetUFMviSxPhGssQS4RAQ6vPvoI446lgtRuJ+5LIozBY7/fS7VrZXg2BqNEV5imCdM0IQYPOKr2OxPh9evXUkCXC8asUGEp4CyI/2E3Ctu0c7WaticvwaBe8A92s+SwBUltHoaAXAqOHDHsPBbP4H3A4qUUHyLAKcF7yR0AAKKgqrRHKdJTjhyy8lJwYaTCcPA1tqIUVPamCSzX3ZscmqJenMMMSMFkAK7sACLkHvhkyRJlBrLboYSMOBfEDAx+lGOq7VwDtqraDaSi2EHYrU0GxRGGnWw/LgmLY6QBiADmIPd+iguOacFUEhALdij4ye1rXO0PGEYP5ATyDqeySIo2hNDFkVvHyjgVqABmBYydCikCQF6ifDNrjgcYTgl0eSlV2GYCnCf4wWPhDF8khXwkj1KAIWgRJhAoMeAcCAOclyS9mDOcjkV2jOw+wx0SloEQDwF5ToAnpORAXqrCSUKaLV4jmDOIsybuqTBmYbp6X3H4QQgKRsF+GHA47GHuKHKP1MJUhNLiA4ha4RfL3x+GAR9//DFevtwra7Vvk8IK2ZSEOOsKrIDjkoSw1qjmLTMw54zMBbevXyPGBMNJTeW+vhYy2uvr62rjDl2qNgBEdbGllEQwqbdiSVL163h/j8F5vHjxAlc313j1cgCgg9A5zTYljdVwNVALaKt184qwpiW3UAdL9QaAcfDVJDIVtxHVjHj58qVknpaC+/mN9rFwQJLGLhAAODGH7HdEBM7zGXUaK/3YtqeSkVVDSF2Q0XJ3C2ZG1HKGrPp1ASPFiNM0IXLBkhNSzHj9m38b/Jty78EpMxcR/E7GkPce3/jGN3B1pZXGtC0pNr5KXmM2cATKzXwyLc4A5RUv6EZzMpPDMCwrFu241xRa+PwKjCxFaQIGOBL6u+MUa10VgiQy9vF+mTKoZJBS6RIxcma40Ihz3qd9EILCkcPP/ux3ah4CIAE3vU272n901e62dtjtqx1uwuZwOGDnNZ1bmYwBkd5CaX9qwTsaJ0FDUKYkoW8raPkKS4pISxIXHcQTYtyG5p4zxijgoartu8FQsYUiAz/NUgcjO2HfXpYFRGNFtY2TzjQCkWBS1o/Kw8jLKiE6tdUIagxv6Yv/Wl6BTXjLKO1NLEZjqHaOhAwGbWLIakhay2kDGvbzoPOUZBa/R2atfKZm16KV0fPGlmcipFKQWbwYBSwEwzkhpajh8JrgSkCZpBu8BxA8liL8n/VabEfXgdJuPeGrG5bXKeWtv88O05XAaPbBWqA4IiU3JjWZVbNVASn9qloik2TtchG3+EZY2LNnQEByfHEECh+GoHAiKGyg1gI2F4wv7uxOtoAUrX3RawfEjLQ0d2cpBSkvmE+TgG4xIsco5fsscSkmCRNW1dlKAXApQNaKToAG2aBOuC0w2F/7OTCvb5X/kUXLmFPEnKKop3qvgmtQVSLJAYVkFeTQvAfCgNYALwHplBEJkhhGRIg5AY7ABHgWdjDv1NQKvk7CAjEz5FhoY15N6q29LULKiXDaDFPWwSuRSrIPnEdhuRcxERwSWCpfEWDBcgwIBYAjFAckCDgJpyS2jqSylmWasl5fEu80ReDzN3dYUsH10ipmZaPbZ1pNhurBIWpBbroQbdm8H2oSWy3Das60ACuPM/uyYBglw+LO6vNiFpB6GIb6rHuPEScthl3a+EgFQOIaI/Q+7YMQFEDBN17dYJom7PcjlmWBNzv3TEvcheYmRs7A6f5NfXhBsYopiuuRU0aMc/1NirOu5u34wugEzDG2aEJmNWEJEaIRXGkdTwYQxkY1vw55XoNnD0A8RnPV6fLHjuCGgBwjpnnGaZrgwgAUh0yE0QcQAbkk9dwEOKWQY3LwNlFzVrqNpkkYBlCZr+Ew7vb1uhmoK7G9X2JENlNAszGNxtKo/KiLXC2krmUGmEvTDrpJlbvJZSbHjIzMGRnCo5FVACTXsJBSS/0RZs4iJDwhxYwZBdkTUmFkB4gbhtXDAykzoIG5r+8nvDlOGN98Xq/j+z/8XS00tMNhN9bFKvgGQlfz1jQyE8SdRnFJWAiGQXBKhZfJwcGLSSHLDZy68YUNyyHnBdNxguOAcdjj+nCDHKVc5W63Ez6TjSaxpCxacYkaBaAcqQTMMeP05nhh7j2tfRCCQlakjJQWpOSQc4RF1p1rcZlWNnnJWSIhzS2nEjSnBM8OiRUx7x548EHUzkrDrquMYgfQXAlSY9yxvLz3zZSwTMzNizuXW9/sfraIN4C6IpYoE87wjKqyEwn1ZxYBRICo+SQAnat2PFmngoikmncNOCL4UgCngWdcQEUKBjvn4IZgDP0aT6LAH1nAO2qfZDCCYkh9qvn2/qqLtHtfA59Yzp1LqbkppsWkkjtPjnYB5Lqy7cNa9JgAPwRJqGOGDx6Dlgw4xrmmAxUWQLfE5mn5/PYNdrud9kEjtSmlwJeCQc1YE6h9fIZBaOfwmPW2Vl6wR90qWTAJOYFjJ4sgc8W70vIJHDxurq7bomT5Q13MT8ylCgrpqwIjOb47TnhzPCK+e+nRD0NQAMDx/hbBe6Q4gwAss3oIOtXWOl/nh8T1L7HWuayp1t1xc9LIQyKUohRxnEXtLSKgrBHW7jKvNjHpZBzHEXu1w51zGA/7uq9RxDG3QKY++AcQrQVoJkgqkh5P3iGz2OURBUtJoMEj7PdI84ww7BCLJEbtgrowXQC0ngR8UCALIBVkDqK5yIvrNQpoB7ATzcARwQnyhzkuSCVjWuaqIXnvkSED92bcNZsdQFYB06+6wgHhNMuTKw8os0x+6w9T3zNBAsZUG2A2wmCvQqMJJyInZlGKiDmLEHUklhZEcFMI8MTIJSHGBJBXUxawdacPAfzJZ6+rRnF7EA3x6uqqllY87KQI0mG/74L3qDJg91jMeZenmWMkJhQRAjyInVS0Iy/bmSXalxwO+2uMYcAnH32C9F0B1odhwDAasRBX7UYWh6Ku/Ii4SJFqclw9dQWE/+Av/sePzL63tw9GUPQZcoLab5icsH4wRKQmhQiKrOLyAaqhgqJwqgQuKFmDhYywBgCZBtEK67CCTYJKK7A0hLrCDGqGAKjHFuIUXyfTOd/3No7fwLzUCRbDaQAF0AqDucCX5nN3QGO07/pOPBRUafACmg1c+xtrDIiZFUBdZ45674HgUVgZvslpdW9Ue9jMi361rdpA99kyPlcck/6ChnWhrb087RzUe72caA45Z6BuzyKEWhY/AKmYVopWZKNUPW1LCBiX0OqRuEZCvK2r8ZRrX5md1j8ZALJocSwagCqOEl8Tdhj8iMMBuLrai7eG+tD7JnhjnEGcUbIssA4FmRnjOCJmIXJ+HxfpByMocuLqEyYixJirUNhqFF47K8aEtCTEOdYHCmxSxbZhvFpFzOlxWg5Y143qufCWlGW1IXzjkrSVtB13LQDkkJu/9VQPJxUzCyYA0TRmxSqG4OG8R04RSWuQOhIUX4SEKaJ6bGqagwkKR64GW1Vh6RqdvBXyWXJaXbMJCjcEpJJrGrcQ36iw0PtIJWvkpOSR5MyIhWtV9gJ5X1Pr7HokwgHGtE3s1GNjiIm4eUnt+d7bVYHrXMBevBbyXCxpj0Ba48P7gpxP4jam9jCYTagwpmVp9IApIXbP1zsHHkaNi2nYzFb490Q97dn33hFdTLpnn8Fg4loNkIvTsg1cwexSRIN2zoGcLZidR81LvvJ+2FchUkrB9fU15phWwvFd2gchKAjAdD/VwemcUz6CtfpeBYUSlcao3ATKPL1iodCBsFR7rm10rFmI3tcBVVhBqjBUATEMQ7W/yTgdFNTrr82+t2119cBaWFjocM+LyaxuwS49ao4LXt++wd3dER999AK73QFTKcjzjHmW+qQ+DCCyc3sUXV0kf6W5+VJKKBAwVvhqNUBHn3y/mhfu6oX0UahO0rqXnDBorY/BB1xfSYnB6XjEUV9EhGXp+D47O7oHM6vbGWtyFgZXj414sfRFbTV2IATnMKVFCjSXjMCAHwcMzmM3SCnCUgqKa6UKS8ogXpDSSrWSSVgypgw4J2p80DEwhxPm/QFcCg6j1Ea92h9U83BVWPSml2EJzISUpOepGDuJiLyMAvJiGoG1PyjDaQnF0TkpOOgaUG7CwjQJCVrTNIOyIOUFwamHq0i6/edvXiNmrl6qd20fhKBgRiXpsNU6xlhjH7aqnaHgKWW1Nx2otOizvk9qFF4HiBUStbkgw0jpbdB6dK6x4GtQF3SbCw3M7F24xvXYA4pbjYK6zyuNog/dhQB2p2XGaZlxWHZg19T2BJZ7Zd5AY1idK0OCchw6in87B6MWGlpd3waxr6HsCvje39+3gDYfRGg5hzjNuL+/r3VPY2w5Mn38xOr6TMjCnkVb4s9hTQYmm7lTj6exKOfYuhwk7sDOa14MI7SRfdfKpMnYzFy1pWVZcDwekWYpBJ0Wq8au1c4cQN4hhJYlK2O3LSbennv3/EXAtH5hE5qZVdgGDIPWPs0ORAXTNKGUJGBuanVdT9MbzNMRJc0Sm5OW2v+swuh92gchKEDi8kw5wcMjTyIksgqPrSsoIq9sXmYWuvON2mweDqDDQEgi4iho8A4AsICA1voHaa0yVJvgABBc677IEaLRl6o59FoGACSe9K+EcKdiTE2mKSnHQgHu39zhd3/0KQoYLw47lLiAnQCisSScSsTBBbjBYU5T1YzgpHJVLhkOwoUBJ8lnWVdVBiPmNbuVmRnsGPAFcVmQkZGQsJSEOROm5bZhD4kxqGpekk1uOc7oA1KM6l3yNSdj3O2QwaplcGXPtnDkWCy4i5GzmDHOG2eFClS/EwHA4nYUQNdLcTg/wvug6I1TT5I9VCCMHnAD/OCAW9FYh10zH9wg5QaSZtYSPJIfENwOx2PGRAXAgtc4SX/tGDc3NwghYJyP2O+lmnoqwDAc4WgAF8kmXtwOUAAyuALnCsgVhKDjNhMKUwXZRyR1Mcui6RcPdiTpBfO8yn8ppeDu/nN8/vnn+Pz1p1WbOE33yEmwp+BHpHSeQuEp7cMQFF1bTVJ7X9pKAaCWV7P9n3v8bduqj+tgmfa7B/EQoIvHfOy8W0yjL6nX37+o8weMDqCcQCgi0Hob3a77zDkzMzhq6C/LJDQbP+ZWFcxe89xCr2vmqnNAEq3k/v7YGKSKqP8Ej3EYhP/Ci5pvK7aAdM2GrnEV9Z/ShWyANhMgPddFZvHQbBmsAHVlXgj33y4e26A3S/ZjZiQTGBa1qsxhKSWAqbJ5ma6TSaJo5ZoSSmHdN2AYdhiHA4K/0lyO2JkmBOdJH6ONbdaAUMFiTFOoAXYkNUZ6HKz/W/fXMdRo9L4Y3s0PRlCsAUcTEKWGOfctlRZw1f99yjmsnRMCzAzaxAQ8+H1/Lu62X7iOrWYCmIBoWEUIXcx/FYai6t9f7XE1BnhXsNNkpNJRtVnt0RowxQwJkVAcJ60FhVXt6vkk+z4x9RxoPJqjEyKVaZoqoYqngFNK8G4Arq6AMFSBVwAcxl1NtrMIwiUnVYWpgni567sqRKrbk2vQl0WR9uZFo/JrQrwPD7e+rm51FRK9sBjHsQrFRflFSKNg7f63mcIm/DgWzPOs5pZHSkKuExfGOO5xdWC8uNmBGcg8w7kggVZeCh8BOpEV0DWT2bkW62FD0QRFT32wFRbOQ00rdBrH7yFBUdXZDkADUNmJzAa1lrm3Mc9P6nOtXz37ehb9354DwK7NrskGZT1eZ4D3Hoz+vuzedCe5/pxRMpRANSOEdm31NwCmacKb21vc7Ecc9gE0yOpnwixzERo8Z6s1an3KkqIGj0UMPoC8hyuihlZNARL0Y+dMMQK+gF1utjaJW9ANQmAb51TzY/wgcSRcCCkzfJBVf0kZVy92CD5gPk2YLbFL+zSxhFhnSL3V2odyQQCASA65ZERu3J4xF+yv9jieTrg7HTHNUZ4j+Qc8l6al9IKlFmV2DsAtAODq6qqlsOu1WExMUFNwFwb0DjITfLEIZaFlJZvQINxjGHa4OtwhRYCZkDsz9Xi6xm63w9XNy0rfVzbX+mKnfUuGF4kJ5t2AECxswEK7Ux1XKcnzBZWqVYhweb8w7g9CUADnNQrbjgcTsDyYyPrN5u9DoK6PoLTt/d9tKPb2Gnst4lIavB1vKzQMwKtsnWJqg9X9aglSrK9YRLW9v78HlwGOCw7j+MD9CwZcJ4yYhX9T6O0lZsSr58Du21yZPcuVpUynlGoQlXyWQXZz9QJ5J96aeY4tv0R5NaqADR6zCpNlmRE1LD4VyWxkAjjo5PebftYJn1AkD9IRitIPLjkD84zT6YT7+3vEVJCcR0ErV/CYdtePgXq6bhyEUaYD9b8tavryxgxii/TUcou+X+AivI+ISwKzB8GjhFCV0evlhP3VAc4BYTcCcA3XIkkCy2FQzGk9PkMIVYvsQdmYZiX4mZFLXOWn0EUasae3D0RQSOJKy/jPasEyUKfOGtVe/brDDy41G8Rb5qDt8YyV6NI5+oHIZwTS9njnhJ/t07+2+5hrdVkW3J2O4KJU/x+9xECuDqCscSGeO22ss1kdqTYE1KhDIlqFk88KjglxT6PDN8/HkorygkoqPRHhxZXds6ur1uk0I5eCOUfc/uhWriGmRjJTJOMT3mF/fQU/DiB1X7ZJoZNAXcZwBG+RrDljnu9xf38vXgEGyAcBoxXTAEQIu66fz/V530zbcK7DOYrhP6xh1ahmp5hNttITgFKFADOrR8lVb0nwI0qJ9dBuknge5wjhFMA1KCyAvBIxs4aOB9OCBpBmuLpifV/EA6JlFrdmiXOicfRj7F3bByIoTG1Kq7L13Kml/QPOiKvfGihU7Tn5NQCG63L+TUhYOvu5AeO4C5Zx3QRW06PPU6BO3V2lY/fCpLMja/wFNArTSbIQOQkykoQr2wdYYgYRI8YRaZDJI+dV9xsYJRcpPpBi7TPRdlgL4Bqvh5h2w36H/X6Pb338qnojTqcTUkq4vb2VhDxyuL29xTzPmKYJt/cSHzEO+y4lvSVMxawC7e5OgDsC7u/vgcK4ubqqq/a430so8mGH3dUBhRm3aRYBwc2UY0cozmGKEeXDBmYAACAASURBVPv9HsN+jxgj3kwnvPnshJiTLB9O2c/JS+YoWZam2PNEBHTap7XejF1NHsXC2gKsx1DTjnVs2VgshWrf2ouIEIJWACOupoB3bcLGyEhpwvH0OcK40+fdPE9XV1eY3BWur68lzN45fOtb34IPA0pJCIND4YSUF8QksTXMMlaG0YMSI8alCvvtPb9L+2AERVO424ttYsinL/Rs25Xm3Pfcf3fJFMFDd+ol6X1udXusec3atHMbqAeI+l6jATvvSwFXW9rSygvp4O+OM01TvQbjZ9jv91Lf83SC9x7H4xHee0yzCPHPPvusRTbChFXTKIrW/3BXIowGH3DdUxFqHIpzomozszJI2SrdPDi19xy1dHfzmrB4jUAwWo4aH+P1viuwe6Hvz7Wthr51yz9oFgVWP2tagH2s+FQCp/bMA0QYengsd0uNWjVeiru7HT66+kQSHYMEb8U0I2vYvSx+QvMnkZtdeUXmapoLLuHB/H5CAvigBMXDtp1s9fPT8cv6ux7TuGQCAKgaBRHV8wg4tlJXVtdzTkDY+/7Y9fJJXGNvExY28XPOiFm4Ip1zIK9YAhekrCq4TiALSGIyb4PmupBEbRZmpJxxf3e3Iq5xzuHqcMB+t8P11RX2ux1OpxOmacI3v/kdHI9HfO9738M8xbrKpQqLyHVa0WP2AR9//DHGMDR+hI05YUntQSniUk3CIAVYJXs1Z0aMYtYcJ1klFw0Fp+rh4CokMoTKrlA7bz8Gtmbq6tlfyK7sx0zh3gR1kAAOM3QKwK5OUmbNwSBC0Vwk5xx8kP6G9j0BVQPIOWM6Orz+8U9we/cJwjhgv98jlwnkPV6+fIndYS9aSpBXGGyccY3UlPciTAzc/dpHZl5qBq7ZY62mBz89X5a39GZ1Oz+Y2I8f53EcxI7VmLQe4hMWfWi8DpVJm85sY9Tw8R6A3SZa2XkstoR0VWfIqmu+d+eEm6EGPylXI4AaLm9BZYCEbLPaybsR2I87KXkY1cQhJzyPzoE6c5GIUJzHXlXq5SQUU4Se4YsAchph26JGeSVUWyJZzgLq5hjV3Sf9Y2N/+1RMeGzbNg/jqc327vv7qcconEHsUKglOcLwIwYWLBUwNi+FYEuM4+kNaHZYlh3G0QHewQdG1t+UUpDLgsILRO74lZBqQvD3lEZhdh6J+kZQtxMAVXffB4zpm3UybQbmA0/F5jdA50PntQfknBli56m+bjTAsQe/LrXKb9GNyTlFEDH8mDrVlhriDhYPh04hUekLULASYkWxm63JZECv9x67neQ2OBoRY0YIo/BbsLg3F8WUgnJrTkvLXeFckJdYJ/NKIKtZh8JYNNtR8J91kpVoUnLM0+mEZSlIvM7+lHtpFWilDyR8fdvOLQp9/7ve3b3RHPv7qJqJQaZcHtVyezwLEBwpM6MUCYhjRyh57WInJ1pG5oKUI4Y3QQUpsJ8P9T4sxHw7ngHxxBCah+992gckKB62Fn+wCVp6xoKw1SLsc18JyzQXk/J1X9cNXB0gVStgRo757Ln6wd6DmVl92RZw9TaAKaUMlIy7krDMhOkYQGAs+xGsxL2FGc55UGlVtYsOtEwQzgMApCQmBnAt3akbSi6JcPYKIWC/3+Ow/wgxRux2uyooCwgxZwCE0zzhdDrVUgfzPCNOUh/FOwdOEqcw7Hc64QgxJcwx4vX961bOQMsFJB304kk5CRv3dMKyAFkFxTkhLtfFXaHoh328FcznJhjQ8AoTGNvF4GkahUbdulbfnguhcAYnrhoUOxJ8o5gnzWGJd1ojpSDnGZ9+OmHJCa/f/BjjXrS13W5XhYXDQagUOmE4DAMIyk/i0BO6Pbt9GIKCgZIWoGSQAyxF1p4W01pQPOd+nZkpRCjwEu0Zs7iscgaREIhIZDShKJgmoFrLChV1EZVerg8QA94+kJgZMcu2JXVMzs5pPAUBnmo95cLycBygzE2Cm6WUkGYH7DOGMAgr0pLAXuj7na5uBOGn8sHBIcMVgs8RcEKOkpgeTBCpBJ5aVqtR6DlZsQTNV+9MSWCoVrNMmE8nJBUO5RTBSbQKBELwA8YglP/FkbB3aa5GjLGaMJE1roE8TtOCaSk4LZJcNacdFsyYwJLG4QB2DqMDBgd4xxCCzKabSus8Z3UItRE0QoLKACC7C4LbiSlD3SQEGJmW2odi4p4pmaiu1VDFuFAVAoIrZWaBNuSBCSjLCT6MAkIrB0YCwYcRS8w4zrfiyk6lEgC5IFnFBVIEKoHAPCi25vG+eOaHISi0bXGEL7M1e3UNaFZcZPN+e53v2556DAOp+uvuU8GpC79aD9Jed6a16nvBjJumqRY9Mu6DEAKW2bSgVJOsSknC08iMN/cnHI+iVQAA5y4Hpeu7PvqwXQutMAAhm2n3afdaQ7m/hPYczOGp+z/4/pmXXkqpGm0PwtbANY2uLdpXA0kJgpL5gRb1XEzmXPsgBAWjRSoWkvcENK8DNv38jE63Tu4BLvGJW2JRe+jbgf1gUG+0hue2niLPHnSvKjZvyPo3wQsHh/MabwIBFT25Gt9flAqPSYlauQWE1VWPZBUvBUicV/fLLC7Tu7s7nE6nWjqRmWtiU+EEC3yLeZGgJwApiVrLUHYwLRzsvbhDzdXMLNR3uQiRLpOoz9ZSzsgFKCXiNCccpxPuTzMAh5QV17lgbl9+Hvzg7cpU6UyPSxOqxw76iWtg8xaf2mICDUvSz91pqsa6lSuKYWzvopSOuLgUWHWlGnWrWqsQLmkMzRcgYJ9SpPjnAfwlAN+BLAq/xsx/joheAfjPAfwCpFDxP83Mn5H09p8D8I8DOAL4w8z8N956JSyktGxxzWs/5Ppmn3Xj54NrnFsLh+1g2b6/GKTzjHbJRXtunxVNv2s8HQrVaUTeAHYO3jfeDq0Mgd4fIDEM5jFi5UZQRilq12PMVCkVEGkGaCkg9dPnEiWwuhTM8wmT0d7TgBBG7EaJ+rSAMOrKGGRmo41UkhbRJizXgZmAYuUUuRZhEhikfBFj/Wx7yrPshcT6N+sV/7GF5Yu+3jYem6dNYixaSLppxZcya5/TnqJRJAB/nJn/BhG9APC/E9F/C+APA/jvmfnPENGfAPAnAPzrAP4xAL+or38YwJ/Xv4+2LegIfDEqkx1HOpZX2/r3zdZ8eB3nBsm7DIBeUJTCcO58TIe8V5YnZZciSMhvKQWJpUAvmMF5Bx65Ji0xFy1OjOZydSQmit5fYgCqUYDbihm5AMEj7HfIiqbnnFHirNcptU1LKTjNk7opJShoCB67q0ONE7DeTbmZDBkCtAqTNmkkqmgbhQuWmBFjQsxZeBcWgY4dEeD4gbejb++qUTxlMveCojc7Lo2VPluzjuULGsVj23oPm2lkKUkGbqUBGKTwVKjaY9NSvQmJr0KjYObfAfA7+v6WiP4mgJ8D8CsAfll3+48A/A8QQfErAP4Sy9X+dSL6mIi+q8e5fB7qVxqoe1S/w7ubHmfuZ/V3uxr0acn9+6+yVYHRmyWd2VSoK9xLETl7BNUYzmvmrnMhGjvWQ0FomAQRVawCUO8LEZzzCCSh4yOPIPJIXODdDuNeXrKqaUFnNTVskAsjtD1nqtGlzIycuJYVtOxa03jIe8mSlav9Qvv60sQ/t99jYPVj+wKXHXV1v3PH6LCkGslKdDZKuZpHaIIqBN8tgu8nL56FURDRLwD4BwH8LwC+bZOfmX+HiH5Gd/s5AL/d/ex7um0lKIjoVwH8qn0221nIWCU6L+d+Cemwgy7a7+0TuXQP9iFgqddfVbmghXm3D7oPoqpHvrDE9ZpJf40WrGRZf/aduS1tsvbHjTFiCAWeoIwFGrFYREBkMrND9y8yFa8PV0qE2+53KalWqErc0rfhSMw+YoRxJyHGNIHJqRtZ3bqQdGXHDApXAByGUjDurnE4XGMcRmQuiGaSFC3gowN3ybkJiSJJYiEExCVjSROOxxOWlJBSwXGx/geSHq+gCQ/vWwDaJa1Mft9lUZ5J+Huq6bH9zMw1QtVe1czq6tFszRZgLRQq9rDRTnpc59w1mElhYCaxJKGxawmQ5lnL6ekBipfakwUFEd0A+C8A/GvM/OaRCXruiwdPg5l/DcCvAYAj4q1692XYd2euAcAaVV7zKT7EK97H9Gi/3UIulzJMUT0ATjVIIVsSs6SkjOyDCJPNakcsJsu4C2DN+AQ5FE0H56K+ffsdEYIWGSYiOHWVyvEsOEszfFn4Rodhh5QzSnHwbkAqJAWKBo3UJOGeMGJt0nofmQumOWrpRGHvFlNjQczc6m+gm0hv6ddzWsGDbfqc+4ncJ049xQR5+BzPUyOce//oMbEWFMy8IuFZCZIulsfuybTf3EVlWq5UuVBI6zntSYKCiAaIkPhPmPm/1M0/MJOCiL4L4Ie6/XsAfr77+e8D8P3Hji/WrxQqyQwrnbkltqptFQr83MQPO+cZ9fG5q8ul/S8J0e3+5zSiS/YvMyQOgROISfgPC2swEwMea4GXB6F1X8QUcIbQOyd5Fs6tpTcBQXM+nHPwOWPQ1dq8s8wZhZqgcBTgcsaS1KRRt6jxdpbOoyR2dVGMRaqQxyxFeiStPas2CfAmoKoJCSGY7busTYrzz3MrRB575o9piOee3xaXOLf93MJyTqMA4cG+5zSKS/fSBErTsByrWf2WwL6ntKd4PQjAXwDwN5n5z3Zf/VUA/wKAP6N//6tu+x8jor8MATFfvw2fAFpnm8r0+KR9f9ygHf+hSlfP0mkU5x76cwVFM1/WGoWZQ/25nHOgIoS9pYjbWALPZHXJnCWlvMMD+nOUIgVtd6NoSJawRd7V2h/b1dciJC1Ksmon9f5FoyjMYGSkAvhSQAsjFck3IS8mY9bJzuq+80SYUkRkrQGSE5YYMU0LliTCImu/sNULxVpIMFncy/YZPt62GgXOTLDHjrfNEdmarOeE/VbTWCWZ4uH7wucFwrl76TUKu2TTGnrNg4rW+/gqBAWAfwTAPwfg/ySi/0O3/UmIgPgrRPRHAPwWgH9Kv/trENfor0Pco//iUy7k4Qr6yCD4EgHGcw/qOULiXdo5Fy0z11lip7IxTvab3uZmq9WpAiNlJCJ41xKNpByBRhASHmgUmQu0sHm7BkZlj5I0b7moDA0dJ0LyBShr0hRAj9MxX7ngQcoFmTSoK1Zux3JWm7B2zitQvzujKZz7rtc++u/fFkp/7nfWZ+e0jD4+pvXH41rm1vSQheOhttQft/+uahQdlmYZ71+JoGDm/wmXl/B/9Mz+DOCPvsvFbCfkV+VxuGTjntvvbfu87Tzyd719KyTO/xb1KWyxE+ccPFlVqzW41yelbc/Zq+a9JvdwYpmgETCTmZGRMY4tscwiKZl5ldzV31cF/YoAuLH0lPOa7MXnBMXTkpqeOmZ6wfAUQpdLq/wl9f/S56e0rZZ3UUhd+l2/yDJW2ub7tA8iMpMAIBd4CMNUShk+hFVV8P6BRAtGQpNgvpNl/eOUOjcEqsVyCLlQF4Sig92cb/6sBbk58FoFBTZCzgFGk04gqabOGYPubp3uucAzCRV/Vm9CN0k5S+VqIW0nFDhkVSl2+wHBa00MIgwsdVB9kKjTmBKWNCOOCzIW7ONOaPCGAgcHKjcgCqJJlALHDh4eDIeSJfXcwEwfG2FMFUxDAGWAnZT5Cw6YsaCUDJ9HlEwSZOUIFDx48Li7v8VxnnA/3eP13RtMS0KeBgBBiBCJa8Rm0uQExwCx1HZxUGEiuKzUhcUm/qVTzQFgIFGdpFsbC9q55+Y262HVYpzGelRBbaPgfKBV3oCHRLQyCQx7q2EAaCaW7ZU3GVx2LeQqgX8tBcmQcpzMMkrYnhW7mtL+vjrFByEo7CEYW1GGcj0yV7jSrYRFr9q+vVEdLA/33qpwX0Q7G3DzyKrQu/n6l2Mgg+DUFhXtQHgTZeIQPHkMnhCKCj4N886UweywTFLl3ZNDHCKuCivznM44QMVQ0/vFBMhgXfEte7eVBhCKvRC8JGFxQSkZYOGi8p5QYkYuGd4FVYOFm/N0OuE0zZhOSTw4RQZ2wXriXGo9RvGcduk5P2fV/zLGSr2GM2bMh9Q+CEEBPMQlLuEUMum7/QhSa5FaanBvETIppV2VxHIMo12zX8h7yRDsB8G7DIgViNUDWzVnWUUdSwEdpeLWSao8BQwYhZkvjEISnkssWpes/1Kf0kE4LEXz0lV/EQo0wSqA6DxSGMA3suLKfibA1iX7iAjQ4r99Cb+tdlcJeqnZ1QHQVS3qfYtgEXSTBYBNuWbJZmRYOBlbvzzS5yYonjrZz03qc/sLSLjZ1r15DOd4Srt0R+fM2UevRQ/2QLh8yVb6ByMooF5zsYdL/Vy/ZVkBmdGl6ep3qlnYPHTdLwkPJ/1WVd1+tn0ewwxk1/ODZYt61yhKY8rurpsJiFnKKa75K4C1vigbaBhA7OAK4LLwLroCOC/p0sEHUUNHRs4jHJ+QYsKUFyADe7+TQKw9V0Yk83iQFBiRF2d9lVqkR/IIRAwXZiwx1+pZzAzykkMT44SMBHKMmCKOk7hDT6dJighNCzjakc5reZeaCYm3PZunHm87Qbc93r9h4J2EBICLwq+f6CsAdLvfmfcrIfP/F0FhBKqkk6dxK0KCjlQnlsGh0v2JpgfwuGbwRauSK88FGqBYz4fmuQBa3H4zP3S1VS0pQ16eCSkVyREpI+Cls8g5JdiViFbvHcZxXwHGtGTknLCAcDxOUisTjUXbe4/ici18Q0TCJaF1MGsova3ktA7WSiV3Ao4xp1lIVADEVHA/nTDPEffHGdM0Y1lKlYG1NgqJCWPP49KDfY6AsGM9ZX+x75+435cAsj9X+HzV5soHIyiIGN4Tco5i45bUMU/1A8TSpVnqG9TVReMQsF6IvamM3baqPtoPgAqAnaNQk+t7qI2c85RsVUkTAILyN42CqNGnZABzEuQ/BC2lxwAFMZnywjgVKYrjy4LoHK6GA8ZxgA8jcsrwwyhBVGGED4SQhNr9QE5iLZJEP/7o089xfX2NgwtC7EoExCSFZWjjXSCp6B5zUhCzq5FKVLk2UxFXp1TaLoh5AbyktL9+c4vjtCDGjNu7BVX+aNe3mICu/+yZcNsuOFWvKz6tPdVEIaIHNW6rIDeX42b7diw87rXqwUzVNrtdV9GWWMdDyLnbdXYW7MV727ra87Ponh62D0ZQvEs7a3/iS9fCntwuucsWQC70dKa6dOp+Y6XWV98DQAF+8jnwk3e8sNcT8P0fv+OPv+h2Bo9fW53d9gIp6fLALvtSWj++6sLyNWprrO25InbdPgxBoeYGo/sLNTeo7QNSydtJZFsD+xILTxEWbRJvuu8LlDKGUVSzonwoIuzvNgD4xuH88H/MvPjSzI4v0Iyo18hf3PV+GIICALrSgU1U9MCR2eBUQUGnvmRHTkyMM5hFLS63ohW6DEi9P8XH2tzoKd1Sydh1hDmGETSa9jXmlZRD8bAbsRsCBh+QTjM4StzFOA7YjzsE75GCVCr33mP0AR+9vBHKfGKM5IULc5lw+/oNYoyYqWU7hhAq6zaAVbp5X4LR6l6mlDAtM47H/6+97w25Lrvu+q197n2ed5J3tI2pMk6DmbQRjCCTGIZApR+qWDtfxoLSUah+KES0hRb1Q2pBI9gPFduCKAkpCbSlOI1JxVAsGDWlFDQxiZNJ0iF20gY6zZConaR5M/M+956zlx/WWnuvvc8+5/59nufO9K6X573nnnvOPuvsP7+9/u21X0KMEfdeekk3DtIcFV2XgqhsAXByVllzqPnDMlcM6iIdoK2vC8kCpzkCCwrAIsdybDMIbKvFJtV9wKkR3qCYxHj3/VhUq6qHgoXn7RjZt41OCCjG5F2kpT0gN2RLN/TNSIVI0r6ieupRefcSRZ12zf+e78tgkfRzIlC3kBDoLsi29gwM6x59BCgwel1e3KnBcR0HPHC5lF3ALi9lD41ugXBxCTCwWq3TnhkGWnfurBNvskRdBqNtwWg2o2EYcH8tWwj2UZLM9P2A++bJGBi6ry+COlFAhF5Pmhisfq4j1/y4HXa4OHtwvDt0Zv3P0firvB5zLuI5KsfBq031gC56cp+1YsAaO+kbaC7+fxPlCq0KOWLj1wFU1v61V2SuXzAjb6c3hLTb1hrAEBkD1uAeChQ6YWvmqfurS9xfrRAffBAdyWa/iwcu0RMjrgeshoirqxVilO0LV0Ne/ORdp5ZG33iOMUq6/asrWQm60hBs5bl3szFzkOhJjRZkIDUqEZq99zZtAUWcRNUotZHz2M89FiU+4SfXw+g0gIKBBV3ganUFhojJwRLIQNbl+5nYgoC60Im3gwOgiV2JyziGwG3xqyW+MjMidSnoJ/j+oHtyMGKyi/TR3y8xIHmX8KxCySWSl2GArLL0+mOcwKylvcsqYtWvsKZ18jQgUHKbggCOBJ/nZ3VvhRfvrXD3gdfgGyvG3bt38cfuvgbLywUWF68FDV9HPwD3VwPW3IN6xr3hqt08E+MiTuQBiD5reFrZNn4/AHipOmeqBkfo8jUhgoR18xoIgbFcjlfbNnlvrOWo2zxJrbX0OWg/0d3Grc1Sev964x+d4VpA0rs6IbuQ86rSrlA9ZFuDgmcnXTbfNFjCJdlGUI7XIOpkvETLt7ofnQZQvEJpSgwdSxL7Ibq4Bxlkcfx7zGSD5p8MIWAROrzmNQ+kgWX2B0veMsXnKVj7aXQgtEm3b9VY/a4tl7f/bB3vzH/DTdvi+VgqjX/mMSSgkwOKNIM4D0bgclbz4qH9TTUmT1SSSA2ldLlr89S5K+oAKltNaZmlTZrdtSP0TlQIk23e/uH+6govr4CXX34ZL937Jh588C5CCPjGy1cSJblapeCpqaJvKKanIDNgAjaL6mZJYbc8pi1Jw7dVARju9/qzBRK7eEa6rlx67rds8GR9eWevS7KtcHWd6Hd+b5h96OSAYhsq1JAa8d3KzpuKWjPyxkkPEmafkGu2t1M5rWZ/nqBdhWUvkHv37oGI8NLKksWcgrywmUxVDIswkgaAaQt/aFSev78A+6pMOz5EkvDUCtCau2bXskX91bIDq0q+TbDAZjoZoPCA12py7yYzK3QrsWlRpgOOmo5qscYmkMhh2UTYWrLIFnj/zKlrJ87DPA+SA+L+/fsAgJ4prc4Njdl1VEiDbhJjzMAKTQNQZ53quglbFBqJZQugcdc6I6u3PZmNiSrkrgNZW2WmRyI6kGD3h9TG+XeJzt+2bCBbH4gIkrEmg8Ux2ulkgAJog0URRmG/WcYkykABaCU5YyYwbYhj+61SPXaR0PzA8tvf+T8PEvY+HizmaLCQYmwvhbQosmQ3JzAWJIPKMpynrj9nA5kEiutHCnGx5niO7kIkipz4d3623yQJFBKpMxhuc+8u5F3gJh3Zee+S3bdOiXWHOEgOFFkqRbANXwLxQUHcJwMUNvinsjEFyqnlrXEtEEh+N5dQvkcqvv28lkHLlyO0ecY3fn3jp7vdDBHCGIQ29YkWQOzadwMoPZgUsPo+FmN/k0Fw1667q9Qz90yCrLXoucedO3cQowEEa73OS0MtjSRGb9PKfOX24eJzCjBsEVsNVk2JwjZcUl4ZDBAjdPa7gnUqpPlI9WrkjGHJc4YFGJIRXblLNpC5cbAtnQxQkNSbrGDUYwvRpuq6XTwAc/VDRHnp8L5809jjUTy/kih2K3t6ditjNQ6TOE6RQsa3rHYgv7eBcm3MrqltWMzxt2336Pa0bT/s3PaK9XN3ic6css+lvVmdihNIkyiDU66WfelkgMJTqgAer8s3ank8psra9Kziur0GdNkBPH/b3T9xHqWhzj+nJXndNO2oqexYNiV0Hc/Yba9Fce8ML5vAfSceZz7La8e8+e8te1eLvBRRfA4DzFJhhs1kr2ASe8oB6+hOBiiyqsBNO4FJFgTZ8HY0OPeYUo+pg3peWo3Mexif59xz28w+r2RqqZ8AVFRnJEQnDWzztgYnibVn0mwgFjUkGxQnuJlishyUhcI09T7krpVM5lqUL3ZSApUcqf49SMsJKUereTpSPRyhn58MUHRE6IhSfglb8EUTA2xbiYKnAw+S2gEgV+awr2tKyKdL9yHa295fn09uwWrfD/OqEGXvxU3TJN8TA2sXTLP3DES4uLjAnYtLXFxcIHZSxxah6r1eLZE8zOyS5VUaIppMa99PbMm3q0TRAvxaokj2rhnbWkF63aIjcFjoRs6+LNl39sAwitMBikPoNmfVVkMfm59asvijQDJDDsnbYStckZIZyZ8ZtIESWO07GkDh69BfWyeL2YU2tUsNZkaTEaU7tnMLpOpo00PodIAiBCwvLxGvrmTmiBGL4BZ9J2mTccGyyjFE11FYG6ELCKErZlzAiaBkqo1a/vU4pOXQfqG536reOpzs0C20VgkigDoCxx7r4T6ge2yagYkZoEBppvWuUiCLv6PZkAI4kEhFJOnnBs02FTvZaHgAAyFnYTIyFofgDFnkhLMdxsS0UJa9OvJpB10xmOfGX5aFWPmV2e9qGBAALJcDwsUKvAB40YNYvVxBXigEoOuym5ECIzjDIQ/zinnkvNFpXJQvavUWWCXFOtGwP25KEfl8lzQAdWNOTShaTE+7GRQoSn/m4IyZiw59P2DRXWD9R1mi2GTgOsSbcQiVhil3LtvnCvJ9rGXseqVJEX6nLG90nVpE1qLFQnKDLBZdUkPW6zWIFileBUCaELJkUMYsHCsjg+j6Y4/bFEBsS7sGuW0si0pj7bH6zisSKLYdTLc5wFph23WfmAIIL0a+kk2Vh6hgMY7HigBC28uR86uW3qB6Ux+7fhdqDbyp9mo9Z592nLW7tTw2o75Vbk9xKL2igcIDhDfyWVCK7d9409SWKMx4mq+rRfb6fUIIhTH2leLhGG9mtHsZREAXgMvLS9kVLXSSSXw1NF3DSZSn/F1LapQ9NXDm4xfqonYBi9lydzhv5OtXTrT48cbdxZ9eCwAAHoZJREFU2eI20kbJjIjeQEQfI6JniejzRPSjev7dRPT7RPS0/j3u7vlxInqOiL5ARN97GIuTfI2OT0lELySK4nz7+rn3uZb3oh3+di165IHAXh3V7DaA1Kdl5LJNjYchpxqsB84xVbZN0uo2EsVc2a2/fWhO6jmUtpEoegD/iJk/TUQPAvgUEX1Uf/tZZv5X/mIieguAJwH8eQB/GsB/IaI/y8yH5QufoFqiGP12IrPwNlycmvq0L03xvItExAzduFiAIPIgS+J1g+Nh8PaIm18pXNO+ksE+0s0+dGg32mY38xcAvKDH3yCiZwE8PHPLEwCeYuYrAL9LRM8BeAzAf9/iWQA2zwZE4v/uNAFsrUMShXQuAUm62Y6LjILp2K/XKGI1KhXAB+iYKNz3Pfo+yp6aLmjGVm8SlW4y/z7+2H/3A8DPrkMcEg/MjdkM/r78abz48sz4Z4NP6sHzMm4Lq1urZ7/m5Viqx3LZ4c6dOwBFyQ/a9wA6XXAnQCJJd4DFYi0ekAWKfJ++bnz9NZ/ZWDfC2pBe9fDxO9sChA85t9/rsG5P0YViT0VtFi75OF5U1nUBIbDWRfMxW9NORmEieiOAtwL4uJ76ESJ6hog+QETfquceBvB77rbn0QAWInonEX2SiD65K9NTg+y2yAaKiMTT13Vd1/yzDjfla/e067t70d8/wz8rn1MwCSVgtUTjuQzPx5jdDVQXi0XKCG6D3jq+gWSMAnL25zOfe362D6mfr/v9pYR2WZv+WgFlRlPxO8dWP7YGCiK6C+DDAH6Mmf8QwHsAfAeARyESx0/bpY3bRy3EzO9j5rcz89tnnjmrvx1SGcfQDf1MWieo2eWZU51iE//5y2ZeW8+ooxpD0J3SZ+p7jk+ZfMuOewhmGFD49Q123HXiQrU/i6fwIGczbP23qQ18/c71uUOuretzH4DwEbo+rUGr/ENpK68HES0hIPFLzPwryuhX3O8/B+BX9evzAN7gbv92AF/eh7k5ZK8bYR/3U0v12IVy/glf7rRZ5BhRm6lO2L37TJneE1B2nAhK5y2arTC7uvfIYc51OeWAlKvzHqL7v6fVaQgBUaW0rgtJCrtz58IBnUtkE8o4hwWPB66Ps/Bk7+E/5Yf8UaizW0xarWta1+1CaYvK9TrVO/cCkotukcqX6FavIu3fINt4PQjA+wE8y8w/484/5C77fgCf0+OPAHiSiC6J6BEAbwbwib05nOftOordmlqz1CF7rtQz3xwVHXOmGrytZNKlOHOfSQm7gJx/5r5ElT0nhIDlcpnCue1vuVymT/u9BrDdnjsvpbbObwKJqVl9SuKp69t/egk2Rk7qVl3nU5LIvrSNRPFdAH4QwGeJ6Gk9908A/C0iehQCU18C8Pf0ZT5PRB8E8FsQj8kPb+PxiBjQLQhdlNnuYmGr4Qz9XS0EBnVLUOhlw2KEJFUQginkEBzURxerYiI4AmRJQFDO1EaEDuYi9PpuakT7i7rxrK70tegNDaQFoQOBQKFcJmzlTTXkUJ1mgu5czmmX32EYgAhY5HnUTXa8IXWARB8vQBiidh54Huz6rI74TmmFx/RS+e0swIkRNN1brmqpV1efjdfkNOOTlqV8M3Dnzh0wMy6Wd6Q+OCLoalFp74hBxY0ccMXJXQoA1OdEL34BWIvCaEEg5Xr3oJPebzMQ2PkOlDJMMfNIOvGZ2CIBFBdgjojcI0b5Y0TEgTEMhKFnDH2n9UcYaI2OdEATgVKGdZF66UDk3sbr8Ztoz1n/aeaenwTwk/swJBW3v7FoFzXkKBIJy0L/rCMeXuQ+lMVlN5aRo0Ll2M9QY0t6ayaeq8/xDOj42fc9Cs72uJ+yoVW8VdcreW4recyt3GilayQizdye7RDyySmjuyvBqZEZnEp70ZYvNEEnE5mZUtkRWW7Q0cwH6GxNuSF8Po50rurcQWdLWxBGRWnbpX+vKx8A4pD1RQsCyten/9QGMD3rHEJElDpaLkteknk86HLnicU7Mc9sojPRy0zXL7ZE3O81CqolEVCOtC3dvqVkZlRk1j5kO7mar8qGMXfdpvunqN7HOnvUZBISrw7A0d7Z2z2Oa8D0dDJAYfp9jKIyUGBt5A4AF8YUDiZeC5J2oLR/B2lXCkQAK9CkOjOwMBGUpXwWI1i6ZoK/2qLvUX61khgKcs+SqyOIFgAyWEx18G3IjJPF7B9Cev/AyqeuwjQxVwCFUkfkYbzXgzdpRmZRp5gxYfsbLfLyNZhnuPn3yXVRgo15LwQkxD26WHTg2LaV2CZG9UDpui5dP2XE3EQeUP1gJwf+c/aJ+vcIF4Oip0cAQXLdoBs9r4fsBhaQQFLLiWRXO7+htAGrgGsnWcr7V8FOYZIsNaADIQZrCEBAoupMNlNXxhu9vGwcA4UsdwBkwOMU+ZQtaF6tMarRXMKJhTVW2Z8Ckmoi93dFWdvMTnPkB4TXdYOfRYfSDpOuq96pZUgzVWpb9uypfll6skDQRDk0HvgZZLg4F0LAENvSnVFdp1OG3F2obqvUD2z7Scd0Xb/pfEMSmhN2RsZjk4T9KQZkFzmLK2lLE4d62oATAooOhI4CuANoCElaEJCICgrZ8JWso1H1UK2LCKT9RwGfCi1nJzawAMywGFOjT7Vda0DLwJc5mFmQfrEMGAbXyCTo0bKAHwISni+ZebOSJmChdRBCuQ2CPnsIMFnVSkIfIwLLvX0cMOjM5TN0lM/exNvma+wdDEwzgPmO7yTOxuwOSDRmi+KGfBTb0hywb2PMtCXqCTE1XmXwtiJ374CISLppd+iATneBDgN069pkXCaKWMDqKHuKJFK4B6FDf4A0AZwQUDAGrPsePEhugWHoK5Bw1zILqDRmVFNDTDroRg0YwGTWfM7n2G2m26BamkhrEGLUxiCEwKp+eL91Nq7VYbybKCVjqWbQyKXrLJWrfBIgHdFLV6qGWIKbqVl5GCL6Phtl02vM1Ek9ey8LPdEdNsrpOSY1I3LOKN51hG4hAWBXV1ep489RSzoKjYdua4DMfI8XnAHIohPlbO5A2c6WNYu82qMgEQnJBhdde0oZHVarlXhwIhAHUwHLmBUDY8+6AcZ6LRnC1qte3PYHrLY6GaAAtDMD8DaJNJaLjD/ya0dOoWDnxvJXclD9zxka9bPsQl7imKZCPI+U/ibfSeXFQ41LZpuwz3QOpZtNniaXRMrSAGunPIYUM0VJ+9uj+DwIszRR0nEkg2ORqLUK4gbI9t6mU7DUOTMXakb9ZnV1RQLAfkUswExJDWQHqLavia8zbz+jqe3MdqSTAQoBCZGlJLFuREgxEbGwXksysSxB2AAwHa5zjRdgiUo7bayIiE4liFgysCVlL4e5q6wh84zj3Y0pc/LMjNWsk2qGjNoBPC7ZTBTlhRE0Em9QIyaC2H0A7b+B4OMsCj6qv+KHGb5HBsQtfB/p3jQ15ufI4i5bXCbBKcwQg/MMKLdoF4DeBKC1xECN1Pl2XNs0fMnRSQDszvnvfRzQqyeNmdAPlLwfRstFUKkrYNnl5D055odUOrPFZ/tPECcDFPYSutFaPk216hHTVQWK2+WVGEgsqoYcA0wBgeNofrLvU/p44tIZ+sptA0uxNO8qLWIkOelnW7KykneF4wgoCvBhltrhfD+z5PhMNotroAyI0i5TncrXE+v3esIzicKS69pAizyvdtwUFXaHzm2B7M5Hne7ZfbbmoblhW8en2KcZzEsbTjsSlRPwHi5VnBBQVDERDE1ISyqrOa+FRcQJIgCo1BCH8iJpdADFJJKzSRzwjbXbzONBYmoznlZD7kMjT8TMFgQgQlSpLDLLnpQkdpsQSz34UDLxVx7rFjNNXO/32XSltF6hyLvpeZ7yQGySyrZ7n+3K8F4P/z3zYnYgUwsBcJ8AfhNHXPT39FQADNZ5slXv9buwTQ6FzWw/Ohmg8Enr5rSAwBLa7I10dZpoAlJNJvDhAFKwsHIiQmX7mKdaB8yRc9PXp7899nTzgyrGiMgSep59ZdOURNrCIJr5P6Zs4SUKIgJPrLUvjHwxDyY5h/TdA4XMylWGcZr2QFwH1SqWHaP+7vjz9wHzGlN0Y7iwZZiUEINIqNqHx+PdXKRjsABeZRLFQIRF14GGNSiusVwCFNdaiaRrOqRSAw0giiDqcnwAR3QIyPHLBCCitzz8DqFjiGIPsV17sb1+apIEIIakvNOURswxY7mEPj+oMUm2Ewjc6+yey9WUMcW5xEvM+23K4JoGCLML8GDvo+dDAFjqxZtzW+87lUvDTpO6lb16JnYhiQKiwaI95yM8/cDrqNM1PQK4Qy+4H/sei8sFOK40qS4wxAEX9Fr13gy5TpANtfbu6S+MX2pacqgVT7e/aURCMQoXCBTEo1J5RAA0PG3AKpRDzQO5P5Hc/OuAANZ+OkifX5SqR6BB9+kN6CPJahvKai8CEInBsUfcMf1/TScDFAk9YYNyeuAWqsWmdSENpG5Rto04o2nVoWpXpf/dGm/EnxfJbUYlN1ig+jjyILKZ0odImwjLhoH20A00GY3YOr3N5MzUFPmYpVPXHpj6mvrY7qnJRxiWZZg3wBZ++Xuz6H9MSUMkwmzIPKatx/qEV19bn9GiMVMfG5cjQYt57Hgbx6F0MkBRkwwuqxCqama8t0IWrf0fwDYfcu5S2eLuu1k5cqb0Y39uE1i0AENYcfxXurd/3tQz2A78yYlRzknCqn9oXr4ltXa83awOtOq0hfNEKHYAq8to1UvrmWxixQwf1S/N77vamZrlT9zWKq8w+hYGzem5wcZLuxw6GCxOCihSQ2z5UrtcC5ghU6IPh+LG9qxbNxaAwojpr/NI3jJiTnUIf+yliDyLbMgDccTZbZpswLYBx8AoidM8X59Aqce3Zsq832os7olDHAHFdXlzPB3DKL2Jam9a7VETPsrPubLSva8miWIsIfgZupQoyBszi8ZrSBTqdzYXKRhgXesRya3T56yeTAFE3/e4f/9+jpjjmudSxfAGPptpvYjOzBgMIMBV5wA4UtbBCTn5q3v/pKpNGlSR6qL8YXxqs7juAcMNYibTpgU0J8rxJhbToznYs0ue016jToyWMvpch8gxCuwC8Nn9a+mc+0gUfqFVfovNktMmSv1Ac3xajM5q3RflBFdPdfHMLCvHAoMjA5aYlxnUycV72NILOtaOa0ej46O1BFa1PCnFy5NPktMWk33YdqszjKzi7ryVs92fdQgbJNNvdxOz6YhsdPsa3HI5d+tVmpL6BinsJrwdNV1nXc/1B3l2Q9tslOHLqo8PoZORKJL4bqid8ji2rwXE6+Ct3S0qB6zNOwRQBBOh08U3RditdtLswpN1BsysgUDRGRjztURULE7K4rNQ13V6n9sbkyhF0sm75XyRSY1XIybHnKBk26afslGkOXGrTmRrSdS4CgY4IqhHx97HVp1PtQY3vkj9OUMtSuCwpdOMmNZ61AOhBhX/fWi4cqZtKOU6GG9v8kZHLyH68upPz0dobPfgv9cqh91f87pclju3F8dERd+S7FZXIAr6/P0B42SAYhe03kVPtNg5H+ySIzXF1xA4d5JYCVke2X0H6XvJ+lzz39JlfWPHGAvVw8T1yFnlkHs2vJgp9k7B3/perRm9epuLq3s06ictod9uncxci5Xg0PAGpJuj8uyflVXN66JshyrPtT7r61pePOtHXt3IyZqn67FlMGZmDLGHqBwRFneSY24Y2GGD6BadDFAASKhoszqSJXfs9Sju2YJMR4saEs6FX3na+FbPDtk9Nm54f12LLx+KnYyXqMFj/j3S74QCLLj6/ZiSOVWgQuiQ62zsAdkkUWQ7xRimiKDbBpRAsY3x7rqoBQitwep/m+KtLqs2Wm7zHq1rpE8NMGuNXAeNQTkcRE8GKFrGzCmgSIvAvD1gY/1qkBSXYGGGt5EZa0JXNPWj62LRKTxA+EzQ/n7bOzM6icKHg201yM1mlz7H/Kf7s33vIMqxKgRLJ4gCLIxMLJ6YESuDZTpdSROWkh9O3ViE+VU40wPs8ArwEqWBVqtfGA3DMLJVDcMYbIAxUGzLj9zrJkwz2hfXQRPExYMX354MUABOisB4xp67Rxpw4ncbfBv6iy1T9+Q7g2V2NvFwEz8e+Kxhk5jJ2WAZIYOjJRFMslyDRcGzu2aOeB/VA1pJ9T2tuIpp8rp/KpbL8+bNkBkx6+WqrDV4uD6gyDy2JYoWWJTAku1dc8ZYmkLPJg/Tl/nfmL2MsT+dFFDsSse2Qs/l766BYhKYZqz1YrRiREYBFKQq9nWoDadKU21HVIL/saz2xyIvUdh3/1kf1+eO6bXZ1F+sjx0DK08GKOIQwF1ARIeBGBh65LUcHQghJcztyUSvLqkdwdYrkxnXckSmdMq8bmKI6oFwob5RO2jgdZq5bKNYjhGrl1dYr9dYr9cwrZ3BII4IZIlMCcuOkNK66b1RPSb9OosCXk+P1TqfqXZNLl4nbaQMSXMzKnsDrR6b3srIPW0KeKk8iMVqjwjQAGgOERDL17lyKL9MiJdgXoMRdQNljRmgiCHKvi1rBoY1EMJS1vOAQcUmvtPAMziDsrc3NW0KXRidM6/LymVrN9VKczynakw1Uo9g9nlMywxVzACxrHDWFS+IHBFS3ywN4ikRG5cAEV0TWv6T5RII1IGoQ78WY+e+dDJAAUQ1MDL8C6nQDoLLo2mVgqiGNejGMMHfJNeM3KwsKznJEteUQ7TscAGsAT5mlZYdv7XDufHnjZ3j3bxPa1Y8jKyObeVtcHpSnFBNpomIQSPhmJOxmZ1OIsfaZlS13QRQ2ABNAVgQcLVjBjuwLT1eLaOkVyeOSR64WrYP3308rtcYTw5sQwgIJAsTD+X3hIBCyPqZrXbTpHcAD0kcNSu8SOwDwNRIbmN3u/0rzPgXox5HxDKVN4ZkKAJWmpx0GBg9R/QcMSBPjJmLoDwFDEwAyQ5jcRgQAQxsNpSbB4scsZrOyIdmJxfGbABNxd/Z9GfdxYEF1B3HISFnnBQpjKc8CDv0yUDKQQQdCjZYBk0RIMdEi8JCUcD/BBCTi49JsRvulbx0tsin9BpGIEr5LklHpn0ODnw81HEjuq8wP1T3MRiRNDERIgYewLB8HI21M43mTF+Ddr9gC9lIhcrD+t7pAEUhTZgFV2YsxqASe21V186eGsYDhHxPSBoc/KrY7fMAGIm6QM63zS5oJwcH1TOnX+lZR2JORXLeBE0BRYud6VmnlLLGVMdTbJ/FlUPOBamOLv0vykA04NaRPuX7mASKrTlx91Qei1qSmLMz7NPO48jdsW3GF+ubKbeZqnLBUi9S9fthdDJAEeHyBlJ5Ptnnbcw3VA/bjUsuzPez5qMwTwqpJCHXZvGVtYPKVpYWKYekcuTGs1u4yJdjgDAMQ5G70O/ydEwL/O2TAeVuHo8xVUqHVpFP9CKJgSXAK5Is54tUK5Rt6qK7YINE0YqEbMVLbAq82oW8C70oKwrA1+eTVG1Gc3skDc4QXCX+2ZM3TxuBgojuAPgNAJd6/YeY+Z8R0SMAngLwOgCfBvCDzLwioksAvwDgLwL4fwB+gJm/tPk5nbp7ScTFoDt4oVNZtUNQiYKjbTwcXCsXBoN0GNT/HrVHMDMQAoYoCeMY5WIsok6liIj10GPgCAbJVgKaEcsMR8GUH01oyzFi3d8fzTqpjdj31KxPTntQNtWa6wCuTF8fsbaU2s8+7+heAFZLcZlCt/380/c9qEs5f9F1QFgu0K8Bjh2oWygqBHAgDFHfi8sYlals05FDaoOYdhnLFVIE0UXA1wUzMKh7trAR6P0xpYovK78VNh5KMSBdK8F2kF3c0icj75dclt1N7F8ig4UEtymIuZyQzOZh0QFX++cd3aZFrwB8DzPfI6IlgN8kol8D8A8B/CwzP0VE7wXwQwDeo58vMvN3EtGTAH4KwA9sesgQgSV1IGYMTKCoWafTjuNZ1O8SumpgCxM49BiFGAPolnLGI2reKi8bHU3f7Fji5ZPq0UdVJ2Qlox9Ytd8/lZ985ig+U2ep+sxt2C52IwWDQtvykkRphN5l8qIge3dI/Q1ACGBLg0+dZLvqxIBBWGQjJzldn6ZrcLG4yOqLtrffH8Sv7g0TQV2Te7JO4Cs3EJ4nQqg5qv2KbdohBCL0E9cPUxv52HoZRBBV++lyuP4NgFhq+Z5+XeofA/geAH9bz/88gHdDgOIJPQaADwH4N0REvEH2CWEBCguEQOBejDpL6gAE3chGU8oBWIx0tJA+S+YDus7PEGL16FiWdZvRbABjYRJFzAu8CD5rUOLUzfRp+naqxdjLkQ4nOtaUHnmtdo1G9OTk04oXGNRWZFJdFFWOIsByLvJ8FGXBBolNiDotcWCsrtZ4abECM2PdG+iLqsfDWlniSqKYqNxY5h0FkBb4yfOzt4rDaoLHifaZAopGu7WkDM+TTy9gmxBvWzZgUg6DelPDGdlhZK75/WkrGZFkOv0UgO8E8G8BfBHA15hTDvXnATysxw8D+D0AYOaeiL4O4E8A+L9Vme8E8E77/uIf3AMQEblHFxgx9hKVx4C4Kc2zQUAoazHlheByQANApzXuDWMsvJXntMMtFp02lrctaPExu9UIhD51Ns/LzIzacHV5FbqmXexQDbPaBENWB7uIocalgUT1wtRnaySAddzemCnFRmCd+fvmS2u8+LWvVxIMsI89pOP7o3PmDvVtSUTop0bnEWiqLXeeCyYkDe/yHT+bwXyYLWkroGBZbfIoEX0LgP8A4M+1LjO+Zn7zZb4PwPsAgIj+z4svvvhNVGByG7RapwH0eoz4cVICuDCIbUO7aohVJ2rw02StcaKxAG6HDpovHeA8GpmfHeth6weOypvt6M36aUNWXfAcuu9NBT/X7fSaC9OWZ8fXA/gz+5a/k9eDmb9GRL8O4B0AvoWIFipVfDuAL+tlzwN4A4DniWgB4I8D+IMN5X4bEX2Smd++6wtcF535maczP/N0ovy8cd/7N2a4IqJvU0kCRPQAgL8C4FkAHwPwN/SyvwvgP+rxR/Q79Pf/tsk+caYznem0aRuJ4iEAP692igDgg8z8q0T0WwCeIqJ/AeB/AXi/Xv9+AL9IRM9BJIknr4HvM53pTDdI23g9ngHw1sb53wHwWOP8fQB/cw9e3rfHPddJZ37m6czPPL2q+KGzVnCmM51pE51cFu4znelMp0e3DhRE9NeI6AtE9BwRveuWePgSEX2WiJ4mok/qudcR0UeJ6Lf181uv8fkfIKKvEtHn3Lnm80noX2t9PUNEb7tBnt5NRL+v9fQ0ET3ufvtx5ekLRPS9R+blDUT0MSJ6log+T0Q/qudvpY5m+LmV+tHy7xDRJ4joM8rTP9fzjxDRx7WOfpmILvT8pX5/Tn9/4+wDWnkTbuoPsvzziwDeBOACwGcAvOUW+PgSgNdX5/4lgHfp8bsA/NQ1Pv+7AbwNwOc2PR/A4wB+DRKv8g4AH79Bnt4N4B83rn2Ltt0lgEe0Tbsj8vIQgLfp8YMA/rc+81bqaIafW6kffQYBuKvHSwAf13f/IIAn9fx7Afx9Pf4HAN6rx08C+OW58m9bongMwHPM/DvMvIIsMnvilnkyegISmg79/OvX9SBm/g2MY02mnv8EgF9gof8BiWd56IZ4mqInADzFzFfM/LsAnkPD0H0ALy8w86f1+BsQ9/zDuKU6muFniq61fpQPZuappRYf0vN1HVndfQjAX6bJOPjbVz1SuLeSDwW/SWIA/5mIPqWh5QDwp5j5BUA6BoA/ecM8TT3/tuvsR1Sc/4BTx26MJxWR3wqZMW+9jip+gFusHyLqiOhpAF8F8FHssNQCgC21aNJtA8VW4d43QN/FzG8D8H0AfpiIvvsWeNiWbrPO3gPgOwA8CuAFAD99kzwR0V0AHwbwY8z8h3OX3hI/t1o/zDww86OQSOnHcISlFka3DRQW7m3kQ8FvjJj5y/r5VchalscAfMXEVf386g2zNfX8W6szZv6KdsYI4OeQxedr54kkxcGHAfwSM/+Knr61Omrxc5v144mZvwbg1+GWWjSem3iiLZZa3DZQ/E8Ab1bL7AXEqPKRm2SAiF5LRA/aMYC/CuBzKEPRfYj6TdHU8z8C4O+oZf8dAL5u4vd1U6Xnfz+knoynJ9WS/giANwP4xBGfS5CI32eZ+WfcT7dSR1P83Fb96LOvd6nFMS2ve1prH4dYjb8I4Cdu4flvglikPwPg88YDRF/7rwB+Wz9fd408/DuIqLqGIP0PTT0fIjLaUv/PAnj7DfL0i/rMZ7SjPeSu/wnl6QsAvu/IvPwliFj8DICn9e/x26qjGX5upX60/L8AWUrxDASg/qnr35+AGFD/PYBLPX9Hvz+nv79prvxzZOaZznSmjXTbqseZznSmVwCdgeJMZzrTRjoDxZnOdKaNdAaKM53pTBvpDBRnOtOZNtIZKM50pjNtpDNQnOlMZ9pIZ6A405nOtJH+P3WqGJqwmE0HAAAAAElFTkSuQmCC\n", 69 | "text/plain": [ 70 | "
" 71 | ] 72 | }, 73 | "metadata": { 74 | "needs_background": "light" 75 | }, 76 | "output_type": "display_data" 77 | } 78 | ], 79 | "source": [ 80 | "def get_bbox(path):\n", 81 | " image = Image.open(path)\n", 82 | " img_w, img_h = image.size\n", 83 | " w = int(img_w / 2.2)\n", 84 | " h = int(img_h / 2.2)\n", 85 | " x1 = img_w // 2 - w // 2\n", 86 | " y1 = img_h // 2 - h // 2\n", 87 | " x2 = img_w // 2 + w // 2\n", 88 | " y2 = img_h // 2 + h // 2\n", 89 | " bbox = (x1, y1, x2, y2)\n", 90 | " return np.array(image), bbox\n", 91 | "\n", 92 | "def viz_bbox(image, bbox):\n", 93 | " x1, y1, x2, y2 = bbox\n", 94 | " color = (255, 0, 0)\n", 95 | " o = 1\n", 96 | " image[y1-o:y1+o, x1:x2, :] = color\n", 97 | " image[y2-o:y2+o, x1:x2, :] = color\n", 98 | " image[y1:y2, x1-o:x1+o, :] = color\n", 99 | " image[y1:y2, x2-o:x2+o, :] = color\n", 100 | " plt.imshow(image)\n", 101 | " plt.show()\n", 102 | " \n", 103 | "image, bbox = get_bbox(root + '/Aaron_Eckhart/0/aligned_detect_0.555.jpg')\n", 104 | "viz_bbox(image, bbox)" 105 | ] 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "metadata": {}, 110 | "source": [ 111 | "Write all bbox info csv file, the header is ``Filename;Width;Height;X1;Y1;X2;Y2``" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": 10, 117 | "metadata": {}, 118 | "outputs": [ 119 | { 120 | "name": "stdout", 121 | "output_type": "stream", 122 | "text": [ 123 | "Abdullah\n", 124 | "Abid_Hamid_Mahmud_Al-Tikriti\n", 125 | "Abraham_Foxman\n", 126 | "Adriana_Lima\n", 127 | "Adriana_Perez_Navarro\n", 128 | "Adrian_Fernandez\n", 129 | "Adrien_Brody\n", 130 | "Ahmed_Qureia\n", 131 | "Ahmet_Necdet_Sezer\n", 132 | "Akbar_Hashemi_Rafsanjani\n", 133 | "Akhmed_Zakayev\n", 134 | "Alanna_Ubach\n", 135 | "Alecos_Markides\n", 136 | "Alex_Zanardi\n", 137 | "Alicia_Keys\n", 138 | "Ali_Khamenei\n", 139 | "Allison_Janney\n", 140 | "Alvaro_Noboa\n", 141 | "Amanda_Marsh\n", 142 | "Amelie_Mauresmo\n", 143 | "Amram_Mitzna\n", 144 | "Anders_Ebbeson\n", 145 | "Andres_DAlessandro\n", 146 | "Andrew_Firestone\n", 147 | "Andrew_Luster\n", 148 | "Andre_Bucher\n", 149 | "Andy_Garcia\n", 150 | "Angela_Merkel\n", 151 | "Angie_Arzola\n", 152 | "Anibal_Ibarra\n", 153 | "Barbara_Becker\n", 154 | "Barry_Hinson\n", 155 | "Beatrice_Dalle\n", 156 | "Benjamin_Bratt\n", 157 | "Bernard_Siegel\n", 158 | "Bertie_Ahern\n", 159 | "Bill_Cartwright\n", 160 | "Bill_Herrion\n", 161 | "Brennon_Leighton\n", 162 | "Brian_Billick\n", 163 | "Cabas\n", 164 | "Calbert_Cheaney\n", 165 | "Candice_Bergen\n", 166 | "Carla_Tricoli\n", 167 | "Carlos_Alberto\n", 168 | "Carlos_Moya\n", 169 | "Carlton_Dotson\n", 170 | "Carmen_Electra\n", 171 | "Carroll_Weimer\n", 172 | "Catherine_Bell\n", 173 | "Catherine_Deneuve\n", 174 | "Catherine_Ndereba\n", 175 | "Chakib_Khelil\n", 176 | "Charles_Mathews\n", 177 | "Charles_Rogers\n", 178 | "Charles_Tannok\n", 179 | "Charlie_Hunnam\n", 180 | "Chelsea_Clinton\n", 181 | "Chen_Kaige\n", 182 | "Chen_Shui-bian\n", 183 | "Cheryl_Hines\n", 184 | "Christian_Gimenez\n", 185 | "Christian_Malcolm\n", 186 | "Christian_Olsson\n", 187 | "Christopher_Matero\n", 188 | "Christoph_Daum\n", 189 | "Chris_Columbus\n", 190 | "Clara_Harris\n", 191 | "Claudia_Cardinale\n", 192 | "Claudia_Pechstein\n", 193 | "Claudia_Schiffer\n", 194 | "Claudio_Ranieri\n", 195 | "Colin_Farrell\n", 196 | "Compay_Segundo\n", 197 | "Corliss_Williamson\n", 198 | "Costas_Simitis\n", 199 | "Craig_MacTavish\n", 200 | "Craig_OClair\n", 201 | "Dale_Earnhardt\n", 202 | "Daniela_Hantuchova\n", 203 | "Danny_Ainge\n", 204 | "Dan_Guerrero\n", 205 | "Darren_Clarke\n", 206 | "Dave_McGinnis\n", 207 | "David_Canary\n", 208 | "Eddie_Fenech_Adami\n", 209 | "Edmund_Hillary\n", 210 | "Edward_Flynn\n", 211 | "Eileen_Coparropa\n", 212 | "Elgin_Baylor\n", 213 | "Elinor_Caplan\n", 214 | "Eliott_Spitzer\n", 215 | "Elizabeth_Hurley\n", 216 | "Elizabeth_Pena\n", 217 | "Elizabeth_Regan\n", 218 | "Elizabeth_Smart\n", 219 | "Ellen_Engleman\n", 220 | "Elodie_Bouchez\n", 221 | "Emily_Robison\n", 222 | "Emmanuelle_Beart\n", 223 | "Enrique_Haroldo_Gorriaran_Merlo\n", 224 | "Eric_Lindros\n", 225 | "Eric_Rosser\n", 226 | "Eric_Vigouroux\n", 227 | "Erin_Brockovich\n", 228 | "Esther_Macklin\n", 229 | "Etta_James\n", 230 | "Eugene_Melnyk\n", 231 | "Evander_Holyfield\n", 232 | "Farouk_Kaddoumi\n", 233 | "Fazal-ur-Rehman\n", 234 | "Federico_Trillo\n", 235 | "Flavia_Delaroli\n", 236 | "Francesco_Totti\n", 237 | "Francis_Mer\n", 238 | "Frank_Griswold\n", 239 | "Franz_Beckenbauer\n", 240 | "Gabrielle_Union\n", 241 | "Gavin_Degraw\n", 242 | "Gene_Keady\n", 243 | "George_Allen\n", 244 | "George_Tenet\n", 245 | "Georgi_Parvanov\n", 246 | "Habib_Hisham\n", 247 | "Hamad_Bin_Jassim\n", 248 | "Hamid_Karzai\n", 249 | "Hamid_Reza_Asefi\n", 250 | "Hannah_Stockbauer\n", 251 | "Hans_Blix\n", 252 | "Harbhajan_Singh\n", 253 | "Hartmut_Mehdorn\n", 254 | "Harvey_Weinstein\n", 255 | "Hasan_Wirayuda\n", 256 | "Hashim_Thaci\n", 257 | "Ian_Thorpe\n", 258 | "Irina_Lobacheva\n", 259 | "Islam_Karimov\n", 260 | "Ivan_Shvedoff\n", 261 | "Jaap_de_Hoop_Scheffer\n", 262 | "Jackie_Chan\n", 263 | "Jacob_Frenkel\n", 264 | "Jacqueline_Obradors\n", 265 | "Jada_Pinkett_Smith\n", 266 | "Jalen_Rose\n", 267 | "James_Brosnahan\n", 268 | "James_Brown\n", 269 | "James_Franco\n", 270 | "James_Kopp\n", 271 | "James_Phelps\n", 272 | "James_Watt\n", 273 | "Jane_Leeves\n", 274 | "Jane_Russell\n", 275 | "Janice_Abreu\n", 276 | "Jaromir_Jagr\n", 277 | "Jason_Alexander\n", 278 | "JC_Chasez\n", 279 | "Katie_Harman\n", 280 | "Katie_Smith\n", 281 | "Keanu_Reeves\n", 282 | "Kelly_Ripa\n", 283 | "Kevin_Keegan\n", 284 | "Kevin_Tarrant\n", 285 | "Kim_Clijsters\n", 286 | "Kurt_Schottenheimer\n", 287 | "Kyra_Sedgwick\n", 288 | "Laila_Ali\n", 289 | "Lara_Logan\n", 290 | "Larenz_Tate\n", 291 | "Larry_Donald\n", 292 | "Larry_Hagman\n", 293 | "Larry_Johnson\n", 294 | "Laura_Hernandez\n", 295 | "Laura_Romero\n", 296 | "Leah_Remini\n", 297 | "Lennart_Johansson\n", 298 | "Leonard_Hamilton\n", 299 | "Leon_Lai\n", 300 | "Leo_Ramirez\n", 301 | "Lew_Rywin\n", 302 | "Liam_Neeson\n", 303 | "Lili_Taylor\n", 304 | "Lima_Azimi\n", 305 | "Lin_Yi-fu\n", 306 | "Lionel_Chalmers\n", 307 | "Lisa_Raymond\n", 308 | "Lisa_Stansfield\n", 309 | "Liu_Xiaoqing\n", 310 | "Liza_Minnelli\n", 311 | "Li_Changchun\n", 312 | "Li_Ka-shing\n", 313 | "LK_Advani\n", 314 | "Lokendra_Bahadur_Chand\n", 315 | "Luca_Cordero_di_Montezemolo\n", 316 | "Ludwig_Ovalle\n", 317 | "Lynn_Abraham\n", 318 | "Lynn_Redgrave\n", 319 | "Madonna\n", 320 | "Mae_Jemison\n", 321 | "Mahmoud_Al_Zhar\n", 322 | "Mariah_Carey\n", 323 | "Marina_Anissina\n", 324 | "Mario_Vasquez_Rana\n", 325 | "Markus_Beyer\n", 326 | "Mark_Shapiro\n", 327 | "Mary_Lou_Retton\n", 328 | "Matt_Doherty\n", 329 | "Nabil_Shaath\n", 330 | "Naomi_Watts\n", 331 | "Natalia_Dmitrieva\n", 332 | "Na_Na_Keum\n", 333 | "Nelson_Acosta\n", 334 | "Nestor_Santillan\n", 335 | "Nicolas_Massu\n", 336 | "Nikki_Cascone\n", 337 | "Nikki_Reed\n", 338 | "Nikolay_Davydenko\n", 339 | "Norm_Coleman\n", 340 | "Omar_Sharif\n", 341 | "Omar_Vizquel\n", 342 | "Oprah_Winfrey\n", 343 | "Oscar_Bolanos\n", 344 | "Oswaldo_Paya\n", 345 | "Owen_Wilson\n", 346 | "Padraig_Harrington\n", 347 | "Parthiv_Patel\n", 348 | "Patrice_Chereau\n", 349 | "Patricia_Medina\n", 350 | "Patricia_Phillips\n", 351 | "Patrick_Coleman\n", 352 | "Patti_Smith\n", 353 | "Pat_Burns\n", 354 | "Paula_Abdul\n", 355 | "Paula_Locke\n", 356 | "Paulie_Ayala\n", 357 | "Paul_Hogan\n", 358 | "Paul_Luvera\n", 359 | "Paul_Newman\n", 360 | "Peter_Bacanovic\n", 361 | "Peter_Hartz\n", 362 | "Peter_Hillary\n", 363 | "Peter_Max\n", 364 | "Pharrell_Williams\n", 365 | "Phillipe_Comtois\n", 366 | "Phillip_Fulmer\n", 367 | "Pierre_Gagnon\n", 368 | "Placido_Domingo\n", 369 | "Porter_Goss\n", 370 | "Portia_de_Rossi\n", 371 | "Prince_Philippe\n", 372 | "Queen_Beatrix\n", 373 | "Queen_Elizabeth_II\n", 374 | "Rachel_Leigh_Cook\n", 375 | "Raghad_Saddam_Hussein\n", 376 | "Ralph_Fiennes\n", 377 | "Randy_Johnson\n", 378 | "Ray_Nagin\n", 379 | "Richard_Gere\n", 380 | "Robert_Blackwill\n", 381 | "Robin_Wagner\n", 382 | "Roger_Suarez\n", 383 | "Roman_Coppola\n", 384 | "Samantha_Daniels\n", 385 | "Samira_Makhmalbaf\n", 386 | "Sarah_Michelle_Gellar\n", 387 | "Sasha_Alexander\n", 388 | "Scott_Blum\n", 389 | "S_Jayakumar\n", 390 | "Taha_Yassin_Ramadan\n", 391 | "Takaloo\n", 392 | "Takashi_Sorimachi\n", 393 | "Takashi_Yamamoto\n", 394 | "Takeshi_Kitano\n", 395 | "Tanya_Holyk\n", 396 | "Tayshaun_Prince\n", 397 | "Teresa_Graves\n", 398 | "Terry_Semel\n", 399 | "Thalia\n", 400 | "Theo_Epstein\n", 401 | "Thomas_Gottschalk\n", 402 | "Thomas_OBrien\n", 403 | "Thor_Pedersen\n", 404 | "Tiago_Splitter\n", 405 | "Tim_Salmon\n", 406 | "Tina_Andrews\n", 407 | "Tina_Fey\n", 408 | "Tomas_Enge\n", 409 | "Tom_McClintock\n", 410 | "Tom_OBrien\n", 411 | "Valeri_Bure\n", 412 | "Vanessa_Incontrada\n", 413 | "Vanessa_Laine\n", 414 | "Victoria_Beckham\n", 415 | "Victor_Kraatz\n", 416 | "Vinnie_Jones\n", 417 | "Vin_Diesel\n", 418 | "Wang_Nan\n", 419 | "Wayne_Brady\n", 420 | "William_Joppy\n", 421 | "William_Morrow\n", 422 | "William_Pryor_Jr\n", 423 | "Will_Self\n", 424 | "Wilma_McNabb\n", 425 | "Win_Aung\n", 426 | "Woody_Allen\n", 427 | "Yao_Ming\n", 428 | "Yasushi_Akashi\n", 429 | "Yoon_Jin-Sik\n", 430 | "Yuvraj_Singh\n", 431 | "Yu_Shyi-kun\n", 432 | "Zalmay_Khalilzad\n", 433 | "Zarai_Toledo\n", 434 | "Zeljko_Rebraca\n" 435 | ] 436 | } 437 | ], 438 | "source": [ 439 | "def write_bbox(subjects):\n", 440 | " for subject, video_frames in subjects.items():\n", 441 | " print(subject)\n", 442 | " csv_path = os.path.join(root, subject, subject+\".csv\")\n", 443 | " with open(csv_path, 'w') as f:\n", 444 | " f.write('Filename;Width;Height;X1;Y1;X2;Y2\\n')\n", 445 | " for video_frame in video_frames:\n", 446 | " image_path = os.path.join(root, subject, video_frame)\n", 447 | " image, bbox = get_bbox(image_path)\n", 448 | " H, W, _ = image.shape\n", 449 | " if bbox is None:\n", 450 | " continue\n", 451 | " entry = [video_frame.replace('\\\\', '/'), W, H, *bbox]\n", 452 | " entry = ';'.join([str(e) for e in entry])\n", 453 | " f.write(entry + '\\n')\n", 454 | " \n", 455 | "write_bbox(subjects)" 456 | ] 457 | } 458 | ], 459 | "metadata": { 460 | "kernelspec": { 461 | "display_name": "Python 3", 462 | "language": "python", 463 | "name": "python3" 464 | }, 465 | "language_info": { 466 | "codemirror_mode": { 467 | "name": "ipython", 468 | "version": 3 469 | }, 470 | "file_extension": ".py", 471 | "mimetype": "text/x-python", 472 | "name": "python", 473 | "nbconvert_exporter": "python", 474 | "pygments_lexer": "ipython3", 475 | "version": "3.7.3" 476 | }, 477 | "toc": { 478 | "base_numbering": 1, 479 | "nav_menu": {}, 480 | "number_sections": true, 481 | "sideBar": true, 482 | "skip_h1_title": false, 483 | "title_cell": "Table of Contents", 484 | "title_sidebar": "Contents", 485 | "toc_cell": false, 486 | "toc_position": {}, 487 | "toc_section_display": true, 488 | "toc_window_display": false 489 | } 490 | }, 491 | "nbformat": 4, 492 | "nbformat_minor": 2 493 | } 494 | --------------------------------------------------------------------------------