├── tidecv ├── __init__.py ├── errors │ ├── qualifiers.py │ ├── main_errors.py │ └── error.py ├── functions.py ├── data.py ├── plotting.py ├── ap.py ├── datasets.py └── quantify.py ├── .gitignore ├── examples ├── minimal_example.py └── coco_instance_segmentation.ipynb ├── CHANGELOG.md ├── LICENSE ├── setup.py └── README.md /tidecv/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '1.0.1' 2 | 3 | from .quantify import * 4 | from .errors.qualifiers import * 5 | from . import datasets 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | .vscode/ 3 | build/ 4 | dist/ 5 | *.egg-info 6 | .ipynb* 7 | examples/mask_rcnn_mask.json 8 | examples/mask_rcnn_bbox.json 9 | examples/mask_rcnn_lvis.json 10 | minimal_example.py 11 | figs/ 12 | -------------------------------------------------------------------------------- /examples/minimal_example.py: -------------------------------------------------------------------------------- 1 | # The bare-bones example from the README. 2 | # Run coco_example.py first to get mask_rcnn_bbox.json 3 | from tidecv import TIDE, datasets 4 | 5 | tide = TIDE() 6 | tide.evaluate(datasets.COCO(), datasets.COCOResult('mask_rcnn_bbox.json'), mode=TIDE.BOX) 7 | tide.summarize() 8 | tide.plot() 9 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # TIDE Changelog 2 | 3 | ## v1.0.1 4 | ### Error Calculation 5 | - **Important**: Fixed an oversight where detections ignored by AP calculation were not allowed to contribute to fixing errors. This caused a lot of error in datasets with large amounts of ignore regions (e.g., LVIS) to significantly overrepresent Missed Error and underrepresent either Classification or Localization error. This fix will also slightly change errors for other datasets (< .4 dAP on COCO), but conclusions on LVIS change dramatically. 6 | ### Plotting 7 | - Fixed issue with pie plots sometimes having negative padding. 8 | - Added automatic rescaling to the bar plots. This scalining will persist for the entire TIDE object, so you can compare between runs. 9 | ### Datasets 10 | - Fixed bug in the LVIS automatic download script that caused it to crash after downloading. 11 | 12 | 13 | ## v1.0.0 14 | - Initial release. 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 Daniel Bolya 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | from setuptools import setup 3 | 4 | # The directory containing this file 5 | HERE = pathlib.Path(__file__).parent 6 | 7 | # The text of the README file 8 | README = (HERE / "README.md").read_text(encoding='utf8') 9 | 10 | # This call to setup() does all the work 11 | setup( 12 | name="tidecv", 13 | version="1.0.1", 14 | description="A General Toolbox for Identifying ObjectDetection Errors", 15 | long_description=README, 16 | long_description_content_type="text/markdown", 17 | url="https://github.com/dbolya/tide", 18 | author="Daniel Bolya", 19 | author_email="dbolya@gatech.edu", 20 | license="MIT", 21 | classifiers=[ 22 | "License :: OSI Approved :: MIT License", 23 | "Programming Language :: Python :: 3", 24 | "Programming Language :: Python :: 3.6", 25 | "Operating System :: OS Independent" 26 | ], 27 | python_requires='>=3.6', 28 | packages=["tidecv", "tidecv.errors"], 29 | include_package_data=True, 30 | install_requires=["appdirs", "numpy", "pycocotools", "opencv-python", "seaborn", "pandas", "matplotlib"], 31 | # entry_points={ 32 | # "console_scripts": [ 33 | # "tidecv=tidecv.__main__:main", 34 | # ] 35 | # }, 36 | ) 37 | -------------------------------------------------------------------------------- /tidecv/errors/qualifiers.py: -------------------------------------------------------------------------------- 1 | # Defines qualifiers like "Extra small box" 2 | 3 | 4 | 5 | def _area(x): 6 | return x['bbox'][2] * x['bbox'][3] 7 | 8 | def _ar(x): 9 | return x['bbox'][2] / x['bbox'][3] 10 | 11 | 12 | 13 | class Qualifier(): 14 | """ 15 | Creates a qualifier with the given name. 16 | 17 | test_func should be a callable object (e.g., lambda) that takes in as input an annotation 18 | object (either a ground truth or prediction) and returns whether or not that object qualifies (i.e., a bool). 19 | """ 20 | 21 | def __init__(self, name:str, test_func:object): 22 | self.test = test_func 23 | self.name = name 24 | 25 | # This is horrible, but I like it 26 | def _make_error_func(self, error_type): 27 | return (lambda err: isinstance(err, error_type) \ 28 | and (self.test(err.gt) if hasattr(err, 'gt') else self.test(err.pred))) \ 29 | if self.test is not None else (lambda err: isinstance(err, error_type)) 30 | 31 | 32 | 33 | 34 | AREA = [ 35 | Qualifier('Small' , lambda x: _area(x) <= 32 ** 2), 36 | Qualifier('Medium', lambda x: 32 ** 2 < _area(x) <= 96 ** 2), 37 | Qualifier('Large' , lambda x: 96 ** 2 < _area(x) ), 38 | ] 39 | 40 | ASPECT_RATIO = [ 41 | Qualifier('Tall' , lambda x: _ar(x) <= 0.75), 42 | Qualifier('Square', lambda x: 0.75 < _ar(x) <= 1.33), 43 | Qualifier('Wide' , lambda x: 1.33 < _ar(x) ), 44 | ] 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /tidecv/errors/main_errors.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | import numpy as np 3 | 4 | from .error import Error, BestGTMatch 5 | 6 | class ClassError(Error): 7 | 8 | description = "Error caused when a prediction would have been marked positive " \ 9 | + "if it had the correct class." 10 | short_name = "Cls" 11 | 12 | def __init__(self, pred:dict, gt:dict, ex): 13 | self.pred = pred 14 | self.gt = gt 15 | 16 | self.match = BestGTMatch(pred, gt) if not self.gt['used'] else None 17 | 18 | def fix(self): 19 | if self.match is None: 20 | return None 21 | return self.gt['class'], self.match.fix() 22 | 23 | 24 | class BoxError(Error): 25 | 26 | description = "Error caused when a prediction would have been marked positive if it was localized better." 27 | short_name = "Loc" 28 | 29 | def __init__(self, pred:dict, gt:dict, ex): 30 | self.pred = pred 31 | self.gt = gt 32 | 33 | self.match = BestGTMatch(pred, gt) if not self.gt['used'] else None 34 | 35 | def fix(self): 36 | if self.match is None: 37 | return None 38 | return self.pred['class'], self.match.fix() 39 | 40 | 41 | class DuplicateError(Error): 42 | 43 | description = "Error caused when a prediction would have been marked positive " \ 44 | + "if the GT wasn't already in use by another detection." 45 | short_name = "Dupe" 46 | 47 | def __init__(self, pred:dict, suppressor: dict): 48 | self.pred = pred 49 | self.suppressor = suppressor 50 | 51 | def fix(self): 52 | return None 53 | 54 | class BackgroundError(Error): 55 | 56 | description = "Error caused when this detection should have been classified as background (IoU < 0.1)." 57 | short_name = "Bkg" 58 | 59 | def __init__(self, pred:dict): 60 | self.pred = pred 61 | 62 | def fix(self): 63 | return None 64 | 65 | 66 | class OtherError(Error): 67 | 68 | description = "This detection didn't fall into any of the other error categories." 69 | short_name = "Both" 70 | 71 | def __init__(self, pred:dict): 72 | self.pred = pred 73 | 74 | def fix(self): 75 | return None 76 | 77 | 78 | class MissedError(Error): 79 | 80 | description = "Represents GT missed by the model. Doesn't include GT corrected elsewhere in the model." 81 | short_name = "Miss" 82 | 83 | def __init__(self, gt:dict): 84 | self.gt = gt 85 | 86 | def fix(self): 87 | return self.gt['class'], -1 88 | 89 | 90 | # These are special errors so no inheritence 91 | 92 | class FalsePositiveError: 93 | 94 | description = "Represents the potential AP gained by having perfect precision" \ 95 | + " (e.g., by scoring all false positives as conf=0) without affecting recall." 96 | short_name = "FalsePos" 97 | 98 | @staticmethod 99 | def fix(score:float, correct:bool, info:dict) -> tuple: 100 | if correct: 101 | return 1, True, info 102 | else: 103 | return 0, False, info 104 | 105 | 106 | class FalseNegativeError: 107 | 108 | description = "Represents the potentially AP gained by having perfect recall" \ 109 | + " without affecting precision." 110 | short_name = "FalseNeg" 111 | -------------------------------------------------------------------------------- /tidecv/functions.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | import os, sys 5 | 6 | def mean(arr:list): 7 | if len(arr) == 0: 8 | return 0 9 | return sum(arr) / len(arr) 10 | 11 | def find_first(arr:np.array) -> int: 12 | """ Finds the index of the first instance of true in a vector or None if not found. """ 13 | if len(arr) == 0: 14 | return None 15 | idx = arr.argmax() 16 | 17 | # Numpy argmax will return 0 if no True is found 18 | if idx == 0 and not arr[0]: 19 | return None 20 | 21 | return idx 22 | 23 | def isiterable(x): 24 | try: 25 | iter(x) 26 | return True 27 | except: 28 | return False 29 | 30 | def recursive_sum(x): 31 | if isinstance(x, dict): 32 | return sum([recursive_sum(v) for v in x.values()]) 33 | elif isiterable(x): 34 | return sum([recursive_sum(v) for v in x]) 35 | else: 36 | return x 37 | 38 | def apply_messy(x:list, func): 39 | return [([func(y) for y in e] if isiterable(e) else func(e)) for e in x] 40 | 41 | def apply_messy2(x:list, y:list, func): 42 | return [[func(i, j) for i, j in zip(a, b)] if isiterable(a) else func(a, b) for a, b in zip(x, y)] 43 | 44 | def multi_len(x): 45 | try: 46 | return len(x) 47 | except TypeError: 48 | return 1 49 | 50 | def unzip(l): 51 | return map(list, zip(*l)) 52 | 53 | 54 | def points(bbox): 55 | bbox = [int(x) for x in bbox] 56 | return (bbox[0], bbox[1]), (bbox[0]+bbox[2], bbox[1]+bbox[3]) 57 | 58 | def nonepack(t): 59 | if t is None: 60 | return None, None 61 | else: 62 | return t 63 | 64 | 65 | class HiddenPrints: 66 | """ From https://stackoverflow.com/questions/8391411/suppress-calls-to-print-python """ 67 | 68 | def __enter__(self): 69 | self._original_stdout = sys.stdout 70 | sys.stdout = open(os.devnull, 'w') 71 | 72 | def __exit__(self, exc_type, exc_val, exc_tb): 73 | sys.stdout.close() 74 | sys.stdout = self._original_stdout 75 | 76 | 77 | 78 | 79 | def toRLE(mask:object, w:int, h:int): 80 | """ 81 | Borrowed from Pycocotools: 82 | Convert annotation which can be polygons, uncompressed RLE to RLE. 83 | :return: binary mask (numpy 2D array) 84 | """ 85 | import pycocotools.mask as maskUtils 86 | 87 | if type(mask) == list: 88 | # polygon -- a single object might consist of multiple parts 89 | # we merge all parts into one mask rle code 90 | rles = maskUtils.frPyObjects(mask, h, w) 91 | return maskUtils.merge(rles) 92 | elif type(mask['counts']) == list: 93 | # uncompressed RLE 94 | return maskUtils.frPyObjects(mask, h, w) 95 | else: 96 | return mask 97 | 98 | 99 | def polyToBox(poly:list): 100 | """ Converts a polygon in COCO lists of lists format to a bounding box in [x, y, w, h]. """ 101 | 102 | xmin = 1e10 103 | xmax = -1e10 104 | ymin = 1e10 105 | ymax = -1e10 106 | 107 | for poly_comp in poly: 108 | for i in range(len(poly_comp) // 2): 109 | x = poly_comp[2*i + 0] 110 | y = poly_comp[2*i + 1] 111 | 112 | xmin = min(x, xmin) 113 | xmax = max(x, xmax) 114 | ymin = min(y, ymin) 115 | ymax = max(y, ymax) 116 | 117 | return [xmin, ymin, (xmax - xmin), (ymax - ymin)] 118 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A General **T**oolbox for **I**dentifying Object **D**etection **E**rrors 2 | ``` 3 | ████████╗██╗██████╗ ███████╗ 4 | ╚══██╔══╝██║██╔══██╗██╔════╝ 5 | ██║ ██║██║ ██║█████╗ 6 | ██║ ██║██║ ██║██╔══╝ 7 | ██║ ██║██████╔╝███████╗ 8 | ╚═╝ ╚═╝╚═════╝ ╚══════╝ 9 | ``` 10 | 11 | An easy-to-use, general toolbox to compute and evaluate the effect of object detection and instance segmentation on overall performance. This is the code for our paper: [TIDE: A General Toolbox for Identifying Object Detection Errors](https://dbolya.github.io/tide/paper.pdf) ([ArXiv](https://arxiv.org/abs/2008.08115)) [ECCV2020 Spotlight]. 12 | 13 | Check out our ECCV 2020 short video for an explanation of what TIDE can do: 14 | 15 | [![TIDE Introduction](https://img.youtube.com/vi/McYFYU3PXcU/0.jpg)](https://youtu.be/McYFYU3PXcU) 16 | 17 | # Installation 18 | 19 | TIDE is available as a python package for python 3.6+ as [tidecv](https://pypi.org/project/tidecv/). To install, simply install it with pip: 20 | ```shell 21 | pip3 install tidecv 22 | ``` 23 | 24 | The current version is v1.0.1 ([changelog](https://github.com/dbolya/tide/blob/master/CHANGELOG.md)). 25 | 26 | # Usage 27 | TIDE is meant as a drop-in replacement for the [COCO Evaluation toolkit](https://github.com/cocodataset/cocoapi), and getting started is easy: 28 | 29 | ```python 30 | from tidecv import TIDE, datasets 31 | 32 | tide = TIDE() 33 | tide.evaluate(datasets.COCO(), datasets.COCOResult('path/to/your/results/file'), mode=TIDE.BOX) # Use TIDE.MASK for masks 34 | tide.summarize() # Summarize the results as tables in the console 35 | tide.plot() # Show a summary figure. Specify a folder and it'll output a png to that folder. 36 | ``` 37 | 38 | This prints evaluation summary tables to the console: 39 | ``` 40 | -- mask_rcnn_bbox -- 41 | 42 | bbox AP @ 50: 61.80 43 | 44 | Main Errors 45 | ============================================================= 46 | Type Cls Loc Both Dupe Bkg Miss 47 | ------------------------------------------------------------- 48 | dAP 3.40 6.65 1.18 0.19 3.96 7.53 49 | ============================================================= 50 | 51 | Special Error 52 | ============================= 53 | Type FalsePos FalseNeg 54 | ----------------------------- 55 | dAP 16.28 15.57 56 | ============================= 57 | ``` 58 | 59 | And a summary plot for your model's errors: 60 | 61 | ![A summary plot](https://dbolya.github.io/tide/mask_rcnn_bbox_bbox_summary.png) 62 | 63 | ## Jupyter Notebook 64 | 65 | Check out the [example notebook](https://github.com/dbolya/tide/blob/master/examples/coco_instance_segmentation.ipynb) for more details. 66 | 67 | 68 | # Datasets 69 | The currently supported datasets are COCO, LVIS, Pascal, and Cityscapes. More details and documentation on how to write your own database drivers coming soon! 70 | 71 | # Citation 72 | If you use TIDE in your project, please cite 73 | ``` 74 | @inproceedings{tide-eccv2020, 75 | author = {Daniel Bolya and Sean Foley and James Hays and Judy Hoffman}, 76 | title = {TIDE: A General Toolbox for Identifying Object Detection Errors}, 77 | booktitle = {ECCV}, 78 | year = {2020}, 79 | } 80 | ``` 81 | 82 | ## Contact 83 | For questions about our paper or code, make an issue in this github or contact [Daniel Bolya](mailto:dbolya@gatech.edu). Note that I may not respond to emails, so github issues are your best bet. 84 | -------------------------------------------------------------------------------- /tidecv/errors/error.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | import cv2 4 | from .. import functions as f 5 | 6 | class Error: 7 | """ A base class for all error types. """ 8 | 9 | def fix(self) -> Union[tuple, None]: 10 | """ 11 | Returns a fixed version of the AP data point for this error or 12 | None if this error should be suppressed. 13 | 14 | Return type is: 15 | class:int, (score:float, is_positive:bool, info:dict) 16 | """ 17 | raise NotImplementedError 18 | 19 | def unfix(self) -> Union[tuple, None]: 20 | """ Returns the original version of this data point. """ 21 | 22 | if hasattr(self, 'pred'): 23 | # If an ignored instance is an error, it's not in the data point list, so there's no "unfixed" entry 24 | if self.pred['used'] is None: return None 25 | else: return self.pred['class'], (self.pred['score'], False, self.pred['info']) 26 | else: 27 | return None 28 | 29 | def get_id(self) -> int: 30 | if hasattr(self, 'pred'): 31 | return self.pred['_id'] 32 | elif hasattr(self, 'gt'): 33 | return self.gt['_id'] 34 | else: 35 | return -1 36 | 37 | 38 | def show(self, dataset, out_path:str=None, 39 | pred_color:tuple=(43, 12, 183), gt_color:tuple=(43, 183, 12), 40 | font=cv2.FONT_HERSHEY_SIMPLEX): 41 | 42 | pred = self.pred if hasattr(self, 'pred') else self.gt 43 | img = dataset.get_img_with_anns(pred['image_id']) 44 | 45 | 46 | if hasattr(self, 'gt'): 47 | img = cv2.rectangle(img, *f.points(self.gt['bbox']), gt_color, 2) 48 | img = cv2.putText(img, dataset.cat_name(self.gt['category_id']), 49 | (100, 200), font, 1, gt_color, 2, cv2.LINE_AA, False) 50 | 51 | if hasattr(self, 'pred'): 52 | img = cv2.rectangle(img, *f.points(pred['bbox']), pred_color, 2) 53 | img = cv2.putText(img, '%s (%.2f)' % (dataset.cat_name(pred['category_id']), pred['score']), 54 | (100, 100), font, 1, pred_color, 2, cv2.LINE_AA, False) 55 | 56 | if out_path is None: 57 | cv2.imshow(self.short_name, img) 58 | cv2.moveWindow(self.short_name, 100, 100) 59 | 60 | cv2.waitKey() 61 | cv2.destroyAllWindows() 62 | else: 63 | cv2.imwrite(out_path, img) 64 | 65 | def get_info(self, dataset): 66 | info = {} 67 | info['type'] = self.short_name 68 | 69 | if hasattr(self, 'gt'): 70 | info['gt'] = self.gt 71 | if hasattr(self, 'pred'): 72 | info['pred'] = self.pred 73 | 74 | img_id = (self.pred if hasattr(self, 'pred') else self.gt)['image_id'] 75 | info['all_gt'] = dataset.get(img_id) 76 | info['img'] = dataset.get_img(img_id) 77 | 78 | return info 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | class BestGTMatch: 88 | """ 89 | Some errors are fixed by changing false positives to true positives. 90 | The issue with fixing these errors naively is that you might have 91 | multiple errors attempting to fix the same GT. In that case, we need 92 | to select which error actually gets fixed, and which others just get 93 | suppressed (since we can only fix one error per GT). 94 | 95 | To address this, this class finds the prediction with the hiighest 96 | score and then uses that as the error to fix, while suppressing all 97 | other errors caused by the same GT. 98 | """ 99 | 100 | def __init__(self, pred, gt): 101 | self.pred = pred 102 | self.gt = gt 103 | 104 | if self.gt['used']: 105 | self.suppress = True 106 | else: 107 | self.suppress = False 108 | self.gt['usable'] = True 109 | 110 | score = self.pred['score'] 111 | 112 | if not 'best_score' in self.gt: 113 | self.gt['best_score'] = -1 114 | 115 | if self.gt['best_score'] < score: 116 | self.gt['best_score'] = score 117 | self.gt['best_id'] = self.pred['_id'] 118 | 119 | def fix(self): 120 | if self.suppress or self.gt['best_id'] != self.pred['_id']: 121 | return None 122 | else: 123 | return (self.pred['score'], True, self.pred['info']) 124 | -------------------------------------------------------------------------------- /tidecv/data.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from collections import defaultdict 4 | import numpy as np 5 | import cv2 6 | 7 | from . import functions as f 8 | 9 | class Data(): 10 | """ 11 | A class to hold ground truth or predictions data in an easy to work with format. 12 | Note that any time they appear, bounding boxes are [x, y, width, height] and masks 13 | are either a list of polygons or pycocotools RLEs. 14 | 15 | Also, don't mix ground truth with predictions. Keep them in separate data objects. 16 | 17 | 'max_dets' specifies the maximum number of detections the model is allowed to output for a given image. 18 | """ 19 | 20 | def __init__(self, name:str, max_dets:int=100): 21 | self.name = name 22 | self.max_dets = max_dets 23 | 24 | self.classes = {} # Maps class ID to class name 25 | self.annotations = [] # Maps annotation ids to the corresponding annotation / prediction 26 | 27 | # Maps an image id to an image name and a list of annotation ids 28 | self.images = defaultdict(lambda: {'name': None, 'anns': []}) 29 | 30 | 31 | def _get_ignored_classes(self, image_id:int) -> set: 32 | anns = self.get(image_id) 33 | 34 | classes_in_image = set() 35 | ignored_classes = set() 36 | 37 | for ann in anns: 38 | if ann['ignore']: 39 | if ann['class'] is not None and ann['bbox'] is None and ann['mask'] is None: 40 | ignored_classes.add(ann['class']) 41 | else: 42 | classes_in_image.add(ann['class']) 43 | 44 | return ignored_classes.difference(classes_in_image) 45 | 46 | 47 | def _make_default_class(self, id:int): 48 | """ (For internal use) Initializes a class id with a generated name. """ 49 | 50 | if id not in self.classes: 51 | self.classes[id] = 'Class ' + str(id) 52 | 53 | def _make_default_image(self, id:int): 54 | if self.images[id]['name'] is None: 55 | self.images[id]['name'] = 'Image ' + str(id) 56 | 57 | def _prepare_box(self, box:object): 58 | return box 59 | 60 | def _prepare_mask(self, mask:object): 61 | return mask 62 | 63 | def _add(self, image_id:int, class_id:int, box:object=None, mask:object=None, score:float=1, ignore:bool=False): 64 | """ Add a data object to this collection. You should use one of the below functions instead. """ 65 | self._make_default_class(class_id) 66 | self._make_default_image(image_id) 67 | new_id = len(self.annotations) 68 | 69 | self.annotations.append({ 70 | '_id' : new_id, 71 | 'score' : score, 72 | 'image' : image_id, 73 | 'class' : class_id, 74 | 'bbox' : self._prepare_box(box), 75 | 'mask' : self._prepare_mask(mask), 76 | 'ignore': ignore, 77 | }) 78 | 79 | self.images[image_id]['anns'].append(new_id) 80 | 81 | def add_ground_truth(self, image_id:int, class_id:int, box:object=None, mask:object=None): 82 | """ Add a ground truth. If box or mask is None, this GT will be ignored for that mode. """ 83 | self._add(image_id, class_id, box, mask) 84 | 85 | def add_detection(self, image_id:int, class_id:int, score:int, box:object=None, mask:object=None): 86 | """ Add a predicted detection. If box or mask is None, this prediction will be ignored for that mode. """ 87 | self._add(image_id, class_id, box, mask, score=score) 88 | 89 | def add_ignore_region(self, image_id:int, class_id:int=None, box:object=None, mask:object=None): 90 | """ 91 | Add a region inside of which background detections should be ignored. 92 | You can use these to mark a region that has deliberately been left unannotated 93 | (e.g., if is a huge crowd of people and you don't want to annotate every single person in the crowd). 94 | 95 | If class_id is -1, this region will match any class. If the box / mask is None, the region will be the entire image. 96 | """ 97 | self._add(image_id, class_id, box, mask, ignore=True) 98 | 99 | def add_class(self, id:int, name:str): 100 | """ Register a class name to that class ID. """ 101 | self.classes[id] = name 102 | 103 | def add_image(self, id:int, name:str): 104 | """ Register an image name/path with an image ID. """ 105 | self.images[id]['name'] = name 106 | 107 | 108 | def get(self, image_id:int): 109 | """ Collects all the annotations / detections for that particular image. """ 110 | return [self.annotations[x] for x in self.images[image_id]['anns']] 111 | -------------------------------------------------------------------------------- /tidecv/plotting.py: -------------------------------------------------------------------------------- 1 | 2 | from collections import defaultdict, OrderedDict 3 | import os 4 | import shutil 5 | 6 | import cv2 7 | from matplotlib.lines import Line2D 8 | import matplotlib.pyplot as plt 9 | import matplotlib as mpl 10 | import numpy as np 11 | import pandas as pd 12 | import seaborn as sns 13 | 14 | from .errors.main_errors import * 15 | from .datasets import get_tide_path 16 | 17 | 18 | 19 | def print_table(rows:list, title:str=None): 20 | # Get all rows to have the same number of columns 21 | max_cols = max([len(row) for row in rows]) 22 | for row in rows: 23 | while len(row) < max_cols: 24 | row.append('') 25 | 26 | # Compute the text width of each column 27 | col_widths = [max([len(rows[i][col_idx]) for i in range(len(rows))]) for col_idx in range(len(rows[0]))] 28 | 29 | divider = '--' + ('---'.join(['-' * w for w in col_widths])) + '-' 30 | thick_divider = divider.replace('-', '=') 31 | 32 | if title: 33 | left_pad = (len(divider) - len(title)) // 2 34 | print(('{:>%ds}' % (left_pad + len(title))).format(title)) 35 | 36 | print(thick_divider) 37 | for row in rows: 38 | # Print each row while padding to each column's text width 39 | print(' ' + ' '.join([('{:>%ds}' % col_widths[col_idx]).format(row[col_idx]) for col_idx in range(len(row))]) + ' ') 40 | if row == rows[0]: print(divider) 41 | print(thick_divider) 42 | 43 | 44 | 45 | 46 | class Plotter(): 47 | """ Sets up a seaborn environment and holds the functions for plotting our figures. """ 48 | 49 | 50 | def __init__(self, quality:float=1): 51 | # Set mpl DPI in case we want to output to the screen / notebook 52 | mpl.rcParams['figure.dpi'] = 150 53 | 54 | # Seaborn color palette 55 | sns.set_palette('muted', 10) 56 | current_palette = sns.color_palette() 57 | 58 | # Seaborn style 59 | sns.set(style="whitegrid") 60 | 61 | self.colors_main = OrderedDict({ 62 | ClassError .short_name: current_palette[9], 63 | BoxError .short_name: current_palette[8], 64 | OtherError .short_name: current_palette[2], 65 | DuplicateError .short_name: current_palette[6], 66 | BackgroundError .short_name: current_palette[4], 67 | MissedError .short_name: current_palette[3], 68 | }) 69 | 70 | self.colors_special = OrderedDict({ 71 | FalsePositiveError.short_name: current_palette[0], 72 | FalseNegativeError.short_name: current_palette[1], 73 | }) 74 | 75 | self.tide_path = get_tide_path() 76 | 77 | # For the purposes of comparing across models, we fix the scales on our bar plots. 78 | # Feel free to change these after initializing if you want to change the scale. 79 | self.MAX_MAIN_DELTA_AP = 10 80 | self.MAX_SPECIAL_DELTA_AP = 25 81 | 82 | self.quality = quality 83 | 84 | def _prepare_tmp_dir(self): 85 | tmp_dir = os.path.join(self.tide_path, '_tmp') 86 | 87 | if not os.path.exists(tmp_dir): 88 | os.makedirs(tmp_dir) 89 | 90 | for _f in os.listdir(tmp_dir): 91 | os.remove(os.path.join(tmp_dir, _f)) 92 | 93 | return tmp_dir 94 | 95 | 96 | def make_summary_plot(self, out_dir:str, errors:dict, model_name:str, rec_type:str, hbar_names:bool=False): 97 | """Make a summary plot of the errors for a model, and save it to the figs folder. 98 | 99 | :param out_dir: The output directory for the summary image. MUST EXIST. 100 | :param errors: Dictionary of both main and special errors. 101 | :param model_name: Name of the model for which to generate the plot. 102 | :param rec_type: Recognition type, either TIDE.BOX or TIDE.MASK 103 | :param hbar_names: Whether or not to include labels for the horizontal bars. 104 | """ 105 | 106 | tmp_dir = self._prepare_tmp_dir() 107 | 108 | high_dpi = int(500*self.quality) 109 | low_dpi = int(300*self.quality) 110 | 111 | # get the data frame 112 | error_dfs = {errtype: pd.DataFrame(data={ 113 | 'Error Type': list(errors[errtype][model_name].keys()), 114 | 'Delta mAP': list(errors[errtype][model_name].values()), 115 | }) for errtype in ['main', 'special']} 116 | 117 | # pie plot for error type breakdown 118 | error_types = list(errors['main'][model_name].keys()) + list(errors['special'][model_name].keys()) 119 | error_sum = sum([e for e in errors['main'][model_name].values()]) 120 | error_sizes = [e / error_sum for e in errors['main'][model_name].values()] + [0, 0] 121 | fig, ax = plt.subplots(1, 1, figsize=(11, 11), dpi=high_dpi) 122 | patches, outer_text, inner_text = ax.pie(error_sizes, colors=self.colors_main.values(), labels=error_types, 123 | autopct='%1.1f%%', startangle=90) 124 | for text in outer_text + inner_text: 125 | text.set_text('') 126 | for i in range(len(self.colors_main)): 127 | if error_sizes[i] > 0.05: 128 | inner_text[i].set_text(list(self.colors_main.keys())[i]) 129 | inner_text[i].set_fontsize(48) 130 | inner_text[i].set_fontweight('bold') 131 | ax.axis('equal') 132 | plt.title(model_name, fontdict={'fontsize': 60, 'fontweight': 'bold'}) 133 | pie_path = os.path.join(tmp_dir, '{}_{}_pie.png'.format(model_name, rec_type)) 134 | plt.savefig(pie_path, bbox_inches='tight', dpi=low_dpi) 135 | plt.close() 136 | 137 | # horizontal bar plot for main error types 138 | fig, ax = plt.subplots(1, 1, figsize = (6, 5), dpi=high_dpi) 139 | sns.barplot(data=error_dfs['main'], x='Delta mAP', y='Error Type', ax=ax, 140 | palette=self.colors_main.values()) 141 | ax.set_xlim(0, self.MAX_MAIN_DELTA_AP) 142 | ax.set_xlabel('') 143 | ax.set_ylabel('') 144 | if not hbar_names: 145 | ax.set_yticklabels([''] * 6) 146 | plt.setp(ax.get_xticklabels(), fontsize=28) 147 | plt.setp(ax.get_yticklabels(), fontsize=36) 148 | ax.grid(False) 149 | sns.despine(left=True, bottom=True, right=True) 150 | hbar_path = os.path.join(tmp_dir, '{}_{}_hbar.png'.format(model_name, rec_type)) 151 | plt.savefig(hbar_path, bbox_inches='tight', dpi=low_dpi) 152 | plt.close() 153 | 154 | # vertical bar plot for special error types 155 | fig, ax = plt.subplots(1, 1, figsize = (2, 5), dpi=high_dpi) 156 | sns.barplot(data=error_dfs['special'], x='Error Type', y='Delta mAP', ax=ax, 157 | palette=self.colors_special.values()) 158 | ax.set_ylim(0, self.MAX_SPECIAL_DELTA_AP) 159 | ax.set_xlabel('') 160 | ax.set_ylabel('') 161 | ax.set_xticklabels(['FP', 'FN']) 162 | plt.setp(ax.get_xticklabels(), fontsize=36) 163 | plt.setp(ax.get_yticklabels(), fontsize=28) 164 | ax.grid(False) 165 | sns.despine(left=True, bottom=True, right=True) 166 | vbar_path = os.path.join(tmp_dir, '{}_{}_vbar.png'.format(model_name, rec_type)) 167 | plt.savefig(vbar_path, bbox_inches='tight', dpi=low_dpi) 168 | plt.close() 169 | 170 | # get each subplot image 171 | pie_im = cv2.imread(pie_path) 172 | hbar_im = cv2.imread(hbar_path) 173 | vbar_im = cv2.imread(vbar_path) 174 | 175 | # pad the hbar image vertically 176 | hbar_im = np.concatenate([np.zeros((vbar_im.shape[0] - hbar_im.shape[0], hbar_im.shape[1], 3)) + 255, hbar_im], 177 | axis=0) 178 | summary_im = np.concatenate([hbar_im, vbar_im], axis=1) 179 | 180 | # pad summary_im 181 | if summary_im.shape[1] object: 25 | """ Makes a new data object where we remove the ids in the pred and gt lists. """ 26 | obj = APDataObject() 27 | num_gt_removed = 0 28 | 29 | for pred_id in self.data_points: 30 | score, is_true, info = self.data_points[pred_id] 31 | 32 | # If the data point we kept was a true positive, there's a corresponding ground truth 33 | # If so, we should only add that positive if the corresponding ground truth has been kept 34 | if is_true and info['matched_with'] not in kept_gts: 35 | num_gt_removed += 1 36 | continue 37 | 38 | if pred_id in kept_preds: 39 | obj.data_points[pred_id] = self.data_points[pred_id] 40 | 41 | # Propogate the gt 42 | obj.false_negatives = self.false_negatives.intersection(kept_gts) 43 | num_gt_removed += (len(self.false_negatives) - len(obj.false_negatives)) 44 | 45 | obj.num_gt_positives = self.num_gt_positives - num_gt_removed 46 | return obj 47 | 48 | def push(self, id:int, score:float, is_true:bool, info:dict={}): 49 | self.data_points[id] = (score, is_true, info) 50 | 51 | def push_false_negative(self, id:int): 52 | self.false_negatives.add(id) 53 | 54 | def add_gt_positives(self, num_positives:int): 55 | """ Call this once per image. """ 56 | self.num_gt_positives += num_positives 57 | 58 | def is_empty(self) -> bool: 59 | return len(self.data_points) == 0 and self.num_gt_positives == 0 60 | 61 | def get_pr_curve(self) -> tuple: 62 | if self.curve is None: 63 | self.get_ap() 64 | return self.curve 65 | 66 | def get_ap(self) -> float: 67 | """ Warning: result not cached. """ 68 | 69 | if self.num_gt_positives == 0: 70 | return 0 71 | 72 | # Sort descending by score 73 | data_points = list(self.data_points.values()) 74 | data_points.sort(key=lambda x: -x[0]) 75 | 76 | precisions = [] 77 | recalls = [] 78 | num_true = 0 79 | num_false = 0 80 | 81 | # Compute the precision-recall curve. The x axis is recalls and the y axis precisions. 82 | for datum in data_points: 83 | # datum[1] is whether the detection a true or false positive 84 | if datum[1]: num_true += 1 85 | else: num_false += 1 86 | 87 | precision = num_true / (num_true + num_false) 88 | recall = num_true / self.num_gt_positives 89 | 90 | precisions.append(precision) 91 | recalls.append(recall) 92 | 93 | # Smooth the curve by computing [max(precisions[i:]) for i in range(len(precisions))] 94 | # Basically, remove any temporary dips from the curve. 95 | # At least that's what I think, idk. COCOEval did it so I do too. 96 | for i in range(len(precisions)-1, 0, -1): 97 | if precisions[i] > precisions[i-1]: 98 | precisions[i-1] = precisions[i] 99 | 100 | # Compute the integral of precision(recall) d_recall from recall=0->1 using fixed-length riemann summation with 101 bars. 101 | resolution = 100 # Standard COCO Resoluton 102 | y_range = [0] * (resolution + 1) # idx 0 is recall == 0.0 and idx 100 is recall == 1.00 103 | x_range = np.array([x / resolution for x in range(resolution + 1)]) 104 | recalls = np.array(recalls) 105 | 106 | # I realize this is weird, but all it does is find the nearest precision(x) for a given x in x_range. 107 | # Basically, if the closest recall we have to 0.01 is 0.009 this sets precision(0.01) = precision(0.009). 108 | # I approximate the integral this way, because that's how COCOEval does it. 109 | indices = np.searchsorted(recalls, x_range, side='left') 110 | for bar_idx, precision_idx in enumerate(indices): 111 | if precision_idx < len(precisions): 112 | y_range[bar_idx] = precisions[precision_idx] 113 | 114 | self.curve = (x_range, y_range) 115 | 116 | # Finally compute the riemann sum to get our integral. 117 | # avg([precision(x) for x in 0:0.01:1]) 118 | return sum(y_range) / len(y_range) * 100 119 | 120 | 121 | 122 | class ClassedAPDataObject: 123 | """ Stores an APDataObject for each class in the dataset. """ 124 | 125 | def __init__(self): 126 | self.objs = defaultdict(lambda: APDataObject()) 127 | 128 | def apply_qualifier(self, pred_dict:dict, gt_dict:dict) -> object: 129 | ret = ClassedAPDataObject() 130 | 131 | for _class, obj in self.objs.items(): 132 | pred_list = pred_dict[_class] if _class in pred_dict else set() 133 | gt_list = gt_dict[_class] if _class in gt_dict else set() 134 | 135 | ret.objs[_class] = obj.apply_qualifier(pred_list, gt_list) 136 | 137 | return ret 138 | 139 | def push(self, class_:int, id:int, score:float, is_true:bool, info:dict={}): 140 | self.objs[class_].push(id, score, is_true, info) 141 | 142 | def push_false_negative(self, class_:int, id:int): 143 | self.objs[class_].push_false_negative(id) 144 | 145 | def add_gt_positives(self, class_:int, num_positives:int): 146 | self.objs[class_].add_gt_positives(num_positives) 147 | 148 | def get_mAP(self) -> float: 149 | aps = [x.get_ap() for x in self.objs.values() if not x.is_empty()] 150 | return sum(aps) / len(aps) 151 | 152 | def get_gt_positives(self) -> dict: 153 | return {k: v.num_gt_positives for k, v in self.objs.items()} 154 | 155 | def get_pr_curve(self, cat_id:int=None) -> tuple: 156 | if cat_id is None: 157 | # Average out the curves when using all categories 158 | curves = [x.get_pr_curve() for x in list(self.objs.values())] 159 | x_range = curves[0][0] 160 | y_range = [0] * len(curves[0][1]) 161 | 162 | for x, y in curves: 163 | for i in range(len(y)): 164 | y_range[i] += y[i] 165 | 166 | for i in range(len(y_range)): 167 | y_range[i] /= len(curves) 168 | else: 169 | x_range, y_range = self.objs[cat_id].get_pr_curve() 170 | 171 | return x_range, y_range 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | # Note: Unused. 184 | class APEval: 185 | """ 186 | A class that computes the AP of some dataset. 187 | Note that TIDE doesn't use this internally. 188 | This is here so you can get a look at how the AP calculation process works. 189 | """ 190 | 191 | def __init__(self): 192 | self.iou_thresholds = [x / 100 for x in range(50, 100, 5)] 193 | self.ap_data = defaultdict(lambda: defaultdict(lambda: APDataObject())) 194 | 195 | 196 | def _eval_image(self, preds:list, gt:list, type_str:str='box'): 197 | data_type = 'segmentation' if type_str == 'mask' else 'bbox' 198 | preds_data = [x[data_type] for x in preds] 199 | 200 | # Split gt and crowd annotations 201 | gt_new = [] 202 | gt_crowd = [] 203 | 204 | for x in gt: 205 | if x['iscrowd']: 206 | gt_crowd.append(x) 207 | else: 208 | gt_new.append(x) 209 | 210 | gt = gt_new 211 | 212 | # Some setup 213 | num_pred = len(preds) 214 | num_gt = len(gt) 215 | num_crowd = len(gt_crowd) 216 | 217 | iou_cache = mask_utils.iou( 218 | preds_data, 219 | [x[data_type] for x in gt], 220 | [False] * num_gt) 221 | 222 | if num_crowd > 0: 223 | crowd_iou_cache = mask_utils.iou( 224 | preds_data, 225 | [x[data_type] for x in gt_crowd], 226 | [True] * num_crowd) 227 | 228 | # Make sure we're evaluating sorted by score 229 | indices = list(range(num_pred)) 230 | indices.sort(key=lambda i: -preds[i]['score']) 231 | 232 | classes = [x['category_id'] for x in preds] 233 | gt_classes = [x['category_id'] for x in gt] 234 | crowd_classes = [x['category_id'] for x in gt_crowd] 235 | 236 | for _class in set(classes + gt_classes): 237 | ap_per_iou = [] 238 | num_gt_for_class = sum([1 for x in gt_classes if x == _class]) 239 | 240 | for iouIdx in range(len(self.iou_thresholds)): 241 | iou_threshold = self.iou_thresholds[iouIdx] 242 | 243 | gt_used = [False] * len(gt_classes) 244 | 245 | ap_obj = self.ap_data[iouIdx][_class] 246 | ap_obj.add_gt_positives(num_gt_for_class) 247 | 248 | for i in indices: 249 | if classes[i] != _class: 250 | continue 251 | 252 | max_iou_found = iou_threshold 253 | max_match_idx = -1 254 | for j in range(num_gt): 255 | if gt_used[j] or gt_classes[j] != _class: 256 | continue 257 | 258 | iou = iou_cache[i][j] 259 | 260 | if iou > max_iou_found: 261 | max_iou_found = iou 262 | max_match_idx = j 263 | 264 | if max_match_idx >= 0: 265 | gt_used[max_match_idx] = True 266 | ap_obj.push(preds[i]['score'], True) 267 | else: 268 | # If the detection matches a crowd, we can just ignore it 269 | matched_crowd = False 270 | 271 | if num_crowd > 0: 272 | for j in range(len(crowd_classes)): 273 | if crowd_classes[j] != _class: 274 | continue 275 | 276 | iou = crowd_iou_cache[i][j] 277 | 278 | if iou > iou_threshold: 279 | matched_crowd = True 280 | break 281 | 282 | # All this crowd code so that we can make sure that our eval code gives the 283 | # same result as COCOEval. There aren't even that many crowd annotations to 284 | # begin with, but accuracy is of the utmost importance. 285 | if not matched_crowd: 286 | ap_obj.push(preds[i]['score'], False) 287 | 288 | def evaluate(self, preds:Data, gt:Data, type_str:str='box'): 289 | for id in gt.ids: 290 | x = preds.get(id) 291 | y = gt.get(id) 292 | 293 | self._eval_image(x, y, type_str) 294 | 295 | def compute_mAP(self): 296 | 297 | num_threshs = len(self.ap_data) 298 | thresh_APs = [] 299 | 300 | for thresh, classes in self.ap_data.items(): 301 | num_classes = len([x for x in classes.values() if not x.is_empty()]) 302 | ap = 0 303 | 304 | if num_classes > 0: 305 | class_APs = [x.get_ap() for x in classes.values() if not x.is_empty()] 306 | ap = sum(class_APs) / num_classes 307 | 308 | thresh_APs.append(ap) 309 | 310 | return round(sum(thresh_APs) / num_threshs * 100, 2) 311 | 312 | -------------------------------------------------------------------------------- /tidecv/datasets.py: -------------------------------------------------------------------------------- 1 | from .data import Data 2 | from . import functions as f 3 | 4 | import zipfile 5 | from pathlib import Path 6 | from appdirs import user_data_dir 7 | import urllib.request 8 | from collections import defaultdict 9 | import shutil 10 | import json 11 | import os 12 | 13 | def default_name(path:str) -> str: 14 | return os.path.splitext(os.path.basename(path))[0] 15 | 16 | def get_tide_path(): 17 | if 'TIDE_PATH' in os.environ: 18 | tide_path = os.environ['TIDE_PATH'] 19 | else: 20 | tide_path = user_data_dir('tidecv', appauthor=False) 21 | 22 | if not os.path.exists(tide_path): 23 | os.makedirs(tide_path) 24 | 25 | return tide_path 26 | 27 | def download_annotations(name:str, url:str, force_download:bool=False) -> str: 28 | tide_path = get_tide_path() 29 | candidate_path = os.path.join(tide_path, name) 30 | finished_file_path = os.path.join(candidate_path, '_finished') 31 | zip_file_path = os.path.join(candidate_path, '_tmp.zip') 32 | 33 | # Check if the file has already been downloaded 34 | # If there isn't a file called _finished, that means we didn't finish downloading last time, so try again 35 | already_downloaded = os.path.exists(candidate_path) and os.path.exists(finished_file_path) 36 | 37 | if not force_download and already_downloaded: 38 | return candidate_path 39 | else: 40 | print('{} annotations not found. Downloading...'.format(name)) 41 | 42 | if os.path.exists(candidate_path): 43 | shutil.rmtree(candidate_path) 44 | os.makedirs(candidate_path) 45 | 46 | urllib.request.urlretrieve(url, zip_file_path) 47 | with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: 48 | zip_ref.extractall(candidate_path) 49 | 50 | os.remove(zip_file_path) 51 | open(finished_file_path, 'a').close() # Make an empty _finished file to mark that we were successful 52 | 53 | print('Successfully downloaded {} to "{}"'.format(name, candidate_path)) 54 | return candidate_path 55 | 56 | 57 | 58 | 59 | 60 | def COCO(path:str=None, name:str=None, year:int=2017, ann_set:str='val', force_download:bool=False) -> Data: 61 | """ 62 | Loads ground truth from a COCO-style annotation file. 63 | 64 | If path is not specified, this will download the COCO annotations for the year and ann_set specified. 65 | Valid years are 2014, 2017 and valid ann_sets are 'val' and 'train'. 66 | """ 67 | if path is None: 68 | path = download_annotations( 69 | 'COCO{}'.format(year), 70 | 'http://images.cocodataset.org/annotations/annotations_trainval{}.zip'.format(year), 71 | force_download) 72 | 73 | path = os.path.join(path, 'annotations', 'instances_{}{}.json'.format(ann_set, year)) 74 | 75 | if name is None: name = default_name(path) 76 | 77 | with open(path, 'r') as json_file: 78 | cocojson = json.load(json_file) 79 | 80 | images = cocojson['images'] 81 | anns = cocojson['annotations'] 82 | cats = cocojson['categories'] if 'categories' in cocojson else None 83 | 84 | 85 | # Add everything from the coco json into our data structure 86 | data = Data(name, max_dets=100) 87 | 88 | image_lookup = {} 89 | 90 | for idx, image in enumerate(images): 91 | image_lookup[image['id']] = image 92 | data.add_image(image['id'], image['file_name']) 93 | 94 | if cats is not None: 95 | for cat in cats: 96 | data.add_class(cat['id'], cat['name']) 97 | 98 | for ann in anns: 99 | image = ann['image_id'] 100 | _class = ann['category_id'] 101 | box = ann['bbox'] 102 | mask = f.toRLE(ann['segmentation'], image_lookup[image]['width'], image_lookup[image]['height']) 103 | 104 | if ann['iscrowd']: data.add_ignore_region(image, _class, box, mask) 105 | else: data.add_ground_truth (image, _class, box, mask) 106 | 107 | return data 108 | 109 | def COCOResult(path:str, name:str=None) -> Data: 110 | """ Loads predictions from a COCO-style results file. """ 111 | if name is None: name = default_name(path) 112 | 113 | with open(path, 'r') as json_file: 114 | dets = json.load(json_file) 115 | 116 | data = Data(name) 117 | 118 | for det in dets: 119 | image = det['image_id'] 120 | _cls = det['category_id'] 121 | score = det['score'] 122 | box = det['bbox'] if 'bbox' in det else None 123 | mask = det['segmentation'] if 'segmentation' in det else None 124 | 125 | data.add_detection(image, _cls, score, box, mask) 126 | 127 | return data 128 | 129 | 130 | 131 | def LVIS(path:str=None, name:str=None, version_str:str='v1', force_download:bool=False) -> Data: 132 | """ 133 | Load an LVIS-style dataset. 134 | The version string is used for downloading the dataset and should be one of the versions of LVIS (e.g., v0.5, v1). 135 | 136 | Note that LVIS evaulation is special, but we can emulate it by adding ignore regions. 137 | The detector isn't punished for predicted class that LVIS annotators haven't guarenteed are in 138 | the image (i.e., the sum of GT annotated classes in the image and those marked explicitly not 139 | in the image.) In order to emulate this behavior, add ignore region labels for every class not 140 | found to be in the image. This is not that inefficient because ignore regions are separate out 141 | during mAP calculation and error processing, so adding a bunch of them doesn't hurt. 142 | 143 | The LVIS AP numbers are slightly lower than what the LVIS API reports because of these workarounds. 144 | """ 145 | if path is None: 146 | path = download_annotations( 147 | 'LVIS{}'.format(version_str), 148 | 'https://s3-us-west-2.amazonaws.com/dl.fbaipublicfiles.com/LVIS/lvis_{}_val.json.zip'.format(version_str), 149 | force_download) 150 | 151 | path = os.path.join(path, 'lvis_{}_val.json'.format(version_str)) 152 | 153 | 154 | if name is None: name = default_name(path) 155 | 156 | 157 | with open(path, 'r') as json_file: 158 | lvisjson = json.load(json_file) 159 | 160 | images = lvisjson['images'] 161 | anns = lvisjson['annotations'] 162 | cats = lvisjson['categories'] if 'categories' in lvisjson else None 163 | 164 | data = Data(name, max_dets=300) 165 | image_lookup = {} 166 | classes_in_img = defaultdict(lambda: set()) 167 | 168 | for image in images: 169 | image_lookup[image['id']] = image 170 | data.add_image(image['id'], image['coco_url']) # LVIS has no image names, only coco urls 171 | 172 | # Negative categories are guarenteed by the annotators to not be in the image. 173 | # Thus we should care about them during evaluation. 174 | for cat_id in image['neg_category_ids']: 175 | classes_in_img[image['id']].add(cat_id) 176 | 177 | if cats is not None: 178 | for cat in cats: 179 | data.add_class(cat['id'], cat['synset']) 180 | 181 | for ann in anns: 182 | image = ann['image_id'] 183 | _class = ann['category_id'] 184 | box = ann['bbox'] 185 | mask = f.toRLE(ann['segmentation'], image_lookup[image]['width'], image_lookup[image]['height']) 186 | 187 | data.add_ground_truth(image, _class, box, mask) 188 | 189 | # There's an annotation for this class, so we should consider the class for evaluation. 190 | classes_in_img[image].add(_class) 191 | 192 | all_classes = set(data.classes.keys()) 193 | 194 | # LVIS doesn't penalize the detector for detecting classes that the annotators haven't guarenteed to be in/out of 195 | # the image. Here we simulate that property by adding ignore regions for all such classes. 196 | for image in images: 197 | ignored_classes = all_classes.difference(classes_in_img[image['id']]) 198 | 199 | # LVIS doesn't penalize the detector for mistakes made on classes explicitly marked as not exhaustively annoted 200 | # We can emulate this by adding ignore regions for every category listed, so add them to the ignored classes. 201 | ignored_classes.update(set(image['not_exhaustive_category_ids'])) 202 | 203 | for _cls in ignored_classes: 204 | data.add_ignore_region(image['id'], _cls) 205 | 206 | return data 207 | 208 | def LVISResult(path:str, name:str=None) -> Data: 209 | """ Loads predictions from a LVIS-style results file. Note that this is the same as a COCO-style results file. """ 210 | return COCOResult(path, name) 211 | 212 | 213 | def Pascal(path:str=None, name:str=None, year:int=2007, ann_set:str='val', force_download:bool=False) -> Data: 214 | """ 215 | Loads the Pascal VOC 2007 or 2012 data from a COCO json. 216 | 217 | Valid years are 2007 and 2012, and valid ann_sets are 'train' and 'val'. 218 | """ 219 | if path is None: 220 | path = download_annotations( 221 | 'Pascal', 222 | 'https://s3.amazonaws.com/images.cocodataset.org/external/external_PASCAL_VOC.zip', 223 | force_download) 224 | 225 | path = os.path.join(path, 'PASCAL_VOC', 'pascal_{}{}.json'.format(ann_set, year)) 226 | 227 | return COCO(path, name) 228 | 229 | 230 | 231 | def Cityscapes(path:str, name:str=None): 232 | """ 233 | Loads the fine cityscapes annotations as instance segmentation masks, and also generates bounding boxes for them. 234 | 235 | Note that we can't automatically download Cityscapes because it requires registration and an agreement to the ToS. 236 | You can get cityscapes here: https://www.cityscapes-dataset.com/ 237 | 238 | Path should be to gtFine/. E.g., /gtFine/val. 239 | """ 240 | if name is None: name = default_name(path) 241 | data = Data(name) 242 | 243 | instance_classes = { 244 | 'person' : 24, 245 | 'rider' : 25, 246 | 'car' : 26, 247 | 'truck' : 27, 248 | 'train' : 31, 249 | 'motorcycle': 32, 250 | 'bicycle' : 33, 251 | 'bus' : 28, 252 | 'caravan' : 29, 253 | 'trailer' : 30, 254 | } 255 | 256 | ignored_classes = set([29, 30]) 257 | 258 | for class_name, class_id in instance_classes.items(): 259 | data.add_class(class_id, class_name) 260 | 261 | for ann in Path(path).glob('*/*.json'): 262 | with open(ann) as json_file: 263 | ann_json = json.load(json_file) 264 | 265 | # Note: a string for an image ID is okay 266 | image_id = os.path.basename(ann).replace('_gtFine_polygons.json', '') 267 | objs = ann_json['objects'] 268 | 269 | data.add_image(image_id, image_id) # The id in this case is just the name of the image 270 | 271 | # Caravan and Trailer should be ignored from all evaluation 272 | for _cls in ignored_classes: 273 | data.add_ignore_region(image_id, _cls) 274 | 275 | for obj in objs: 276 | class_label = obj['label'] 277 | is_crowd = False 278 | 279 | # Cityscapes labelers can label objects without defined boundaries as 'group'. In COCO-land this would be 280 | # a crowd annotation. So in this case, let's make it an ignore region. 281 | if class_label.endswith('group'): 282 | is_crowd = True 283 | class_label = class_label[:-5] # Remove the group at the end 284 | 285 | # We are only considering instance classes 286 | if not class_label in instance_classes: 287 | continue 288 | 289 | class_id = instance_classes[class_label] 290 | 291 | # If the class is not used in evaluation, don't include it 292 | if class_id in ignored_classes: 293 | continue 294 | 295 | poly = [sum(obj['polygon'], [])] # Converts a list of points to a list of lists of ints, where every 2 ints represents a point 296 | box = f.polyToBox(poly) 297 | 298 | if is_crowd: 299 | data.add_ignore_region(image_id, class_id, box, poly) 300 | else: 301 | data.add_ground_truth(image_id, class_id, box, poly) 302 | 303 | return data 304 | 305 | 306 | 307 | 308 | -------------------------------------------------------------------------------- /tidecv/quantify.py: -------------------------------------------------------------------------------- 1 | from .data import Data 2 | from .ap import ClassedAPDataObject 3 | from .errors.main_errors import * 4 | from .errors.qualifiers import Qualifier, AREA 5 | from . import functions as f 6 | from . import plotting as P 7 | 8 | from pycocotools import mask as mask_utils 9 | from collections import defaultdict, OrderedDict 10 | import numpy as np 11 | from typing import Union 12 | import os, math 13 | 14 | class TIDEExample: 15 | """ Computes all the data needed to evaluate a set of predictions and gt for a single image. """ 16 | def __init__(self, preds:list, gt:list, pos_thresh:float, mode:str, max_dets:int, run_errors:bool=True): 17 | self.preds = preds 18 | self.gt = [x for x in gt if not x['ignore']] 19 | self.ignore_regions = [x for x in gt if x['ignore']] 20 | 21 | self.mode = mode 22 | self.pos_thresh = pos_thresh 23 | self.max_dets = max_dets 24 | self.run_errors = run_errors 25 | 26 | self._run() 27 | 28 | def _run(self): 29 | preds = self.preds 30 | gt = self.gt 31 | ignore = self.ignore_regions 32 | det_type = 'bbox' if self.mode == TIDE.BOX else 'mask' 33 | max_dets = self.max_dets 34 | 35 | if len(preds) == 0: 36 | raise RuntimeError('Example has no predictions!') 37 | 38 | 39 | # Sort descending by score 40 | preds.sort(key=lambda pred: -pred['score']) 41 | preds = preds[:max_dets] 42 | self.preds = preds # Update internally so TIDERun can update itself if :max_dets takes effect 43 | detections = [x[det_type] for x in preds] 44 | 45 | 46 | # IoU is [len(detections), len(gt)] 47 | self.gt_iou = mask_utils.iou( 48 | detections, 49 | [x[det_type] for x in gt], 50 | [False] * len(gt)) 51 | 52 | # Store whether a prediction / gt got used in their data list 53 | # Note: this is set to None if ignored, keep that in mind 54 | for idx, pred in enumerate(preds): 55 | pred['used'] = False 56 | pred['_idx'] = idx 57 | pred['iou'] = 0 58 | for idx, truth in enumerate(gt): 59 | truth['used'] = False 60 | truth['usable'] = False 61 | truth['_idx'] = idx 62 | 63 | pred_cls = np.array([x['class'] for x in preds]) 64 | gt_cls = np.array([x['class'] for x in gt]) 65 | 66 | if len(gt) > 0: 67 | # A[i,j] is true iff the prediction i is of the same class as gt j 68 | self.gt_cls_matching = (pred_cls[:, None] == gt_cls[None, :]) 69 | self.gt_cls_iou = self.gt_iou * self.gt_cls_matching 70 | 71 | # This will be changed in the matching calculation, so make a copy 72 | iou_buffer = self.gt_cls_iou.copy() 73 | 74 | for pred_idx, pred_elem in enumerate(preds): 75 | # Find the max iou ground truth for this prediction 76 | gt_idx = np.argmax(iou_buffer[pred_idx, :]) 77 | iou = iou_buffer[pred_idx, gt_idx] 78 | 79 | pred_elem['iou'] = np.max(self.gt_cls_iou[pred_idx, :]) 80 | 81 | if iou >= self.pos_thresh: 82 | gt_elem = gt[gt_idx] 83 | 84 | pred_elem['used'] = True 85 | gt_elem['used'] = True 86 | pred_elem['matched_with'] = gt_elem['_id'] 87 | gt_elem['matched_with'] = pred_elem['_id'] 88 | 89 | # Make sure this gt can't be used again 90 | iou_buffer[:, gt_idx] = 0 91 | 92 | # Ignore regions annotations allow us to ignore predictions that fall within 93 | if len(ignore) > 0: 94 | # Because ignore regions have extra parameters, it's more efficient to use a for loop here 95 | for ignore_region in ignore: 96 | if ignore_region['mask'] is None and ignore_region['bbox'] is None: 97 | # The region should span the whole image 98 | ignore_iou = [1] * len(preds) 99 | else: 100 | if ignore_region[det_type] is None: 101 | # There is no det_type annotation for this specific region so skip it 102 | continue 103 | # Otherwise, compute the crowd IoU between the detections and this region 104 | ignore_iou = mask_utils.iou(detections, [ignore_region[det_type]], [True]) 105 | 106 | for pred_idx, pred_elem in enumerate(preds): 107 | if not pred_elem['used'] and (ignore_iou[pred_idx] > self.pos_thresh) \ 108 | and (ignore_region['class'] == pred_elem['class'] or ignore_region['class'] == -1): 109 | # Set the prediction to be ignored 110 | pred_elem['used'] = None 111 | 112 | if len(gt) == 0: 113 | return 114 | 115 | # Some matrices used just for error calculation 116 | if self.run_errors: 117 | self.gt_used = np.array([x['used'] == True for x in gt])[None, :] 118 | self.gt_unused = ~self.gt_used 119 | 120 | self.gt_unused_iou = self.gt_unused * self.gt_iou 121 | self.gt_unused_cls = self.gt_unused_iou * self.gt_cls_matching 122 | self.gt_unused_noncls = self.gt_unused_iou * ~self.gt_cls_matching 123 | 124 | self.gt_noncls_iou = self.gt_iou * ~self.gt_cls_matching 125 | 126 | self.gt_used_iou = self.gt_used * self.gt_iou 127 | self.gt_used_cls = self.gt_used_iou * self.gt_cls_matching 128 | 129 | 130 | class TIDERun: 131 | """ Holds the data for a single run of TIDE. """ 132 | 133 | # Temporary variables stored in ground truth that we need to clear after a run 134 | _temp_vars = ['best_score', 'best_id', 'used', 'matched_with', '_idx', 'usable'] 135 | 136 | def __init__(self, gt:Data, preds:Data, pos_thresh:float, bg_thresh:float, mode:str, max_dets:int, run_errors:bool=True): 137 | self.gt = gt 138 | self.preds = preds 139 | 140 | self.errors = [] 141 | self.error_dict = {_type: [] for _type in TIDE._error_types} 142 | self.ap_data = ClassedAPDataObject() 143 | self.qualifiers = {} 144 | 145 | # A list of false negatives per class 146 | self.false_negatives = {_id: [] for _id in self.gt.classes} 147 | 148 | self.pos_thresh = pos_thresh 149 | self.bg_thresh = bg_thresh 150 | self.mode = mode 151 | self.max_dets = max_dets 152 | self.run_errors = run_errors 153 | 154 | self._run() 155 | 156 | 157 | def _run(self): 158 | """ And awaaay we go """ 159 | 160 | for image in self.gt.images: 161 | x = self.preds.get(image) 162 | y = self.gt.get(image) 163 | 164 | # These classes are ignored for the whole image and not in the ground truth, so 165 | # we can safely just remove these detections from the predictions at the start. 166 | # However, since ignored detections are still used for error calculations, we have to keep them. 167 | if not self.run_errors: 168 | ignored_classes = self.gt._get_ignored_classes(image) 169 | x = [pred for pred in x if pred['class'] not in ignored_classes] 170 | 171 | self._eval_image(x, y) 172 | 173 | # Store a fixed version of all the errors for testing purposes 174 | for error in self.errors: 175 | error.original = f.nonepack(error.unfix()) 176 | error.fixed = f.nonepack(error.fix()) 177 | error.disabled = False 178 | 179 | self.ap = self.ap_data.get_mAP() 180 | 181 | # Now that we've stored the fixed errors, we can clear the gt info 182 | self._clear() 183 | 184 | 185 | 186 | 187 | def _clear(self): 188 | """ Clears the ground truth so that it's ready for another run. """ 189 | for gt in self.gt.annotations: 190 | for var in self._temp_vars: 191 | if var in gt: 192 | del gt[var] 193 | 194 | def _add_error(self, error): 195 | self.errors.append(error) 196 | self.error_dict[type(error)].append(error) 197 | 198 | def _eval_image(self, preds:list, gt:list): 199 | 200 | for truth in gt: 201 | if not truth['ignore']: 202 | self.ap_data.add_gt_positives(truth['class'], 1) 203 | 204 | if len(preds) == 0: 205 | # There are no predictions for this image so add all gt as missed 206 | for truth in gt: 207 | if not truth['ignore']: 208 | self.ap_data.push_false_negative(truth['class'], truth['_id']) 209 | 210 | if self.run_errors: 211 | self._add_error(MissedError(truth)) 212 | self.false_negatives[truth['class']].append(truth) 213 | return 214 | 215 | ex = TIDEExample(preds, gt, self.pos_thresh, self.mode, self.max_dets, self.run_errors) 216 | preds = ex.preds # In case the number of predictions was restricted to the max 217 | 218 | for pred_idx, pred in enumerate(preds): 219 | 220 | pred['info'] = {'iou': pred['iou'], 'used': pred['used']} 221 | if pred['used']: pred['info']['matched_with'] = pred['matched_with'] 222 | 223 | if pred['used'] is not None: 224 | self.ap_data.push(pred['class'], pred['_id'], pred['score'], pred['used'], pred['info']) 225 | 226 | # ----- ERROR DETECTION ------ # 227 | # This prediction is a negative (or ignored), let's find out why 228 | if self.run_errors and (pred['used'] == False or pred['used'] == None): 229 | # Test for BackgroundError 230 | if len(ex.gt) == 0: # Note this is ex.gt because it doesn't include ignore annotations 231 | # There is no ground truth for this image, so just mark everything as BackgroundError 232 | self._add_error(BackgroundError(pred)) 233 | continue 234 | 235 | # Test for BoxError 236 | idx = ex.gt_cls_iou[pred_idx, :].argmax() 237 | if self.bg_thresh <= ex.gt_cls_iou[pred_idx, idx] <= self.pos_thresh: 238 | # This detection would have been positive if it had higher IoU with this GT 239 | self._add_error(BoxError(pred, ex.gt[idx], ex)) 240 | continue 241 | 242 | # Test for ClassError 243 | idx = ex.gt_noncls_iou[pred_idx, :].argmax() 244 | if ex.gt_noncls_iou[pred_idx, idx] >= self.pos_thresh: 245 | # This detection would have been a positive if it was the correct class 246 | self._add_error(ClassError(pred, ex.gt[idx], ex)) 247 | continue 248 | 249 | # Test for DuplicateError 250 | idx = ex.gt_used_cls[pred_idx, :].argmax() 251 | if ex.gt_used_cls[pred_idx, idx] >= self.pos_thresh: 252 | # The detection would have been marked positive but the GT was already in use 253 | suppressor = self.preds.annotations[ex.gt[idx]['matched_with']] 254 | self._add_error(DuplicateError(pred, suppressor)) 255 | continue 256 | 257 | # Test for BackgroundError 258 | idx = ex.gt_iou[pred_idx, :].argmax() 259 | if ex.gt_iou[pred_idx, idx] <= self.bg_thresh: 260 | # This should have been marked as background 261 | self._add_error(BackgroundError(pred)) 262 | continue 263 | 264 | # A base case to catch uncaught errors 265 | self._add_error(OtherError(pred)) 266 | 267 | for truth in gt: 268 | # If the GT wasn't used in matching, meaning it's some kind of false negative 269 | if not truth['ignore'] and not truth['used']: 270 | self.ap_data.push_false_negative(truth['class'], truth['_id']) 271 | 272 | if self.run_errors: 273 | self.false_negatives[truth['class']].append(truth) 274 | 275 | # The GT was completely missed, no error can correct it 276 | # Note: 'usable' is set in error.py 277 | if not truth['usable']: 278 | self._add_error(MissedError(truth)) 279 | 280 | 281 | 282 | def fix_errors(self, condition=lambda x: False, transform=None, false_neg_dict:dict=None, 283 | ap_data:ClassedAPDataObject=None, 284 | disable_errors:bool=False) -> ClassedAPDataObject: 285 | """ Returns a ClassedAPDataObject where all errors given the condition returns True are fixed. """ 286 | if ap_data is None: 287 | ap_data = self.ap_data 288 | 289 | gt_pos = ap_data.get_gt_positives() 290 | new_ap_data = ClassedAPDataObject() 291 | 292 | # Potentially fix every error case 293 | for error in self.errors: 294 | if error.disabled: 295 | continue 296 | 297 | _id = error.get_id() 298 | _cls, data_point = error.original 299 | 300 | if condition(error): 301 | _cls, data_point = error.fixed 302 | 303 | if disable_errors: 304 | error.disabled = True 305 | 306 | # Specific for MissingError (or anything else that affects #GT) 307 | if isinstance(data_point, int): 308 | gt_pos[_cls] += data_point 309 | data_point = None 310 | 311 | if data_point is not None: 312 | if transform is not None: 313 | data_point = transform(*data_point) 314 | new_ap_data.push(_cls, _id, *data_point) 315 | 316 | # Add back all the correct ones 317 | for k in gt_pos.keys(): 318 | for _id, (score, correct, info) in ap_data.objs[k].data_points.items(): 319 | if correct: 320 | if transform is not None: 321 | score, correct, info = transform(score, correct, info) 322 | new_ap_data.push(k, _id, score, correct, info) 323 | 324 | # Add the correct amount of GT positives, and also subtract if necessary 325 | for k, v in gt_pos.items(): 326 | # In case you want to fix all false negatives without affecting precision 327 | if false_neg_dict is not None and k in false_neg_dict: 328 | v -= len(false_neg_dict[k]) 329 | new_ap_data.add_gt_positives(k, v) 330 | 331 | return new_ap_data 332 | 333 | def fix_main_errors(self, progressive:bool=False, error_types:list=None, qual:Qualifier=None) -> dict: 334 | ap_data = self.ap_data 335 | last_ap = self.ap 336 | 337 | if qual is None: 338 | qual = Qualifier('', None) 339 | 340 | if error_types is None: 341 | error_types = TIDE._error_types 342 | 343 | errors = {} 344 | 345 | for error in error_types: 346 | _ap_data = self.fix_errors(qual._make_error_func(error), 347 | ap_data=ap_data, disable_errors=progressive) 348 | 349 | new_ap = _ap_data.get_mAP() 350 | # If an error is negative that means it's likely due to binning differences, so just 351 | # Ignore the negative by setting it to 0. 352 | errors[error] = max(new_ap - last_ap, 0) 353 | 354 | if progressive: 355 | last_ap = new_ap 356 | ap_data = _ap_data 357 | 358 | if progressive: 359 | for error in self.errors: 360 | error.disabled = False 361 | 362 | return errors 363 | 364 | def fix_special_errors(self, qual=None) -> dict: 365 | return { 366 | FalsePositiveError: self.fix_errors(transform=FalsePositiveError.fix).get_mAP() - self.ap, 367 | FalseNegativeError: self.fix_errors(false_neg_dict=self.false_negatives).get_mAP() - self.ap} 368 | 369 | def count_errors(self, error_types:list=None, qual=None): 370 | counts = {} 371 | 372 | if error_types is None: 373 | error_types = TIDE._error_types 374 | 375 | for error in error_types: 376 | if qual is None: 377 | counts[error] = len(self.error_dict[error]) 378 | else: 379 | func = qualifiers.make_qualifier(error, qual) 380 | counts[error] = len([x for x in self.errors if func(x)]) 381 | 382 | return counts 383 | 384 | 385 | def apply_qualifier(self, qualifier:Qualifier) -> ClassedAPDataObject: 386 | """ Applies a qualifier lambda to the AP object for this runs and stores the result in self.qualifiers. """ 387 | 388 | pred_keep = defaultdict(lambda: set()) 389 | gt_keep = defaultdict(lambda: set()) 390 | 391 | for pred in self.preds.annotations: 392 | if qualifier.test(pred): 393 | pred_keep[pred['class']].add(pred['_id']) 394 | 395 | for gt in self.gt.annotations: 396 | if not gt['ignore'] and qualifier.test(gt): 397 | gt_keep[gt['class']].add(gt['_id']) 398 | 399 | new_ap_data = self.ap_data.apply_qualifier(pred_keep, gt_keep) 400 | self.qualifiers[qualifier.name] = new_ap_data.get_mAP() 401 | return new_ap_data 402 | 403 | 404 | 405 | class TIDE: 406 | """ 407 | ████████╗██╗██████╗ ███████╗ 408 | ╚══██╔══╝██║██╔══██╗██╔════╝ 409 | ██║ ██║██║ ██║█████╗ 410 | ██║ ██║██║ ██║██╔══╝ 411 | ██║ ██║██████╔╝███████╗ 412 | ╚═╝ ╚═╝╚═════╝ ╚══════╝ 413 | """ 414 | 415 | 416 | # This is just here to define a consistent order of the error types 417 | _error_types = [ClassError, BoxError, OtherError, DuplicateError, BackgroundError, MissedError] 418 | _special_error_types = [FalsePositiveError, FalseNegativeError] 419 | 420 | # Threshold splits for different challenges 421 | COCO_THRESHOLDS = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95] 422 | VOL_THRESHOLDS = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] 423 | 424 | # The modes of evaluation 425 | BOX = 'bbox' 426 | MASK = 'mask' 427 | 428 | def __init__(self, pos_threshold:float=0.5, background_threshold:float=0.1, mode:str=BOX): 429 | self.pos_thresh = pos_threshold 430 | self.bg_thresh = background_threshold 431 | self.mode = mode 432 | 433 | self.pos_thresh_int = int(self.pos_thresh * 100) 434 | 435 | self.runs = {} 436 | self.run_thresholds = {} 437 | self.run_main_errors = {} 438 | self.run_special_errors = {} 439 | 440 | self.qualifiers = OrderedDict() 441 | 442 | self.plotter = P.Plotter() 443 | 444 | 445 | def evaluate(self, gt:Data, preds:Data, pos_threshold:float=None, background_threshold:float=None, 446 | mode:str=None, name:str=None, use_for_errors:bool=True) -> TIDERun: 447 | pos_thresh = self.pos_thresh if pos_threshold is None else pos_threshold 448 | bg_thresh = self.bg_thresh if background_threshold is None else background_threshold 449 | mode = self.mode if mode is None else mode 450 | name = preds.name if name is None else name 451 | 452 | run = TIDERun(gt, preds, pos_thresh, bg_thresh, mode, gt.max_dets, use_for_errors) 453 | 454 | if use_for_errors: 455 | self.runs[name] = run 456 | 457 | return run 458 | 459 | def evaluate_range(self, gt:Data, preds:Data, thresholds:list=COCO_THRESHOLDS, pos_threshold:float=None, 460 | background_threshold:float=None, mode:str=None, name:str=None) -> dict: 461 | 462 | if pos_threshold is None: pos_threshold = self.pos_thresh 463 | if name is None: name = preds.name 464 | 465 | self.run_thresholds[name] = [] 466 | 467 | for thresh in thresholds: 468 | 469 | run = self.evaluate(gt, preds, pos_threshold=thresh, background_threshold=background_threshold, 470 | mode=mode, name=name, use_for_errors=(pos_threshold == thresh)) 471 | 472 | self.run_thresholds[name].append(run) 473 | 474 | def add_qualifiers(self, *quals): 475 | """ 476 | Applies any number of Qualifier objects to evaluations that have been run up to now. 477 | See qualifiers.py for examples. 478 | """ 479 | raise NotImplementedError('Qualifiers coming soon.') 480 | # for q in quals: 481 | # for run_name, run in self.runs.items(): 482 | # if run_name in self.run_thresholds: 483 | # # If this was a threshold run, apply the qualifier for every run 484 | # for trun in self.run_thresholds[run_name]: 485 | # trun.apply_qualifier(q) 486 | # else: 487 | # # If this had no threshold, just apply it to the main run 488 | # run.apply_qualifier(q) 489 | 490 | # self.qualifiers[q.name] = q 491 | 492 | def summarize(self): 493 | """ Summarizes the mAP values and errors for all runs in this TIDE object. Results are printed to the console. """ 494 | main_errors = self.get_main_errors() 495 | special_errors = self.get_special_errors() 496 | 497 | for run_name, run in self.runs.items(): 498 | print('-- {} --\n'.format(run_name)) 499 | 500 | # If we evaluated on all thresholds, print them here 501 | if run_name in self.run_thresholds: 502 | thresh_runs = self.run_thresholds[run_name] 503 | aps = [trun.ap for trun in thresh_runs] 504 | 505 | # Print Overall AP for a threshold run 506 | ap_title = '{} AP @ [{:d}-{:d}]'.format(thresh_runs[0].mode, 507 | int(thresh_runs[0].pos_thresh*100), int(thresh_runs[-1].pos_thresh*100)) 508 | print('{:s}: {:.2f}'.format(ap_title, sum(aps)/len(aps))) 509 | 510 | # Print AP for every threshold on a threshold run 511 | P.print_table([ 512 | ['Thresh'] + [str(int(trun.pos_thresh*100)) for trun in thresh_runs], 513 | [' AP '] + ['{:6.2f}'.format(trun.ap) for trun in thresh_runs] 514 | ], title=ap_title) 515 | 516 | # Print qualifiers for a threshold run 517 | if len(self.qualifiers) > 0: 518 | print() 519 | # Can someone ban me from using list comprehension? this is unreadable 520 | qAPs = [ 521 | f.mean( 522 | [trun.qualifiers[q] for trun in thresh_runs if q in trun.qualifiers] 523 | ) for q in self.qualifiers 524 | ] 525 | 526 | P.print_table([ 527 | ['Name'] + list(self.qualifiers.keys()), 528 | [' AP '] + ['{:6.2f}'.format(qAP) for qAP in qAPs] 529 | ], title='Qualifiers {}'.format(ap_title)) 530 | 531 | # Otherwise, print just the one run we did 532 | else: 533 | # Print Overall AP for a regular run 534 | ap_title = '{} AP @ {:d}'.format(run.mode, int(run.pos_thresh*100)) 535 | print('{}: {:.2f}'.format(ap_title, run.ap)) 536 | 537 | # Print qualifiers for a regular run 538 | if len(self.qualifiers) > 0: 539 | print() 540 | qAPs = [run.qualifiers[q] if q in run.qualifiers else 0 for q in self.qualifiers] 541 | P.print_table([ 542 | ['Name'] + list(self.qualifiers.keys()), 543 | [' AP '] + ['{:6.2f}'.format(qAP) for qAP in qAPs] 544 | ], title='Qualifiers {}'.format(ap_title)) 545 | 546 | 547 | 548 | print() 549 | # Print the main errors 550 | P.print_table([ 551 | ['Type'] + [err.short_name for err in TIDE._error_types], 552 | [' dAP'] + ['{:6.2f}'.format(main_errors[run_name][err.short_name]) for err in TIDE._error_types] 553 | ], title='Main Errors') 554 | 555 | 556 | 557 | print() 558 | # Print the special errors 559 | P.print_table([ 560 | ['Type'] + [err.short_name for err in TIDE._special_error_types], 561 | [' dAP'] + ['{:6.2f}'.format(special_errors[run_name][err.short_name]) for err in TIDE._special_error_types] 562 | ], title='Special Error') 563 | 564 | print() 565 | 566 | def plot(self, out_dir:str=None): 567 | """ 568 | Plots a summary model for each run in this TIDE object. 569 | Images will be outputted to out_dir, which will be created if it doesn't exist. 570 | """ 571 | 572 | if out_dir is not None: 573 | if not os.path.exists(out_dir): 574 | os.makedirs(out_dir) 575 | 576 | errors = self.get_all_errors() 577 | 578 | max_main_error = max(sum([list(x.values()) for x in errors['main'].values()], [])) 579 | max_spec_error = max(sum([list(x.values()) for x in errors['special'].values()], [])) 580 | dap_granularity = 5 # The max will round up to the nearest unit of this 581 | 582 | # Round the plotter's dAP range up to the nearest granularity units 583 | if max_main_error > self.plotter.MAX_MAIN_DELTA_AP: 584 | self.plotter.MAX_MAIN_DELTA_AP = math.ceil(max_main_error / dap_granularity) * dap_granularity 585 | if max_spec_error > self.plotter.MAX_SPECIAL_DELTA_AP: 586 | self.plotter.MAX_SPECIAL_DELTA_AP = math.ceil(max_spec_error / dap_granularity) * dap_granularity 587 | 588 | # Do the plotting now 589 | for run_name, run in self.runs.items(): 590 | self.plotter.make_summary_plot(out_dir, errors, run_name, run.mode, hbar_names=True) 591 | 592 | 593 | 594 | def get_main_errors(self): 595 | errors = {} 596 | 597 | for run_name, run in self.runs.items(): 598 | if run_name in self.run_main_errors: 599 | errors[run_name] = self.run_main_errors[run_name] 600 | else: 601 | errors[run_name] = { 602 | error.short_name: value 603 | for error, value in run.fix_main_errors().items() 604 | } 605 | 606 | return errors 607 | 608 | def get_special_errors(self): 609 | errors = {} 610 | 611 | for run_name, run in self.runs.items(): 612 | if run_name in self.run_special_errors: 613 | errors[run_name] = self.run_special_errors[run_name] 614 | else: 615 | errors[run_name] = { 616 | error.short_name: value 617 | for error, value in run.fix_special_errors().items() 618 | } 619 | 620 | return errors 621 | 622 | def get_all_errors(self): 623 | """ 624 | returns { 625 | 'main' : { run_name: { error_name: float } }, 626 | 'special': { run_name: { error_name: float } }, 627 | } 628 | """ 629 | return { 630 | 'main': self.get_main_errors(), 631 | 'special': self.get_special_errors() 632 | } 633 | 634 | 635 | -------------------------------------------------------------------------------- /examples/coco_instance_segmentation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# TIDE COCO Example\n", 8 | "This example downloads some Mask R-CNN results files from [detectron](https://github.com/facebookresearch/Detectron/blob/master/MODEL_ZOO.md) and then evaluates them using the COCO dataset." 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "metadata": {}, 14 | "source": [ 15 | "### Grabbing Mask R-CNN results\n", 16 | "For demonstration purposes, we'll be using Mask R-CNN results (Model ID 36229740), but you'll want to use your own results in practice." 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 1, 22 | "metadata": {}, 23 | "outputs": [ 24 | { 25 | "name": "stdout", 26 | "output_type": "stream", 27 | "text": [ 28 | "Results Downloaded!\n" 29 | ] 30 | } 31 | ], 32 | "source": [ 33 | "# Download the Mask R-CNN results to test on. Only for demonstration purposes.\n", 34 | "import urllib.request # For downloading the sample Mask R-CNN annotations\n", 35 | "\n", 36 | "bbox_file = 'mask_rcnn_bbox.json'\n", 37 | "mask_file = 'mask_rcnn_mask.json'\n", 38 | "\n", 39 | "urllib.request.urlretrieve('https://dl.fbaipublicfiles.com/detectron/35861795/12_2017_baselines/e2e_mask_rcnn_R-101-FPN_1x.yaml.02_31_37.KqyEK4tT/output/test/coco_2014_minival/generalized_rcnn/bbox_coco_2014_minival_results.json', bbox_file)\n", 40 | "urllib.request.urlretrieve('https://dl.fbaipublicfiles.com/detectron/35861795/12_2017_baselines/e2e_mask_rcnn_R-101-FPN_1x.yaml.02_31_37.KqyEK4tT/output/test/coco_2014_minival/generalized_rcnn/segmentations_coco_2014_minival_results.json', mask_file)\n", 41 | "\n", 42 | "print('Results Downloaded!')" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "### Running TIDE" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 2, 55 | "metadata": {}, 56 | "outputs": [], 57 | "source": [ 58 | "# Import the TIDE evaluation toolkit\n", 59 | "from tidecv import TIDE\n", 60 | "\n", 61 | "# Import the datasets we want to use\n", 62 | "import tidecv.datasets as datasets" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 3, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# Load the dataset\n", 72 | "# This will automatically download COCO's 2017 annotations for you. You can specify your own COCO-style dataset too!\n", 73 | "# See datasets.py for a list of all the supported datasets.\n", 74 | "gt = datasets.COCO()" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 4, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "# Load the results in COCO format\n", 84 | "bbox_results = datasets.COCOResult(bbox_file) # These files were downloaded above.\n", 85 | "mask_results = datasets.COCOResult(mask_file) # Replace them with your own in practice." 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": 5, 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [ 94 | "# Create a TIDE object to use for evaluation\n", 95 | "tide = TIDE()" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": 6, 101 | "metadata": {}, 102 | "outputs": [], 103 | "source": [ 104 | "# Run the evaluations on the standard COCO metrics (i.e., a range of AP with IoU thresholds [50:95])\n", 105 | "tide.evaluate_range(gt, bbox_results, mode=TIDE.BOX ) # Several options are available here, see the functions\n", 106 | "tide.evaluate_range(gt, mask_results, mode=TIDE.MASK) # evaluate and evaluate_range for more details." 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": 7, 112 | "metadata": {}, 113 | "outputs": [ 114 | { 115 | "name": "stdout", 116 | "output_type": "stream", 117 | "text": [ 118 | "-- mask_rcnn_bbox --\n", 119 | "\n", 120 | "bbox AP @ [50-95]: 40.01\n", 121 | " bbox AP @ [50-95]\n", 122 | "===================================================================================================\n", 123 | " Thresh 50 55 60 65 70 75 80 85 90 95 \n", 124 | "---------------------------------------------------------------------------------------------------\n", 125 | " AP 61.80 59.76 57.07 53.72 49.59 43.66 35.87 25.77 11.68 1.19 \n", 126 | "===================================================================================================\n", 127 | "\n", 128 | " Main Errors\n", 129 | "=============================================================\n", 130 | " Type Cls Loc Both Dupe Bkg Miss \n", 131 | "-------------------------------------------------------------\n", 132 | " dAP 3.40 6.65 1.18 0.19 3.96 7.53 \n", 133 | "=============================================================\n", 134 | "\n", 135 | " Special Error\n", 136 | "=============================\n", 137 | " Type FalsePos FalseNeg \n", 138 | "-----------------------------\n", 139 | " dAP 16.28 15.57 \n", 140 | "=============================\n", 141 | "\n", 142 | "-- mask_rcnn_mask --\n", 143 | "\n", 144 | "mask AP @ [50-95]: 35.92\n", 145 | " mask AP @ [50-95]\n", 146 | "===================================================================================================\n", 147 | " Thresh 50 55 60 65 70 75 80 85 90 95 \n", 148 | "---------------------------------------------------------------------------------------------------\n", 149 | " AP 58.30 55.61 52.48 48.82 44.05 37.95 30.33 20.96 9.54 1.17 \n", 150 | "===================================================================================================\n", 151 | "\n", 152 | " Main Errors\n", 153 | "=============================================================\n", 154 | " Type Cls Loc Both Dupe Bkg Miss \n", 155 | "-------------------------------------------------------------\n", 156 | " dAP 3.05 9.38 0.58 0.32 4.31 7.43 \n", 157 | "=============================================================\n", 158 | "\n", 159 | " Special Error\n", 160 | "=============================\n", 161 | " Type FalsePos FalseNeg \n", 162 | "-----------------------------\n", 163 | " dAP 15.50 18.03 \n", 164 | "=============================\n", 165 | "\n" 166 | ] 167 | } 168 | ], 169 | "source": [ 170 | "# Summarize the evaluations run so far in the console\n", 171 | "tide.summarize()" 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": 8, 177 | "metadata": {}, 178 | "outputs": [ 179 | { 180 | "data": { 181 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZYAAAJ1CAYAAAD6wNYnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAXEQAAFxEByibzPwAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hUVf4/8HcqRXoRJSJVLomAgFQVEYgNdXdBF6xrbHxDGRfEVVAUFYHFxiIQIohERbGwov42qBiKKIZeQ5IBkS6995Bkfn/gDFPuzNyZufeeW96v5+F5mFvO/UAy855zzi1xLpfLBSIiIpXEiy6AiIishcFCRESqYrAQEZGqGCxERKQqBgsREamKwUJERKpisBARkaoYLEREpCoGCxERqYrBQkREqmKwEBGRqhgsRESkKgYLERGpisFCRESqYrAQEZGqGCxERKQqBgsREakqUXQBZAySJAUsczqdAiohI9Pq94S/f9bCHgsREamKwUJERKpisBARkaoYLEREpCoGCxERqYrBQkREquLpxn/yP91x9erVqFKliqJtR44ciUceeSSm43mL9DTLnJwcjBs3TnZdqH+HVXj/X8bFxaG4uFjx9t7UOL116tSp+M9//qN6u2PGjMFHH32kertq8/+/bdWqFebMmaN6uxUqVMCGDRtUaw8ARowYgYyMjKjbpEviXC6XS3QRevD/RXK/KUN9wHtvF+m2cnbu3Ilbb7015DaRtBeuHm9t2rTB559/HlFbcsfX8kM5lEh/fpHU7q+4uBhxcXGK6nAfq1OnTjh27FjIdkP9HwVrNysrCxMnToy6XbUFq1PJ/200//5Y2/X23HPP4ZtvvlG0baS1BttH1PtFNFsPhSn5pXVvE8m2cvLy8iIKlXDtRRIqALBu3bqI91F6TFFvkmD1ZGdn+7x2uVwR/dtbtGiB2bNnR1RHuFBxbxcJSZLChko07apN6fGj+fertZ0kSYpDJVyb9957r6I2/ve//8kut3qoADYOlkh+ydXYdtCgQYrbCNdebm5uVG3FwiyhAgDdu3f3ed2iRYuI23/llVci3keJXr16adKuVvWqTasQVPNLWLj9xo4dq2j7YcOGBWzzwgsvRFWL2dh+jsX/gzGS4a527drh9OnTYY+h9EN5//79uPnmm8O298wzz4Rty72dfwhJkhRxGBgtVCKhtHa57SL9v1LS5tatWxW3F0m7s2fPFh4ukb6X9GhX6Vxmjx49sGfPHtn95X4Hgg3TubcPdtxHH31USdmmZ9seCwBs2rQpYFmoDxL/dWvWrFF0HP9J12DHqVevnqL2/FWqVEl2+TvvvOP5+zXXXAOn02m5UHH/m7z/KNlHybJI61C7TS3bVVskdUYSDJG027ZtW0VttmjRIqCNhQsXRvz/Gmz78ePHR7S9Fdk6WBITlXfYYvml6NSpU8QffpE4e/ZsyABwOp1Bx3tDMUOohCJX//fffx90+5SUFABAUlJSRD8nrf4/jPL/HE6oOv/73/9q0q7cujNnzvi8DtY7CDXXcvfddwcsi7Tn9cEHHwQsM8vPUi22DhYj2bFjhyqT6+4/kydPVqkyX2Z/gzRu3DjoOve31oKCAh0rsraWLVsKO/ayZcsClq1evTrkPm+//XZExzD7+0Ertp9jESWWEFFyGuakSZMwadIkn33UEM0cDZFRaHFNV7j3ox3fL+yx6Mi7RxEr/1NqlR5bDcePH1elHSKraNOmjezyoqIinSsxBgaLTpR8qC9cuFBxe927d4/qm5Aa4dKxY8eY2yCyknXr1skuT01N1bkSY+BQmA6CfZgvXboUderUiantSO4MEG37HTt2DOiltG3bFmvXrlX9eERmE+59Z8fhY/ZYBHE6nTGHilybSs48GzhwoKL28vPzAQArVqwIWOd/Bo5VuIcMR40aJboU0kBmZqaq7Sm98HnKlCmqHtfoGCwm9f7774edNwkWMAsWLFB0jFq1ann+/tVXXwWsF30rkXBuuummgGVKr9D+7LPPIEkSDhw4oEltVnLttdcGXRfL70ioWyBNmDAh7P5yt8NZtGhRyH0irTcvLy9gWaNGjQKWvfvuuxG1a3YMFpNxh8mbb77psywS06ZNi/i4wT48fv/994jb0suMGTNklx85ckRxG5dffrla5VhWaWlpRNsrHRbauXNn0HVyJ6/4t3vHHXfI7hvs/TJ06FDZ5ZFe6PnDDz9EtL0VMVgEGTFihM/rL7/8MqY7uQbbV255t27dFFSo7Nh33nlnVG2J1KVLF89NCbds2aLqGXN25f9/2LJlS1X+T/3bfeSRRyJq133Ra7B2Dxw4gJdffhmSJGHevHkB211xxRVB95fjfo8Ee5+mpaUpKdv0OHkvyFdffSU7vKTEqlWr0L59+4DlSt5wFSpUiOqYobRo0SLsM1BECXWNwXPPPRd2XwrtxhtvxNKlSz2vY320hFvPnj19hmyjbXfhwoUh9+3atWvIdn/66aeQ60PVkJycjJKSEp9lZWVl2Lp1K5o2baq4XTNij0UHkXxAyV31feHCBZ/XVatWDfq8kHBieTgSIP9vMfojfaIJCIaKMpF8QAa7p52cPn36KN62YcOGIddH+7OM9V5nGzdulF2u1R2ujYTBohOlN0dMSkoKWC53W4zi4uKI3jDum1BqxehDSbHcpZhCczqduOWWW0Juk5ubG/Raj1DthguulStXYv78+YraUvpzHThwYMShEmx7NW7EaUa2eYKkkXj/UqWkpER0YWQwX375JUaOHOmz7LrrrsMXX3wRc9tWs2HDBvz973/3Wfbzzz9zol4FW7Zs8bmRo1ohXVpa6nMCSaztfvTRRxgzZozndeXKlXldlooYLEREpCoOhRERkaoYLEREpCqebkyq0epeZRQZ/hxINPZYiIhIVQwWIiJSFc8KIyIiVbHHQkREqmKwEBGRqhgsRESkKgYLERGpisFCRESqYrAQEZGqGCxERKQqBgsREamKwUJERKriTSjJ9g6cKcGqfadRcOg0zpeFvhHF4wXf44TXM97lVE5LQ+0770SVVq3ULJPINBgsZFk/bDuKFftO6X7cM4WFOFNYqGjbio0aofErr2hbEJHOGCxkaot3HcfPu0+ILiNq57ZvR1FGhuy6Cg0aoMno0foWRKQCBguZxuv5u2CnO6ae37VLNnSk7GzEV6yof0FECjFYyJBW7D2JH7YfE12GITkzM31eJ9asiWsmTBBUDVEgBgsZgqj5ECsoPXrUt2cTF4fUmTOF1UPEYCEhth0/h1mFB0WXYU0ul0/QJNerh6bjx4urh2yHwUK6eWPF7rCn85L6Svbv9wma1JwcYbWQPTBYSFOj83eJLoH8MGRIawwWUh3DxDwYMqQFBgup4psth7Hh0BnRZVAMGDKkFgYLRe18WTneWLFHdBmkAXfIJFSvjuYTJ4othkyHwUIR41CXfZQdP+4JGfZiSCkGCynGQLE3d8DUuPlmXPn442KLIUNjsFBIC3Ycw69/nBRdBhnIsSVLcGzJEgDsxZA8BgvJGrNsF8p5yQmFwWEyksNgIR8c7qJoMGDIG4OFADBQSB0MGAIYLLbHQCEtMGDsjcFiUxsOnsY3vx0RXQZZHAPGnhgsNlN0+AzmbD4sugyyGQaMvTBYbITDXiRaUUYGKjZqhMavvCK6FNJQvOgCSHuj83cxVMgwzm3fjqKMDLjKy0WXQhphj8XCGCZkZMV/Xr3P4THrYbBY0PnScryxkjeHJHMoysjg45QthkNhFjM6fxdDhczH73HKZG7ssVhE1tq9OHyuVHQZRDHh2WPWwB6LBYzO38VQIUspyshA6YkTosugKLHHYmJvr9yDM6U8s4asacvTTwNg78WM2GMxqdH5uxgqZAtFGRnY/+mnosugCLDHYjJjl+1CGW9nTzZzZP58HJk/n70Xk2CPxURG5zNUyN54YaU5sMdiAidKSjFx9V7RZRAZAi+sND72WAxudP4uhgqRDF73YlwMFgPjLVmIQivKyICrlKfaGw2DxaAYKkTKFD/5JHsvBsNgMZg3VuxmqBBFgeFiHAwWAxmdvwvnedoXUdQYLsbAYDEI9lKI1MFwEY/BYgAMFSJ1cVJfLAaLYAwVIm0UP/kkdr71lugybInBIkhJWTlDhUhjpwsKODQmAINFgHm/H8H4FXwYF5FeGC76YrDobHT+Lqzef1p0GUS2w3DRD4NFRxz6IhKL4aIPBotOGCpExsBw0R6DRQcMFSJjYbhoi8GiMYYKkTExXLTDYNEQQ4XI2Bgu2mCwaIShQmQODBf1MVg0wFAhMheGi7oYLCpjqBCZE8NFPQwWFTFUiMyN4aIOBotKGCpE1sBwiR2DRQUMFSJrYbjEhsESI4YKkTUxXKLHYIlBuYuPESayMoZLdBgsMRizbLfoEohIY5sdDtElmA6DJUocAiOyh7KTJ1F64oToMkyFwRIFhgqRvWx5+mnRJZgKgyVCDBUie+J8i3IMlggwVIjsjeGiDINFoYmr/xBdAhEZAMMlPAaLQidKykSXQEQGwXAJLVF0AWbAITB5/3upP47t+j3kNg/n5AVdt69oHfLGP6t4eyIjOb93LypceaXoMgyJPZYwGCryZmWkhw0V93azMtJ1qIhIX7+PGCG6BMNisITAUJEXTVAwXMiKOCQmj0NhQZSXX0BfqTa+cB4WXYqhBAsI7yGs3WvzsXjiS3qVRCRUUUYGUnNyRJdhKOyxBFG80IGyNc/gpS4NRJdiGLMeuzVg2cM5eQHzIle17SI7V8JeC1lVyaFDokswFAaLjMK8TJ+/M1z+5HfTzXAT7WpOxJ86uNczX+P+U/C/2aq1TxSLrc8+G34jG+FQmJ/NPwdOyBXmZeLR2tfi6rYOzrtE6L53v0TFajWj3v+b4Rk4uU/+Zp/r5szAujkzAPBsMhKPQ2KXsMfip/T8Udnlpw5vsnXvZe6zD0e1XyyhsixnQtBQ8cdhNjKC3/71L9ElGAJ7LF68h8BCbfPCLRMwef0RnCgp16EqYzh9aJ/P67rXXKv5MX9bnOvz2r9X4h8mx//Yger1G2peF1EwFw4eFF2CIbDH8icloeLmXDwUtx4dbdveCwC0uLW3rsfr/fanAcu8g+au0dMYKmQIPAWZwRITOw+NIU7fX525wx6UHe5yn5VWs0ETXeshCqXoscdElyAUgwWR9Vbk9rVjuPyxYbmQ43qfFXb+FB++RAZl88eW2z5Y9m3+MuY2CvMy0bdkIkZ2vkqFisxh688/aH6McGd6fTm4D28ZQ4Zl5yEx2wfLkZ0LVGmn7MIpFC0YYNneS+fHh0W1X6wf/EpPI56VkY5TB/dGfRwiLdj1wklbB0ssQ2Ch2rRiuDS7+c6Y9o8lYNzzKOFC5ut/PRJV+0RaseuFk7YOFq0U5mXi3tNjLBkw3sIFhRZDVN4hc/eY91Vvn0htdhwSs22waNFbkTuGlcIlkvt/BTuDSyn/27fsK1oXsE2NlEaK2yMi/dgyWAoXDNTvWHmZGNk5xdL/0f4hoEZPxT+E/B8IBgA7V/8S83GI9GC3Xos9r7x36XvFfNGCgegNIC092/T3Gns4Jy+q4FDjXl7hjsv7hZGRHVuyBDVuvll0Gbqw8hdpWXoMgYU6thWGxh7OyUOPZ/8d0fbRHichKVnRtv2mfhvVMYj0sveDD0SXoBt79lgEKszLxMPVm6B+u2fxxoo9osuJWv2W7T2BIdeTaHPfE2h59wMxH+eB6fMAACs+ehebFwaGx0Mzf0RcXFzMxyHSg13ugBznctnnElGRvRU5Vhgas5vHC77HiaVLRZdBJmaHYLHNUFhZ6VnRJQSwytAYESlnh4l82wSLc/FQ0SXIKszLxN/PvW2r28EQ2V3ZWeN90VWTLYLFaENg/srLzln6djBE5GvzgAGiS9CULYLFLDg0RmQfpwsLRZegGcsHi9F7K/7scjsYIrvb+cYbokvQjOWDxazYeyGyvsM/aP/4CREsHSxm6634K8zLxMhOVyLZ0j8lIvs6MHu26BI0wY8sgytaOBj3nOTQGJFVHbPgdVGWDZbCPGuddcGhMSJr2jt9uugSVGfZYAGsd0MBd7hwaIyIjMySH1E7108VXYJmCvMyOTRGZDFWuxrfksFy6uB60SVojkNjRGRUlguWC+eOii5BN4V5mXiuVSKa16wouhQiipGVei2WC5Ytv4wQXYKuti57Fa12v8TeCxEZhuWCxa44NEZkflbptVgqWMx+QWSsCvMyMajBNvRuVkt0KURkY3yCpMXsK/4U8fgUL/EhYhSFv+Xnyy7/ukuXiPeR289/21Dt2tXJNWtQtV070WXExDI9FudPw0SXYCgcGiM1ff3HH6JLsI3d774ruoSYWSZYyi6cFl2C4RTmZSKjci6GXF9fdClkcjk7dogugUyEQ2EWd/LgOpw8OIhDY6SJMcXFEW3PoS9lijIykJqTI7qMqFmix2L3SXslODRGWlh51D7XjZFy7LHYSGFeJvpVqIFmN43DmGW7RZdDJvP4qlX4oH37oOvfaNkSzxUUhGxD6eR9sBMC5nTqhMT48N+Hg+3/eceOqJCQEHZ/Iyg5eBDJdeuKLiMqpu+xHNj6/0SXYCql54+heMEA9l4oYkcuXAi5vnnVqjEf42/5+SHPMrtv+fKQ68Pt32/FipDrjWTrv/4luoSomT5YDm3LFV2CKRXmZeLZa11oVaeS6FLIwN5q1Srouj4qf0Dft2yZ4m3lwuGhFSti2p/Uw6EwG9u2YiyaA/gbJ/YpiGZVqgRdV67ysUpdvo+6CHcNzNjiYrzQooXn9emysoj2H7J+Pf5z3XVR16uH3ZMn46rBg0WXETFT91is9jAvUTixT0oF+6Y/pU0bVY9zX0pKwDJ3UDzUoAG+7tLFJ1T83VynTtD9+111Fb7u0sXwoQIAJ1etEl1CVEwdLFZ8mJcoDBeKRUoldYdU5+zZIxtiX3fpgr9fdVXY/ZccOhR0/wca8PdcaxwKI4/CvEzcizikpU/l0Bh59G/cGNO2bfNZptccRay3gLHCLWT25uTgSpPdnNK0PZbfl48VXYJFudh7IR+9rrhCl+Mo+dB3n/W1+eTJmPZfY6Lrb44tXiy6hIiZNljOndwpugRLK8zLxMjOKeb9BSHN+PcCnr3mGtXaVtqjeK6gAE+sXi27v5Lf2deKi3H/8uURVkdKcSiMgipaMBC9AaTxrDEK4SaZifJYeIfL68XFWBWkd3G4pER2+Vde+7+xeTN+PXxYdrtz5Wqf16adYz/9hBrduokuQzFTBsuR3UtEl2ArhXmZvNeYzTmaNsWkrVt1P+5IvzO/Ip3bea5585j2N4q9M2cyWLS2r/hT0SXYTmFeJoZ3/TdynOew73ToK7DJenpefrmmweL/gV+/YkVktW0b9f5VEhIwq2NHVWqjyHEInRTb8vNw3HjgFU7sk8eo1FRV2vGfW/nj3DkM27DBZ9m/nU7F+58qK8PAtWt9lk367bcYqySlTNljIbE4NEZubWvUUK2t9jVqYNWxY57XW0+fjujplLfXq4cf9u/3vP7j3LmI9jc6M91K33Q9Ft4i3xgK8zIxpNlRjOwc/mI1sgY1z/6SMzI1FdMUPpJXLhQGNGmCj0LcfTnc/qSeOJfLZarL1xksxmOns8YeL/geJ5YuFV2GLfTOz/e5t0bd5GRMv/56xfv3yc/3uZ9ZzaQkzFQYPEZllh4Lg4VUYZdwYbCQSLV79cLlffuKLiMsUw2Fndi/RnQJFERhXibuPT2GE/tEGjo8b57oEhQxVbDs3jhNdAkUBm8HQ0SmChYyh8K8TIzsVB+J/O0isiW+9UkTRQsH4a8nOTRGpLY9770nuoSwTBMsJWcOii6BosChMSJ1nTDBbWlMEyy//fqS6BIoSoV5mXi+TSVcXTVZdClEpAPTBAuZ229LR6LDvlHsvRDZAIOFdMWhMaLY/TF9uugSQmKwkO4YLkSxOW7wi3RNESwHt5njoiBSrjAvE/edGc97jRFZkDmCZeu3oksgDbhcpShaMIC9FyKLMUWwkLUV5mXC0XA37mpSU3QpRKQCBgsZwh+FH6LixmfZeyFS6ODcuaJLCIrBQobCiX0iZQ59843oEoIyfLAULxoiugTSWWFeJh6r8gOebnel6FKIKAqGD5bysnOiSyABTuxfhT0/D2bvhciEDB8sZG8cGiMyHwYLGV5hXibuL8vGi7zmhcgUGCxkChfOHUYxr3kh8nFixQrRJchisJCpcGiM6JI9WVmiS5Bl6GC5cO6o6BLIgArzMnHvmXG8HQyRQRk6WH5bymewUBCuct4OhsigDB0sLlep6BLI4NxDY/c2ry26FCL6k6GDhUiJwrxMYO0z7L0QGQSDhSyDE/tExsBgIUspzMvEo5W+xdDr64suhci2GCxkOacObcTOJYPYeyEShMFClsWhMbKDI3l5oksIwGAhSyvMy8SI66ujbqVE0aUQaWL/rFmiSwjAYCHL27zkOdx86FX2Xoh0wmAh2+DQGJE+DBssh3cYb9yQzK8wLxNDm5/EjfWrii6FyLIMGyyHtv8gugSyqJ1r38UVW4az90KkEcMGS9mFk6JLIIvj0BiRNgwbLER6KMzLxFO1l+OJVvVEl0JkGQwWsr0jO/NwbNnT7L0QqYTBQvSnwrxMjOycwjcFUYz4HiLyUrRgIHqfHsPeC1EMGCxEMjixTxQ9BgtREIV5mRjetirqX5YkuhQiU2GwEIWw5Zfh6HLgFfZeiCLAYCFSgENjRMoxWIgUKswbiF+PpiI1J0d0KUSGxmAhUigtPQtH951EliMXVV94iwFDFAQfUkGk0D/XDIB04W4AwLeTlgMABsycibLjx7FlyBCRpREZCnssRAqk9pyKUldpwPKpT8/DtJeWsvdC5IXBQqRAXFxcyPVZjlwsqvp3BgwRGCxEigxa/RT6XvVA2O0YMEQMFqKw0tKzAQCd6tygeJ8sRy5+qs6AIXtisBCFMa7wNQBAxYSKEe1XXn4xYMqffJUBQ7bCYCEKIS09G7vP7oqpjZ8+L0CWIxfSjA8gTZumUmVExmXY040TkquhrOSE6DKIVJM95DsAwMCcHBz+7jsc+PxzwRURacOwPZZaV3UTXQIRBq1+SvU2sxy5+HxeOVJzcpBQpYrq7ROJZthgqdvkLtElkM25J+21kuXIRV7cnZx/IcsxbLAQifbcOn2upucpymQ1DBYiGWnp2ThddlrXYzJgyCoMO3lvZr0HLfJ5PXdKd0GVkJpc5S5djpPlyAWq/h0DJ92FoowMXY5J5lWnd2/RJQRgj4XIT3xCRdlJ+xMr/9C1jixHLk71fYE9GAqp7l//KrqEAAwWIj8tuv9Hdvnx5Xt0rgRY+d0WZDlycc30GWjxwQe6H58oGhwKI/IzeHV/2eUXDp/VuZJLpj3zPYCL18AcW7IEexkyZGDssRB5SUvPhgv6zKVEI8uRi0+/PInUnBwk1q4tuhwiWeyxmMTWnSfx7PhVPssiPSngs9xt+Hzedp9l8fFx+O+kW2KsjvSW5cgF0AMDczjBT8Zj6GBJvuxKlJzeK7oMofoN+QklF8pl17nPPvvvpFsQHx/8eSHL1x/Ev6cVyK4rL3d52rH72WsVqzXS5Ep7LfEMMjIiQw+FNek4QnQJQvUetChoqHi717EY50vKZNeVlbuChoq/75foPzltJE06Dg+6rn7FFB0riVyWIxcN332PZ5CRIRg6WOITkkWXIMyi5fsi2v7+oUtkl9/nWByw7IXMVpgxNvDZIu99vjmiY1rNKwUvBl13f8OHdawkOjkv5iHLkYvUnBwGDAll6GCxs3c/KvJ57XikBeZO6e7zx5//hZl9Bi8K2GbulO7o0KoOalWvINtG5qj8GCs3p7T0bBw8fyDo+iaXNdWxmthkOXI9AXP188+LLoc01HDkSNElyGKwGNCxkyUBy3p0vjJgWbg5EZffyU1y27uXvTyoNeZO6Y7sV7tEUKl1fLtnbsj14Z55b0RZjlx8mPU7UnNyUKmpeYKRlKvcrJnoEmQxWAzoseFLfV5HMqm+bN3BiI83d0p3tE2z76mrLXpMxg/75okuQzNZjlzMO9COw2OkGwaLyb37Ukef1+OnK5uop0vi4w19cqRqeJNL0ovhg6VStcaiSzC0qpWTRJdgemY7xThWDBjSmuGDpXFHTj6G4n/R5CN/bSKoEnPS+mFeRpblyEVxm/4MGFKdPcYATOam6y/HL6svnaHUe9CioPMsh4+d93nd57aGER/P+2yyuDjgq8n2uVBy2tYs0SUItXfrUWQ5cvHQm1NQve5lvMjSRC6//37RJQRl+B6LHQ17/FpF2y1dHfz0WDn+pyMDwOz/bfN57X8mmZWl9pyK9cfWKt8hQbtaRPvktcXIcuSixcyZaPzKK6LLIQVq33GH6BKCYo9FB3If6HJCnf3lf9sVuTb99587pXvAdt69nz6DFyk6JdmqIjmFuOTAaSTWqAQcPKVhReJNffri2XEDc3Kw8+23cXrjRsEVkRmZoseS0vJx0SXoLtgHfO9Bi2RDpW7NCrLb9+oWeCsSdxv+oZKUaIpfB9VEMml/fPkeJNWoqGE1xpLlyMX/trdAak4OLmvdWnQ5ZDKm+CSpfkXH8BtZkNLewzvD22Pa64G3aAGAp/o2xzOPpYVtIzkpHl9M7BZRfWYW6aT9iVV7kVjVfrcYynLk4n/bJE7wU0Q4FGZw7nB56T9rUbDlmM+6559qic5t6oZto2v7eujavh6mfb4Z3/ndaLJe7YrIfs1+V9u/vmlUxPsk2DBY3HgXZWOpdeedoksIKc7lMsd0bWFepugSyCLS0rMjvnblmW8fRu07m+LTT9ZrVJW5MGDEMnoP0hRDYURGkHAZL0Z1y3LkYl2zDMN/wJEYpgkWO1/IRuqK9kr7hEoMFm9H959GliMXdUZPZMCQD86xkK2kpWcDEQbL31LuAwDEV+LbRc6Xb/wCABgwcyYuHDiArbxVv6Yajx4tuoSwTNNjIVLDsLVPR7zPjXW6AgDiKzBYQpn69Dy8//oqpObkoFrnzqLLsayKDRqILiEsBgvZRmrPLJwrPxvxfpUTKwMA4pL4dlEiy5GLbzY14PCYjZnqndLsxtdFl0AmFhcX2697fJKF7+migSxHLpbWfYABYzv/QJMAACAASURBVEOmCpbkSnVEl0AmFuvt8eNsdmcCNZScK0WWIxc/12HAqOGKRx8VXYIifKeQLahxVmFcgvkeT2wUpecvBkzyP8cxYGJQs7s57uXH2UiyhXc3vxN7I/EMllj98MEaAMCAD2ai/NxZbB44UHBFpAXT9Vh4PQtFKrXnVDhPFokug7xM/ec8vPf8IqTm5KD2PfeILodUZrpgIYpUJLfH16MduiTLkYs5iysiNScHcRXk79BNF5lpCJHBQpZnt2fam1GWIxcLk/9iqg9PCs6UwdLgOo7LkjJqDJ2eKojsSZ0UvSxHLhZV/TsDxuRMGSxV6/LBQ6TMSxtjv73I8eV/qFAJRSLLkYufqjNg3Bq9/LLoEiJiymAhUiItPRtHSo7E3M65ncdVqIYiVV5+MWDKnnzV9gFTqUkT0SVExLTBclmt8E9FJCLzW/J5AbIcuSh9/BXbB4xZmDZYGraL/GaCZCfxnLS3mJ+/3IQsRy6kGR9Amj5ddDm6ubxfP9ElRMy0wUIUSlp6lugSSCPZQ75D9jPzkZqTg8vvv190OZqrbfDHEMsxdbDUbnib6BLIoIasUefMwVrJtVVph9SX5cjF57llSM3JQULVqqLLIS+mDpZ61/QRXQIZUFp6Ni64LqjS1sMNzXHTPzvLcuQiD3dYcv6l0UsviS4hKqYOFiKtXVNVEl0CKWTFa2AqNW0quoSomD5YWnSfKLoEMpDEirVVnbSPj/EZLqQ/KwaM2Zj+XROfwPsL0SXNbxojugQyiCxHLk71fcG0AWPWugELBAuRt/FFDBa6ZOV3W5DlyEVqTg6ucjhEl2MblggW3kqfgIu3x995ZrvoMsiAshy5+DhnH1JzcpBcv77ocizPEsFCBPC29hReliMXP5y80fDDTEavLxzLBEul6uY8e4LUwyvtSSlO8GvLMsHSuMO/RJdAAnE4lKJhxICR3ntPdAkxs0ywkL09v36oJu2WnS7RpF0ylixHLrbfONgQARNvgSdpWipY+K3VntLSs3Gq9JQmbfNZLPaxbcN+ZDlycfXEbHEBY5F5wkTRBRDF6pMdH2nW9vEVezRrm4zpw5ELAAADc3JwuqgIO8eP1+3YqTNn6nYsLVmqxwKw12I3qT2z8OuhnzVrv/xsqWZtk7FlOXLxYdbvSM3JQUWT3lpFFMsFC9lLHG+5QhrLcuTiuwPtkJqTg0rNmml2HCPM76jFku/Kmld1E10C6YSnGJNeshy5mLe/raUCQCuWDJYrWzwgugTSAYc9SQQtTlG2WlhZMlgAIKkiH9BkdROcb2jaftc67PlScEa8BsYoLHtW2DU3jUFhXqboMkgjqT2nYsqa/poe47YreuEoNmp6jGAmzn8m5Pr4uHhUrVQLzeu1wQ3X9Iq4vX/e9k5M9dElWY5c1GvVH/cOuwFFGRkR72/FYLJssJC16XFfsFoVauGo5keJTrmrHMfPHMLKbXlYuS3Ps5yBIcb+7UeR5cjFg29MRo3Lq0QVMFZi2aEwgGPwVsZJe3nhejqkrU9H/4QsRy5azJyJxq+9FnZ7K/ZWAIsHCwDEJ1QUXQKpjF8YQmO4iDf16Xn44M2NSM3JwWWtW4suR3eWHwpr0f0/nGuxmBc22O+Go6GGuN6dPwwuuHSshpTKcuQCkDAw55mA4TGr9lYAG/RYAKDCZXywj1WkpWfj+IVjosswlKdve1t0CRSG3c4gs3yPBQCadnmZvRaiEErLSjBlwXCfZQN6jENyYuCdduWG2lqmdEbPa/sib9Pn2LRnuWc5TybwleXIReWUh5Ax5lbRpWjKFj0WgOPyVsFJe234h8pd12UoDhUAKNizjHM7Cp05Yf1HMdiix0LWkJaeDTBYfHy34WNs3rfWZ1mkvQT/QLiqZlM0qxc44awkOLx7KyRv4KS7RJegOVsFS1p6NofETGzG7/o+We/sNuPM5SjtDcQaKglxCbi3w6CA7SbnPRf2WOyxhBcXb43nrYRjm6EwMrfUnlOx5ugqXY95fLm1n8UiFwSDb31Tdtuyct/HB8gFGOdTwhswMfxdEqzAdsHCuRZz0uNKe3+niw7pfsxYTZz/DKYteknRdv4YDNp6bJy1J+y92WoozK1Clatw/tRu0WVY3sL8vZg0qziifeZO6S67XMSkfa/PfO9F5nzVqXsNbqE+9P1D4uyF05i3/kP0uu5RrcuiCFSqkiy6BN3YrscCAE07jxRdAgXRe9Ai9B60yGeZlr3MvAeXIO/BJZq1r4d/3vYOqlSo7rNsy/71EbfDORLt2GHC3pstgwXgkJjReYfL2MJXVW/fCoHi7Yluo0SXQORh22Ah4/vjwBmkpWdjz1kOW2rhn7e9wzO7dGC33gpg0zkWN55+rL9gcygAAobABr26HM4Hta5I3qONnkAezNOjiTQQQs3ZLC7+Cre06BNy/+2HitCoTqrPsjkrJ0dUgx3YMVQA9ljQ4LqBokugP8mFjpJJ+42TizxDWzty1endtK15fcj15eVlmLHkNXy1SsyQanl5GY6c2o/5BbNj7mX4h8z6nb8EbNO1+V98Xn+zZjpm/Xrp1OSJ85/BnqO/x1QHWYeteywAULWu/W5pbQUlJy5gSWZ+wPItn/yOLZ9c/IBL//TmgPXB5lW8l6d/ejOS4pNkt1uzfTF+3vyt5/Wpc8c8H+xanq6rZo9EjnRFOzj3rfE5nncb7Rrd4vPvBoDDp/Zy6CwEu/ZWAPZYAHAi3yj8h8Lu+fwO2e3yHlwiGypy25WcvKBKbQDQtm3bgA9Xb0b5kI0m4O5o/XDM7T7U5dmIj2tVf326s+gShGKw/OkKqZ/oEmzBfTqx3B9vlStXxtmyMwH7Hy2K7DYrS/4vfAApdebMxXqSEirAkS5/hbrIcLk2pVNMvaanb33L53WwiyhrVK4ru7xOVT6ewi3lmtqiSxAqzuVy8QlBf+JEvrqiuUASAGb++0bc0PsD2fkV/6GsJvc1RJM+DUNuAwQOi/lv479+yvXTIUlSQDv+H9xHTu/Hx0vHh9zGTvzDyI7/F3YeAnOz/RyLN54lZgyPDV+Kyu+2wQ1vdfBZXl4W+B3IP1SAiyGhxTUqVapUCVhW67J6qh/HiPwDo13DW9BV+kuQre0rY2y66BIMgUNhfqRbJogugQCc+eNsQDgsfORnn9dyk/MefrcW2zEv9rPFfl70a8xtWMWaHYsDlhlljkmkylUDn2FjR+yx+ElIrCS6BEsLdR3L9C82Y95P6txR+IZ3OuDXoSs9r7fM+h0Ne10VU5sl59Q7EcBs/nnbOwHBES5I7DYMxiGwS9hjkcGzxMR4qm/zgGXRDmlVrqf+F4TzZ+wbLAAQH6f84+KyCtU0rMR4GCq+GCxBMFzEeOGFF1Rpp2jGFp/XtVvXjGj/4ysCe07nz5bKbGkfjlvfUtQL+edt7+DJbq9oXxAZFofCQooDwJPm9DR27Nig66o1rYoTW096Xi945Gf0/Lir7LZ7Fuz1ed12eKuI6pB7yJfdeyxudhviCoe9lUDssYSQlj5VdAm2ItdLrNasqufvHUe39VnnkjlLTC0XDp0NWMZgIX8MFXnssYTBU5DV5X8hpK/A60Y6vtZWZrtL8h5cgk5j26Fqo4unAq8YuQYnfj/ls02bf7VUXN/a8RvR9nn53s35swwWuoShEhyDRQGGixh12tYKWCZ3jcryF9YEbBeuHX9KThJgj4Xc4uP1f1S2mXAoTKEaKfJj+aSNuh1qB+1phLx+xVtc8G3jkyP/1WewkFvmxF6iSzA09lgUqp/6EE4dXI/SkhOiS7G0m97tiIp1Kobdzh0YwXoa4cKnR85N2Prldmybu9NneVw8IFVNld2HQ2EEcAhMCd4rLEIcElNfWnq2oueu6GVE6ss4Pz7w2SIFdSpgz+bDAioio2CoKMOhsAjx+hb1DVkzSHQJPlIqyV+hz6Ewe2OoKMdgiQLDRT1p6dm44CoRXYaPuDj5iVkOhdkXQyUyDJYoMVzU8d5vU0SXoBh7LPbUoEUd0SWYDoMlBgyX2KT2nIoNx9eJLkOxEpvf0sWu7hnUSXQJpsNgiVHDdkNEl2BawYaciIyCQ2DRYbDE6LJaLZB82ZWiyzAlI50JRuSPoRI9BosKmnUZJboE0+EwIhkZQyU2DBaV8IMyMq9tekl0CUSyGCqxY7CoiOGiTFp6Nvaf2ye6jKDO7eLdFeyKoaIOBovKGC7md3L9ftElkAAMFfUwWDTAcAnN6JP2J1b+IboE0hlDRV0MFo0wXOTx/4WMhqGiPgaLhvghGuizHbNEl0DkwVDRBoNFYwyXS1J7ZuHnQz+JLoMIAENFSwwWHTBcLoqL468bGQNDRVt8p+uE4WL8SXsA+Ev93qJLII0xVLTHYNFRWno2kirWFl2GEGYJ1pvqdhNdAmmIoaIPBovOrrlpDBq1f050GbqbuPlt0SUoclniZaJLII0wVPTDYBGgco0mpvkGr4bUnlOx+WSx6DLIxhgq+mKwCGSXcOHt8UmU9EfbMFQEYLAIZodwMcOkPVnPwEl3oXn7FNFl2BKDxQCsHC5W/reRcbGXIhaDxSDS0rMRF58kugzVjdzwvOgSyGYYKuIxWAwktcckS33DT0vPxtELR0SXQTbCUDGGRNEFUKC09GwU5mWKLoPINJ5663YkVeDHmVGwx2JQ5u+5xJty0v7Emr2iS6AIDZx0F0PFYBgsBpaWno36af8QXUZU0tKzRJcQlePL94gugSLAoS9jYswbXI36N6BG/RtMNzT2zzUDRJcQlZJ9p0WXQAo8MLIbatarIroMCoI9FpMw09BYas+pKHWVii6DLGrgpLsYKgbHYDGRtPRsXFarhegywuKV9qQVDn2ZA4fCTKZhuyEAYOihMTNO2pOxPfJqd1StVVl0GaQQeywmlZaejZpX3SK6jABmGrLzVz2xhugSSMbASXcxVEwmzuVyuUQXQbExUu/l25QU7DqzU3QZURncbAiS3zkkuy6+WjK+286LPfXEYS/zYo/FAtLSs1HvmntFl4G09GzThgoASNVSg65Lql5Rx0qIoWJunGOxiNoNb0XthrcaqvdiNvFxwb9nJVZP1rES+2KgWAODxWLccxwiAsbKk/YJVSqILsHSqtSshH+81kN0GaQSDoVZVFp6NhKS9HvMrpkn7ZVIrMIei1YGTrqLoWIx7LFYmNTt4nPm9ei9PLduqObHECmBwaI6DntZF4PFBrQeHktLz8ZpCw+DAUBCZb5V1NLzkesgdbxKdBmkIb5bbCQtPRuu8lIULRwsuhTTia9ovYew6S25UiKefON20WWQDhgsNhMXn4i09GyUlZ6Dc/GQmNuLj69o6Ul7t/iKCaJLMDUOe9kLg8WmEhIrIi09G2ePb8e2lf+Oup0WPf4DWCRYLhw5G3RdfDKDJRoMFHtisNhcpeqNkJaejRMH1mH3hsjP7Bq8ur8GVYlxfMUfQdfFMVgiwkCxNwYLAQCqXd4GaenZuHDuCLb88oKifdLSs+GySG8FCP2Qr/hEnpkfVhww8F0GCjFYyE9SxVqKzyJ7uzj6ITRDKg9+27y4BAZLMKldGqD7g61Fl0EGwmChoEIFTGrPqZiyxjrDYInh3grxfMaMv4wx6ahcjXckoEAMFgpLLmCs9jCvjCZPATgZfANr/XNjwvkTCofBQoq5A6ZoocNypxinVbsWu7Es6HqrBWk0GCikFIOFIpbaYxKm/Pl3qwRMhQQO6ci55YFWSLvhatFlkMkwWCgmU66fDgDYdHwjsn57V3A1pIbE5AT0f/sO0WWQiTFYSBXXVm/lCZnXN43C3nPBrwkhY+JQF6mFwUKqG3ntq56/P7/+GZwqDTEpTkIxTEgLDBbS1Pjr3vH8/Yudn+Kng4sEVkMAw4S0x2Ah3fS9+kH0vfpBz2urTPwbXde/X4tWNzcSXQbZCIOFhHHPyQDA7jO7MK7oNYHVWEfTNlfg9ieuF10G2RiDhQzhqsoNfILm5IWTGL7hGd2O7yot1+1Yarvhb6lo07OJ6DKIPBgsZEhVk6r6BA0AfLhtBlYcCX4RYyyOrwh+A0ojSUyKR/937hRdBlFIDBYyjUcbP4FHGz/hs+xYyTG8uPFfMbcd6s7GonR/qDVSOzcQXQZRxBgsZGo1kmsE9Gzc9p/bh9GbXoYLwe9a7FZ67LzapSlya0ZbXHN9fSHHJtIKg4Usq17FKzD5+mmKtj1b5zhObzqIk+v2ofx8WdTHbNrmCrTs2ggpzWtH3QaR2cW5XK7wX+eIiIgU4tOLiIhIVQwWIiJSFYOFiIhUxWAhIiJVMViIiEhVDBYiIlIVg4WIiFTFYCEiIlUxWIiISFUMFiIiUhWDhYiIVMVgISIiVTFYiIhIVQwWIiJSFYOFiIhUxWAhIiJVMViIiEhVDBYiIlIVg4WIiFTFYCEiIlUxWIiISFWJogsgIlJCkiSf106n0+d1nz59sGnTpoD95syZg1atWmlaG/lij4WIDG3nzp2QJAnx8fFwOp2eQPEPGrlQITHiXC6XS3QRRETBuAPEv4ciSRJ69OiBqVOnel63b98en3zyie41ki/2WIjItBYuXOjzmqFiDJxjISJD8++peHM4HAHLvIfIQu1L2mGPhYhMxx0egwcPBgCsW7fOs/yTTz7B66+/7rMd6YtzLERkKnJzLmVlZbjxxhuxbNmysNuS9thjISLTCBYUCQkJAaFC4jBYiMgUoul9NGzYUKtyKAQGCxEZXrhQkSRJdj5lx44dmtZF8hgsRGRoSnoq69evBwBs3brVs2zy5Mlh9yNt8HRjIjKFYGd4OZ1OVKxYEa1bt0avXr181v366696lEZ+eFYYERGpikNhRESkKg6FRSE7fzPOXigTXYZhDL05VXQJRGQg7LEQEZGqTNtjkZvIC3UqYqj1RESkHtMFS6h7/zBAiIjEM1WwhLtrqXu9JEkMFyIiQUwzx6LkVtgMEyIi8UwTLG7hwuORRx4BAAwZMkRRe+5bQUiShIMHD8ZcHxGR3ZnmAslY5k/k9r3pppuCBkm4Y/B0Y1883ZiIvJmix9K3b18AQMWKFVVr0x0qTqfT8+eKK64AwIcDERHFwhTB4r7BXHZ2tirtjR07FkBgz+Snn35SpX0iIjszRbC4JSQkqNLOCy+8ACD4tTA8CYCIKHqmCJZmzZoBAMaNG6d6296T97t27VK9fSIiuzFFsOTm5gIACgsLVWtTrmeSnp4OSZIwfPhw1Y5DRGQ3pgiWSJSVlQV9mpwc78l7t7lz52pVHhGR5ZkuWPr16xdyfVpaGoDQpwxPmDAh6PxKYqKpbkZARGQ4pgkWd1CsW7fO88hRf0p7KTNmzAAAfPHFFwHrSktLo6yQiIgAE10g6aYkPPx7K3IXSHq3s2HDBhw4cADp6elB2/DGCyR98QJJIvJmmh6LW6gP/N69eys+Vdh7u9atW3tCpW7dujzdmIgoBqacUIj0g583rSSyj1WrVgEA2rdvL7gS+zJlsBCRvcgNgSckJPhcgjB//nw4HA6fbSZPnoxbb71V8/rIF4OFiAwt2A1o/cPG4XAgPz8ftWrVAgBs3rwZ99xzD0cmBGCwRCGzS3PRJRARLj3Ur1evXgDgCRUAaN784vv0gQcewOzZs4XUZ1cMFiIyNCU9jq1btwZdt2bNGjXLIQVMd1YYEdHvv/8OgCfgGBWDhYhM584770RSUpLoMigIBgsRmcb58+c9k/YFBQWCq6FgGCxEZArHjx9H69atAXAIzOg4eR+FVT++itKS06LLIAPpfNcbokuwtAkTJiA7Oxs333wzpk+fLrocCoPBQkSG9uKLL2LOnDmYN28emjZtKruN0+kMeh9B9m70x2AhIkObM2cOAHiuVfHnf3PZoqIiAEBqKm+OKoph51gieVgXEZHT6UStWrWQmpqK1NRU1K5dm70VQdhjISJDiyQc8vPzNayElDJsj4WIiMyJwUJERKqy1FDY6NGjMWvWrIDltWrVku0ib9++HbfffnvA8o4dO+Ljjz/WpEYiIquzTLDceeedsvcPkiQJR44c8dwF1e3MmTOeUPHffsWKFTh69Chq1qypU/VERNZhmaGwYDelCzbx17Zt25Dbd+7cWe0SiYhswRLBUlxcDAC44447ZNevXLkSgPxT6OQ4nU6epkhEFCVLBMtf//pXAMDEiRNl11erVk12+b333qtZTUREdmWJYImUuwfTo0cPwZUQEVmPLYOlQ4cOAC7dKoKIiNRjiWBxP5fBPSHvT+4UZABYtGiR7HLeToaIKHqWCBb3k+TOnDkju3706NEAeJdTIiI9GD5Y3L2HYH/c3L0SSZKwe/duAIDL5Qp7K21JkrBp0yYAQHl5uWd7hhARUXQsc4Fkhw4d0LZtW6xduxY9e/b0WVe5cmWsXbs2YJ+5c+eid+/e6NOnj8/ymTNnalorEZGVxblcLpfoIsyGT5Akf3yCJNElhh8KIyIic2GwEBGRqjgURkREqrLM5D0RUawefHmxLsf59LVbdDmOKBwKIyIiVTFYiMhUbrjhBkyaNClg+ccffyx7rdvGjRsFVGlvDBYiMpXDhw/LLn/99dd1roSC4RwLEZnC4MGD8eOPP4bcJtjF0KQvBksUXl36Ek5f0OYCycuSLsOoG0dr0jaRWXnfainUDWKtEiqb38rQ5TjNn83RpF0OhRGR4bVs2VLx/fvccyv/93//p3FVFAyDhYgM77///W/I9WfPngXg+/jxxYsX8/EXgggZClPyw1br7sKSJAW0xTsYE1lL+/btAQS+p929F77X9WXYHosa3zT4bYXIHjZt2sTwMBChk/fBfhEYCESkhrS0NBQWFoouw3YM22MhIlIq2OPEGSpimC5Yvvzyy5BPknTzXhZsm3nz5ilqi4iMbdiwYUHXFRcX61gJASYLFkmSMHLkSAAXh9GcTqfnF8o/ELyH2dzb+hs6dCg6deoUsD47O1uL8olII/379wcQ+Cjzhg0bIi4uTnB19iN0jiVU7yDURJz3uv79+6N///5Rnf1RUFCApKQkn3YlScKECROQmZmpuB0i0k+w97h7eXp6OurWrYvZs2frWRZ5MeyV92VlZUhISPC87ty5MwBgyZIlqh3DO1SIyBry8vJEl2B7QofC3ENQ3n8WL14M4OLZHJs3b/Zse/ToUQBAvXr1ZNvKzc0FAEyYMEHboomIKCTDzbFceeWVni7tPffco3i/Zs2aAQDef/99TeoiIiJlDBcssWrXrp3oEoiIbM0ywZKamgrg4sN+iIhIHEMGy2OPPRawzD08FuxMsvLyck1rIiIiZQx7ujEQ/LTC1NRUFBUVBbSTmGjYk9yIiGzDkD0WQD5U3MvKy8sDrpRPSEjApk2bfLb3v2iKiIi0J+Qrfix3IXXvO2bMGMyePRuzZs1CmzZtZLcdNmyY7K0elF58SUREkTNsjyWcF198EQUFBUFDhYiIxDBtsBARkTFxtjsKo24cLboEIiLDYo+FiIhUxWAhIiJVMViIiEhVDBYiIlIVg4WIiFTFs8KiUDD5J5SdvQAASKiUhJaDuwmuiIjIONhjISIiVTFYiIhIVZoMhX311VcYMWJEyG369u2L0aN5oSERRUaSJAwePBgOh0N2nTfe+08MYT2WL774gnccJqKIhPrMcK9zOp1hn99E2tJ08j4lJQULFy6UXef+gUuSxG8VRBRWqJBYu3YtAN8eitPphCRJ2LZtGxo3bqx5fXSJsB6L9y/AvffeK6oMIjIBSZIQFxcX9Evo/fffH3TfO+64Q6uyKAihpxu7v1EUFBT4LPfu0vqTWydJEh566CG8/PLLEY2x+m9brVo1rFy5MrJ/BBFpjqMa5mKps8LcQVG9evWAZcG29XbixAmOyRIRxcgyF0h+8sknqFKlClavXu1Ztnz5cvzjH/8ImMcJ1iMqKSlBq1atOO9DRBQDS/VYvEMFADp16hSwTVlZGQAgKSkpYF1ycrI2hRER2YilgiWU999/HwCQlpYGAAHzOm4pKSm61UREZEWWCZZ+/frJLn/xxRcBAG+++abPckmSZP/s2bMHALB161ZtCyYi1YQauuawtv4sM8dy/vx52eU7duwAAFSuXNlnOX/ZiKzHe36UJ+KII7TH0q2bencF/vrrr2WXz5o1CwCwZs0a1Y5FRMbjHSihLlkg7Qntsezbtw+APj/8uLg4z7Hcv3hKr5MhIuPgsJfx6dpjOXbsGIYOHerzjWL8+PGK9w/XtfVf7349YMAA2e379u3r8/qBBx5QXAsREcnTtMeyZ8+ekGEQ7NvFiBEjMG7cuKjGSOX2GTJkSMBxJUnC+vXrZbfntx4iouhpEix9+vRBnz59ot4/IyMDGRkZPh/6Sj7snU4n7r//ftkb0sltCyDiYxARUWiGPissmg/6zz77TPNjEBFRcJa5joWIiIzB0D0Wo2o5WL3TpImIrIY9FiIiUpUleiycJyEiMg72WIiISFUMFiIiUhWDhYiIVGWJORa9zXj1W5w7XSK6DFMa9MZ9oksgIo2xx0JERKpisBARkaoMNxQW7saTQ4cORWZmZtD9eOoxEZFYpuuxTJgwgU+GIyIyMMP1WNyC9TwYKkRExma6HgufZ01EZGyG7bGoJdTzVvzDyf0AMLltiYhIGdP1WCL54Hdv27FjR5/t16xZI9vjYS+IiCh2hu2xhPqQT01NVbx/v3798Nprr/mscz/bPlwPhojM48MPP8TYsWNl13EEQl+GDZZQPvzww5Dr3QHx8ssv46GHHvJZN3ToUADwPL7Ym/dQGBGZiztUGCLiGXYozOl0ev4UFxcjOzvbs65jx4545513ZPdzB0OFChUCQgUA5s2bBwCoXLmyBlUTEZFhg8VbXFwcunfv7gkaAHjvvfdC7nP+/Hk9SiMiA2FvxRhMORQWSv369bFo0SJIkgRJkviLRmQjckPZ/AzQnyl6LJFYtGiRz+sJEyYIqoSI9OY9hM5r3sSxXLC4uX+pvOdmvJfzl43IWuR6Jh07J0jDTQAACKJJREFUdhRQCZkqWCI9a8vdewm2T05Ojs9rhg2RtXz88ceiS7Alw86xhPuQVzJuWr9+fZ/23Pu4A2rcuHEYN25cbIUSkXDBLpz++uuvRZRje6bpsTRp0gQLFizwGTtVIti27nauvvpq3HDDDRG3S0TGMWvWLNnlzz//vM6VEGDAHku0H+6h9vNet3PnTlx99dWe1z/++GNUxyMi4+jQoQOAiz2XxMREdO/e3fPe5hdG/Zmmx6KWq6++GpIk4fbbbw9Yd9dddwEA+vTpo3dZRBQjp9OJ/Px8lJaW4scff4TD4WCoCGK4Hotetm/fDkmSsHLlShw/fhzp6emedeHmXZ4Y9RetyyOiKNSqVYthYgC2DBbvs8vcXWjvdUREFD1bBgvAACEi0ort5liIiEhbDBYiIlIVg4WIiFRl2zmWWKwbOxalp06JLoMsqH2QJyASmQmDJQqlp06h9PRp0WUQERkSh8KIiEhVQnos3jeYDHfab8uWLXHhwoWAbYPddI6IiMQyfI/FHSpERGQOpp1jYU+FiMiYhPVYUlJSAIR+7gofvEVEZD6m7bGE4h1I06ZNQ7du3RRtu3HjRiQnJ2taGxGR1QmdY/n000/DbhNsyEuSpIAejdyy/v37Q5IklJeXh922VatW7CUREcVIaLBcf/31AOSHvGL5gHc/DdL7qZCpqakB27lvse3+M3r06JiPTURkd4Y/KyxSwXo47hADgGuvvRYAkJ+f77NN3759AQBPP/20RtUREVmf8GCZP39+0HXRnPklSRLmzZsX0I73sNumTZs8257yuzWL0+nEoEGDIj4uERFdJDxYGjZsCMB3+CnWoaihQ4d65lDCtXX99dd7tmvVqlVMxyUiIgMEi5rccyWVKlXyWS4XMN7zL24lJSWQJMkzVEZERJEzRLAsX748YFksF0CuW7fOExwbN270LJ8+fbrscdx/3n77bQBAaWlp1McmInHuvvtuxaMVpB1DBEuNGjUAyPcslAq2b3JyMj788EMAwFtvvRVy27vvvjuqYxOReJIkYcuWLT6jEQwXMQwRLP6SkpIi3sd9hpfcL9Kjjz4KAMjLy/Np3+FwRFsiERmQ90iH++/ukQjSj2GuvN+4caNn8rygoCDi/WvVquX5e7BvKQ0aNPC0L0kS5s+fD0mSkJCQgLKyMs920QQbEYnzwQcfBF03bdo0DBs2TMdqyDA9FjVupeJ0OlGvXr2A5Tk5OQFzNt6vvUPF6XRGFWxEJM748eNll9922206V0KAoB5LsIn5SJYH23bJkiUx10FE1vDoo4+GvFaOtGGYoTAiIrX5X3oQzqev3aJJHZFq/myO6BJiwmCJQmKVKqJLICIFli1bJroEW4pzuVwu0UUQEcUi2KPK+QhzMQwzeU9EFC0Gh7EwWIjIMuQuNWDo6I9DYURkGf7BcuWVV2Lx4sViirExBgsRkZdIbgMT6a1jlPaeYq0h1HH0mHfiUBgRkU7scu8ynm5MRCQjmm/0SnoKkiQpbjvaXkUkx9ACeyxERDrQ+4NeZO+IwRIBl8vlueX+X/7yF9HlyDLLsyiMWuOMGTM8tY0aNUp0OQE6dOjgqW/t2rWiy/EI9bM0y++kVRjhLDgGi0KSJKFFixa46aab8PHHH8PpdBrqjeL9xh08eLBnGSknSRLeeOMNvPXWW3A4HPjss88M9X8oSRJOnDiBWbNmoVevXrj//vsNUV+4UAF8H2tx4MABXeqyM9HPo+EcSwQKCwuRkJAAAJ5g2bFjBxo2bCi4sou8v6k4HA5P2BjhG4w3I3wY+pM7U2bw4MGGqbWwsBDApfo6dOiACRMmCK9PSai4a3Y6ncjMzETXrl0N9zspJ9S/LSUlBQsXLoyqvTp16uhag4jPAAaLAu4frjtUvN12223C3ySLFi2SXW60XhVgzFBxa9KkScAy0T9bt+HDh4suIYB3cAT7uX7//fc+r7Ozsw39OxArJf+2pUuX6lCJ2Pc/gyUGbdu2NcQ4d/fu3WU/AI32Br5w4QIAYwYeAHz33XcAgHHjxmHx4sX44YcfBFd0ybfffgtJklBSUuJ5dlFaWprQmpSEbuPGjWWXjxo1Cq+++qraJalK7S8V+fn5Pg8k1KMG93tN714LgyUGgwYNwpNPPim6jJDatm0rugSPli1bGqYHEIx34EmShEqVKmHdunUCK7pELpCN/v8ZzGeffWb4YImGkX8eeoYLJ+9jcPnll4suISj3B9Bnn30muJKLjNhD8ed+47n/fPjhhzh79qzosjy8h542bdrks4woGBFhx2CJgRGGweQY7Vbhc+bMAWCceoJp0KCBz+vOnTsDMMaHt//PNDEx0fN30UNi0VDjUeSknN5niTFYYmC06xwWLlwISZIQFxdnqA/xF198EYD89QySJGH58uWiSvPx1ltviS4hpLFjx8ouLysr07mS2H3yySeiSyANMVgUyMzMFF1CWHPnzsWAAQPw9ttvo7i4WHQ5PryHl9x/vNd16tRJYHWX9OvXT3a5UX7+L730kugSIvbEE0/ILm/durXOlZCeXzYZLAoMHToUgLGf9TB8+HA89dRTuPvuu0WXYhm9evUCcOnnL1qwnsnGjRt1rkSZmjVr4pdffvFZZoRhRTvT6/OKt81XqG/fvli/fn3AciMES7g3qxFq9Ge0eSBA/v/xgQcewCuvvKJ/MTKC/ZyN8H8Y7IwjI38ZI+0wWCI0YsQILFmyRLeLnEh/6enpaN68ObKyskSXIqtr165o3LgxPvroI9GlKNahQwdkZGRg0KBBokshHTBYiIhIVZxjISIiVTFYiIhIVQwWIiJSFYOFiIhUxWAhIiJVMViIiEhVDBYiIlIVg4WIiFTFYCEiIlUxWIiISFUMFiIiUhWDhYiIVMVgISIiVTFYiIhIVQwWIiJSFYOFiIhUxWAhIiJVMViIiEhVDBYiIlIVg4WIiFTFYCEiIlX9f6BeADww5i2yAAAAAElFTkSuQmCC\n", 182 | "text/plain": [ 183 | "
" 184 | ] 185 | }, 186 | "metadata": {}, 187 | "output_type": "display_data" 188 | }, 189 | { 190 | "data": { 191 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZYAAAJ1CAYAAAD6wNYnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAXEQAAFxEByibzPwAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hUVf7H8W8INTTpUhVBLomAAtLsaETFsoor6M+Ga1kEoiKuq6KiomJHxILY4oqKguK6i6sYAUWNgAJSEgZEaUE6UgOk/f7AGWYy7c7Mvffc8n49D8/D3HLuFzIzn5xzbkmrqKioEAAADFJFdQEAAHchWAAAhiJYAACGIlgAAIYiWAAAhiJYAACGIlgAAIYiWAAAhiJYAACGIlgAAIYiWAAAhiJYAACGIlgAAIYiWAAAhiJYAACGIlgAAIYiWAAAhqqqugCoo2la2DKfz6egEsC5+ByFo8cCADAUwQIAMBTBAgAwFMECADAUwQIAMBTBAgAwlKdPN658muBPP/0kderU0bXt/fffL9dee21KxwuW6OmJubm5Mnbs2IjrYv073CL4/7Jr164yZcoU3dsHM+K00EsvvVQKCwsNb/exxx6Tf/3rX4a3a7TK/7dff/21HH300bq2HTJkiIwYMSKl4wVL9P9nzZo1ct5550Vc9+STT8qll16aUHs4LK2ioqJCdRFmqfwG9L/pYr0xg7dLdNtI1q1bJ+eee27MbRJpL149wU466ST54IMPEmor0vHN/FKOJdGfXyK169k3Vhs+n0969eolf/zxh2Ht1q1bV3788Ud5+eWXZfz48Um3azQ7fI70tJFIewMGDJDly5cb0p7dP0cqeG4oTM+b079NIttGkpeXl1CoxGsvkQ+WiMjixYsT3kfvMVV9GKLVM3HixJDXFRUVCf3bNU2T999/P6Ht44WKf7tEaJoWN1SSaddoVn6O9Lahd3tN0xIKlWSOr3d/N4aKiMeCJdEvmlS3HTZsmO424rWn4ovEbh+GWP8Hffv2DXndsWPHhNt/6KGHEt5Hj/79++vabs+ePQm1a1a98Vj9OUr2vW/0Z+b11183tA63hoqIR+dYKv9AY70Bu3fvLu+9917gdbdu3WTfvn1xjxGpzVatWslXX30Vsmzz5s1yxhlnxG0vkkhvzDvvvFNmzJgRVkuib2Infxj01h4twBP5N+ppc/Xq1brbS6Td999/X1m4iIgsX75cqlY98hWSyHDXRRddJKtWrYp7jP3798dtS+/xo20T7eddedunn35abrrpprjH0FOTEz5HqfBUj0VEInaBY/2Qg0NFRGThwoW6jlN50lVEwkJFRKRZs2a62qusVq1aEZc/99xzgb8ff/zx4vP5XBcq/n9T8B89++hZlmgdRrdpZrtG+uqrr0JCRSR2jZXX/fe//9V1nCpVwr+iVPxfuPFzZCbP9VgqfxhiSeUN0KtXL1PfQMXFxVF/u07luHb/MCRzcsPnn38et11N0+TTTz81rI5k2eX/OZ5WrVrp3jaVf1PNmjUtOUnE6M/RkCFDIi53ys83VZ4LFjtZu3at9OvXL6U2gr9Ic3JyZPjw4amWFcbpH4a2bdtGXef0fxsO0zufkpaWJpFOhA3e/8QTT5QPP/wwpXpmz54dtmz69OkptekkBIuFUplMzMvLk+zs7JjbTJgwQSZMmBB4bdSXZjJzNIBZLr74Ylm5cmVS+65YsSLu5/Dnn38O2cao9/5ll13mmc+R5+ZYrKZpWuBPKlq3bi3Vq1dP6thG2LVrlyHtAMl46qmnAu/nZEPFL5m5EqM+R6pPE7cKwWIiPW+iWbNm6W5v6dKlSf3GY8SbuWfPnim3ASRD0zR54403Ym7zj3/8I6E2VX2OvIJgMUm0N+F3330XcjZTy5YtE247kbOhkuHz+aR+/fphy7t27WrK8YBoon2Ohg4dGvI5SPQ0YJHkPkeJhEunTp1SuhuEkxEsFvL5fNK4cWPD29TzARk6dKiu9vLz80VEZP78+WHrol1T4GRFRUWBoY7Ro0erLgc6+Hw+uf322w1vM/hPpNOcE5GRkSEfffRR1PV6zlR0MoLFIkZcyDZq1Ki4473RAibSNTSRNGzYMPD3jz/+OGy93X/bOu2008KWxar57LPPDvx9ypQpommabNmyxZTaYA/ff/994HP02GOPRdymsLAwpRGBRYsWBf4eqR2jg9FuCBaLpBIs/g/BtGnTQpYlYtKkSQkf94QTTkh4H9WijcXv2LFDdxtNmzY1qhzYjKZpcsMNNwReR7qQ2cpa3IpgsVBpaWnI66lTp+p6c+m95USs5WeeeaaOCvUd24kfiD59+oimafLvf/9bVq1aZeiZPrCW0ffSS+Rz9NJLLyV1DK+cZuzHdSwWSqUH0KZNG1m3bl3Ycj0fqBo1aiR93GiWLVsmnTp1MrxdI/h8vqj/L3fffXfcfWF/yQZJtPeG3vbiXUuWKLdeI0aPxSSJvFmWLVsWtqykpCTk9Zdffpl0LUuWLEl6X5HI/5bLL788pTbNlsyH1Y0fcKdL5Gdy/PHHhy2LFBi5ubmm12LG/k5CsJhI780Rq1WrFrY8Um/A5/MldKsJ/00ozZKTk2Na20ZI5S7FsA+9nyO9N7bs06dPwj9vo94fjRo1ClvmxiFZVz9B0i4qKipCng/SsmXLhC6MjGbq1Kly//33hywz4j5HbrRkyRK54oorQpbNnTuXiXqHycrKkrKyssBrI77wi4qKQs4OFBGpXr26LF26NOW2vYpgAQAYiqEwAIChCBYAgKE43RgpMWPikYn0xPFzgJ3QYwEAGIpgAQAYirPCAACGoscCADAUwQIAMBTBAgAwFMECADAUwQIAMBTBAgAwFMECADAUwQIAMBTBAgAwFMECADAUwQIAMBS3zYcn7DhQItN822Xz/pKU2vnrqw+kXEvrkSOlTufOKbcD2BXBAtd4Z/kWWbP7oOoy4lr/7LNR11XJyJD2zzwj6RkZFlYEGItggeM4JUCSUb5/v6wcOjTiuszcXGuLAZJEsMDWZq7ZKfN+36u6DFsoHDw45HVa9erScdIkNcUAMRAssJUn5m2QknIeEaRHxaFDYWFDrwZ2QLBAqWcWFElxabnqMlyDoIEdECyw3Jj89apL8IzgoGl1221St1s3dcXAMwgWmG7VzmKZsmKb6jI8b8MLLwT+XqNlSznusccUVgM3I1hgitV/FMt7hYSJXR0sKgr0Zqo1biztn3lGbUFwFYIFhmKYy3lKtm0LhEzzm26So047TW1BcLy0iooKTsFBSrwUJkZcee8UTPwjWfRYkJQ3l26Wor2HVJcBEwVP/BMySATBgoR4qXeCI/whQ8BAD4IFcR0sK5en5hepLgM24A+Yuj16SKthw9QWA9tijgVR/fuX7bJk637VZdiKl+ZY9KIXg8rosSAMw11IBMNkqIxgQQCBglQQMPAjWCDLtu2T6at2qC4DLkHAgGDxsD8OlsqEhb+rLgMuVTh4sKTXri0dXnpJdSmwGMHiQduLS+TlxZtUlwEPKNu373DA1K0rHSZMUF0OLFJFdQGw1pj89YQKLFe2Z48UDh4sO2fPVl0KLMDpxh7BxLwxON3YGMy/uBtDYS5HoMCOmOB3N4bCXKqkrJxQge0VDh4sv9x1l+oyYDCCxYXG5K+XJ7gFCxwi+Lb9cAeGwlxk0s+bZPP+EtVlAElheMw96LG4xJj89YQKXIHei/PRY3G4J+ZtkJJyTuyDu9B7cTZ6LA42Jn89oQJXo/fiTPRYHOiDFVtl5c4DqssALEHvxXnosTjMmPz1hAo8id6Lc9BjcYjl2/bLx6u2qy4DUIreizPQY3GAMfnrCRUgCL0XeyNYbI6r54HICgcPlorSUtVlIAKGwmxq76EyGffTRtVlALa24qabRKpUkcw331RdCoLQY7Gh74t2EyqAXuXlDI3ZDMFiM2Py18tX63apLgNwHMLFPggWG2E+BUgN4WIPBItNECqAMQoHD5aKsjLVZXgawWIDhApgrBU33ii/jhqlugzPIlgU8u0oJlQAkxwsKmJoTBGCRZEn5m2QD33bVJcBuB7hYj2CRQHuSgxYi3CxFsFiMYa+ADUIF+sQLBYiVAC1CBdrECwAPIVwMR/BYoGCvCFSkDdEHujTWnUpAIRwMRvBYrKCvCEhfydcAHsgXMxDsJgoOFSClxEugD0QLuYgWEwSKVSC1xEugD0QLsYjWEwQK1SCt7m/dysLqgEQD+FiLILFYHpCxa/wq1vlvl6EC2AHhItxCBYDJRIqfr5Zt8rt3ZqbUA2ARBEuxiBYDJJMqPhtmDtcLm3f0MBqACSLcEkdwWKAFbNvT7mNViWLDagEgBEIl9QQLCnaWfSdlJcdTLmd3wvekX905scB2AXhkjy+yVL0e+E7hrX16w9jZFTPpoa1ByA1K265RXUJjkSwpCCVeZVoVsy+nWtcAJuoOHRIDm3ZoroMxyFYkmRGqAS3TbgA9rD67rtVl+A4aRUVFTxxKkFmhkqwrOyJjr/V/uTB2VHXdTjnL9Lz2pyE9r8mN8+QupL111cfUHp8qJOZm6u6BMegx5Igq0LFfyyn9lwmD86OGSoiIiu/+rdMHpwt636ca1FVQPKYzNePYEnA5lUfW35MJ4ZLvECp7JsXH5b/3Pc3k6oBjEO46EOw6HRw32bZvnamkmM7KVyihUr12nXlihc/lp7XRb7mZ9fGdWaWBRiGcImPYNFpdf5opcd3UrgEq1G3vlyTmycDX5ouNerUkw5nXyzX5OZFnCtJtKcDqFK8erXqEmyNYNHBynmVWOweLpGC4YoJH0Xd/qS/3mhmOYBp1owZo7oEW6uqugC7s0uo+BXkDZEHHHK2WLwzuDpddJUsnvaGHNPzTDl9aPJnW8Xr6TTVuki/e59Lun0gksLBgzlTLAp6LDH8sfF71SVEZPeeSyKuyc0zNVRERLb4ljDMBlMw3xIZwRLDxoJ/qS4hKruFS/4bz1h+zEhhcf6DL8rlz39oeS3wruI1a1SXYDsESxR2GwKLxE7hsnru56pLkGty86TxcR2l1lENI54g8NEdgxRVBjdb89BDqkuwHeZYInBCqPg5ac5FBdVX6sMbmG8JRY/FBezUc1HJf7X/5MHZ8u4N/VSXA4/55a67VJdgGwRLJU7qrQRTHS7NO3W3/JgDxk2Juq6iojwkaACzlWzbproE2yBYgjg1VPxUhss5dz2Z1H6TB2fL/HcmJLVvRoPGuoe6CBdYgbPEDiNYXKYgb4jc1aOF6jIS4r8ZZbJf/tGu5K+McIEVduQxr0ew/MnpvZVgv80ZJtdkNVFdRtwvcqO/6P0BozdoADNsnjxZdQnKESwisnLuvapLMFzxgjukx9F1LD1mpC/zT+6+LuK2P015NWxZWnq67mPtWLsqZA5l7YJvdNUDWMHrQ2IEi4iUHtypugRTHLvmYWlUU+0Z5Xu3bJTJg7PlP/fdKKUHD8iP774skwdnS+HnU8O2vfqNL3S3W7Neg5DXc196JOVaASOV7t6tugRlPH8di5uGwCorLzsg5x96S96Vay075jW5eRGHuHZtXCtT/n5RzP0SkdGgcdiyeENrtRs1S+gYQCpW3XabZ69t8XSPpeTgLtUlmO7AnnVyU8PvLD1moiGR7JBVovtd9uy7SR0HSJZXh8Q83WNZNfefqkuwxM71c+SOEzPl+V8aWnZM/5d+rF7EXyd8JDXr1k/5OHu2bJR/R5nLCa4FgDXSKioqKlQXoYKbh8CiaX/qY/Lk4v2qy3C0v76a/J2Y4V1eGxLz9FCY1/zy3Si5v5ezrnEB3ODQpk2qS7CUJ4PFi70Vv8JZw7ivGGCx1ffco7oES3kyWLyuIG+I3HVCueoyAE8pvOEG1SVYxnPB4uXeSrDf5o+V+3qEn7ILwCQems72VLDs3V6ougRb8c0ZwbAYYCGvnH7sqWBZt2i86hJsR/Xt9gG4j2eC5bcFyd3W3QsIF8A6Xui1eCZYinf9proEWyNcAOuU7nL3XT88ESxM2OtDuADWWHX77apLMJUnggX6ES6ANUq2b1ddgmlcHyz0VhJHuADm+2XkSNUlmMbTN6FEdAV5Q+SB7IkyJn+96lJggkvz8yMu/6RPn4T3yWnXTs5p2jTqdpXXw/1c3WOht5Iaei6Audx6hpirgwWpI1y8ZV9pqeoS4AKuDZbV+Q+rLsE1CBfvuHrBgojLow2DIXVu7LW4do7l4L7fVZfgKgV5Q+T+c16RR3/YoLoUGKxL/fqyxMDrKmLN08AbXNljOVTs3tP4VCr86la5r1cr1WXAYI9kZSW0vZHB8d327XJpfn7In/k7dujef/6OHWH7f+/A03jd1mtxZY/ll+9GqS7BtXyzbpXbT39Rxi+kR+hWH6xfL4NaHxn6LEvwrrx6zgqLNbT2uM8X+Hu0EIu1/1MrV8bdH+ZyZY8F5towd7hc2r6h6jJgkvc3hA53Xv7DD4a2n8h8TaRtU93frn4bM0Z1CYZxXY+FU4ytkf7zXZLVeowU7ChWXQoM0KNBA1mwc6eSYwf3KsoqKhIOssq9EieFSbADq1erLsEw9FiQpArptPkpubtnS9WFwACjOnbUtZ3RQ0uV20tPSwtZFu94kdYnsj/M4apgWTnXW8+VVq3s0G5ZPXuo3NiZq6rd5pr580VETJ8IvzQ/XwZE6GF80qePrlDwT9gnu7/duGUS31VDYaUH/1Bdgif98cPtcnqHJ2Vu0W7VpcAge8vKRCR0Itws5RI6fHVMRoaMP/HEhNoI3r95zZrySteuRpWHJLiqxwJ1mv3ygDSvXU11GUjBaY0aWXKceD2Jtfv3R+2J6Nn/9wMHYu5vd4U33qi6hJS5JliYtFerovyQ9N33qnveUB50V4cOMdcbObSkt61kwyXe/rb2Z2/RyfgegGEO7vtdBtebpboMGMTsL2X/PMgnffpIRnp6wnUE71+/avRRfUeGi8O5Yo7lj9+NPc8eydv1e77c1ilTXvituepS4CDv9ewZ8jrRMHi7R4+U9rebwsGDJTM3V3UZSXNFsGxcnqu6BAQpWvam3N3nYXlqySHVpSBB5zRpIl9t3Rq23Kihjcpf+B/17i3paWlJ7/9ejx6SEaO3AjUYCoMpVuePllE9j1ZdBhKU0759xOUfGzS/Unlu5PIffpCFlS7MrBwewbFTef//W7BA5m7bFnN/pyrdu1d1CUlzfNRv++1/qktAFCtm5/AUSsT1yIoVMddPjxNqz65aJc+uWhV1vROvZxERWTV8uGOHwxzfY9my+t+qS0AMPMsFlSXyRR/vynojjwXjOL7HAvsryBtCz8VB2tSqJeuKzb0HnP8LP2fxYlkf4Vh6b+UycskSWb1vX8L7w1xpFRUJ3hPbRrb+OkO2/vof1WVApywXhMtfX31AdQnwGCcOhzl6KIxQcRaGxQBvcHSwwHkIF8D9HBssZSXh46pwBsIF0M+Jdzx2bLD4vh6pugSkgHAB3MuxwQLnI1wAdyJYoBThAsT3yz//qbqEhDgyWLhFvrsQLkBsJZs3qy4hIY4MFrgP4QK4B8EC2yBcAHcgWGArhAsQmZNOO3ZcsDC/4n4FeUPk/t6tVJcBIEmOCxZ4Q+FXt8oowgVwJIIFtrXiq1vlzpNbqC4DQIIIFtja2q+HyZUdG6suA7CFwhtuUF2CLo4KloKvhqouAQqU/DRCTmpaW3UZgHoOecqJo4JFKspVVwBFshvvVl0CAJ2cFSzwrN8WPCH39WBIDHACggWO4ZszgmtcAAcgWOAoXEAJr3PChZKOCZbCWcNVlwCbIFwAe3NMsFSUl6ouATZCuAD25ZhgASojXAB7IljgaIQLvGjnnDmqS4iJYIHjES7wmk25uapLiMkRwVJWsk91CbA5wgWwD0cEi+/rkapLgAMQLoA9OCJYAL0IF0A9ggWuQ7gAahEscCXCBVCHYIFrES5ws41vvKG6hKgIFrga4QK32jV3ruoSorJ9sKxfMkl1CXA4wgWwlu2DZc+WhapLgAsQLoB1bB8sgFEIF8AaBAs8pSBviNzYuZnqMgBXI1jgOX/8cJuc3rKe6jIA1yJY4EnNfnlAmteuproMwJUIFnhSRfkh6bvvVT4AgAn4XMGzDu77XQbXm6W6DMB1CBZ42q7f8+W2Y39XXQaQlIO/2/O9S7DA84qWvyl3d6kmdarxcYCz7Jw9W3UJEfFJAkRkdf5DMiDtQ9VlAAn5g2AB7G3fTp8MabZEdRmAbhUlJapLiMjWwVJRXqa6BHjM1l//I3d2PKC6DMDRbB0s+/9YpboEeNDan56Ve7sfpboMwLFsHSz7dhIsUGPlN//gvmJAkmwdLIf2bVJdAjyMm1YCybF1sJQc2KG6BHgc4QIkzt7BcnCn6hIAwgVIkK2DpaLcnqfSwXsIF0C/qqoLiCnN1rkHjynIGyJ17n5K2mQ1FRGRwsGD1RYE2JStgyUtLV11CUCINllN5eWcGYdf1L1Chk64UEQIGSCYrYOlao36UnrwD9VlAFEFh8wNY7OlVp0asm/FCln3xBNqCwMUsnWwVKvZSA7sXqu6DECXt+7NC/x98AuvSka9GrJl6lTZPmOGwqoA69k6WKrXaqK6BCApuaP8IZMh1ZpcJTc/fZ6IMGQGb7B1sNSqf6zqEoAQJeUl0vPCDjJ/xkr9+xwoZV4GnpJWUVFRobqIaMpKD4hvzh2qywACXm2QJuO7v3IkKFJ09eizpH7j2rJtxgzZOnWqIW3CWzJzc1WXEMbWPZb0qjVVlwCEKJVSQ9t79+E5R17Uv0KGjqc3A+ezdbAAdrR/tUl3hCgPPcvs1hf6S1pamvw6apQcLCoy55iACQgWQKcqVTNEZI9UlJZbcrxXbvvsz7+dIt0GtJPel3SUkh075Jc777Tk+LC/+qeeqrqEiAgWQKd6zbqJ7P9aamuN5MwrO8vXU5ZaduyFX66WhV+uPvyCEwDwp6P69lVdQkTcMwXQqXYDLfD3E05to7CSw0NmL+fMkNl1r5COb74lmbm5Uuv445XWBOtltG+vuoSI6LEAOtWs00p1CRG9crt/yOwk0bIvlHOuOUlE6M1AHYIF0KlazYaBv+/5ebPCSqLzzSsS37w/J/oZMoMiBAugU1p6dRERydv0hbT92BmnwgefZTbwn6dJ41b1Zefs2bLp7bfVFgZXI1gAndLS0kREZHrRNLlTrpHzb+oun7/+k+Kq9PvwyW+PvKA3AxMRLECSjjvxaNUlpCTibWb+9jeRcmtOp4Z7ESxAEuqe2Ex1CYYKhEzty+Uvt/WWlsc3kt3z5knRK6+oLQyOZPtgqdu0m+zZslB1GUCIpgM6ys657nykw79f+OHIC4bMkATbB0vrLrdIQd4Q1WUAYXbkrVFdgiWCh8z+/vwFkp5eRdY//7zsXbxYbWGwLdsHC2Bnl43oI9PH5asuwzKv3vG/P/92vLQ6ubdcMry3lJeUiO/mm5XW5UX1+vRRXUJUBAuQgubHNYy/kUtt8G0P9Gb63f64tO/aQvYVFMi6p55SXJk3tPz731WXEBXBAiThQNkB1SXYysw3F8lMWXT4BfMynkewmOyFfxXK7HmbQpZNf8meN46DfiMX58idco3sLdiquhRbCp6Xuenp86R6zaqy8Y03ZNfcuWoLgyUIFiAFmz8oUF2C7b3+jy/+/NvRUv+46+XqBw//YkVvxr0ccXfjY7uPVF0CEFX29SepLsExdm3dH3Jn5szcXFs+WhepcUSwZDTgduCwnxaDTxQRkQ4nt1RciXMFh8yxL74qmbm50vC881SXhRQxFAYkqVbbo1SX4Cpv3Zv359/qSY1mV8mNTx4OGIbMwjU8/3zVJcREsDjQ+t/3yW2Pzg9bXjU9Taa+cJauNtYU7ZURjy+IuI6TCxKzax7Pozfawf2lIScA3PpCf0lLS5NVt98upbt2qS3OBppdeaXqEmIiWBzmsmGzo64rLauQy4bNlodyTpQTO0a/viJWG8HrCRh9tn32i+oSXO+V2/wPM+snp//tBOl8xrFyYO1a+W30aKV1ITJHzLHgsHiB4PfQhJ9lyozfUmpDROTt6Xxh6nXx8F6qS/CMuVOXy8s5M+TNZ5ZxAoBNOabHkpU90dP3DIsUCME9isrrP/hsjVx5YduE2vh9a7EMfejIDQg/yVsv119mz2dq28F/iqZLlhzuGbbWGiuuxrsiDZmtvuceObRpU+wdYRp6LA7wynu+sGWVh6mmv9RXqlRJC1kWr3dSuY3mTWolWaE3rN+/LuT155s+i7IlVHnlts/k5ZwZ8sW+02XvwPskMzdX2j39tOqyDNXh5ZdVlxCXY3osXjbzu40hr6PNfXw04ayEhroiYV4lug2VgiXYtpm/WlgJ9Fjwv1Wy4H+rDr9w0W1m0jMyVJcQF8EC6LRh//qwZfV6tpDd8zfKru/C18FegofMrnmor9RrlCHbPv1Utn78sdrCXMhZwZJWRaSCx6Ym49ufNqsuwfE2FIeHR5MLj5fd8w/3KAfec7p8+AT3wnKCyQ/5e/bpUqX+FTJkvDt6M3bhqGDJOudlT0/gp6J189qqS3C8ynMslTVuWc+iSmCk8vLQ3oxbhsxUclSwIHnHtKije9tFBdula1YjE6txpoPlB1WXAAsEh8yge8+QRi3qyo68PNk8ebLawkQcc1o1weJAlw2bbdok+5vTVsl/Zm8IvO7eqZHcf2sXU47lNps+WK66BBjsg7HfHHlBb0Y3Tjd2gEghcteTP4YtC7uW5fkzYrYb6Qyy4FAREUIlji0Hjsxd7SvYJt3P47ofNwu+aWbHt96SzNxcqdmuneqybCetoqKiQnURiSgvOyQrZt+mugzdIj3oS69YF0Amun+y7XD68REvNSiLuPzOT68J/L3dw2ceGUqBZwR6Mn/72+FJG5MwFGaSKunVVZegxPSX+iYUCtEC4diWtWVN0b6U2gAQKvDLRO3L5S+395aW7RvJ7nnzpOiVVww7hlNCRcSBweJl01/qKzfe953s2HUo7nbRjLuvp5SUlMvAO75Oug1EV7Yv9s8G7vfv8Udui+TVeRnHDYWJCKcc/+nzb4rkrY9/keOPrSeP3tE1qTa+/WmzTPpgldSvW03Gj+oZdlsYHBFtKOzZo5+TokkLA6+/2LHXqpLgMP6QWf/882Dst2kAACAASURBVLJ38eKE9nVSj8WRwSJCuMB60YLlpe6vyerRR3qAv7WrLysX8IwWxNZKaySXDO8t5SUl4rv55rjbOylYGAoDDJZ93UkEC+La4Nuu+8JMJ4WKCMECALYQHDI3P3u+VKueLkWTJsnu779XW1gSHBssR3e8SjateF91GfCIuk1OEin9Sde2JTuKTa4GbvfayM///FtLadDhBslUWk3iHHuBZMNWZ6ouAR5Su1FW1HUfrH035PW68fPNLgcecuU9sS90tiPHBgtgpVr1jom67pttc8KWnXh22/ANgSSkOfBMTUcHS7VaPA4W1qie4Hvt1Mui93AAt3N0sBx/6qOqS4BHVKka+6l9R53a2qJK4CVX3ue8YTARhwcLYJW0tNjDEY36HRfy+sDGPWaWA49o2Lyu6hKS4vhgYTgMdlT06sL4GwEu5fhgYTgMdtX7Ek11CXCw6x89R3UJSXN8sAB21e1cns2C5NWuX1N1CUlzRbAcrQ1SXQI8bsP+9apLAGzDFcHSsDW3eIdaYwsfCVu2b9V2BZXADfz3DXMqVwQLYEebJi9TXQKghGuCJfPsl1SXAK+LcIX0CadHv2IfcCvH3oSysrQq6apLgMe1HnqyrH9xQciyMwd2kuVz11pey/iZdya8z+39ntPdXqxtkRqnD4OJuKjHAqhWvUnsq/PtbvzMO+WNrx9WXQZcwFXBkpU9UXUJQIiDm531mOK9B3epLsHTrhvj3GtXgrlmKAywow0v63uGi9miDV0V7fxVpi14MWTZ+Jl3MtSlSJ2jnHvtSjBX9VhE6LXAfk6+4HjVJUTVssFxhAgMR48FMMibv06SMyX8Ni49+3eQH/+3SkFF1op0wkC00Ip2MsDSDfmyYceR/6sLulxnYIX25oZJez9XBku9pt1l9xZ7DEHAO37auSBisHiB3lCJdrba+Jl3SvtmJ0qNqjVl5abFgeVeChY3cd1QmIhIqy43qy4BHtVsUPgDvorX/KGgEv1SPZU41VDx+2Xzz7Jr/7aEju0WbuqtiLi0xwKoUieriWyutGzjWz8rqSWY3uta0qsk9pWgN1T2HNgZtqzbMWfJ6dolIiLywbznZdOudbJh5+qEjg97cmWPRYRJfBhr64EtKe1/+hUnGFSJeW456xEZnv2U7u0TmVN585sxIa9vPGN0IFRERAb1ukMu6z5E97HdxG29FREXBwtgpA3Fqd29uPMZxxpTiIkmzXlQd8/mox9fCVuWyBBanZr1w5a1adRB9/6wN1cPhWVlT5SCPG/+FgRjrd+/TnUJKQn+0q+oKJd9B/fIr1uXy+zCaWHb6rmOJfjMLSTPLRdEVkaPBdBB7/NWvtkyJ+LyPctSG0ozUlpaFalTs750aX2K3N7vuaQm2iNJZh+vc8sFkZW5PliYa4ERNhTr67F8sP7diMu3TC00shzDcZGk9a66/0zVJZjG9cECGGFXSer30Op3Q1cDKrGPSL0dei36NWhWR3UJpvFEsNBrgZVqn9Ak4vL23VpYXIl+yzbkJ7Q9PZzU/PWuU1WXYCpPBIuISI3azVWXAI84emD4RZJ29t/Fb8pXBVOT3j+ZXkukbbzU22l6zFGqSzCVq88KC9auz2jOEINSf3y/QdmxE/3STrVHMv/XPOl5XHZIe5Vr8FKQBHPjdSuVeabHIiJSrVZj1SXAw7Z/4YyrymtUrZXwPpWDKP+Xz8K24b5f3uGpYDn+1EdVlwCPO+XSTNUlxHTtqf+UIWc/Zkhbr80ZHfK6w9EnybBznoy6vRfmbbzQWxHx0FCYX9se98hvC55QXQZc7FD5oajrTjrnOPn+E/NPPTb6S1pPe3q2qZpezRMB4nWe6rGIiNSqf6zqEuByIxflqC7BdibOGiX7DznrMc1G80pvRcSDPRYRbvUCc5VLedR1+3zevC38wdJieW3OgyHLvNRzad6uoeoSLOXJYAFU2fTectUlKJF9wpWSt3xKyLJ4Z4X9X5+RZpZkqcvu6KO6BEt5bijMj4smoVet+sclvE/zaztHXXf2NSemUo4jndCyZ8L7NKnb0oRKrOelITA/zwaLSHJfGPCeuo2jh0Q0Ge2jD3107NUqlXIc6/DQV1rc7WrXqOepYTI38vRQWNsedzPXgrhqHdVOZIfqKtzh9n7Pqi7BUl7srYh4vMciwpAY4que0czQ9nb/uNHQ9mBP7bt59zZSng8WIJ6q1Yy9C+3W//CQLC/od0M31SUoQ7AIvRbEllYlPeF9vvh9Rsz1F97aI9ly4ABeHQLzI1j+RLjASJ9u/CTm+mOymlpUCSwX//wE1yNYAMBAQ1/wdm9FhGAJQa8FRqrb7eio63Z89ZuFlcAqXh8C8yNYKiFcYJSmf9Girtv5zToLK4EVqtfy9NUbIQgWQJG//sPdj6f1mpueOk91CbZBsERArwVWaNrG3Y+n9RKGwEIRLFEQLgD0aNSiruoSbIdgiYnzBpG84tLimOv3LNlsUSUw06B7z1Bdgu0QLDFkZb+iugQ42F0/3xZz/ZaPVlhUCczCEFhkBEscDInBTOff1F11CUgSoRIdwaID4QKzHHdi9GtdYF91jqqpugRbI1h0qlqDM3iQuBY3nqS6BJjgujHnqC7B1ggWnTqc/oTqEuBAtdrUj7l+57dcKOk0DIHFR7AkgCExGG3Hl9zaxUkIFX0IlgQRLjDapXf0UV0CdCBU9CNYkkC4wEgt2jVUXQLiuOlpbteSCIIlSYSLd+wu2Z30vtM3TDOwEqhQq251qV6TG0wmgmBJQdN2f1FdAiywYf/6pPfN2/xF3G22/pdHFdvZDY+fq7oExyFYUtC47QWqS4AFNhSbe+bW7gUbTW0fyWNeJTkES4oYEnO/VHosIiL1e7eMu81V95+Z0jFgPEIleQSLAQgXd1u/P7UeS+ML2sfdpkGzOikdA8YiVFJDsBiEcHGvLQe5C7GXECqpI1gMRLggWRvfWaK6BAihYhSCxWCEC5JR/MtO1SV4HqFiHILFBIQLgv1erO+sr789wWmtqhAqxiJYTEK4wO/RgtG6tqtZu7rJlSCS3pdoqktwHYLFRIQLYG+9L+ko3c6Nf9YeEkOwmIxwgYiIpMXfZPuXv5pfBwLOv6m7dDu3neoyXIlgsQDhglZ/j/8I4j++Te1CTOj313+cytM7TUSwWIRw8bYazfVdADnwn6eZXAmue+RsadqGJ8KaiWCxEOGCeBq3iv3ESaRm6IQLpU6DWqrLcD2CxWKEi7NUrdlYdQkwCKcUW4dgUYBwcY76TU+y9HibPlxu6fG8glCxFsGiCOHiDLUaHG9IO5PXvK1ru33Lt0n3fpz+aiRCxXoEi0JZ2RPlqBanqi4DMdSsE/+W93rkb/9W97a9LuaCPaMQKmoQLIq1yLqW3ouNVa3OZLoTNWxel1BRiGCxCcLFnqqkVzOsraPOaKNru+J1uww7phfd/Mx5cuV9Z6guw9MIFhshXNyt0TltdW238Y3FJlfiXkMnXCjValRVXYbnESw2Q7hAROTUAVmqS3Achr7sg2CxoazsidK2xz9VlwGFTuyrr3eDwwgVe6HPaFO16reVrOyJUpA3RHUpgG0NfixbMurVUF0GKqHHYnMMjbnH2n1rdG+7t3CreYW4xNAJFxIqNkWwOEBW9kRp2n6A6jKQoqdWPKZ7281TCkysxPkY+rI3gsUhGh/bj96Lx2RfZ+3tZJxgyPj+hIoDECwOQ7g4W1pV/R+5Dj2MuerfLYZOuFCqVNHxxDQoR7A4UFb2RMk852XVZSAJrYeerLoEx2nRviG9FIfhrDCHSkurwlljDlStkf5ngRzYuMfESpyBQHEmeiwOl5U9UbSznlddBkxQ9OpC1SUoU6tudULFwQgWF0ivWpO5F5fq7cE7HQ+dcKHc8Pi5qstACggWF8nKnkjA2FxFRUVC23fz0LNZel/SkV6KSxAsLpSVPVEkLV11GYhg0mpOuohk6IQLpdu57VSXAYMwee9SWee8JCLC5L7NLNm1WLKlk+7t9/2yw8Rq1KOH4k4Ei8v5h8YIGGfa9M5S1SWYostZx8ppl5+gugyYhKEwjzj8GORTVJfhOIfKDxreZsPsxO5c3Pf/uhheg0pDJ1xIqLgcweIhLbKuY3I/QRv2rze8zQan63uSpF9mn9aG16DC0AkXMvTlEQyFeRDDY/qtNyFYvIYw8R6CxcMImPg2FKsPlt2LNqkuISkEincRLCBgYjBjKCxRWz/xqS4hIQQKCBYEEDDhNuxfZ3ibq/asTHhy84JbTpb/TfrR8FqMRKDAj2BBGH/A/DrvMTmwR/1v7CqVS7nhbT6/8mm5U65JaJ+2nZsZXodRCBRURrAgquN6jRIRkUP7t8ov3z+guBrYyVFNa8v/PXCW6jJgUwQL4qqe0YRhMoNVP7qOHNq0V/f2O75ea2I1+tE7gR5pFYneFQ8QEd/Xd0lZif4vRqd6qUGZOe12f01Wj/46oX2+2KHu/5tAQSLosSAp2pnPiIhI6cHdsnLu3Yqr8YbTrzhB5k5dbtnxMnu3lr5Xu+uqf1iDYEFKqtaoFxgmK5x1m1SUH1JckXt1PuNYS4KF3glSRbDAMJlnvxD4O3MxzkKYwEgEC0wRfE8yQiZcMlObe5ZvMbQGwgRmYfIelirIu1VEnPOWM2vy/rja7eXS93snvF+qE/iECaxAsECZ4t3r5Lf5j6suI7oqVeWl+sbfNt/vzk8Tu0hSRKTiL8fLzLcW6d7+xLPbyqmXZSV8HCAVDIVBmVr12oQMme0s+lZ+L5yssKJQ9Zv1EDnwreoyQrTv1iJmsNRvkiFXP9jXwoqAcAQLbKNBy9OkQcvTAq8P7N0ov/7wiLJ6ajfURDaaFyyN+7eXbZ/9klIb3fu1l14XawZVBBiDYIFt1azTIuzBZJt8H8iO9bOtOX7dxB7Ilaj6vVomFCxVah3+uDJPArsjWOAoR2uD5GhtUNjy1T+MkYN7iww9VrWaDQ1tLxGtc3pI9cYZyo4PpIJggSu06x39Jplbf/2vbP31vwm3WSW9RiolxdS5/okiItLu4TNNOwagCmeFAREc2LNBatZtJVPXT5FtB7dKcVmxFJful7KKMsmomiEZ6RnSoHpDaZXRRlpntJFja7dVXTJgGwQLAMBQiT7IDgCAmAgWAIChCBYAgKEIFgCAoQgWAIChCBYAgKEIFgCAoQgWAIChCBYAgKEIFgCAoQgWAIChCBYAgKEIFgCAoQgWAIChCBYAgKEIFgCAoQgWAIChCBYAgKEIFgCAoQgWAIChCBYAgKEIFgCAoaqqLgAA9NA0LeS1z+cLeT1gwABZvnx52H7Tpk2Tzp07m1obQtFjAWBr69atE03TpEqVKuLz+QKBUjloIoUK1EirqKioUF0EAETjD5DKPRRN0+Tss8+WV155JfD65JNPlnfffdfyGhGKHgsAx5o1a1bIa0LFHphjAWBrlXsqwXJycsKWBQ+RxdoX5qHHAsBx/OExfPhwERFZvHhxYPm7774rjz76aMh2sBZzLAAcJdKcS1lZmZx66qnyww8/xN0W5qPHAsAxogVFenp6WKhAHYIFgCMk0/s45phjzCoHMRAsAGwvXqhomhZxPmXt2rWm1oXICBYAtqanp/Lzzz+LiMjq1asDy1588cW4+8EcnG4MwBGineHl8/mkZs2a0qVLF+nfv3/Iuu+//96K0lAJZ4UBAAzFUBgAwFAMhSVhYv5KKS4pU12GpUackam6BAAOQY8FAGAox/ZYIk3kxToVMdZ6AIBxHBcsse79Q4AAgHqOCpZ4dy31r9c0jXABAEUcM8ei51bYhAkAqOeYYPGLFx7XXnutiIjccccdutrz3wpC0zTZunVryvUBgNc55gLJVOZPIu172mmnRQ2SeMfgdGMAiM4RPZaBAweKiEjNmjUNa9MfKj6fL/Dn6KOPFhEeDgQAqXBEsPhvMDdx4kRD2nv88cdFJLxn8vXXXxvSPgB4mSOCxS89Pd2Qdu677z4RiX4tDCcBAEDyHBEs7du3FxGRsWPHGt528OT9+vXrDW8fALzGEcEyY8YMEREpKCgwrM1IPZPs7GzRNE3uuecew44DAF7jiGBJRFlZWdSnyUUSPHnvN336dLPKAwDXc1ywDBo0KOb6rKwsEYl9yvC4ceOizq9UreqomxEAgO04Jlj8QbF48eLAI0cr09tLeeONN0RE5MMPPwxbV1pammSFAAARB10g6acnPCr3ViJdIBnczpIlS2TLli2SnZ0dtY1gXCAJANE5psfiF+sL/7LLLtN9qnDwdl26dAmESpMmTTjdGABS4MgJhUS/+LlpJeAdP/74o4iInHzyyYor8S5HBgsAb4k0BJ6enh5yCcLMmTMlJycnZJsXX3xRzj33XNPrQyiCBYCtRbsBbeWwycnJkfz8fGnYsKGIiKxcuVIuvvhiRiYUIFiSMKRPB9UlAJAjD/Xr37+/iEggVEREOnQ4/Dm96qqr5P3331dSn1cRLABsTU+PY/Xq1VHXLVy40MhyoIPjzgoDgF9//VVEOAHHrggWAI5zwQUXSLVq1VSXgSgIFgCOcfDgwcCk/bJlyxRXg2gIFgCOsGvXLunSpYuIMARmd0zeJ+HHLx+W0kP7VJcBWKL3hU+pLkHGjRsnEydOlDPOOENee+011eUgDoIFgK2NGjVKpk2bJp999pm0a9cu4jY+ny/qfQTp3ViPYAFga9OmTRMRCVyrUlnlm8sWFhaKiEhmJjdOVcW2cyyJPKwLAHw+nzRs2FAyMzMlMzNTGjVqRG9FEXosAGwtkXDIz883sRLoZdseCwDAmQgWAIChXDUUNmbMGJk8eXLY8oYNG0bsIq9Zs0bOO++8sOU9e/aUd955x5QaAcDtXBMsF1xwQcT7B2maJjt27AjcBdVv//79gVCpvP38+fNl586d0qBBA4uqBwD3cM1QWLSb0kWb+OvatWvM7Xv37m10iQDgCa4IlhUrVoiIyPnnnx9x/YIFC0Qk8lPoIvH5fJymCABJcsVQ2F/+8hcRERk/fnzE9fXq1Yu4/PLLLzetJgBI1spnBltynA535ZrSrit6LIny92DOPvtsxZUAgPt4Mlh69OghIkduFQEAMI4rgsX/XAb/hHxlkU5BFhGZPXt2xOXcTgYAkueKYPE/SW7//v0R148ZM0ZEuMspAFjB9sHi7z1E++Pn75VomiYbNmwQEZGKioq4t9LWNE2WL18uIiLl5eWB7QkhAEiOK84KEzk8b9K1a1dZtGiRnHPOOSHrMjIyZNGiRWH7TJ8+XS677DIZMGBAyPK33nrL1FoBwM1sGyzJ9BimTJmS0PZZWVn0TADAYLYfCgMAOAvBAgAwlG2Hwuzs5HNHqy4BAGyLHgsAwFAECwDAUAQLAEc55ZRTZMKECWHL33nnnYjXui1dulRBld5GsABwlO3bt0dc/uijj1pcCaJh8h6AIwwfPly+/PLLmNtEuxga1iJYkvDwdw/IvpJ9urZ96qznTK4GcL/gWy3FukFsqqHyfw/OSWl/vd575CxLjqMKQ2EAbK9Tp06675Lhn1v5+9//bnJViIZgAWB7H330Ucz1xcXFIhL6+PE5c+bw+AtFlAyF6flhG3UPL03TwtriDsaAu5x88skiEv6Z9vde+Kxby7Y9FiN+0+C3FcAbli9fTnjYiNLJ+2hvBAIBgBGysrKkoKBAdRmeY9seCwDoFe1x4oSKGo4LlqlTp8Z8kqRf8LJo23z22We62gJgbyNHjoy6bsWKFRZWAhGHBYumaXL//feLyOFhNJ/PF3hDVQ6E4GE2/7aVjRgxQnr16hW2fuLEiWaUD8Akt9xyi4iEP8r8mGOOkbS0NMXVeY/SOZZYvYNYE3HB62655Ra55ZZbkjr7Y9myZVKtWrWQdjVNk3HjxsmQIUN0twPAOtE+4/7l2dnZ0qRJE3n//fetLAtBbHvlfVlZmaSnpwde9+7dW0REvvnmG8OOERwqANwhLy9PdQmep3QozD8EFfxnzpw5InL4bI6VK1cGtt25c6eIiDRr1ixiWzNmzBARkXHjxplbNAAgJtvNsTRv3jzQpb344ot179e+fXsREXn99ddNqQsAoI/tgiVV3bp1U10CAHiaa4IlMzNTRA4/7AcAoI4tg+WGG24IW+YfHot2Jll5ebmpNQEA9LHt6cYi0U8rzMzMlMLCwrB2qla17UluAOAZtuyxiEQOFf+y8vLysCvl09PTZfny5SHbV75oCgBgPiW/4qdyF1L/vo899pi8//77MnnyZDnppJMibjty5MiIt3rQe/ElACBxtu2xxDNq1ChZtmxZ1FABAKjh2GABANgTs91JGH3qGNUlAIBt0WMBABiKYAEAGIpgAQAYimABABiKYAEAGIqzwpKw7MWvpay4JGz5if/IVlANANgLPRYAgKEIFgCAoUwZCvv444/l3nvvjbnNwIEDZcwYLjQEkBhN02T48OGSk5MTcV0w7v2nhrIey4cffsgdhwEkJNZ3hn+dz+eL+/wmmMvUyfuWLVvKrFmzIq7z/8A1TeO3CgBxxQqJRYsWiUhoD8Xn84mmafLbb79J27ZtTa8PRyjrsQS/AS6//HJVZQBwAE3TJC0tLeovoVdeeWXUfc8//3yzykIUSk839v9GsWzZspDlwV3ayiKt0zRNrr76annwwQcTGmOtvG29evVkwYIFif0jAJiOUQ1ncdVZYf6gqF+/ftiyaNsG2717N2OyAJAi11wg+e6770qdOnXkp59+CiybN2+eXHfddWHzONF6RIcOHZLOnTsz7wMAKXBVjyU4VEREevXqFbZNWVmZiIhUq1YtbF316tXNKQwAPMRVwRLL66+/LiIiWVlZIiJh8zp+LVu2tKwmAHAj1wTLoEGDIi4fNWqUiIg8/fTTIcs1TYv4p6ioSEREVq9ebW7BAAwTa+iaYW3ruWaO5eDBgxGXr127VkREMjIyQpbzZgPcJ3h+lBNx1FHaYznzzDMNa+uTTz6JuHzy5MkiIrJw4ULDjgXAfoIDJdYlCzCf0h7Lpk2bRMSaH35aWlrgWP43nt7rZADYB8Ne9mdpj+WPP/6QESNGhPxG8eSTT+reP17XtvJ6/+tbb7014vYDBw4MeX3VVVfprgUAEJmpPZaioqKYYRDtt4t7771Xxo4dm9QYaaR97rjjjrDjapomP//8c8Tt+a0HAJJnSrAMGDBABgwYkPT+gwcPlsGDB4d86ev5svf5fHLllVdGvCFdpG1FJOFjAABis/VZYcl80U+ZMsX0YwAAonPNdSwAAHuwdY/FrjoNN+40aQBwG3osAABDuaLHwjwJANgHPRYAgKEIFgCAoQgWAIChXDHHYrU3Hv5UDuw7pLoM16lZu7rcOPoS1WUASBE9FgCAoQgWAIChbDcUFu/GkyNGjJAhQ4ZE3Y9TjwFALcf1WMaNG8eT4QDAxmzXY/GL1vMgVADA3hzXY+F51gBgb7btsRgl1vNWKoeT/wFgkbYFAOjjuB5LIl/8/m179uwZsv3ChQsj9njoBQFA6mzbY4n1JZ+Zmal7/0GDBskjjzwSss7/bPt4PRgAzvH222/L448/HnEdIxDWsm2wxPL222/HXO8PiAcffFCuvvrqkHUjRowQEQk8vjhY8FAYAGfxhwohop5th8J8Pl/gz4oVK2TixImBdT179pTnnnsu4n7+YKhRo0ZYqIiIfPbZZyIikpGRYULVAADbBkuwtLQ06du3byBoREReffXVmPscPHjQitIA2Ai9FXtw5FBYLC1atJDZs2eLpmmiaRpvNMBDIg1l8x1gPUf0WBIxe/bskNfjxo1TVAkAqwUPoXPNmzquCxY//5sqeG4meDlvNsBdIvVMevbsqaASOCpYEj1ry997ibZPbm5uyGvCBnCXd955R3UJnmTbOZZ4X/J6xk1btGgR0p5/H39AjR07VsaOHZtaoQCUi3bh9CeffKKiHM9zTI/luOOOk6+++ipk7FSPaNv622nTpo2ccsopCbcLwD4mT54ccfk///lPiyuBiA17LMl+ucfaL3jdunXrpE2bNoHXX375ZVLHA2AfPXr0EJHDPZeqVatK3759A59tfmG0nmN6LEZp06aNaJom5513Xti6Cy+8UEREBgwYYHVZAFLk8/kkPz9fSktL5csvv5ScnBxCRRHb9VissmbNGtE0TRYsWCC7du2S7OzswLp48y43jr7E7PIAJKFhw4aEiQ14MliCzy7zd6GD1wEAkufJYBEhQADALJ6bYwEAmItgAQAYimABABjKs3MsqVj8+ONSunev6jLgIidHefIh4EQESxJK9+6V0n37VJcBALbEUBgAwFBKeizBN5iMd9pvp06dpKSkJGzbaDedAwCoZfseiz9UAADO4Ng5FnoqAGBPynosLVu2FJHYz13hwVsA4DyO7bHEEhxIkyZNkjPPPFPXtkuXLpXq1aubWhsAuJ3SOZb33nsv7jbRhrw0TQvr0URadsstt4imaVJeXh53286dO9NLAoAUKQ2W7t27i0jkIa9UvuD9T4MMfipkZmZm2Hb+W2z7/4wZMyblYwOA19n+rLBERevh+ENMROSEE04QEZH8/PyQbQYOHCgiIrfddptJ1QGA+ykPlpkzZ0Zdl8yZX5qmyWeffRbWTvCw2/LlywPb7q10axafzyfDhg1L+LgAgMOUB8sxxxwjIqHDT6kORY0YMSIwhxKvre7duwe269y5c0rHBQDYIFiM5J8rqVWrVsjySAETPP/id+jQIdE0LTBUBgBInC2CZd68eWHLUrkAcvHixYHgWLp0aWD5a6+9FvE4/j/PPvusiIiUlpYmfWwA6lx00UW6RytgHlsEy1FHHSUikXsWekXbt3r16vL222+LiMgzzzwTc9uLLrooqWMDUE/TNFm1alXIaAThooYtgqWyatWqJbyP/wyvSG+k66+/XkRE8vLyQtrPyclJtkQANhQ80uH/u38kAtaxzZX3S5cuDUyeL1u2LOH9GzZsGPh7tN9SWrdukrkX4AAABUpJREFUHWhf0zSZOXOmaJom6enpUlZWFtgumWADoM6bb74Zdd2kSZNk5MiRFlYD2/RYjLiVis/nk2bNmoUtz83NDZuzCX4dHCo+ny+pYAOgzpNPPhlxeb9+/SyuBCKKeizRJuYTWR5t22+++SblOgC4w/XXXx/zWjmYwzZDYQBgtMqXHsTz3iNnmVJHojrclau6hJQQLEmoWqeO6hIA6PDDDz+oLsGT0ioqKipUFwEAqYj2qHIeYa6GbSbvASBZBIe9ECwAXCPSpQaEjvUYCgPgGpWDpXnz5jJnzhw1xXgYwQIAQRK5DUyit47R23tKtYZYx7Fi3omhMACwiFfuXcbpxgAQQTK/0evpKWiaprvtZHsViRzDDPRYAMACVn/Rq+wdESwJqKioCNxy/5JLLlFdTkROeRaFXWt84403ArWNHj1adTlhevToEahv0aJFqssJiPWzdMp70i3scBYcwaKTpmnSsWNHOe200+Sdd94Rn89nqw9K8Ad3+PDhgWXQT9M0eeqpp+SZZ56RnJwcmTJliq3+DzVNk927d8vkyZOlf//+cuWVV9qivnihIhL6WIstW7ZYUpeXqX4eDXMsCSgoKJD09HQRkUCwrF27Vo455hjFlR0W/JtKTk5OIGzs8BtMMDt8GVYW6UyZ4cOH26bWgoICETlSX48ePWTcuHHK69MTKv6afT6fDBkyRE4//XTbvScjifVva9mypcyaNSup9ho3bmxpDSq+AwgWHfw/XH+oBOvXr5/yD8ns2bMjLrdbr0rEnqHid9xxx4UtU/2z9bvnnntUlxAmODii/Vw///zzkNcTJ0609XsgVXr+bd99950Flaj9/BMsKejatastxrn79u0b8QvQbh/gkpISEbFn4ImI/O9//xMRkbFjx8qcOXPkiy++UFzREZ9++qlomiaHDh0KPLsoKytLaU16Qrdt27YRl48ePVoefvhho0sylNG/VOTn54c8kNCKGvyfNat7LQRLCoYNGyY33XST6jJi6tq1q+oSAjp16mSbHkA0wYGnaZrUqlVLFi9erLCiIyIFst3/P6OZMmWK7YMlGXb+eVgZLkzep6Bp06aqS4jK/wU0ZcoUxZUcZsceSmX+D57/z9tvvy3FxcWqywoIHnpavnx5yDIgGhVhR7CkwA7DYJHY7Vbh06ZNExH71BNN69atQ1737t1bROzx5V35Z1q1atXA31UPiSXDiEeRQz+rzxIjWFJgt+scZs2aJZqmSVpamq2+xEeNGiUika9n0DRN5s2bp6q0EM8884zqEmJ6/PHHIy4vKyuzuJLUvfvuu6pLgIkIFh2GDBmiuoS4pk+fLrfeeqs8++yzsmLFCtXlhAgeXvL/CV7Xq1cvhdUdMWjQoIjL7fLzf+CBB1SXkLAbb7wx4vIuXbpYXAms/GWTYNFhxIgRImLvZz3cc889cvPNN8tFF12kuhTX6N+/v4gc+fmrFq1nsnTpUosr0adBgwby7bffhiyzw7Cil1n1fcVt83UaOHCg/Pzzz2HL7RAs8T6sdqixMrvNA4lE/n+86qqr5KGHHrK+mAii/Zzt8H8Y7YwjO/8yBvMQLAm699575ZtvvrHsIidYLzs7Wzp06CAvv/yy6lIiOv3006Vt27byr3/9S3UpuvXo0UMGDx4sw4YNU10KLECwAAAMxRwLAMBQBAsAwFAECwDAUAQLAMBQBAsAwFAECwDAUAQLAMBQBAsAwFAECwDAUAQLAMBQBAsAwFAECwDAUAQLAMBQBAsAwFAECwDAUAQLAMBQBAsAwFAECwDAUAQLAMBQBAsAwFAECwDAUP8P4otwOi9kLtAAAAAASUVORK5CYII=\n", 192 | "text/plain": [ 193 | "
" 194 | ] 195 | }, 196 | "metadata": {}, 197 | "output_type": "display_data" 198 | } 199 | ], 200 | "source": [ 201 | "# Plot summaries for the evaluations run so far\n", 202 | "tide.plot()" 203 | ] 204 | } 205 | ], 206 | "metadata": { 207 | "kernelspec": { 208 | "display_name": "Python 3", 209 | "language": "python", 210 | "name": "python3" 211 | }, 212 | "language_info": { 213 | "codemirror_mode": { 214 | "name": "ipython", 215 | "version": 3 216 | }, 217 | "file_extension": ".py", 218 | "mimetype": "text/x-python", 219 | "name": "python", 220 | "nbconvert_exporter": "python", 221 | "pygments_lexer": "ipython3", 222 | "version": "3.7.4" 223 | } 224 | }, 225 | "nbformat": 4, 226 | "nbformat_minor": 2 227 | } 228 | --------------------------------------------------------------------------------