├── .gitignore ├── figs └── eye.png ├── frontend ├── public │ ├── eye.ico │ └── favicon.ico ├── src │ ├── main.js │ ├── components │ │ ├── LeftPannel.vue │ │ ├── Header.vue │ │ ├── graphs │ │ │ └── graph_config.js │ │ ├── left_controls │ │ │ └── Config.vue │ │ └── Graph.vue │ ├── utils.js │ └── App.vue ├── jsconfig.json ├── index.html ├── README.md ├── vite.config.js ├── package.json └── .gitignore ├── roofline_model.py ├── utils.py ├── backend_settings.py ├── LICENSE ├── model_params └── DiT.py ├── backend_app.py ├── analyze_cli.py ├── analyze_gen_cli.py ├── configs ├── chatglm3.py ├── DiT.py ├── gpt-j-6B.py ├── opt.py └── Llama.py ├── hardwares └── hardware_params.py ├── README.md ├── get_model_graph.py ├── model_analyzer.py └── examples ├── plot_hardware.ipynb └── plot_memory.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | output 2 | tmp 3 | *.pyc 4 | .vscode -------------------------------------------------------------------------------- /figs/eye.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hahnyuan/LLM-Viewer/HEAD/figs/eye.png -------------------------------------------------------------------------------- /frontend/public/eye.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hahnyuan/LLM-Viewer/HEAD/frontend/public/eye.ico -------------------------------------------------------------------------------- /frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hahnyuan/LLM-Viewer/HEAD/frontend/public/favicon.ico -------------------------------------------------------------------------------- /frontend/src/main.js: -------------------------------------------------------------------------------- 1 | import { createApp } from 'vue' 2 | import App from './App.vue' 3 | 4 | const app=createApp(App) 5 | app.mount('#app') -------------------------------------------------------------------------------- /frontend/jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "paths": { 4 | "@/*": ["./src/*"] 5 | } 6 | }, 7 | "exclude": ["node_modules", "dist"] 8 | } 9 | -------------------------------------------------------------------------------- /frontend/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | LLM-Viewer 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /frontend/README.md: -------------------------------------------------------------------------------- 1 | # LLMViewer Frontend 2 | 3 | This project uses Vue 3 in Vite. 4 | 5 | ## Project Setup 6 | 7 | ```sh 8 | npm install 9 | ``` 10 | 11 | ### Compile and Hot-Reload for Development 12 | 13 | ```sh 14 | npm run dev 15 | ``` 16 | or 17 | ```sh 18 | npm run build 19 | npm run preview 20 | ``` 21 | 22 | ### Compile and Minify for Production 23 | 24 | ```sh 25 | npm run build 26 | ``` 27 | -------------------------------------------------------------------------------- /frontend/vite.config.js: -------------------------------------------------------------------------------- 1 | import { fileURLToPath, URL } from 'node:url' 2 | 3 | import { defineConfig } from 'vite' 4 | import vue from '@vitejs/plugin-vue' 5 | 6 | // https://vitejs.dev/config/ 7 | export default defineConfig({ 8 | plugins: [ 9 | vue(), 10 | ], 11 | resolve: { 12 | alias: { 13 | '@': fileURLToPath(new URL('./src', import.meta.url)) 14 | } 15 | }, 16 | define: { 17 | llm_viewer_frontend_version: JSON.stringify(process.env.npm_package_version) 18 | } 19 | 20 | }) 21 | -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "llmviewer", 3 | "version": "0.4.0", 4 | "private": true, 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "vite build", 9 | "preview": "vite preview" 10 | }, 11 | "dependencies": { 12 | "@antv/g6": "^4.8.24", 13 | "@vitejs/plugin-vue": "^5.0.4", 14 | "@vue/cli-service": "^5.0.8", 15 | "axios": "^1.6.7", 16 | "chart.js": "^4.4.2", 17 | "chartjs-plugin-annotation": "^3.0.1", 18 | "numeral": "^2.0.6", 19 | "vite": "^5.1.5", 20 | "vue": "^3.4.21" 21 | }, 22 | "devDependencies": { 23 | "gh-pages": "^6.1.1" 24 | } 25 | } -------------------------------------------------------------------------------- /roofline_model.py: -------------------------------------------------------------------------------- 1 | def roofline_analyze(bandwidth, max_OPS, OPs, memory_access): 2 | # bandwidth is bytes/s 3 | # memory_access in byte 4 | # x axis is OPS/byte 5 | # y axis is OPS/s 6 | y_max = max_OPS 7 | memory_access_bytes = memory_access 8 | turning_point = y_max / bandwidth 9 | arithmetic_intensity = OPs / memory_access_bytes 10 | if arithmetic_intensity < turning_point: 11 | bound = "memory" 12 | performance = arithmetic_intensity * bandwidth 13 | else: 14 | bound = "compute" 15 | performance = y_max 16 | if performance==0: 17 | 1==1 18 | pass 19 | return arithmetic_intensity, performance, bound 20 | -------------------------------------------------------------------------------- /frontend/src/components/LeftPannel.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 18 | 19 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | 2 | def str_number(num): 3 | if num > 1e14: 4 | return f"{num/1e12:.0f}T" 5 | elif num > 1e12: 6 | return f"{num/1e12:.1f}T" 7 | elif num>1e11: 8 | return f"{num/1e9:.0f}G" 9 | elif num > 1e9: 10 | return f"{num/1e9:.1f}G" 11 | elif num > 1e8: 12 | return f"{num/1e6:.0f}M" 13 | elif num > 1e6: 14 | return f"{num/1e6:.1f}M" 15 | elif num > 1e5: 16 | return f"{num/1e3:.0f}K" 17 | elif num > 1e3: 18 | return f"{num/1e3:.1f}K" 19 | elif num >= 1: 20 | return f"{num:.1f}" 21 | else: 22 | return f"{num:.2f}" 23 | 24 | def str_number_time(num): 25 | if num >= 1: 26 | return f"{num:.1f}" 27 | elif num > 1e-3: 28 | return f"{num*1e3:.1f}m" 29 | elif num > 1e-6: 30 | return f"{num*1e6:.1f}u" 31 | elif num > 1e-9: 32 | return f"{num*1e9:.1f}n" 33 | else: 34 | return f"{num:.0f}" -------------------------------------------------------------------------------- /backend_settings.py: -------------------------------------------------------------------------------- 1 | from hardwares.hardware_params import hardware_params 2 | 3 | avaliable_model_ids_sources = { 4 | "meta-llama/Llama-2-7b-hf": {"source": "huggingface"}, 5 | "meta-llama/Llama-2-13b-hf": {"source": "huggingface"}, 6 | "meta-llama/Llama-2-70b-hf": {"source": "huggingface"}, 7 | "EleutherAI/gpt-j-6B":{"source": "huggingface"}, 8 | "THUDM/chatglm3-6b": {"source": "huggingface"}, 9 | "facebook/opt-125m": {"source": "huggingface"}, 10 | "facebook/opt-1.3b": {"source": "huggingface"}, 11 | "facebook/opt-2.7b": {"source": "huggingface"}, 12 | "facebook/opt-6.7b": {"source": "huggingface"}, 13 | "facebook/opt-30b": {"source": "huggingface"}, 14 | "facebook/opt-66b": {"source": "huggingface"}, 15 | # "DiT-XL/2": {"source": "DiT"}, 16 | # "DiT-XL/4": {"source": "DiT"}, 17 | } 18 | avaliable_model_ids = [_ for _ in avaliable_model_ids_sources.keys()] 19 | avaliable_hardwares = [_ for _ in hardware_params.keys()] 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Zhihang Yuan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /frontend/src/utils.js: -------------------------------------------------------------------------------- 1 | export function strNumber(num) { 2 | if (num > 1e14) { 3 | return `${(num / 1e12).toFixed(0)}T`; 4 | } else if (num > 1e12) { 5 | return `${(num / 1e12).toFixed(1)}T`; 6 | } else if (num > 1e11) { 7 | return `${(num / 1e9).toFixed(0)}G`; 8 | } else if (num > 1e9) { 9 | return `${(num / 1e9).toFixed(1)}G`; 10 | } else if (num > 1e8) { 11 | return `${(num / 1e6).toFixed(0)}M`; 12 | } else if (num > 1e6) { 13 | return `${(num / 1e6).toFixed(1)}M`; 14 | } else if (num > 1e5) { 15 | return `${(num / 1e3).toFixed(0)}K`; 16 | } else if (num > 1e3) { 17 | return `${(num / 1e3).toFixed(1)}K`; 18 | } else if (num >= 1) { 19 | return `${num.toFixed(1)}`; 20 | } else { 21 | return `${num.toFixed(2)}`; 22 | } 23 | } 24 | 25 | export function strNumberTime(num) { 26 | if (num >= 1) { 27 | return `${num.toFixed(1)}s`; 28 | } else if (num > 1e-3) { 29 | return `${(num * 1e3).toFixed(1)}ms`; 30 | } else if (num > 1e-6) { 31 | return `${(num * 1e6).toFixed(1)}us`; 32 | } else if (num > 1e-9) { 33 | return `${(num * 1e9).toFixed(1)}ns`; 34 | } else { 35 | return `${num.toFixed(0)}s`; 36 | } 37 | } -------------------------------------------------------------------------------- /model_params/DiT.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict 2 | 3 | 4 | model_params={ 5 | "DiT-XL/2":EasyDict( 6 | depth=28, hidden_size=1152, patch_size=2, num_heads=16 7 | ), 8 | "DiT-XL/4":EasyDict( 9 | depth=28, hidden_size=1152, patch_size=4, num_heads=16 10 | ), 11 | "DiT-XL/8":EasyDict( 12 | depth=28, hidden_size=1152, patch_size=8, num_heads=16 13 | ), 14 | "DiT-L/2":EasyDict( 15 | depth=24, hidden_size=1024, patch_size=2, num_heads=16 16 | ), 17 | "DiT-L/4":EasyDict( 18 | depth=24, hidden_size=1024, patch_size=4, num_heads=16 19 | ), 20 | "DiT-L/8":EasyDict( 21 | depth=24, hidden_size=1024, patch_size=8, num_heads=16 22 | ), 23 | "DiT-B/2":EasyDict( 24 | depth=12, hidden_size=768, patch_size=2, num_heads=12 25 | ), 26 | "DiT-B/4":EasyDict( 27 | depth=12, hidden_size=768, patch_size=4, num_heads=12 28 | ), 29 | "DiT-B/8":EasyDict( 30 | depth=12, hidden_size=768, patch_size=8, num_heads=12 31 | ), 32 | "DiT-S/2":EasyDict( 33 | depth=12, hidden_size=384, patch_size=2, num_heads=6 34 | ), 35 | "DiT-S/4":EasyDict( 36 | depth=12, hidden_size=384, patch_size=4, num_heads=6 37 | ), 38 | "DiT-S/8":EasyDict( 39 | depth=12, hidden_size=384, patch_size=8, num_heads=6 40 | ) 41 | 42 | } -------------------------------------------------------------------------------- /backend_app.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, request 2 | from flask import render_template 3 | from flask_cors import CORS 4 | from get_model_graph import get_model_graph 5 | from backend_settings import avaliable_hardwares,avaliable_model_ids 6 | import argparse 7 | 8 | app = Flask(__name__) 9 | cors = CORS(app, resources={r"/*": {"origins": "*"}}) 10 | 11 | 12 | @app.route("/") 13 | def index(): 14 | return "backend server ready." 15 | 16 | 17 | @app.route("/get_graph", methods=["POST"]) 18 | def get_graph(): 19 | inference_config = request.json["inference_config"] 20 | nodes, edges, total_results, hardware_info = get_model_graph( 21 | request.json["model_id"], 22 | request.json["hardware"], 23 | None, 24 | inference_config, 25 | ) 26 | return { 27 | "nodes": nodes, 28 | "edges": edges, 29 | "total_results": total_results, 30 | "hardware_info": hardware_info, 31 | } 32 | 33 | @app.route("/get_avaliable", methods=["GET"]) 34 | def get_avaliable(): 35 | return { 36 | "avaliable_hardwares": avaliable_hardwares, 37 | "avaliable_model_ids": avaliable_model_ids, 38 | } 39 | 40 | if __name__ == "__main__": 41 | parser=argparse.ArgumentParser() 42 | parser.add_argument("--port", type=int, default=5000) 43 | parser.add_argument("--local", action="store_true") 44 | parser.add_argument("--debug", action="store_true") 45 | args=parser.parse_args() 46 | host="127.0.0.1" if args.local else "0.0.0.0" 47 | app.run(debug=args.debug,host=host,port=args.port) 48 | -------------------------------------------------------------------------------- /analyze_cli.py: -------------------------------------------------------------------------------- 1 | from model_analyzer import ModelAnalyzer 2 | import torch.nn as nn 3 | import numpy as np 4 | import os 5 | import importlib 6 | import argparse 7 | 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument("model_id", type=str, help="model id") 10 | parser.add_argument( 11 | "hardware", 12 | type=str, 13 | help="name of hardware, for example nvidia_V100 or nvidia_A6000", 14 | ) 15 | parser.add_argument( 16 | "--source", 17 | type=str, 18 | default="huggingface", 19 | help="source of model, if not huggingface, will use local model in model_params.", 20 | ) 21 | parser.add_argument("--config_file", type=str, default=None, help="config file") 22 | parser.add_argument("--batchsize", type=int, default=1, help="batch size") 23 | parser.add_argument("--seqlen", type=int, default=1024, help="sequence length") 24 | parser.add_argument("--w_bit", type=int, default=16, help="weight bitwidth") 25 | parser.add_argument( 26 | "--a_bit", type=int, default=16, help="temporary activation bitwidth" 27 | ) 28 | parser.add_argument("--kv_bit", type=int, default=16, help="kv cache bitwidth") 29 | parser.add_argument( 30 | "--use_flashattention", action="store_true", help="use flash attention" 31 | ) 32 | parser.add_argument( 33 | "--tp-size", 34 | type=int, 35 | default=1, 36 | help="the number of devices for tensor parallelism to use" 37 | ) 38 | args = parser.parse_args() 39 | 40 | analyzer = ModelAnalyzer(args.model_id, args.hardware, args.config_file,source=args.source) 41 | results = analyzer.analyze( 42 | batchsize=args.batchsize, 43 | seqlen=args.seqlen, 44 | w_bit=args.w_bit, 45 | a_bit=args.a_bit, 46 | kv_bit=args.kv_bit, 47 | use_flashattention=args.use_flashattention, 48 | tp_size=args.tp_size 49 | ) 50 | analyzer.save_csv() 51 | -------------------------------------------------------------------------------- /analyze_gen_cli.py: -------------------------------------------------------------------------------- 1 | from model_analyzer import ModelAnalyzer 2 | import torch.nn as nn 3 | import numpy as np 4 | import os 5 | import importlib 6 | import argparse 7 | 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument("model_id", type=str, help="model id") 10 | parser.add_argument( 11 | "hardware", 12 | type=str, 13 | help="name of hardware, for example nvidia_V100 or nvidia_A6000", 14 | ) 15 | parser.add_argument("--config_file", type=str, default=None, help="config file") 16 | parser.add_argument("--batchsize", type=int, default=1, help="batch size") 17 | parser.add_argument("--seqlen", type=int, default=1024, help="sequence length") 18 | parser.add_argument("--promptlen", type=int, default=128, help="prompt sequence length") 19 | parser.add_argument("--w_bit", type=int, default=16, help="weight bitwidth") 20 | parser.add_argument("--a_bit", type=int, default=16, help="temporary activation bitwidth") 21 | parser.add_argument("--kv_bit", type=int, default=16, help="kv cache bitwidth") 22 | parser.add_argument("--use_flashattention", action="store_true", help="use flash attention") 23 | parser.add_argument( 24 | "--tp-size", 25 | type=int, 26 | default=1, 27 | help="the number of devices for tensor parallelism to use" 28 | ) 29 | args = parser.parse_args() 30 | 31 | analyzer=ModelAnalyzer(args.model_id,args.hardware,args.config_file) 32 | ret = analyzer.analyze_generate_task( 33 | args.promptlen, 34 | args.seqlen, 35 | args.batchsize, 36 | args.w_bit, 37 | args.a_bit, 38 | args.kv_bit, 39 | args.use_flashattention, 40 | tp_size=args.tp_size 41 | ) 42 | elapse = ret["inference_time"] 43 | prefill_elapse = ret["prefill_time"] 44 | print(f"{args.hardware}: 1st token latency {prefill_elapse:.2f}, total latency {elapse:.2f}, throughput {args.seqlen * args.batchsize / elapse:.2f} Token/sec") 45 | -------------------------------------------------------------------------------- /frontend/src/App.vue: -------------------------------------------------------------------------------- 1 | 40 | 41 | 53 | 54 | 88 | -------------------------------------------------------------------------------- /configs/chatglm3.py: -------------------------------------------------------------------------------- 1 | def get_num_attention_heads(model_params): 2 | return getattr(model_params, "num_attention_heads") 3 | 4 | def get_hidden_size(model_params): 5 | return getattr(model_params, "hidden_size") 6 | 7 | def get_num_key_value_heads(model_params): 8 | if getattr(model_params,"multi_query_attention"): 9 | return getattr(model_params, "multi_query_group_num") 10 | else: 11 | return getattr(model_params, "num_attention_heads") 12 | 13 | def get_num_hidden_layers(model_params): 14 | return getattr(model_params, "num_layers") 15 | 16 | def get_intermediate_size(model_params): 17 | return getattr(model_params, "ffn_hidden_size") 18 | 19 | def get_vocab_size(model_params): 20 | return getattr(model_params, "padded_vocab_size") 21 | 22 | def get_norm_layers(model_params): 23 | return ["attn_norm", "mlp_norm"] 24 | 25 | def post_process(model_params,args): 26 | hiddensize=get_hidden_size(model_params) 27 | vocab_size=get_vocab_size(model_params) 28 | layers=[] 29 | for stage in ["prefill", "decode"]: 30 | layers.append({ 31 | 'name': 'lm_head', 32 | 'stage':stage, 33 | 'OPs':args['batchsize']*hiddensize*vocab_size*1, 34 | 'load_weight':hiddensize*vocab_size *args['w_byte'], 35 | 'load_act':hiddensize*args['a_byte'], 36 | 'store_act':vocab_size*args['a_byte'], 37 | }) 38 | return layers 39 | 40 | def get_linear_layers(model_params, tp_size: int): 41 | hidden_size=get_hidden_size(model_params) 42 | intermediate_size=get_intermediate_size(model_params) 43 | key_value_heads=get_num_key_value_heads(model_params) 44 | attention_heads=get_num_attention_heads(model_params) 45 | 46 | if tp_size > 1: 47 | assert hidden_size % tp_size == 0 48 | assert intermediate_size % tp_size == 0 49 | assert key_value_heads % tp_size == 0 50 | 51 | return { 52 | "q_proj": [hidden_size, hidden_size // tp_size], 53 | "k_proj": [hidden_size, hidden_size * key_value_heads // attention_heads // tp_size], 54 | "v_proj": [hidden_size, hidden_size * key_value_heads // attention_heads // tp_size], 55 | "out_proj": [hidden_size // tp_size, hidden_size], 56 | "gate_proj": [hidden_size, intermediate_size // tp_size], 57 | "up_proj": [hidden_size, intermediate_size // tp_size], 58 | "down_proj": [intermediate_size // tp_size, hidden_size] 59 | } 60 | 61 | from configs.Llama import flashattention_transformer_layer_graph,transformer_layer_graph -------------------------------------------------------------------------------- /hardwares/hardware_params.py: -------------------------------------------------------------------------------- 1 | # the OPS = sparse OPS/2 2 | 3 | hardware_params = { 4 | # NOTICES: For GPU, we use Register File Size as on-chip buffer size 5 | # https://images.nvidia.com/content/volta-architecture/pdf/volta-architecture-whitepaper.pdf 6 | # NOTICE: V100 not support INT8 in tensor core, so INT8 performance is not good 7 | "nvidia_V100": {"bandwidth": 900e9, "FP16": 112e12, "INT8": 62e12, "onchip_buffer": 20480e3}, 8 | # https://images.nvidia.com/aem-dam/en-zz/Solutions/technologies/NVIDIA-ADA-GPU-PROVIZ-Architecture-Whitepaper_1.1.pdf 9 | "nvidia_A6000": {"bandwidth": 768e9, "FP16": 154.8e12, "INT8": 309.7e12, "onchip_buffer": 21504e3}, 10 | # https://images.nvidia.com/aem-dam/en-zz/Solutions/technologies/NVIDIA-ADA-GPU-PROVIZ-Architecture-Whitepaper_1.1.pdf 11 | "nvidia_A6000_Ada": {"bandwidth": 960e9, "FP16": 364.2e12, "INT8": 728.5e12, "onchip_buffer": 36352e3}, 12 | # https://images.nvidia.com/aem-dam/en-zz/Solutions/data-center/nvidia-ampere-architecture-whitepaper.pdf 13 | # Ampere's SM has 256KB RF, max 164KB Shared Mem 14 | "nvidia_A100": {"bandwidth": 1555e9, "FP16": 312e12, "INT8": 624e12, "onchip_buffer": 27648e3}, # use 40G data 15 | "nvidia_A100_40G": {"bandwidth": 1555e9, "FP16": 312e12, "INT8": 624e12, "onchip_buffer": 27648e3}, 16 | "nvidia_A100_80G": {"bandwidth": 2039e9, "FP16": 312e12, "INT8": 624e12, "onchip_buffer": 27648e3}, 17 | "nvidia_A800_80G_SXM": {"bandwidth": 2039e9, "FP16": 312e12, "INT8": 624e12, "onchip_buffer": 27648e3}, 18 | "nvidia_A40": {"bandwidth": 696e9, "FP16": 149.7e12, "INT8": 299.3e12, "onchip_buffer": 21504e3}, 19 | # https://resources.nvidia.com/en-us-tensor-core/gtc22-whitepaper-hopper 20 | "nvidia_H100": { 21 | "bandwidth": 3072e9, 22 | "FP16": 1979e12 / 2, 23 | "INT8": 3958e12 / 2, 24 | "onchip_buffer": 33792e3, 25 | }, # use SXM data 26 | "nvidia_H100_SXM": {"bandwidth": 3072e9, "FP16": 1979e12 / 2, "INT8": 3958e12 / 2, "onchip_buffer": 33792e3}, 27 | "nvidia_H100_PCIe": {"bandwidth": 2048e9, "FP16": 1513e12 / 2, "INT8": 3026e12 / 2, "onchip_buffer": 29184e3}, 28 | # https://images.nvidia.com/aem-dam/Solutions/Data-Center/l4/nvidia-ada-gpu-architecture-whitepaper-v2.1.pdf 29 | # Ada SM has 256 KB Register File, and 128 KB of L1/Shared Memory 30 | "nvidia_L40": {"bandwidth": 864e9, "FP16": 181e12, "INT8": 362e12, "onchip_buffer": 36352e3}, 31 | # Intel Skylake-X (Skylake-X, Cascade Lake) Intel Xeon Phi (Knights Landing, Knights Mill) Intel Ice Lake, Tiger Lake and Rocket Lake 32 | # support AVX-512 & FMA (512-bit), they has throughput of 1 cycle 33 | # https://www.intel.com/content/www/us/en/products/sku/230496/intel-core-i913900k-processor-36m-cache-up-to-5-80-ghz/specifications.html 34 | "intel_13900k": {"bandwidth": 89.6e9, "FP16": 8 * 5.4e9 * (512 / 16), "onchip_buffer": 36e6}, 35 | } 36 | -------------------------------------------------------------------------------- /configs/DiT.py: -------------------------------------------------------------------------------- 1 | 2 | def get_num_attention_heads(model_params): 3 | return getattr(model_params, "num_heads") 4 | 5 | 6 | def get_hidden_size(model_params): 7 | return getattr(model_params, "hidden_size") 8 | 9 | 10 | def get_num_key_value_heads(model_params): 11 | return getattr(model_params, "num_heads") 12 | 13 | def get_norm_layers(model_params): 14 | return ["attn_norm", "mlp_norm"] 15 | 16 | def get_num_hidden_layers(model_params): 17 | return getattr(model_params, "depth") 18 | 19 | def get_intermediate_size(model_params): 20 | mlp_ratio=getattr(model_params, "mlp_ratio", 4.0) 21 | return getattr(model_params, "hidden_size")*mlp_ratio 22 | 23 | def get_linear_layers(model_params, tp_size: int): 24 | hidden_size=get_hidden_size(model_params) 25 | intermediate_size=get_intermediate_size(model_params) 26 | key_value_heads=get_num_key_value_heads(model_params) 27 | attention_heads=get_num_attention_heads(model_params) 28 | 29 | if tp_size > 1: 30 | assert hidden_size % tp_size == 0 31 | assert intermediate_size % tp_size == 0 32 | assert key_value_heads % tp_size == 0 33 | 34 | return { 35 | "q_proj": [hidden_size, hidden_size // tp_size], 36 | "k_proj": [hidden_size, hidden_size * key_value_heads // attention_heads // tp_size], 37 | "v_proj": [hidden_size, hidden_size * key_value_heads // attention_heads // tp_size], 38 | "out_proj": [hidden_size // tp_size, hidden_size], 39 | "gate_proj": [hidden_size, intermediate_size // tp_size], 40 | "up_proj": [hidden_size, intermediate_size // tp_size], 41 | "down_proj": [intermediate_size // tp_size, hidden_size], 42 | } 43 | 44 | def post_process(model_params,args): 45 | return [] 46 | 47 | transformer_layer_graph = { 48 | "input": [], 49 | "attn_norm": ["input"], 50 | "q_proj": ["attn_norm"], 51 | "k_proj": ["attn_norm"], 52 | "v_proj": ["attn_norm"], 53 | "qk_matmul": ["q_proj", "k_proj"], 54 | "softmax": ["qk_matmul"], 55 | "sv_matmul": ["softmax", "v_proj"], 56 | "out_proj": ["sv_matmul"], 57 | "attn_add": ["input", "out_proj"], 58 | "mlp_norm": ["attn_add"], 59 | "up_proj": ["mlp_norm"], 60 | "mlp_act": ["up_proj"], 61 | "down_proj": ["mlp_act"], 62 | "mlp_add": ["attn_add", "down_proj"], 63 | "output": ["mlp_add"], 64 | } 65 | 66 | flashattention_transformer_layer_graph = { 67 | "input": [], 68 | "attn_norm": ["input"], 69 | "q_proj": ["attn_norm"], 70 | "k_proj": ["attn_norm"], 71 | "v_proj": ["attn_norm"], 72 | "fused_attention": ["q_proj", "k_proj", "v_proj"], 73 | "out_proj": ["fused_attention"], 74 | "attn_add": ["input", "out_proj"], 75 | "mlp_norm": ["attn_add"], 76 | "up_proj": ["mlp_norm"], 77 | "mlp_act": ["up_proj"], 78 | "down_proj": ["mlp_act"], 79 | "mlp_add": ["attn_add", "down_proj"], 80 | "output": ["mlp_add"], 81 | } 82 | -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | //.gitignore 2 | 3 | .DS_Store 4 | node_modules 5 | /dist 6 | 7 | # local env files 8 | .env.local 9 | .env.*.local 10 | 11 | # Log files 12 | npm-debug.log* 13 | yarn-debug.log* 14 | yarn-error.log* 15 | 16 | # Editor directories and files 17 | .idea 18 | .vscode 19 | *.suo 20 | *.ntvs* 21 | *.njsproj 22 | *.sln 23 | *.sw? 24 | 25 | # Logs 26 | logs 27 | *.log 28 | npm-debug.log* 29 | yarn-debug.log* 30 | yarn-error.log* 31 | lerna-debug.log* 32 | .pnpm-debug.log* 33 | 34 | # Diagnostic reports (https://nodejs.org/api/report.html) 35 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 36 | 37 | # Runtime data 38 | pids 39 | *.pid 40 | *.seed 41 | *.pid.lock 42 | 43 | # Directory for instrumented libs generated by jscoverage/JSCover 44 | lib-cov 45 | 46 | # Coverage directory used by tools like istanbul 47 | coverage 48 | *.lcov 49 | 50 | # nyc test coverage 51 | .nyc_output 52 | 53 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 54 | .grunt 55 | 56 | # Bower dependency directory (https://bower.io/) 57 | bower_components 58 | 59 | # node-waf configuration 60 | .lock-wscript 61 | 62 | # Compiled binary addons (https://nodejs.org/api/addons.html) 63 | build/Release 64 | 65 | # Dependency directories 66 | node_modules/ 67 | jspm_packages/ 68 | 69 | # Snowpack dependency directory (https://snowpack.dev/) 70 | web_modules/ 71 | 72 | # TypeScript cache 73 | *.tsbuildinfo 74 | 75 | # Optional npm cache directory 76 | .npm 77 | 78 | # Optional eslint cache 79 | .eslintcache 80 | 81 | # Optional stylelint cache 82 | .stylelintcache 83 | 84 | # Microbundle cache 85 | .rpt2_cache/ 86 | .rts2_cache_cjs/ 87 | .rts2_cache_es/ 88 | .rts2_cache_umd/ 89 | 90 | # Optional REPL history 91 | .node_repl_history 92 | 93 | # Output of 'npm pack' 94 | *.tgz 95 | 96 | # Yarn Integrity file 97 | .yarn-integrity 98 | 99 | # dotenv environment variable files 100 | .env 101 | .env.development.local 102 | .env.test.local 103 | .env.production.local 104 | .env.local 105 | 106 | # parcel-bundler cache (https://parceljs.org/) 107 | .cache 108 | .parcel-cache 109 | 110 | # Next.js build output 111 | .next 112 | out 113 | 114 | # Nuxt.js build / generate output 115 | .nuxt 116 | dist 117 | 118 | # Gatsby files 119 | .cache/ 120 | # Comment in the public line in if your project uses Gatsby and not Next.js 121 | # https://nextjs.org/blog/next-9-1#public-directory-support 122 | # public 123 | 124 | # vuepress build output 125 | .vuepress/dist 126 | 127 | # vuepress v2.x temp and cache directory 128 | .temp 129 | .cache 130 | 131 | # Docusaurus cache and generated files 132 | .docusaurus 133 | 134 | # Serverless directories 135 | .serverless/ 136 | 137 | # FuseBox cache 138 | .fusebox/ 139 | 140 | # DynamoDB Local files 141 | .dynamodb/ 142 | 143 | # TernJS port file 144 | .tern-port 145 | 146 | # Stores VSCode versions used for testing VSCode extensions 147 | .vscode-test 148 | 149 | # yarn v2 150 | .yarn/cache 151 | .yarn/unplugged 152 | .yarn/build-state.yml 153 | .yarn/install-state.gz 154 | .pnp.* -------------------------------------------------------------------------------- /configs/gpt-j-6B.py: -------------------------------------------------------------------------------- 1 | 2 | def get_num_attention_heads(model_params): 3 | return getattr(model_params, "num_attention_heads") 4 | 5 | 6 | def get_hidden_size(model_params): 7 | return getattr(model_params, "n_embd") 8 | 9 | def get_norm_layers(model_params): 10 | return ["attn_norm"] 11 | 12 | # no group query attention 13 | def get_num_key_value_heads(model_params): 14 | return getattr(model_params, "num_attention_heads") 15 | 16 | def get_num_hidden_layers(model_params): 17 | return getattr(model_params, "num_hidden_layers") 18 | 19 | def get_intermediate_size(model_params): 20 | return 16384 21 | 22 | def get_vocab_size(model_params): 23 | return getattr(model_params, "vocab_size") 24 | 25 | def post_process(model_params,args): 26 | hiddensize=get_hidden_size(model_params) 27 | vocab_size=get_vocab_size(model_params) 28 | layers=[] 29 | for stage in ["prefill", "decode"]: 30 | layers.append({ 31 | 'name': 'lm_head', 32 | 'stage':stage, 33 | 'OPs':args['batchsize']*hiddensize*vocab_size*1, 34 | 'load_weight':hiddensize*vocab_size *args['w_byte'], 35 | 'load_act':hiddensize*args['a_byte'], 36 | 'store_act':vocab_size*args['a_byte'], 37 | }) 38 | return layers 39 | 40 | def get_linear_layers(model_params, tp_size: int): 41 | hidden_size=get_hidden_size(model_params) 42 | intermediate_size=get_intermediate_size(model_params) 43 | key_value_heads=get_num_key_value_heads(model_params) 44 | attention_heads=get_num_attention_heads(model_params) 45 | 46 | if tp_size > 1: 47 | assert hidden_size % tp_size == 0 48 | assert intermediate_size % tp_size == 0 49 | assert key_value_heads % tp_size == 0 50 | 51 | return { 52 | "q_proj":[hidden_size, hidden_size // tp_size], 53 | "k_proj":[hidden_size, hidden_size * key_value_heads // attention_heads // tp_size], 54 | "v_proj":[hidden_size, hidden_size * key_value_heads // attention_heads // tp_size], 55 | "out_proj":[hidden_size // tp_size, hidden_size], 56 | #"gate_proj":[hidden_size, intermediate_size], 57 | "up_proj":[hidden_size, intermediate_size // tp_size], 58 | "down_proj":[intermediate_size // tp_size, hidden_size], 59 | } 60 | 61 | # name, input_names 62 | transformer_layer_graph={ 63 | "input":[], 64 | "attn_norm": ["input"], 65 | "q_proj":["attn_norm"], 66 | "k_proj":["attn_norm"], 67 | "v_proj":["attn_norm"], 68 | "qk_matmul":["q_proj","k_proj"], 69 | "softmax":["qk_matmul"], 70 | "sv_matmul":["softmax","v_proj"], 71 | "out_proj":["sv_matmul"], 72 | "attn_add":["input","out_proj"], 73 | "up_proj":["input"], 74 | "mlp_act":["up_proj"], 75 | "down_proj":["mlp_act"], 76 | "mlp_add":["attn_add","down_proj"], 77 | "output":["mlp_add"] 78 | } 79 | 80 | flashattention_transformer_layer_graph={ 81 | "input":[], 82 | "attn_norm": ["input"], 83 | "q_proj":["attn_norm"], 84 | "k_proj":["attn_norm"], 85 | "v_proj":["attn_norm"], 86 | "fused_attention":["q_proj","k_proj","v_proj"], 87 | "out_proj":["fused_attention"], 88 | "attn_add":["input","out_proj"], 89 | "mlp_norm":["attn_add"], 90 | "gate_proj":["mlp_norm"], 91 | "up_proj":["mlp_norm"], 92 | "mlp_act":["up_proj","gate_proj"], 93 | "down_proj":["mlp_act"], 94 | "mlp_add":["attn_add","down_proj"], 95 | "output":["mlp_add"] 96 | } 97 | -------------------------------------------------------------------------------- /configs/opt.py: -------------------------------------------------------------------------------- 1 | def get_num_attention_heads(model_params): 2 | return getattr(model_params, "num_attention_heads") 3 | 4 | 5 | def get_hidden_size(model_params): 6 | return getattr(model_params, "hidden_size") 7 | 8 | 9 | def get_num_key_value_heads(model_params): 10 | return getattr(model_params, "num_attention_heads") 11 | 12 | def get_norm_layers(model_params): 13 | return ["attn_norm", "mlp_norm"] 14 | 15 | def get_num_hidden_layers(model_params): 16 | return getattr(model_params, "num_hidden_layers") 17 | 18 | 19 | def get_intermediate_size(model_params): 20 | return getattr(model_params, "ffn_dim") 21 | 22 | 23 | def get_vocab_size(model_params): 24 | return getattr(model_params, "vocab_size") 25 | 26 | def post_process(model_params,args): 27 | hiddensize=get_hidden_size(model_params) 28 | vocab_size=get_vocab_size(model_params) 29 | layers=[] 30 | for stage in ["prefill", "decode"]: 31 | layers.append({ 32 | 'name': 'lm_head', 33 | 'stage':stage, 34 | 'OPs':args['batchsize']*hiddensize*vocab_size*1, 35 | 'load_weight':hiddensize*vocab_size *args['w_byte'], 36 | 'load_act':hiddensize*args['a_byte'], 37 | 'store_act':vocab_size*args['a_byte'], 38 | }) 39 | return layers 40 | 41 | def get_linear_layers(model_params, tp_size: int): 42 | hidden_size = get_hidden_size(model_params) 43 | intermediate_size = get_intermediate_size(model_params) 44 | key_value_heads = get_num_key_value_heads(model_params) 45 | attention_heads = get_num_attention_heads(model_params) 46 | 47 | if tp_size > 1: 48 | assert hidden_size % tp_size == 0 49 | assert intermediate_size % tp_size == 0 50 | assert key_value_heads % tp_size == 0 51 | 52 | return { 53 | "q_proj": [hidden_size, hidden_size // tp_size], 54 | "k_proj": [hidden_size, hidden_size * key_value_heads // attention_heads // tp_size], 55 | "v_proj": [hidden_size, hidden_size * key_value_heads // attention_heads // tp_size], 56 | "out_proj": [hidden_size // tp_size, hidden_size], 57 | "gate_proj": [hidden_size, intermediate_size // tp_size], 58 | "up_proj": [hidden_size, intermediate_size // tp_size], 59 | "down_proj": [intermediate_size // tp_size, hidden_size], 60 | } 61 | 62 | 63 | transformer_layer_graph = { 64 | "input": [], 65 | "attn_norm": ["input"], 66 | "q_proj": ["attn_norm"], 67 | "k_proj": ["attn_norm"], 68 | "v_proj": ["attn_norm"], 69 | "qk_matmul": ["q_proj", "k_proj"], 70 | "softmax": ["qk_matmul"], 71 | "sv_matmul": ["softmax", "v_proj"], 72 | "out_proj": ["sv_matmul"], 73 | "attn_add": ["input", "out_proj"], 74 | "mlp_norm": ["attn_add"], 75 | "up_proj": ["mlp_norm"], 76 | "mlp_act": ["up_proj"], 77 | "down_proj": ["mlp_act"], 78 | "mlp_add": ["attn_add", "down_proj"], 79 | "output": ["mlp_add"], 80 | } 81 | 82 | flashattention_transformer_layer_graph = { 83 | "input": [], 84 | "attn_norm": ["input"], 85 | "q_proj": ["attn_norm"], 86 | "k_proj": ["attn_norm"], 87 | "v_proj": ["attn_norm"], 88 | "fused_attention": ["q_proj", "k_proj", "v_proj"], 89 | "out_proj": ["fused_attention"], 90 | "attn_add": ["input", "out_proj"], 91 | "mlp_norm": ["attn_add"], 92 | "up_proj": ["mlp_norm"], 93 | "mlp_act": ["up_proj"], 94 | "down_proj": ["mlp_act"], 95 | "mlp_add": ["attn_add", "down_proj"], 96 | "output": ["mlp_add"], 97 | } 98 | -------------------------------------------------------------------------------- /configs/Llama.py: -------------------------------------------------------------------------------- 1 | 2 | def get_num_attention_heads(model_params): 3 | return getattr(model_params, "num_attention_heads") 4 | 5 | def get_hidden_size(model_params): 6 | return getattr(model_params, "hidden_size") 7 | 8 | def get_num_key_value_heads(model_params): 9 | return getattr(model_params, "num_key_value_heads") 10 | 11 | def get_norm_layers(model_params): 12 | return ["attn_norm", "mlp_norm"] 13 | 14 | def get_num_hidden_layers(model_params): 15 | return getattr(model_params, "num_hidden_layers") 16 | 17 | def get_intermediate_size(model_params): 18 | return getattr(model_params, "intermediate_size") 19 | 20 | def get_vocab_size(model_params): 21 | return getattr(model_params, "vocab_size") 22 | 23 | def post_process(model_params,args): 24 | hiddensize=get_hidden_size(model_params) 25 | vocab_size=get_vocab_size(model_params) 26 | layers=[] 27 | for stage in ["prefill", "decode"]: 28 | layers.append({ 29 | 'name': 'lm_head', 30 | 'stage':stage, 31 | 'OPs':args['batchsize']*hiddensize*vocab_size*1, 32 | 'load_weight':hiddensize*vocab_size *args['w_byte'], 33 | 'load_act':hiddensize*args['a_byte'], 34 | 'store_act':vocab_size*args['a_byte'], 35 | }) 36 | return layers 37 | 38 | def get_linear_layers(model_params, tp_size: int): 39 | hidden_size=get_hidden_size(model_params) 40 | intermediate_size=get_intermediate_size(model_params) 41 | key_value_heads=get_num_key_value_heads(model_params) 42 | attention_heads=get_num_attention_heads(model_params) 43 | 44 | if tp_size > 1: 45 | assert hidden_size % tp_size == 0 46 | assert intermediate_size % tp_size == 0 47 | assert key_value_heads % tp_size == 0 48 | 49 | return { 50 | "q_proj":[hidden_size, hidden_size // tp_size], 51 | "k_proj":[hidden_size, hidden_size * key_value_heads // attention_heads // tp_size], 52 | "v_proj":[hidden_size, hidden_size * key_value_heads // attention_heads // tp_size], 53 | "out_proj":[hidden_size // tp_size, hidden_size], 54 | "gate_proj":[hidden_size, intermediate_size // tp_size], 55 | "up_proj":[hidden_size,intermediate_size // tp_size], 56 | "down_proj":[intermediate_size // tp_size, hidden_size], 57 | } 58 | 59 | # name, input_names 60 | transformer_layer_graph={ 61 | "input":[], 62 | "attn_norm": ["input"], 63 | "q_proj":["attn_norm"], 64 | "k_proj":["attn_norm"], 65 | "v_proj":["attn_norm"], 66 | "qk_matmul":["q_proj","k_proj"], 67 | "softmax":["qk_matmul"], 68 | "sv_matmul":["softmax","v_proj"], 69 | "out_proj":["sv_matmul"], 70 | "attn_add":["input","out_proj"], 71 | "mlp_norm":["attn_add"], 72 | "gate_proj":["mlp_norm"], 73 | "up_proj":["mlp_norm"], 74 | "mlp_act":["up_proj","gate_proj"], 75 | "down_proj":["mlp_act"], 76 | "mlp_add":["attn_add","down_proj"], 77 | "output":["mlp_add"] 78 | } 79 | 80 | flashattention_transformer_layer_graph={ 81 | "input":[], 82 | "attn_norm": ["input"], 83 | "q_proj":["attn_norm"], 84 | "k_proj":["attn_norm"], 85 | "v_proj":["attn_norm"], 86 | "fused_attention":["q_proj","k_proj","v_proj"], 87 | "out_proj":["fused_attention"], 88 | "attn_add":["input","out_proj"], 89 | "mlp_norm":["attn_add"], 90 | "gate_proj":["mlp_norm"], 91 | "up_proj":["mlp_norm"], 92 | "mlp_act":["up_proj","gate_proj"], 93 | "down_proj":["mlp_act"], 94 | "mlp_add":["attn_add","down_proj"], 95 | "output":["mlp_add"] 96 | } 97 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LLM-Viewer 2 | 3 | LLM-Viewer 4 | 5 | 6 | LLM-Viewer is a tool for visualizing Language and Learning Models (LLMs) and analyzing the performance on different hardware platforms. It enables network-wise analysis, considering factors such as peak memory consumption and total inference time cost. With LLM-Viewer, you can gain valuable insights into LLM inference and performance optimization. 7 | You can use LLM-Viewer in a web browser or as a command line interface (CLI) tool. The web version provides a user-friendly interface for easy configuration and visualization, you can access it at [LLM-Viewer Web](http://llm-viewer.com). 8 | 9 | We invite you to read our paper [LLM Inference Unveiled: Survey and Roofline Model Insights](https://arxiv.org/pdf/2402.16363.pdf). 10 | In this paper, we provide a comprehensive analysis of the latest advancements in efficient LLM inference using LLM-Viewer. 11 | 12 | This ongoing project will be updated. TODO list: 13 | - Show shape of tensors. 14 | - Pre-process and post-process for non-transformer layers. 15 | - Show the whole network. 16 | - Expand hardware platform compatibility and allow manual configuration of hardware parameters. 17 | - Increase support for more LLMs and enable manual configuration of model graphs. 18 | 19 | ## Workflow 20 | 21 | ![LLM-Viewer Workflow](figs/workflow.svg) 22 | 23 | As shown in the Figure, the workflow consists of the following steps: 24 | 25 | 1. Input the LLM and gather essential information about each layer, including the computation count, input and output tensor shapes, and data dependencies. 26 | 2. Provide input for the hardware and generate a roofline model that takes into account the computation capacity and memory bandwidth of the hardware. 27 | 3. Configure the inference settings, such as the batch size, prompt token length, and generation token length. 28 | 4. Configure the optimization settings, such as the quantization bitwidth, utilization of FlashAttention, decoding methods, and other system optimization techniques. 29 | 5. Use the LLM-Viewer Analyzer to analyze the performance of each layer based on the roofline model and layer information. It also tracks the memory usage of each layer and calculates the peak memory consumption based on data dependencies. The overall network performance of the LLM can be obtained by aggregating the results of all layers. 30 | 6. Generate a report that provides information such as the maximum performance and performance bottlenecks of each layer and the network, as well as the memory footprint. The report can be used to analyze curves, such as batch size-performance and sequence length-performance curves, to understand how different settings impact performance. 31 | 7. Access the LLM-Viewer web viewer for convenient visualization of the network architecture and analysis results. This tool facilitates easy configuration adjustment and provides access to various data for each layer. 32 | 33 | ## Web Usage 34 | 35 | To use LLM-Viewer in a web browser, go to the web-site [LLM-Viewer Web](http://llm-viewer.com). 36 | You can click the node to get the detailed analysis of the layer. 37 | 38 | ## CLI Usage 39 | 40 | Clone the LLM-Viewer repository from GitHub: 41 | ```git clone https://github.com/hahnyuan/LLM-Viewer.git ``` 42 | 43 | Install requirements 44 | ```pip install transformers flask flask_cors easydict``` 45 | 46 | To analyze an LLM using LLM-Viewer in command line interface (cli), run the following command: 47 | 48 | ```bash 49 | python3 analyze_cli.py facebook/opt-125m nvidia_A6000 50 | python3 analyze_cli.py meta-llama/Llama-2-7b-hf nvidia_A6000 --batchsize 1 --seqlen 2048 51 | python3 analyze_cli.py meta-llama/Llama-2-13b-hf nvidia_A6000 --batchsize 16 --seqlen 2048 52 | python3 analyze_cli.py meta-llama/Llama-2-13b-hf nvidia_A6000 --batchsize 1 --seqlen 8192 53 | 54 | # DiT models 55 | python3 analyze_cli.py DiT-XL/2 nvidia_A6000 --batchsize 1 --seqlen 256 --source DiT 56 | ``` 57 | 58 | NOTE: The time estimated by the roofline model represents the theoretical performance that the hardware can achieve. 59 | The purpose of creating this tool is to help readers gain a clearer understanding of the key factors that influence LLM inference. 60 | Only the relative relationships can be referenced. 61 | 62 | ## Citation 63 | 64 | If you are using LLM-Viewer in your research, please cite our paper: 65 | 66 | ``` 67 | @misc{yuan2024llm, 68 | title={LLM Inference Unveiled: Survey and Roofline Model Insights}, 69 | author={Zhihang Yuan and Yuzhang Shang and Yang Zhou and Zhen Dong and Chenhao Xue and Bingzhe Wu and Zhikai Li and Qingyi Gu and Yong Jae Lee and Yan Yan and Beidi Chen and Guangyu Sun and Kurt Keutzer}, 70 | year={2024}, 71 | eprint={2402.16363}, 72 | archivePrefix={arXiv}, 73 | primaryClass={cs.CL} 74 | } 75 | ``` -------------------------------------------------------------------------------- /get_model_graph.py: -------------------------------------------------------------------------------- 1 | from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM 2 | import importlib 3 | import os 4 | from hardwares.hardware_params import hardware_params 5 | from model_analyzer import ModelAnalyzer 6 | from utils import str_number 7 | import numpy as np 8 | import re 9 | from backend_settings import avaliable_model_ids_sources 10 | 11 | config_cache = {} 12 | 13 | 14 | def get_analyer(model_id, hardware, config_path) -> ModelAnalyzer: 15 | config = f"{model_id}_{hardware}_{config_path}" 16 | if config not in config_cache: 17 | config_cache[config] = ModelAnalyzer( 18 | model_id, 19 | hardware, 20 | config_path, 21 | source=avaliable_model_ids_sources[model_id]["source"], 22 | ) 23 | return config_cache[config] 24 | 25 | 26 | # def get_model_config(model_id,config_path): 27 | # if model_id not in config_cache: 28 | # model_config = AutoConfig.from_pretrained(model_id, trust_remote_code=True) 29 | # config = importlib.import_module(config_path.replace("/", ".").replace(".py", "")) 30 | # config_cache[model_id] = model_config,config 31 | # return config_cache[model_id] 32 | 33 | 34 | def get_quant_bit(dtype): 35 | if dtype == "FP16": 36 | return 16 37 | elif dtype == "INT8": 38 | return 8 39 | elif dtype == "INT4": 40 | return 4 41 | elif "bit" in dtype: 42 | bitwidth = int(re.findall(r"\d+", dtype)[0]) 43 | return bitwidth 44 | else: 45 | raise ValueError(f"Unsupported dtype:{dtype}") 46 | 47 | 48 | def get_model_graph(model_id, hardware, config_path, inference_config): 49 | 50 | # Roofline model 51 | w_bit = get_quant_bit(inference_config["w_quant"]) 52 | a_bit = get_quant_bit(inference_config["a_quant"]) 53 | kv_bit = get_quant_bit(inference_config["kv_quant"]) 54 | seq_length = int(inference_config["seq_length"]) 55 | batch_size = int(inference_config["batch_size"]) 56 | use_flashattention = bool(inference_config["use_flashattention"]) 57 | gen_length = int(inference_config["gen_length"]) 58 | tp_size = int(inference_config["tp_size"]) 59 | 60 | analyzer = get_analyer(model_id, hardware, config_path) 61 | result = analyzer.analyze( 62 | seqlen=seq_length, 63 | batchsize=batch_size, 64 | w_bit=w_bit, 65 | a_bit=a_bit, 66 | kv_bit=kv_bit, 67 | use_flashattention=use_flashattention, 68 | tp_size=tp_size 69 | ) 70 | bandwidth, max_OPS, onchip_buffer = analyzer.get_hardware_info() 71 | GQA = analyzer.get_model_info()["GQA"] 72 | hardware_info = { 73 | "bandwidth": bandwidth, 74 | "max_OPS": max_OPS, 75 | "onchip_buffer": onchip_buffer, 76 | } 77 | 78 | nodes = [ 79 | { 80 | "label": "input", 81 | "id": "input", 82 | } 83 | ] 84 | edges = [] 85 | 86 | def write_to_node(name, OPs, memory_access, info, input_names=[]): 87 | node = { 88 | "label": name, 89 | "id": name, 90 | "description": f"OPs:{str_number(OPs)}, Access:{str_number(memory_access)}", 91 | "info": info, 92 | } 93 | if GQA and name in ["qk_matmul", "sv_matmul"]: 94 | node["label"] += "(GQA)" 95 | nodes.append(node) 96 | for input_name in input_names: 97 | edge = {"source": input_name, "target": name} 98 | edges.append(edge) 99 | 100 | if use_flashattention: 101 | layer_graph = analyzer.config.flashattention_transformer_layer_graph 102 | else: 103 | layer_graph = analyzer.config.transformer_layer_graph 104 | stage = inference_config["stage"] 105 | total_results = result["total_results"] 106 | if stage != "chat": 107 | result = result[stage] 108 | else: 109 | result = result["prefill"] 110 | 111 | for name, input_names in layer_graph.items(): 112 | if name in ["input", "output"]: 113 | OPs = 0 114 | memory_access = 0 115 | info = {} 116 | else: 117 | OPs = result[name]["OPs"] 118 | memory_access = result[name]["memory_access"] 119 | info = result[name] 120 | write_to_node(name, OPs, memory_access, info, input_names) 121 | if stage == "chat": 122 | # seq_length:seq_length+gen_length 123 | total_results["chat"] = total_results["prefill"] 124 | n_divide = min(10, gen_length) 125 | for lengthi in np.linspace(seq_length + 1, seq_length + gen_length, n_divide): 126 | gen_result = analyzer.analyze( 127 | seqlen=lengthi, 128 | batchsize=batch_size, 129 | w_bit=w_bit, 130 | a_bit=a_bit, 131 | kv_bit=kv_bit, 132 | use_flashattention=use_flashattention, 133 | ) 134 | for k, v in gen_result["total_results"]["decode"].items(): 135 | total_results["chat"][k] += v * gen_length / n_divide 136 | for name, input_names in layer_graph.items(): 137 | if name in gen_result["decode"]: 138 | result[name]["OPs"] += ( 139 | gen_result["decode"][name]["OPs"] * gen_length / n_divide 140 | ) 141 | result[name]["memory_access"] += ( 142 | gen_result["decode"][name]["memory_access"] 143 | * gen_length 144 | / n_divide 145 | ) 146 | for name, input_names in layer_graph.items(): 147 | if name in ["input", "output"]: 148 | OPs = 0 149 | memory_access = 0 150 | info = {} 151 | else: 152 | OPs = result[name]["OPs"] 153 | memory_access = result[name]["memory_access"] 154 | info = {} 155 | write_to_node(name, OPs, memory_access, info, input_names) 156 | return nodes, edges, total_results, hardware_info 157 | -------------------------------------------------------------------------------- /frontend/src/components/Header.vue: -------------------------------------------------------------------------------- 1 | 66 | 67 | 121 | 122 | -------------------------------------------------------------------------------- /frontend/src/components/graphs/graph_config.js: -------------------------------------------------------------------------------- 1 | import G6 from "@antv/g6" 2 | 3 | const ICON_MAP = { 4 | normal: 'https://gw.alipayobjects.com/mdn/rms_8fd2eb/afts/img/A*0HC-SawWYUoAAAAAAAAAAABkARQnAQ', 5 | b: 'https://gw.alipayobjects.com/mdn/rms_8fd2eb/afts/img/A*sxK0RJ1UhNkAAAAAAAAAAABkARQnAQ', 6 | }; 7 | 8 | G6.registerNode( 9 | 'card-node', 10 | { 11 | drawShape: function drawShape(cfg, group) { 12 | const color = cfg.is_linear ? '#F4664A' : '#30BF78'; 13 | const r = 2; 14 | const shape = group.addShape('rect', { 15 | attrs: { 16 | x: 0, 17 | y: 0, 18 | width: 150, 19 | height: 60, 20 | stroke: color, 21 | radius: r, 22 | }, 23 | // must be assigned in G6 3.3 and later versions. it can be any string you want, but should be unique in a custom item type 24 | name: 'main-box', 25 | draggable: true, 26 | }); 27 | 28 | group.addShape('rect', { 29 | attrs: { 30 | x: 0, 31 | y: 0, 32 | width: 150, 33 | height: 21, 34 | fill: color, 35 | radius: [r, r, 0, 0], 36 | }, 37 | // must be assigned in G6 3.3 and later versions. it can be any string you want, but should be unique in a custom item type 38 | name: 'title-box', 39 | draggable: true, 40 | }); 41 | 42 | // left icon 43 | group.addShape('image', { 44 | attrs: { 45 | x: 4, 46 | y: 2, 47 | height: 16, 48 | width: 16, 49 | cursor: 'pointer', 50 | img: ICON_MAP[cfg.nodeType || 'app'], 51 | }, 52 | // must be assigned in G6 3.3 and later versions. it can be any string you want, but should be unique in a custom item type 53 | name: 'node-icon', 54 | }); 55 | 56 | // title text 57 | group.addShape('text', { 58 | attrs: { 59 | textBaseline: 'top', 60 | y: 5, 61 | x: 24, 62 | lineHeight: 20, 63 | text: cfg.title, 64 | fill: '#fff', 65 | }, 66 | // must be assigned in G6 3.3 and later versions. it can be any string you want, but should be unique in a custom item type 67 | name: 'title', 68 | }); 69 | 70 | // if (cfg.nodeLevel > 0) { 71 | // group.addShape('marker', { 72 | // attrs: { 73 | // x: 184, 74 | // y: 30, 75 | // r: 6, 76 | // cursor: 'pointer', 77 | // symbol: cfg.collapse ? G6.Marker.expand : G6.Marker.collapse, 78 | // stroke: '#666', 79 | // lineWidth: 1, 80 | // }, 81 | // // must be assigned in G6 3.3 and later versions. it can be any string you want, but should be unique in a custom item type 82 | // name: 'collapse-icon', 83 | // }); 84 | // } 85 | 86 | // The content list 87 | cfg.panels.forEach((item, index) => { 88 | // name text 89 | group.addShape('text', { 90 | attrs: { 91 | textBaseline: 'top', 92 | y: 27, 93 | x: 24 + index * 60, 94 | lineHeight: 20, 95 | text: item.title, 96 | fill: 'rgba(0,0,0, 0.4)', 97 | }, 98 | // must be assigned in G6 3.3 and later versions. it can be any string you want, but should be unique in a custom item type 99 | name: `index-title-${index}`, 100 | }); 101 | 102 | // value text 103 | group.addShape('text', { 104 | attrs: { 105 | textBaseline: 'top', 106 | y: 45, 107 | x: 24 + index * 60, 108 | lineHeight: 20, 109 | text: item.value, 110 | fill: '#595959', 111 | }, 112 | // must be assigned in G6 3.3 and later versions. it can be any string you want, but should be unique in a custom item type 113 | name: `index-value-${index}`, 114 | }); 115 | }); 116 | return shape; 117 | }, 118 | }, 119 | 'single-node', 120 | ); 121 | 122 | 123 | export const graph_config = { 124 | container: 'graphContainer', // String | HTMLElement，必须，在 Step 1 中创建的容器 id 或容器本身 125 | width: window.innerWidth, // Number，必须，图的宽度 126 | height: window.innerHeight, // Number，必须，图的高度 127 | defaultEdge: { 128 | // type: 'line', 129 | type: 'polyline', 130 | // type: 'quadratic', 131 | sourceAnchor: 1, 132 | // // 该边连入 target 点的第 0 个 anchorPoint， 133 | targetAnchor: 0, 134 | style: { 135 | 136 | endArrow: { 137 | path: G6.Arrow.triangle(5, 10), // 使用内置箭头路径函数，参数为箭头的 宽度、长度、偏移量（默认为 0，与 d 对应） 138 | fill: "#aaaaaa", 139 | opacity: 50, 140 | }, 141 | stroke: "#000000", 142 | }, 143 | }, 144 | defaultNode: { 145 | // ... 其他属性 146 | // type: 'card-node', 147 | type: 'modelRect', 148 | // type: 'rect', 149 | // hight: 200, 150 | size: [190, 60], // 设置节点的默认宽度和高度 151 | anchorPoints: [ 152 | [0.5, 0], 153 | [0.5, 1] 154 | ], 155 | // anchorPoints: [ 156 | // [0, 0.5], 157 | // [1, 0.5], 158 | // ], 159 | logoIcon: { 160 | show: false, 161 | }, 162 | stateIcon: { 163 | show: false, 164 | img: 165 | 'https://gw.alipayobjects.com/zos/basement_prod/c781088a-c635-452a-940c-0173663456d4.svg', 166 | }, 167 | 168 | // style: { 169 | // radius: 5, 170 | // // fill: '#C6E5FF', 171 | // // stroke: '#5B8FF9', 172 | // }, 173 | labelCfg:{ 174 | offset: 15, 175 | style: { 176 | fill: '#000000', 177 | fontSize: 20, 178 | stroke: '#E7E7E7', 179 | } 180 | }, 181 | descriptionCfg: { 182 | style: { 183 | fill: '#656565', 184 | fontSize: 14, 185 | }, 186 | }, 187 | 188 | }, 189 | // fitView: true, 190 | // plugins: [minimap], // 将 minimap 实例配置到图上 191 | modes: { 192 | // default: ['drag-canvas', 'zoom-canvas', 'drag-node', 'lasso-select'], // 允许拖拽画布、放缩画布、拖拽节点 193 | default: ['drag-canvas', 'zoom-canvas', 'lasso-select'], // 允许拖拽画布、放缩画布、拖拽节点 194 | }, 195 | layout: { 196 | type: 'dagre', 197 | // rankdir: 'LR', // The center of the graph by default 198 | // align: 'UR', 199 | nodesep: 10, 200 | ranksep: 20, 201 | controlPoints: true, 202 | }, 203 | } -------------------------------------------------------------------------------- /frontend/src/components/left_controls/Config.vue: -------------------------------------------------------------------------------- 1 | 125 | 126 | 201 | 202 | -------------------------------------------------------------------------------- /frontend/src/components/Graph.vue: -------------------------------------------------------------------------------- 1 | 31 | 32 | 335 | 336 | -------------------------------------------------------------------------------- /model_analyzer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import importlib 3 | from hardwares.hardware_params import hardware_params 4 | from roofline_model import roofline_analyze 5 | from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM 6 | from utils import str_number, str_number_time 7 | import math 8 | 9 | ALL_DATA_NAMES = [ 10 | "OPs", 11 | "memory_access", 12 | "load_weight", 13 | "load_act", 14 | "store_act", 15 | "load_kv_cache", 16 | "store_kv_cache", 17 | "inference_time", 18 | ] 19 | 20 | 21 | class ModelAnalyzer: 22 | def __init__(self, model_id, hardware, config_file=None, source="huggingface"): 23 | """ 24 | source: 'huggingface' or 'DiT' 25 | """ 26 | self.model_id = model_id 27 | self.hardware = hardware 28 | if config_file is None: 29 | # get the current file directory 30 | current_dir = os.path.dirname(os.path.abspath(__file__)) 31 | # auto search the config 32 | for file in os.listdir(current_dir + "/configs"): 33 | if file.endswith(".py") and file.replace(".py", "") in model_id: 34 | config_file = "configs/" + file 35 | # print(f"auto search config file {config_file} {file} {model_id}") 36 | assert config_file is not None, "config file is not found, please specify it manually." 37 | print(f"use config file {config_file} for {model_id}") 38 | if source == "huggingface": 39 | self.model_params = AutoConfig.from_pretrained(model_id, trust_remote_code=True) 40 | else: 41 | if not os.path.exists(f"model_params/{source}.py"): 42 | raise Exception(f"model_params/{source}.py is not found") 43 | # from model_params.DiT import model_params 44 | module = importlib.import_module(f"model_params.{source}") 45 | self.model_params = module.model_params[model_id] 46 | self.config = importlib.import_module(config_file.replace("/", ".").replace(".py", "")) 47 | 48 | # temporary variables 49 | self.results = None 50 | self.w_bit = None 51 | self.a_bit = None 52 | self.kv_bit = None 53 | self.batchsize = None 54 | self.seqlen = None 55 | 56 | def _analyze_to_results( 57 | self, 58 | stage, 59 | name, 60 | OPs=0, 61 | load_weight=0, 62 | load_act=0, 63 | store_act=0, 64 | load_kv_cache=0, 65 | store_kv_cache=0, 66 | ): 67 | 68 | bandwidth, max_OPS, onchip_buffer = self.get_hardware_info() 69 | memory_access = load_weight + load_act + store_act + load_kv_cache + store_kv_cache 70 | arithmetic_intensity, performance, bound = roofline_analyze(bandwidth, max_OPS, OPs, memory_access) 71 | inference_time = OPs / performance 72 | self.results[stage][name] = { 73 | "OPs": OPs, 74 | "memory_access": memory_access, 75 | "arithmetic_intensity": arithmetic_intensity, 76 | "performance": performance, 77 | "bound": bound, 78 | "load_weight": load_weight, 79 | "load_act": load_act, 80 | "store_act": store_act, 81 | "load_kv_cache": load_kv_cache, 82 | "store_kv_cache": store_kv_cache, 83 | "inference_time": inference_time, 84 | } 85 | 86 | def save_csv(self, save_path=None): 87 | if save_path is None: 88 | save_path = f"output/{self.model_id[:self.model_id.rfind('/')]}" 89 | if not os.path.exists(save_path): 90 | os.makedirs(save_path) 91 | save_path += f"{self.model_id[self.model_id.rfind('/'):]}" 92 | 93 | decode_file_name = f"{save_path}_decode.csv" 94 | prefill_file_name = f"{save_path}_prefill.csv" 95 | print(f"save to {decode_file_name} and {prefill_file_name}") 96 | 97 | for file_name, stage in [ 98 | (decode_file_name, "decode"), 99 | (prefill_file_name, "prefill"), 100 | ]: 101 | with open(file_name, "a+") as f: 102 | 103 | f.write( 104 | f"\n\n=== {self.model_id} {self.hardware} w_bit={self.w_bit} a_bit={self.a_bit} kv_bit={self.kv_bit} batchsize={self.batchsize} seqlen={self.seqlen} tp_size={self.tp_size} ===\n" 105 | ) 106 | # legend 107 | f.write( 108 | f"layer_name,OPs,Access,arithmetic_intensity,performance,bound,load_weight,load_act,store_act,load_kv_cache,store_kv_cache,inference_time\n" 109 | ) 110 | with open(file_name, "a+") as f: 111 | for layer_name, result in self.results[stage].items(): 112 | f.write( 113 | f"{layer_name},{str_number(result['OPs'])},{str_number(result['memory_access'])}B,{str_number(result['arithmetic_intensity'])},{str_number(result['performance'])}," 114 | f"{result['bound']},{str_number(result['load_weight'])}B,{str_number(result['load_act'])}B,{str_number(result['store_act'])}B,{str_number(result['load_kv_cache'])}B," 115 | f"{str_number(result['store_kv_cache'])}B,{str_number_time(result['inference_time'])}s\n" 116 | ) 117 | 118 | def analyze( 119 | self, 120 | seqlen, 121 | batchsize, 122 | w_bit=16, 123 | a_bit=16, 124 | kv_bit=None, 125 | use_flashattention=False, 126 | kv_token_ratio=1, 127 | tp_size: int = 1 128 | ): 129 | """ 130 | seqlen: sequence length 131 | batchsize: batch size 132 | w_bit: weight bit 133 | a_bit: activation bit 134 | kv_bit: key and value bit. if it is None, it will be the same as a_bit 135 | use_flashattention: use flash attention/flash decoding 136 | kv_token_ratio: use this for KV compression 137 | tp_size: the number of devices for tensor parallelism to use 138 | 139 | return is a dict with the following format: 140 | { 141 | "decode": { 142 | "layer_name": { 143 | "OPs": "", 144 | "memory_access": "", 145 | "arithmetic_intensity": "", 146 | "performance": "", 147 | "bound": "", 148 | "load_weight": "", 149 | "load_act": "", 150 | "store_act": "", 151 | "load_kv_cache": "", 152 | "store_kv_cache": "", 153 | "inference_time": "" 154 | } 155 | }, 156 | "prefill": { 157 | "layer_name": { 158 | "OPs": "", 159 | "memory_access": "", 160 | "arithmetic_intensity": "", 161 | "performance": "", 162 | "bound": "", 163 | "load_weight": "", 164 | "load_act": "", 165 | "store_act": "", 166 | "load_kv_cache": "", 167 | "store_kv_cache": "", 168 | "inference_time": "" 169 | } 170 | }, 171 | "total_results": { 172 | "decode": {}, 173 | "prefill": {} 174 | } 175 | } 176 | """ 177 | assert seqlen > 0 178 | assert batchsize > 0 179 | self.results = {"decode": {}, "prefill": {}} 180 | if kv_bit is None: 181 | kv_bit = a_bit 182 | self.w_bit = w_bit 183 | self.a_bit = a_bit 184 | self.kv_bit = kv_bit 185 | self.batchsize = batchsize 186 | self.seqlen = seqlen 187 | self.tp_size = tp_size 188 | 189 | w_byte = self.w_bit / 8 190 | a_byte = self.a_bit / 8 191 | kv_byte = self.kv_bit / 8 192 | 193 | config = self.config 194 | model_params = self.model_params 195 | num_attention_heads = config.get_num_attention_heads(model_params) 196 | hidden_size = config.get_hidden_size(model_params) 197 | num_key_value_heads = config.get_num_key_value_heads(model_params) 198 | num_hidden_layers = config.get_num_hidden_layers(model_params) 199 | 200 | for name, (ic, oc) in config.get_linear_layers(model_params, tp_size).items(): 201 | # for linear layers 202 | is_kv_proj = name in ["k_proj", "v_proj"] 203 | is_normal_proj = not is_kv_proj 204 | self._analyze_to_results( 205 | "decode", 206 | name, 207 | OPs=ic * oc * batchsize * 2, 208 | load_weight=ic * oc * w_byte, 209 | load_act=ic * batchsize * a_byte, 210 | store_act=0 if is_kv_proj else oc * batchsize * a_byte, 211 | load_kv_cache=0, 212 | store_kv_cache=(0 if is_normal_proj else oc * batchsize * kv_byte), 213 | ) 214 | # for prefill 215 | self._analyze_to_results( 216 | "prefill", 217 | name, 218 | OPs=ic * oc * batchsize * seqlen * 2, 219 | load_weight=ic * oc * w_byte, 220 | load_act=ic * batchsize * seqlen * a_byte, 221 | store_act=(0 if is_kv_proj else oc * batchsize * seqlen * a_byte), 222 | load_kv_cache=0, 223 | store_kv_cache=(0 if is_normal_proj else oc * batchsize * seqlen * kv_byte), 224 | ) 225 | 226 | # for attention 227 | head_size = hidden_size // num_attention_heads 228 | # for decode 229 | qk_matmul_OPs = seqlen * head_size * num_attention_heads * batchsize * 2 230 | sv_matmul_OPs = 1 * head_size * seqlen * num_attention_heads * batchsize * 2 231 | # the softmax operation takes five steps: 232 | # max_x=max(x) 233 | # x=x-max_x 234 | # x_exp=exp(x) 235 | # sum_x_exp=sum(x_exp) 236 | # y=x_exp/sum(x_exp) 237 | softmax_OPs = batchsize * num_attention_heads * seqlen * 1 * 5 238 | if use_flashattention: 239 | name = f"fused_attention" 240 | bandwidth, max_OPS, onchip_buffer = self.get_hardware_info() 241 | # flashattention-2 https://arxiv.org/pdf/2307.08691.pdf 242 | block_size_r = min(math.ceil(onchip_buffer / (kv_byte * head_size)), head_size) 243 | n_blocks_r = math.ceil(1 / block_size_r) 244 | q_numel = (1) * head_size * batchsize * num_attention_heads * a_byte 245 | o_numel = 1 * seqlen * batchsize * num_attention_heads * a_byte 246 | self._analyze_to_results( 247 | "decode", 248 | name, 249 | OPs=qk_matmul_OPs + sv_matmul_OPs + softmax_OPs, 250 | load_weight=0, 251 | load_act=q_numel, 252 | store_act=o_numel * 2, # initialize O and save O 253 | load_kv_cache=n_blocks_r * (seqlen) * head_size * batchsize * num_key_value_heads * kv_byte * 2, 254 | store_kv_cache=0, 255 | ) 256 | 257 | else: 258 | name = f"qk_matmul" 259 | self._analyze_to_results( 260 | "decode", 261 | name, 262 | OPs=qk_matmul_OPs, 263 | load_weight=0, 264 | load_act=(1) * head_size * batchsize * num_attention_heads * a_byte, 265 | store_act=1 * seqlen * batchsize * num_attention_heads * a_byte, 266 | load_kv_cache=(seqlen) * head_size * batchsize * num_key_value_heads * kv_byte, 267 | store_kv_cache=0, 268 | ) 269 | name = f"sv_matmul" 270 | self._analyze_to_results( 271 | "decode", 272 | name, 273 | OPs=sv_matmul_OPs, 274 | load_weight=0, 275 | load_act=(1 * seqlen * batchsize * num_attention_heads) * a_byte, 276 | store_act=1 * head_size * batchsize * num_attention_heads * a_byte, 277 | load_kv_cache=(seqlen * head_size * batchsize * num_key_value_heads) * kv_byte, 278 | store_kv_cache=0, 279 | ) 280 | 281 | name = f"softmax" 282 | # max sub exp sum div 283 | self._analyze_to_results( 284 | "decode", 285 | name, 286 | OPs=softmax_OPs, 287 | load_weight=0, 288 | load_act=batchsize * num_attention_heads * seqlen * 1 * a_byte, 289 | store_act=batchsize * num_attention_heads * seqlen * 1 * a_byte, 290 | load_kv_cache=0, 291 | store_kv_cache=0, 292 | ) 293 | 294 | for name in config.get_norm_layers(model_params): 295 | # sum sub pow sum div mul add 296 | self._analyze_to_results( 297 | "decode", 298 | name, 299 | OPs=batchsize * hidden_size * 1 * 7, 300 | load_weight=0, 301 | load_act=batchsize * hidden_size * 1 * a_byte, 302 | store_act=batchsize * hidden_size * 1 * a_byte, 303 | load_kv_cache=0, 304 | store_kv_cache=0, 305 | ) 306 | 307 | for name in ["attn_add", "mlp_add"]: 308 | self._analyze_to_results( 309 | "decode", 310 | name, 311 | OPs=batchsize * hidden_size * 1, 312 | load_weight=0, 313 | load_act=batchsize * hidden_size * 1 * a_byte, 314 | store_act=batchsize * hidden_size * 1 * a_byte, 315 | load_kv_cache=0, 316 | store_kv_cache=0, 317 | ) 318 | for name in ["mlp_act"]: 319 | self._analyze_to_results( 320 | "decode", 321 | name, 322 | OPs=batchsize * hidden_size * 1 * 2, 323 | load_weight=0, 324 | load_act=batchsize * hidden_size * 1 * a_byte * 2, 325 | store_act=batchsize * hidden_size * 1 * a_byte, 326 | load_kv_cache=0, 327 | store_kv_cache=0, 328 | ) 329 | 330 | # for prefill 331 | qk_matmul_OPs = seqlen * seqlen * head_size * num_attention_heads * batchsize * 2 332 | sv_matmul_OPs = seqlen * head_size * seqlen * num_attention_heads * batchsize * 2 333 | softmax_OPs = batchsize * num_attention_heads * seqlen * seqlen * 5 334 | if use_flashattention: 335 | name = f"fused_attention" 336 | bandwidth, max_OPS, onchip_buffer = self.get_hardware_info() 337 | # flashattention-2 https://arxiv.org/pdf/2307.08691.pdf 338 | block_size_r = min(math.ceil(onchip_buffer / (kv_byte * head_size)), head_size) 339 | n_blocks_r = math.ceil(seqlen / block_size_r) 340 | q_numel = seqlen * head_size * batchsize * num_attention_heads * a_byte 341 | o_numel = seqlen * seqlen * batchsize * num_attention_heads * a_byte 342 | self._analyze_to_results( 343 | "prefill", 344 | name, 345 | OPs=qk_matmul_OPs + sv_matmul_OPs + softmax_OPs, 346 | load_weight=0, 347 | load_act=q_numel, 348 | store_act=o_numel * 2, # initialize O and save O 349 | load_kv_cache=n_blocks_r * (seqlen) * head_size * batchsize * num_key_value_heads * kv_byte * 2, 350 | store_kv_cache=0, 351 | ) 352 | else: 353 | name = f"qk_matmul" 354 | self._analyze_to_results( 355 | "prefill", 356 | name, 357 | OPs=qk_matmul_OPs, 358 | load_weight=0, 359 | load_act=seqlen * head_size * batchsize * num_key_value_heads * a_byte, 360 | store_act=seqlen * seqlen * batchsize * num_attention_heads * a_byte, 361 | load_kv_cache=seqlen * head_size * batchsize * num_key_value_heads * kv_byte, 362 | store_kv_cache=0, 363 | ) 364 | name = f"sv_matmul" 365 | self._analyze_to_results( 366 | "prefill", 367 | name, 368 | OPs=sv_matmul_OPs, 369 | load_weight=0, 370 | load_act=seqlen * seqlen * batchsize * num_attention_heads * a_byte, 371 | store_act=seqlen * head_size * batchsize * num_attention_heads * a_byte, 372 | load_kv_cache=seqlen * head_size * batchsize * num_key_value_heads * kv_byte, 373 | store_kv_cache=0, 374 | ) 375 | name = f"softmax" 376 | self._analyze_to_results( 377 | "prefill", 378 | name, 379 | OPs=softmax_OPs, 380 | load_weight=0, 381 | load_act=batchsize * num_attention_heads * seqlen * seqlen * a_byte, 382 | store_act=batchsize * num_attention_heads * seqlen * seqlen * a_byte, 383 | load_kv_cache=0, 384 | store_kv_cache=0, 385 | ) 386 | for name in config.get_norm_layers(model_params): 387 | self._analyze_to_results( 388 | "prefill", 389 | name, 390 | OPs=batchsize * hidden_size * seqlen * 7, 391 | load_weight=0, 392 | load_act=batchsize * hidden_size * seqlen * a_byte, 393 | store_act=batchsize * hidden_size * seqlen * a_byte, 394 | load_kv_cache=0, 395 | store_kv_cache=0, 396 | ) 397 | for name in ["attn_add", "mlp_add"]: 398 | self._analyze_to_results( 399 | "prefill", 400 | name, 401 | OPs=batchsize * hidden_size * seqlen * 1, 402 | load_weight=0, 403 | load_act=batchsize * hidden_size * seqlen * a_byte, 404 | store_act=batchsize * hidden_size * seqlen * a_byte, 405 | load_kv_cache=0, 406 | store_kv_cache=0, 407 | ) 408 | for name in ["mlp_act"]: 409 | self._analyze_to_results( 410 | "prefill", 411 | name, 412 | OPs=batchsize * hidden_size * seqlen * 1 * 2, 413 | load_weight=0, 414 | load_act=batchsize * hidden_size * seqlen * a_byte * 2, 415 | store_act=batchsize * hidden_size * seqlen * a_byte, 416 | load_kv_cache=0, 417 | store_kv_cache=0, 418 | ) 419 | 420 | # compute total 421 | total_results = {"decode": {}, "prefill": {}} 422 | for data_name in ALL_DATA_NAMES: 423 | total_results["decode"][data_name] = 0 424 | total_results["prefill"][data_name] = 0 425 | for stage in ["decode", "prefill"]: 426 | for layer_name, result in self.results[stage].items(): 427 | for data_name in ALL_DATA_NAMES: 428 | total_results[stage][data_name] += result[data_name] * num_hidden_layers 429 | 430 | # memory footprint 431 | weight_kv_footprint = total_results["prefill"]["load_weight"] + total_results["prefill"]["store_kv_cache"] 432 | decode_tmp_act = 0 433 | for layer_name, result in self.results["decode"].items(): 434 | decode_tmp_act += result["store_act"] 435 | total_results["decode"]["memory_consumption"] = decode_tmp_act + weight_kv_footprint 436 | total_results["decode"]["memory_consumption_tmp_act"] = decode_tmp_act 437 | total_results["decode"]["memory_consumption_weight"] = total_results["prefill"]["load_weight"] 438 | total_results["decode"]["memory_consumption_kv_cache"] = total_results["prefill"]["store_kv_cache"] 439 | prefill_tmp_act = 0 440 | for layer_name, result in self.results["prefill"].items(): 441 | prefill_tmp_act += result["store_act"] 442 | total_results["prefill"]["memory_consumption"] = prefill_tmp_act + weight_kv_footprint 443 | total_results["prefill"]["memory_consumption_tmp_act"] = prefill_tmp_act 444 | total_results["prefill"]["memory_consumption_weight"] = total_results["prefill"]["load_weight"] 445 | total_results["prefill"]["memory_consumption_kv_cache"] = total_results["prefill"]["store_kv_cache"] 446 | 447 | # lm_head 448 | name = "lm_head" 449 | args = {"batchsize": batchsize, "a_byte": a_byte, "w_byte": w_byte} 450 | for layer_info in self.config.post_process(self.model_params, args): 451 | self._analyze_to_results(**layer_info) 452 | for data_name in ALL_DATA_NAMES: 453 | total_results[layer_info["stage"]][data_name] += self.results[layer_info["stage"]][layer_info["name"]][ 454 | data_name 455 | ] 456 | # for stage in ["prefill", "decode"]: 457 | # self._analyze_to_results( 458 | # stage, 459 | # name, 460 | # OPs=batchsize * hidden_size * vocab_size * 1, 461 | # load_weight=hidden_size * vocab_size, 462 | # load_act=hidden_size * a_byte, 463 | # store_act=vocab_size * a_byte, 464 | # load_kv_cache=0, 465 | # store_kv_cache=0, 466 | # ) 467 | # for data_name in ALL_DATA_NAMES: 468 | # total_results[stage][data_name] += self.results[stage][name][data_name] 469 | 470 | self.results["total_results"] = total_results 471 | return self.results 472 | 473 | def analyze_generate_task( 474 | self, 475 | prompt_len, 476 | gen_len, 477 | batchsize, 478 | w_bit=16, 479 | a_bit=16, 480 | kv_bit=None, 481 | use_flashattention = False, 482 | tp_size: int = 1 483 | ): 484 | prefill_result = self.analyze( 485 | prompt_len, 486 | batchsize, 487 | w_bit, 488 | a_bit, 489 | kv_bit, 490 | use_flashattention=use_flashattention, 491 | tp_size=tp_size 492 | ) 493 | prefill_time = inference_time = prefill_result["total_results"]["prefill"]["inference_time"] 494 | 495 | for i in range(prompt_len, prompt_len + gen_len): 496 | result = self.analyze(i, batchsize, w_bit, a_bit, kv_bit, use_flashattention=use_flashattention, tp_size=tp_size) 497 | inference_time += result["total_results"]["decode"]["inference_time"] 498 | return {"inference_time": inference_time, "prefill_time": prefill_time} 499 | 500 | def get_hardware_info(self): 501 | bandwidth = hardware_params[self.hardware]["bandwidth"] 502 | if self.w_bit <= 8 and self.a_bit <= 8 and self.kv_bit <= 8: 503 | max_OPS = hardware_params[self.hardware]["INT8"] 504 | else: 505 | max_OPS = hardware_params[self.hardware]["FP16"] 506 | onchip_buffer = hardware_params[self.hardware]["onchip_buffer"] 507 | return bandwidth, max_OPS, onchip_buffer 508 | 509 | def get_model_info(self): 510 | if self.config.get_num_attention_heads(self.model_params) != self.config.get_num_key_value_heads( 511 | self.model_params 512 | ): 513 | GQA = True 514 | else: 515 | GQA = False 516 | 517 | info = {"GQA": GQA} # group query attention 518 | return info 519 | -------------------------------------------------------------------------------- /examples/plot_hardware.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import sys\n", 10 | "sys.path.append('..')\n", 11 | "import numpy as np\n", 12 | "import matplotlib.pyplot as plt\n", 13 | "from hardwares.hardware_params import hardware_params\n", 14 | "from model_analyzer import ModelAnalyzer\n", 15 | "%load_ext autoreload\n", 16 | "%autoreload 2" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 30, 22 | "metadata": {}, 23 | "outputs": [ 24 | { 25 | "name": "stdout", 26 | "output_type": "stream", 27 | "text": [ 28 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 29 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 30 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 31 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 32 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 33 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 34 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 35 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 36 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 37 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 38 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 39 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 40 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 41 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 42 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 43 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 44 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 45 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 46 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n", 47 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n" 48 | ] 49 | } 50 | ], 51 | "source": [ 52 | "import os\n", 53 | "os.environ[\"https_proxy\"]=\"127.0.0.1:7890\"\n", 54 | "\n", 55 | "model_id=\"meta-llama/Llama-2-13b-hf\"\n", 56 | "bandwidths=[]\n", 57 | "performances=[]\n", 58 | "for bandwidth_scale in np.linspace(1,20,20):\n", 59 | " hardware_name=f\"h{bandwidth_scale}\"\n", 60 | " bandwidth=200e9*bandwidth_scale\n", 61 | " hardware_params[hardware_name] = {\"bandwidth\": bandwidth, \"FP16\": 200e12, \"onchip_buffer\": 10240e3}\n", 62 | " analyzer=ModelAnalyzer(model_id,hardware_name)\n", 63 | " result=analyzer.analyze(1024,1)\n", 64 | " \n", 65 | " bandwidths.append(bandwidth)\n", 66 | " performances.append(result[\"total_results\"][\"decode\"][\"inference_time\"])\n", 67 | "throughput=1/np.array(performances)" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": 43, 73 | "metadata": {}, 74 | "outputs": [ 75 | { 76 | "data": { 77 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk0AAADwCAYAAAAdMVDaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAACMJklEQVR4nOzdd1gTyRsH8G8oCb0jRQUs2BVUBLGBigL2jh27nnrqcdbzFL3zzt7O7qlgQbE37KJgQ8UCNkRBFAuIIL2TvL8/OPZnBDTBQCjzeZ48kN3Z2TdLsryZnZ3hERGBYRiGYRiG+SYFeQfAMAzDMAxTEbCkiWEYhmEYRgIsaWIYhmEYhpEAS5oYhmEYhmEkwJImhmEYhmEYCbCkiWEYhmEYRgIsaWIYhmEYhpEAS5oYhmEYhmEkwJImhmEYhmEYCbCkiSmRly9fomvXrtDW1gaPx8OJEyfg7e0NHo+H169fc+UcHR3h6Ogotzjl4Udes4WFBUaNGiXTeBiGYSRhYWGBHj16yDuMco0lTRVUQYJS8FBSUkL16tUxatQovH//vtT37+7ujsePH+Ovv/7C3r17YWNjU+r7lMbr16+5Y7NkyZIiywwbNgw8Hg8aGhplHB3DMF8rOKfdu3evyPUFn+lVq1bJZH+Ojo7g8XiwtLQscv2lS5e4c8iRI0eKLLN582bweDzY2dlJtW9/f3+MGTMG9erVg5qaGmrXro1x48YhJiZG4joOHjyI4cOHw9LSEjwer9gvak+fPsXAgQNRu3ZtqKmpwcDAAB06dMDp06elipnJpyTvAJgf88cff6BWrVrIysrC7du34e3tjRs3buDJkydQUVEplX1mZmYiKCgI8+fPx9SpU79Z9uLFi6USg6RUVFRw4MAB/P7772LL09PTcfLkyVI7RgzDlH8qKiqIiIjA3bt3YWtrK7bOx8cHKioqyMrKKnZ7Hx8fWFhY4O7du4iIiEDdunUl2u+cOXPw+fNnDBw4EJaWlnj16hU2btwIPz8/hISEwNjY+Lt1bNmyBffv30erVq2QkJBQbLk3b94gNTUV7u7uMDU1RUZGBo4ePYpevXph27ZtmDBhgkQxM/lY0lTBubq6cq0848aNg4GBAZYvX45Tp05h0KBBpbLPT58+AQB0dHS+W5bP55dKDJLq1q0bjh07htDQUFhZWXHLT548iZycHLi4uODKlStyjJBhGHmpU6cO8vLycODAAbGkKSsrC8ePH0f37t1x9OjRIreNiorCrVu3cOzYMUycOBE+Pj7w9PSUaL9r1qxBu3btoKDw/4s9Li4ucHBwwMaNG4ttHf/S3r17Ub16dSgoKKBJkybFluvWrRu6desmtmzq1Klo2bIl1qxZw5ImKbHLc5VM+/btAQCRkZFiy69cuYL27dtDXV0dOjo66N27N8LCwgpt//DhQ7i6ukJLSwsaGhro3Lkzbt++za1ftGgRzM3NAQCzZs0Cj8eDhYVFsfF83b8nICAAPB4Phw4dwl9//YUaNWpARUUFnTt3RkRERKHt79y5AxcXF2hra0NNTQ0ODg64efOmxMfD3t4etWrVwv79+8WW+/j4wMXFBXp6ekVut3nzZjRu3BgCgQCmpqaYMmUKkpKSCpXbvn076tSpA1VVVdja2uL69etF1pednQ1PT0/UrVsXAoEANWvWxOzZs5GdnS3xa2EY5vu8vLzQqVMnVKtWDQKBAI0aNcKWLVuKLT9kyBAcPHgQIpGIW3b69GlkZGR884unj48PdHV10b17dwwYMAA+Pj4Sx9ihQwexhKlgmZ6eXpHn5aLUrFmzUB2SUlRURM2aNYs8pwH5Vwisra2hoqKCRo0a4dixYyXaT2XEkqZKpqATtq6uLrfs8uXLcHZ2RlxcHBYtWgQPDw/cunULbdu2Feu0/fTpU7Rv3x6hoaGYPXs2FixYgKioKDg6OuLOnTsAgH79+mHt2rUA8k82e/fuxbp166SOc9myZTh+/DhmzpyJefPm4fbt2xg2bJhYmStXrqBDhw5ISUmBp6cn/v77byQlJaFTp064e/euxPsaMmQIfH19QUQAgPj4eFy8eBFDhw4tsvyiRYswZcoUmJqaYvXq1ejfvz+2bduGrl27Ijc3lyu3c+dOTJw4EcbGxlixYgXatm2LXr164e3bt2L1iUQi9OrVC6tWrULPnj2xYcMG9OnTB2vXroWbm5vEr4NhmO/bsmULzM3N8dtvv2H16tWoWbMmJk+ejE2bNhVZfujQoYiJiUFAQAC3bP/+/ejcuTOqVatW7H58fHzQr18/8Pl8DBkyBC9fvkRwcHCJ405LS0NaWhoMDAxKXMe3pKenIz4+HpGRkVi7di3OnTuHzp07Fyr38uVLuLm5wdXVFUuXLoWSkhIGDhyIS5culUpcFQ4xFZKXlxcBoMuXL9OnT5/o7du3dOTIETI0NCSBQEBv377lylpbW1O1atUoISGBWxYaGkoKCgo0cuRIblmfPn2Iz+dTZGQkt+zDhw+kqalJHTp04JZFRUURAFq5cmWRMUVFRXHLHBwcyMHBgXt+9epVAkANGzak7Oxsbvn69esJAD1+/JiIiEQiEVlaWpKzszOJRCKuXEZGBtWqVYu6dOnyzePzZYxPnjwhAHT9+nUiItq0aRNpaGhQeno6ubu7k7q6OrddXFwc8fl86tq1KwmFQm75xo0bCQDt2rWLiIhycnKoWrVqZG1tLfY6tm/fTgDEXvPevXtJQUGB23+BrVu3EgC6efMmt8zc3Jzc3d2/+doYpjIqOH8EBwcXub64887XMjIyCi1zdnam2rVriy1zcHCgxo0bExGRjY0NjR07loiIEhMTic/n0+7du7nz1eHDh8W2vXfvHgGgS5cuEVH++apGjRo0ffp0iV5rUf78808CQP7+/lJv27hxY7FzTlEmTpxIAAgAKSgo0IABA+jz589iZczNzQkAHT16lFuWnJxMJiYm1Lx5c6njqoxYS1MF5+TkBENDQ9SsWRMDBgyAuro6Tp06hRo1agAAYmJiEBISglGjRoldimrWrBm6dOmCs2fPAgCEQiEuXryIPn36oHbt2lw5ExMTDB06FDdu3EBKSorM4h49erRYf6eCy4qvXr0CAISEhODly5cYOnQoEhISEB8fj/j4eKSnp6Nz5864du2aWHP6tzRu3BjNmjXDgQMHAOR/i+zduzfU1NQKlb18+TJycnIwY8YMsabv8ePHQ0tLC2fOnAEA3Lt3D3FxcZg0aZLY6xg1ahS0tbXF6jx8+DAaNmyIBg0acK8jPj4enTp1AgBcvXpVotfBMMz3qaqqcr8nJycjPj4eDg4OePXqFZKTk4vcZujQoTh27BhycnJw5MgRKCoqom/fvsXuw8fHB0ZGRujYsSMAgMfjwc3NDb6+vhAKhVLHfO3aNSxevBiDBg3izguyNmPGDFy6dAm7d++Gq6srhEIhcnJyCpUzNTUVe+1aWloYOXIkHj58iNjY2FKJrSKp0knTtWvX0LNnT5iamnJjDUkjKysLo0aNQtOmTaGkpIQ+ffp8s/zNmzehpKQEa2vrEsf8tU2bNuHSpUs4cuQIunXrhvj4eAgEAm79mzdvAAD169cvtG3Dhg25ROTTp0/IyMgotpxIJCp02elHmJmZiT0vuJyYmJgIIL+JGMgf2sDQ0FDssWPHDmRnZxd7AizK0KFDcfjwYURERODWrVvFXpor7njx+XzUrl2bW1/w8+vblZWVlcWSzoLX8vTp00Kvo169egCAuLg4iV8HwzDfdvPmTTg5OXH9Nw0NDfHbb78BQLHnjMGDByM5ORnnzp2Dj48PevToAU1NzSLLCoVC+Pr6omPHjoiKikJERAQiIiJgZ2eHjx8/wt/fHwCQk5OD2NhYsUdRCdXz58/Rt29fNGnSBDt27BBb9/nzZ7HtpTnnfa1BgwZwcnLCyJEj4efnh7S0NPTs2ZPrtlCgbt264PF4YssKzlVfdueoqqr03XPp6emwsrLCmDFj0K9fP6m3FwqFUFVVxbRp04q9w6JAUlISRo4cic6dO+Pjx48lDbkQW1tb7u65Pn36oF27dhg6dCjCw8PL9fhDioqKRS4v+AAXtCKtXLmy2CRTmtc3ZMgQzJs3D+PHj4e+vj66du0qXcA/QCQSoWnTplizZk2R62vWrFlmsTBMZRYZGYnOnTujQYMGWLNmDWrWrAk+n4+zZ89i7dq1xbZOm5iYwNHREatXr8bNmze/eT6/cuUKYmJi4OvrC19f30LrfXx80LVrV9y6dYtriSoQFRUlduPM27dvuUGCz549WyhR69evHwIDA7nn7u7u8Pb2luBIfN+AAQMwceJEvHjxosgvy0zRqnTS5OrqCldX12LXZ2dnY/78+Thw4ACSkpLQpEkTLF++nLsbTF1dnbsr4+bNm8XeiQAAkyZNwtChQ6GoqCh1i5akFBUVsXTpUnTs2BEbN27E3LlzuTvdwsPDC5V//vw5DAwMoK6uDhUVFaipqRVbTkFBoUz/udepUwdAftOwk5PTD9dnZmaGtm3bIiAgAD/99BOUlIp+6395vL5sMcrJyUFUVBQXS0G5ly9fijWn5+bmIioqSmx4gzp16iA0NBSdO3cu9A2OYRjZOX36NLKzs3Hq1Cmx1mxJLoEPHToU48aNg46OTqFb9L/k4+ODatWqFdmx/NixYzh+/Di2bt0KKyurQp2nvxx/KSEhAV27dkV2djb8/f1hYmJSqL7Vq1dzre9A/qUzWcnMzARQuPUtIiICRCR2rnrx4gUAfPNO6aqiSidN3zN16lQ8e/YMvr6+MDU1xfHjx+Hi4oLHjx8XO4psUby8vPDq1Svs27dPovE3foSjoyNsbW2xbt06zJgxAyYmJrC2tsbu3bsxb948bmylJ0+e4OLFixg+fDiA/ISra9euOHnyJF6/fs19OD5+/Ij9+/ejXbt20NLSKtXYv9SyZUvUqVMHq1atwtChQwu1Kn369AmGhoZS1blkyRJcvXr1m3esOTk5gc/n459//oGLiwt34ti5cyeSk5PRvXt3AICNjQ0MDQ2xdetWsf5Z3t7ehZLnQYMG4ezZs/j3338LjYmSmZkJkUgEdXV1qV4LwzCFFbRgf3nJKTk5GV5eXt/ddsCAAXj79i3q169f7PhymZmZOHbsGAYOHIgBAwYUWm9qaooDBw7g1KlTcHNzK/YLX3p6Orp164b379/j6tWrxf4/admy5Xfj/p64uLhCdwHm5uZiz549UFVVRaNGjcTWffjwAcePH+euvqSkpGDPnj2wtraWaNDNyo4lTcWIjo6Gl5cXoqOjuex+5syZOH/+PLy8vPD3339LVM/Lly8xd+5cXL9+vdjWDVmbNWsWBg4cCG9vb0yaNAkrV66Eq6sr7O3tMXbsWGRmZmLDhg3Q1tbGokWLuO2WLFmCS5cuoV27dpg8eTKUlJSwbds2ZGdnY8WKFWUSewEFBQXs2LEDrq6uaNy4MUaPHo3q1atzJxktLS2ppwFwcHCAg4PDN8sYGhpi3rx5WLx4MVxcXNCrVy+Eh4dj8+bNaNWqFZdkKisrY8mSJZg4cSI6deoENzc3REVFwcvLq1CfphEjRuDQoUOYNGkSrl69irZt20IoFOL58+c4dOgQLly4UO6moWEYedm1axfOnz9faHnv3r0B5E9BUtQo3X369EHXrl3B5/PRs2dPTJw4EWlpafj3339RrVq1705R8vX5sCinTp1CamoqevXqVeT61q1bw9DQED4+Pt/8cjZs2DDcvXsXY8aMQVhYmNjYTBoaGt/tHwvk98m9du0agPwvkenp6dyX8g4dOqBDhw4AgIkTJyIlJQUdOnRA9erVERsbCx8fHzx//hyrV68u9IW0Xr16GDt2LIKDg2FkZIRdu3bh48ePEiWeVYJ8b94rPwDQ8ePHued+fn4EgNTV1cUeSkpKNGjQoELbu7u7U+/evcWW5eXlkY2NDW3ZsoVb5unpSVZWVj8c77duzxUKhVSnTh2qU6cO5eXlERHR5cuXqW3btqSqqkpaWlrUs2dPevbsWaFtHzx4QM7OzqShoUFqamrUsWNHunXrllgZWQw58PUtvAV1enl5iS1/+PAh9evXj/T19UkgEJC5uTkNGjTou7flSnp78tdDDhTYuHEjNWjQgJSVlcnIyIh++uknSkxMLFRu8+bNVKtWLRIIBGRjY0PXrl0r9JqJ8ocoWL58OTVu3JgEAgHp6upSy5YtafHixZScnMyVY0MOMFVVwfmjuMf169e/uX7v3r1ERHTq1Clq1qwZqaiokIWFBS1fvpx27dpV5LmpYMiB4nx9vurZsyepqKhQenp6sduMGjWKlJWVKT4+vtgyBbf2F/UwNzeX6Hh5enoWW4enpydX7sCBA+Tk5ERGRkakpKREurq65OTkRCdPniwyru7du9OFCxeoWbNmJBAIqEGDBoXO11UZj+irrvNVFI/Hw/Hjx7kM/+DBgxg2bBiePn1aqNOyhoZGoWbKUaNGISkpSay/UlJSEnR1dcW2F4lEICIoKiri4sWLpXZ7KcMwDMMwssUuzxWjefPmEAqFiIuL48YQkpaWlhYeP34stmzz5s24cuUKjhw5glq1askiVIZhGIZhykCVTprS0tLE5juLiopCSEgI9PT0UK9ePQwbNgwjR47E6tWr0bx5c3z69An+/v5o1qwZ1yH42bNnyMnJwefPn5GamoqQkBAAgLW1dZETKVarVg0qKirfnGCRYRiGYZjyp0onTffu3RMbR8PDwwPA/8fC8PLywpIlS/Drr7/i/fv3MDAwQOvWrdGjRw9um27dunEDHQL5LVQACg0YxjAMwzBMxcb6NDEMwzAMw0igSk+jwjAMwzAMIymWNDEMwzAMw0igyvVpEolE+PDhAzQ1NdmUFgxTSogIqampMDU1hYIC+272o9h5i2FKl6TnrCqXNH348IFNkMowZeTt27eoUaOGvMOo8Nh5i2HKxvfOWVUuaSqYRfrt27dlOpcaw1QlKSkpqFmzZqFZ25mSYecthildkp6zqlzSVNC0raWlxU4+DFPK2KUk2WDnLYYpG987Z1W5pKmiEQqFuH79OgCgffv2haZ0YRiGYRimbLCkqZzLysriBuBMS0uDurq6nCNiGIZhmKqJ3dbCMAzDMAwjAZY0MQzDVAK7bkTBfdddXH0eJ+9QGKbSYkkTwzBMJfAsJgWBLz7hyftkeYfCMJUWS5oYhmEqAQt9NQDA64QMOUfCMJUXS5oYhmEqAXP9/JtE3iSkyzkShqm8WNLEMAxTCVj8lzSxliaGKT1syIFyTllZGStWrOB+ZxiGKYrZf5fn4tOykZqVC00Vdr5gGFljSVM5x+fzMWvWLHmHwTBMOaetqgw9dT4+p+fgTUIGmlTXlndIDFPplChpys3NRWxsLDIyMmBoaAg9PT1Zx8XISHJyMmbPno2EhAR5h8KUksTEREREREAoFMo7FI5IJJJ3CFWSub4aS5oYphRJnDSlpqZi37598PX1xd27d5GTkwMiAo/HQ40aNdC1a1dMmDABrVq1knjn165dw8qVK3H//n3ExMTg+PHj6NOnT7HlAwICuNGxvxQTEwNjY2OJ91uRCIVCPHjwAADQokULqadRWbNmDbZv314aoTEMU85Y6KvjYXQSXrPO4AxTKiRKmtasWYO//voLderUQc+ePfHbb7/B1NQUqqqq+Pz5M548eYLr16+ja9eusLOzw4YNG2BpafndetPT02FlZYUxY8agX79+EgcdHh4uNmlltWrVJN62osnKyoKtrS0A6adRyc7OxtatWwEAv/zyi0R/E6ZiyMnJga+vL27fvg0AaNq0KRo3biznqP4vNzcXR48elXcYVY75f/2a2B10DFM6JEqagoODce3atWJPyra2thgzZgy2bt0KLy8vXL9+XaJ/0K6urnB1dZUuYuQnSTo6OlJvV9UcPHgQcXFxqFGjBpYvX846klcSERERGDBgAEJDQ6GgoIClS5di1qxZ352duyylpKSwpEkO2B10DFO6JEqaDhw4IFFlAoEAkyZN+qGAJGFtbY3s7Gw0adIEixYtQtu2bUt9nxUNEeGff/4BAEyZMoUlTJXEqVOnMHLkSCQnJ6NatWo4ePAgHB0d5R0WU06wliaGKV0/PE7Tmzdv8OzZszLp+GliYoKtW7fi6NGjOHr0KGrWrAlHR0euz09RsrOzkZKSIvaoCoKCgnD//n2oqKhg3Lhx3y3v6OiIGTNmFLuex+PhxIkTEu8/ICAAPB4PSUlJEm/DFC8vLw+//fYbevfujeTkZLRt2xYPHz5kCRMjpqCl6WNKNjJy8uQcDcNUPhJ3BN+1axeSkpLg4eHBLZswYQJ27twJAKhfvz4uXLiAmjVryj7K/9SvXx/169fnnrdp0waRkZFYu3Yt9u7dW+Q2S5cuxeLFi0stpvKqoJVp2LBhMDAw+OH6YmJioKur+8P1MNKLi4vDkCFDcOXKFQDAjBkzsGLFCtZ6yBSiq86HtqoykjNzEf05Aw2Mtb6/EcMwEpO4pWn79u1i/zTPnz8PLy8v7NmzB8HBwdDR0ZFLcmJra4uIiIhi18+bNw/Jycnc4+3bt2UYnXy8e/cOR44cAQD8/PPPMqnT2NgYAoFAJnUxkrt16xaaN2+OK1euQF1dHQcPHsTatWtZwsQUi5uDLp71a2IYWZM4aXr58iVsbGy45ydPnkTv3r0xbNgwtGjRAn///Tf8/f1LJchvCQkJgYmJSbHrBQIBtLS0xB6V3ZYtWyAUCuHg4AArKyuJtxOJRJg9ezb09PRgbGyMRYsWceu+vjx369YtWFtbQ0VFBTY2Njhx4gR4PB5CQkLE6rx//z5sbGygpqaGNm3aIDw8/AdfXdVQ0CfNwcEBHz58QMOGDREcHIxBgwbJOzSmnGNz0DFM6ZH48lxmZqZYwnHr1i2MHTuWe167dm3ExsZKtfO0tDSxVqKoqCiEhIRAT08PZmZmmDdvHt6/f489e/YAANatW4datWqhcePGyMrKwo4dO3DlyhVcvHhRqv1WJMrKyvD09OR+/57MzExs27YNADB9+nSp9rV79254eHjgzp07CAoKwqhRo9C2bVt06dJFrFxKSgp69uyJbt26Yf/+/Xjz5k2x/aHmz5+P1atXw9DQEJMmTcKYMWNw8+ZNqeKqatLS0jB+/Hj4+voCANzc3LBjxw5oaGjIObLyKzo6Gm/evOEG3G3cuHGVbRnlWprYHXQMI3skoQYNGtDRo0eJiOjTp0+kqKhI9+7d49bfuXOHjIyMJK2OiIiuXr1KAAo93N3diYjI3d2dHBwcuPLLly+nOnXqkIqKCunp6ZGjoyNduXJFqn0mJycTAEpOTpZqu4pi165dBIDMzc0pNzdX4u0cHByoXbt2YstatWpFc+bMISIiAHT8+HEiItqyZQvp6+tTZmYmV/bff/8lAPTw4UMi+v/f9vLly1yZM2fOEACx7RhxYWFh1LBhQwJASkpKtH79ehKJRPIOS2pl8TmLioqi2bNnk5mZGSkoKBCPx+MeAoGAnJyc6NChQyQUCksthrIizfE8cu8tmc/xoyHbg8ogMoapHCT9jEnc0uTu7o4pU6bg6dOnuHLlCho0aICWLVty62/duoUmTZpIlbA5OjqCiIpd7+3tLfZ89uzZmD17tlT7qEroq2EGlJSkmyWnWbNmYs9NTEwQFxdXqFx4eDiaNWsGFRUVblnBAJzfqrPgMmpcXBzMzMykiq0qOHz4MMaMGYO0tDSYmpri0KFDbDiNYkybNg27d++Gs7MzlixZAltb2yIH3F24cCEWL14MLy8vqWYrqMgsDAqGHWAtTQwjaxL/V509ezYyMjJw7NgxGBsb4/Dhw2Lrb968iSFDhsg8wKpOJBIhLCwMANCwYUMoKBTfDe3GjRsICQmBqqqq2KVTSX19+Y/H4/3wUBJf1lkw+CKbl0xcbm4u5syZg7Vr1wIAOnbsiAMHDsDIyEjOkZVf6urqePXqFfT19Qutq1atGjp16oROnTrB09MT58+fx9u3b6tM0lTQp+lDciaycoVQUZZu6iWGYYoncdKkoKCAP/74A3/88UeR679OohjZyMzM5FrwvjeNyvr16wEAI0aMKNVJlOvXr499+/YhOzub6zcSHBxcavurzD58+IBBgwZx/bzmzJmDJUuWSN1KWNUsXbpU4rIuLi6lGEn5o6/Oh4ZACWnZeXiXmIG61TTlHRLDVBpSDW558OBBDBs2DAMHDuTmNGPKh+joaBw/fhyA7IYZKM7QoUMhEokwYcIEhIWF4cKFC1i1ahUAlKupPMq7wMBAtGjRAjdv3oSWlhaOHz+OZcuWsYRJSpmZmcjI+P+lqDdv3mDdunW4cOGCHKOSHx6Px40MzoYdYBjZkjhp2rJlC4YMGYJ79+7h5cuXmDJlCmbNmlWasTFS2Lx5M0QiETp37ix13zJpaWlp4fTp0wgJCYG1tTXmz5+PhQsXAoBYPyemaESElStXonPnzvj48SOaNWuG+/fvo0+fPvIOrULq3bs3d4dtUlIS7OzssHr1avTp0wdbtmyRc3Ty8f856NiwAwwjU5L2LG/UqBEtWrSIe753715SU1MrYT91+alod8+lpaVxdxWmpaUVWSY9PZ309PQIAJ08ebKMI8y3b98+UlZWpoyMDLnsv6JISkqiPn36cH/TkSNHUnp6urzDkrmy/Jzp6+vTkydPiCj/Ls5mzZqRUCikQ4cOUYMGDUp9/2VB2uO5/FwYmc/xo/nHH5VyZAxTOUj6GZO4penVq1dwd3fnng8dOhR5eXmIiYmRZQ7HlMD+/fvx+fNn1KpVC927dy+Tfe7Zswc3btxAVFQUTpw4gTlz5mDQoEFQVVUtk/1XRI8ePeIGAuXz+di2bRu8vb2hpqYm79AqtIyMDGhq5vfbuXjxIvr16wcFBQW0bt0ab968kXN08mHBDXDJLs8xjCxJnDRlZ2eLdUJWUFAAn89HZmZmqQTGSIaIuA7gU6dOhaJi2dwpExsbi+HDh6Nhw4b45ZdfMHDgQGzfvr1M9l0R7du3D61bt0ZERATMzMxw48YNTJgwgfUBk4G6devixIkTePv2LS5cuICuXbsCyB/aoirMAFAUrk8TuzzHMDIlVY/TBQsWiH0rzsnJwV9//QVtbW1u2Zo1a2QXHfNdAQEBePLkCdTV1TFmzJgy2y8bM0sy2dnZ+OWXX7i+Nc7OzvDx8SnyVnmmZBYuXIihQ4fil19+QefOnWFvbw8gv9WpefPmco5OPiwM8r/gvk/MRE6eCHwlqe75YRimGBInTR06dCg0b1ibNm3w6tUr7jn71ix7ysrKmDlzJvf71woGs3R3d4eOjk5ZhsZ8R3R0NAYOHIi7d++Cx+Nh4cKFWLBgQZm1BlYVAwYMQLt27RATEyM212Lnzp3Rt29fOUYmP9U0BVBRVkBWrgjvEjNQ25BNwcMwMlE2XazKj4rWEfxbXr16RQoKCgSAnj179kN1OTg40PTp02UTmBQ8PT3JysqqRNsWTNWSmJhYbBkvLy/S1tb+bl34YpoYWbhw4QLp6+sTANLT06OzZ8/KrO6KoDJ9zsqDkhxP57WBZD7Hj66EfSzFyBimcpB5R/CvxcfHIz4+XhZ5G1NCBcMMdO3aFQ0bNpR3OGWuTZs2iImJEbs8/D2LFi2CtbV1qcUkEonw559/wsXFBQkJCWjZsiXu378PV1fXUttnVTRp0iS8e/dOorIHDx6Ej49PKUdU/rB+TQwje1L1aUpKSsL8+fNx8OBBJCYmAgB0dXUxePBgLFmyhF0eKgUikQjR0dEAADMzM24alfT0dOzYsQNA/jxcVRGfz4exsbG8w+B8/vwZI0aMwNmzZwEAEyZMwPr169nYVaXA0NAQjRs3Rtu2bdGzZ0/Y2NjA1NQUKioqSExMxLNnz3Djxg34+vrC1NS0St6kUNCvid1BxzCyI3FL0+fPn2FnZ4fdu3ejf//+WL16NVavXo1+/frB29sb9vb2XCLFyE5mZiZq1aqFWrVqid2puHfvXiQlJaFOnToya8XIy8vD1KlToa2tDQMDAyxYsICbUHnv3r2wsbGBpqYmjI2NMXToULHJfAMCAsDj8eDv7w8bGxuoqamhTZs2hfrBLVu2DEZGRtDU1MTYsWORlZXFrXvy5AkUFBTw6dMnAPnvOQUFBQwePJgrs2TJErRr105sn0lJSdx6b29vmJmZQU1NDX379kVCQoLYusWLFyM0NBQ8Hg88Hk9sUuj4+Hj07dsXampqsLS0xKlTpyQ+dvfv30fLli1x9uxZqKiowMvLC9u2bWMJUyn5888/8eLFC7Rt2xabN29G69atYWZmhmrVqqF+/foYOXIkXr16he3bt+P27duFJqOuCtgAlwxTCiS93jd9+nRq0qQJxcbGFloXExNDTZs2pRkzZkh9HbGsVbS+FkUNbikSiahhw4YEgNatWyeT/Tg4OJCGhgZNnz6dnj9/Tvv27SM1NTXavn07ERHt3LmTzp49S5GRkRQUFET29vbk6urKbV/Qv8jOzo4CAgLo6dOn1L59e2rTpg1X5uDBgyQQCGjHjh30/Plzmj9/PmlqanJ9mkQiERkYGNDhw4eJiOjEiRNkYGBAxsbGXB1OTk40f/58sX0W9Gm6ffs2KSgo0PLlyyk8PJzWr19POjo6XJ+mjIwM+vXXX6lx48YUExNDMTEx3GCcAKhGjRq0f/9+evnyJU2bNo00NDQoISHhu8fu33//JYFAQACoTp06FBISUrI/QiVS1p+zz58/U0hICAUFBdHLly9JJBKVyX7LSkmO582IT2Q+x48cV14tvcAYppKQ9DMmcdJkbm5O58+fL3b9uXPnyNzcXOIA5aUyJE2XLl0iAKShoSGz1+Hg4EANGzYU+2czZ84catiwYZHlg4ODCQClpqYS0f8TmMuXL3Nlzpw5QwAoMzOTiIjs7e1p8uTJYvXY2dmJdQTv168fTZkyhYiIZsyYQbNmzSJdXV0KCwujnJwcUlNTo4sXL4rtsyBpGjJkCHXr1k2sfjc3N7GO4MV1PAdAv//+O/e84LifO3euyNdPlJ+EjR49mvv79OrV65ud0quSivY5+9LGjRvJ3NycBAIB2dra0p07d4ot++TJE+rXrx+Zm5sTAFq7du0P11mUkhzP94kZZD7Hj+rMO0O5eUKp9scwVY3MO4LHxMSgcePGxa5v0qQJYmNjJW/iYkqsYJiB0aNHy3TwvtatW4sNG2Fvb4+XL19CKBTi/v376NmzJ8zMzKCpqQkHBwcA4PpbFfjyMoiJiQkAcJfxwsLCYGdnJ1a+YEydAg4ODggICACQP6Ftp06d0KFDBwQEBCA4OBi5ublo27ZtkfFLUv+3fBm7uro6tLS0xC5BfunVq1do06YNvLy8oKCggKVLl+L48eOsX18Fd/DgQXh4eMDT0xMPHjyAlZUVnJ2di30fZGRkoHbt2li2bFmx/eukrVNWjLVUwFdSQJ6I8CEp6/sbMAzzXRInTQYGBnj9+nWx66OioqCnpyeLmJhviIyMhJ+fH4D8EcDLQlZWFpydnaGlpQUfHx8EBwfj+PHjAPIHOP3Sl2NJFSRgIpFI4n05Ojri2bNnePnyJZ49e4Z27drB0dERAQEBCAwM5PpLlYavx8Hi8XhFxn769Gm0bNkSISEhMDQ0xKVLlzB37lyukz5Tca1Zswbjx4/H6NGj0ahRI2zduhVqamrYtWtXkeVbtWqFlStXYvDgwRAIBDKpU1YUFHgw12N30DGMLEl8lnd2dsb8+fML/ZME8kc9XrBgAVxcXGQaHFPYxo0bQURwdXVFvXr1ZFr3nTt3xJ7fvn0blpaWeP78ORISErBs2TK0b98eDRo0KNG35IYNGxa5jy81bdoUurq6WLJkCaytraGhoQFHR0cEBgYiICAAjo6OP1Q/n8+HUCiUOnYAEAqFmD9/Pnr16oWkpCTY29vj4cOH6NSpU4nqY8qXnJwc3L9/H05OTtwyBQUFODk5ISgoqEzrzM7ORkpKitijJMy5OehY0sQwsiBx0vTHH38gPDwclpaWWLFiBU6dOoWTJ09i2bJlsLS0RFhYGBYvXlyasVZ5qamp3LfT0hhmIDo6Gh4eHggPD8eBAwewYcMGTJ8+HWZmZuDz+diwYQNevXqFU6dO4c8//5S6/unTp2PXrl3w8vLCixcv4OnpiadPn4qV4fF46NChA3x8fLgEqVmzZsjOzoa/vz93WbAo06ZNw/nz57Fq1Sq8fPkSGzduxPnz58XKWFhYICoqCiEhIYiPj0d2drZEsX/69AnOzs74+++/uX0FBASgevXqUhwBpjyLj4+HUCiEkZGR2HIjI6MSdz0oaZ1Lly6FtrY296hZs2aJ9m/BjdXEhh1gGFmQOGmqUaMGgoKC0KhRI8ybNw99+vRB3759MX/+fDRq1Ag3b94s8QebKZ6SkhImT56MyZMnw9fXFykpKahfvz43KSkAICsLmDQJ2L+/6EqIgJAQ4N69b+5r5MiRyMzMhK2tLaZMmYLp06djwoQJMDQ0hLe3Nw4fPoxGjRph2bJlWLVqldSvxc3NDQsWLMDs2bPRsmVLvHnzBj/99FOhcg4ODhAKhVzSpKCggA4dOoDH4xXbnwnI75P177//Yv369bCyssLFixfx+++/i5Xp378/XFxc0LFjRxgaGuLAgQPfjfv27dto0aIF/P39oa6ujgMHDmD9+vXg8/nSHQCm1OTl5eHy5cvYtm0bUlNTAQAfPnxAWlqanCMrmXnz5iE5OZl7vH37tkT1mBuwliaGkamS9DL//Pkz3blzh+7cuSPRLdnlSUW9q0coFFL9+vUJAG3cuPH/K1JSiDp1IlJRIQoI+P/y1FSi48eJxo0jMjUlAojati3zuCsykUhEGzduJGVlZQJA9evXpydPnsg7rAqhLD9nr1+/pgYNGpCamhopKipSZGQkERFNmzaNJk6cKHE92dnZpKioWGg6nZEjR1KvXr2+u725uXmhu+d+tM4CJT2e117EkfkcP+q06qpU2zFMVVOq06jo6urC1tYWtra2rPN3Gbl06RLCw8OhpaWFkSNH5i+Mjwc6d85vQbpwAaheHVi/HujaFdDXB/r2Ba5fB9zcgMuXgStX5PsiKpD09HQMHz4cU6dORW5uLgYMGIDg4OBv3kHKyMf06dNhY2ODxMREqKqqcsv79u0Lf39/ievh8/lo2bKl2DYikQj+/v5S3YVZ2nVKo2CAy7efMyEUUanvj2EqO4mmUZk0aRJ+//131KhR47tlDx48iLy8PAwbNuyHg2MAIkJ8fDxWrlwJABgzZgw0NTWBd++ALl2A2FigRw9g/HjgxQuAzwccHYGVK4Fu3YC6deX7Aiqg8PBw9O/fH0+fPoWSkhJWrlyJ6dOniw3HwJQf169fx61btwpdLrWwsMD79++lqsvDwwPu7u6wsbGBra0t1q1bh/T0dIwePRpA/iXs6tWrY+nSpQDyO3o/e/aM+/39+/cICQmBhoYG6v732ftenaXJRFsFyoo85AhFiEnORA3d0rnzlGGqComSJjbPk/xkZGSgWrVq3PMpU6YAkZFAs2ZAxn+dO69ezU+cVqzIb3nS0JBTtBXf0aNHMXr0aKSmpsLExASHDh3ipm1hyieRSFTkHZHv3r3L/4IhBTc3N3z69AkLFy5EbGwsrK2tcf78ea4jd3R0tNjQEh8+fEDz5s2556tWrcKqVavExhv7Xp2lSUlRATV11fAqPh1vEjJY0sQwP4hHRBK12X78+BE7duyAr68v982qgKamJpycnDBu3LhyP+xASkoKtLW1kZycLNOBIUtLeno6NP5LglxcXHDu3Dng7l3A2RlITQUK/lmYmQFWVkDbtsCvvwJKUs3FXOXl5uZi3rx5WL16NYD8zui+vr7lakLgiqQsP2dubm7Q1tbG9u3boampiUePHsHQ0BC9e/eGmZkZvLy8SnX/ZeFHjudor7u4Gv4Jf/VtgmF25qUUIcNUbJJ+xiT+z2pkZIT58+dj/vz5SExMRHR0NDIzM2FgYIA6deqwSxel5MvxWSZPnpz/i60tkJgI5OYC4eFAaCjw6FH+48gRYPJkQMpv2FVZTEwM3NzccP36dQDA7Nmz8ddff0GJJZ4VwurVq+Hs7IxGjRohKysLQ4cOxcuXL2FgYCDR3ZGVXf5YTZ/whg07wDA/rET/FXR1daGrqyvrWJgi7Nu3j/u9Y8eO4iuVlYEmTfIfrA9ZiVy7dg1ubm6IjY2FpqYmdu/ejb59+8o7LEYKNWrUQGhoKHx9ffHo0SOkpaVh7NixGDZsmFjH8KqKG6spng07wDA/in2VLsdEIhG2bt3KPWetebJDRFizZg3mzJkDoVCIJk2a4OjRozIfZZ0pG0pKShg+fLi8wyiXLLixmlhLE8P8KJY0lWPnzp3Dq1ev5B1GpZOSkoIxY8bg6NGjAIDhw4dj69atUFdXl3NkTEl9+PABN27cQFxcXKH5Aktj9PyKpGDYgTef0yESERQU2JcvhikpljSVY//884+8Q6h0njx5gv79++PFixdQVlbG+vXrMWnSJNaKV4F5e3tj4sSJ4PP50NfXF/tb8ni8Kp80VddVhaICD1m5IsSlZsNYW0XeITFMhcWSpnIqLCwMFy9eBI/HQ79+/aChocE6Jv8gHx8fTJgwARkZGahZsyaOHDkCW1tbeYfF/KAFCxZg4cKFmDdvnthwAEw+ZUUF1NBVxZuEDLxOSGdJE8P8gBKdYSrbPE/l0caNGwEAvXv3xpEjR+Dt7Q2BQCDnqCqmnJwcTJ06FcOHD0dGRga6dOmCBw8esISpksjIyMDgwYNZwvQN5vpsDjqGkQWpzzJv3rxB06ZN0bt3b0yZMgWfPn0CACxfvhwzZ86UeYBVUVJSEnbv3g2A9cf4UW/fvkWHDh2wadMmAPmtEufOnYOBgYGcI2NkZezYsTh8+LC8wyjXuDvoWGdwhvkhUl/vKZjnKTQ0FPr6+tzyvn37Yvz48TINrqratWsX0tPT0aRJEzg4OCA9Pf/boZqaGut7I4XLly9jyJAhiI+Ph66uLvbu3Yvu3bvLOyxGxpYuXYoePXrg/PnzaNq0KZSVlcXWr1mzRk6RlR+spYlhZEPqpEmW8zwxhQmFQu7S3LRp05CZmcmNCJ6Wlsbu8JKASCTC0qVLsWDBAhARWrRogSNHjqBWrVryDo0pBUuXLsWFCxdQv359ACjUEZz5cqwm1tLEMD9C6stzspzn6dq1a+jZsydMTU3B4/Fw4sSJ724TEBCAFi1aQCAQoG7duvD29pZqn+XdmTNnEBUVBT09PTbpcQkkJiaid+/e+P3330FEGDduHG7evMkSpkps9erV2LVrF8LCwhAQEICrV69yjytXrsg7vHLhy5YmCWfOYhimCFInTV27dsW6deu45zweD2lpafD09ES3bt2kqis9PR1WVlZcf5PviYqKQvfu3dGxY0eEhIRgxowZGDduHC5cuCDVfsuzgmEGxo8fDzU1NrmmNB4+fIiWLVvCz88PKioq2LlzJ/7991+oqLC7hSozgUCAtm3byjuMcq2mnip4PCA9R4j4tBx5h8MwFZbUl+dkOc+Tq6srXF1dJS6/detW1KpVi5tUtWHDhrhx4wbWrl0LZ2dnqfZdHj19+hT+/v5QUFD4/zxzjER27dqFyZMnIzs7G7Vq1cLRo0fFZp9nKq/p06djw4YNbFyzbxAoKcJUWxXvkzLxOiEdhprsTlyGKQmpk6aCeZ4OHjyI0NDQMp3nKSgoCE5OTmLLnJ2dMWPGjGK3yc7ORnZ2Nvf8ywlwy5sNGzYAyO9Ub2ZmJudoKoasrCz8/PPP2LFjBwCgR48e2LNnD5sbsQq5e/curly5Aj8/PzRu3LhQR/Bjx47JKbLyxcJALT9pik9HKws9eYfDMBVSiUZLVFJSwrBhw8q8z01sbCyMjIzElhkZGSElJQWZmZlFJm1Lly7F4sWLyyrEEvv8+TP27NkDgA0zIKmoqCgMGDAADx48gIKCAv7880/MnTuXjddTxejo6KBfv37yDqPcM9dXx82IBDYHHcP8AKmTpqVLl8LIyAhjxowRW75r1y58+vQJc+bMkVlwsjBv3jx4eHhwz1NSUlCzZk05RlS0nTt3IjMzE1ZWVmjfvr28wyn3zpw5g+HDhyMpKYm7NPx1KyRTNXh5eck7hArh/2M1sWEHGKakpP5Kvm3bNjRo0KDQ8saNG2Pr1q0yCao4xsbG+Pjxo9iyjx8/QktLq9hLgwKBAFpaWmKP8iYvL09smIEvb5NWVFTEgAEDMGDAACgqKsorxHJDKBRi4cKF6NGjB5KSkmBnZ4cHDx6whIlhvuP/d9CxliaGKSmpW5piY2NhYmJSaLmhoSFiYmJkElRx7O3tcfbsWbFlly5dgr29fanut7SdPn0a0dHRMDAwwNChQ8XWqaiosNGO/xMfH4+hQ4fi0qVLAICpU6di9erVhcYMYyq/Fi1awN/fH7q6umjevPk3x2N68OBBGUZWfln8lzS9/m/YATaGFcNIT+qkqWbNmkWOe3Pz5k2YmppKVVdaWhoiIiK451FRUQgJCYGenh7MzMwwb948vH//nuvrM2nSJGzcuBGzZ8/GmDFjcOXKFRw6dAhnzpyR9mWUKwV3/UyYMIHdHl+MO3fuYODAgXj79i3U1NSwfft2No5VFda7d29uLsY+ffrIN5gKwkwv//JcalYeEjNyoafOvmwwjNRISsuXLyd9fX3atWsXvX79ml6/fk07d+4kfX19+vvvv6Wq6+rVqwSg0MPd3Z2IiNzd3cnBwaHQNtbW1sTn86l27drk5eUl1T6Tk5MJACUnJ0u1XWkJDQ0lAKSoqEhv376Vdzjljkgkok2bNpGysjIBoHr16tHjx4/lHRbzHWXxORs9ejSlpKSUWv3liayOZ+u/L5P5HD+6/+azjCJjmMpB0s+Y1C1Ns2bNQkJCAiZPnoycnPxB0lRUVDBnzhzMmzdPqrocHR2/OTptUaN9Ozo64uHDh1LtpzwraGXq378/atSoUWh9enp6lZ1GJT09HZMmTcK+ffsA5B+jXbt2lct+aUzZ2717N5YtWyb1TARVmbm+GmKSs/AmIR0tzNiwHAwjLamTJh6Ph+XLl2PBggUICwuDqqoqLC0tuaZyRnLx8fHw8fEBwIYZ+NqLFy/Qv39/PHnyBIqKili+fDk8PDxYPwyG860vXEzRLPTVcfvVZzYHHcOUUInGaQIADQ0NtGrVSpaxVDk7duxAVlYWWrZsiTZt2sg7nHLj+PHjGDVqFFJSUmBsbIyDBw+iQ4cO8g6LKYdSU1O/2w+QtUz+35dz0DEMIz2pk6b09HQsW7YM/v7+iIuLg0gkElv/6tUrmQVXmeXl5XFz7n09zEBVlZeXh99++w0rV64EALRv3x4HDx4s8m5NhgGAevXqFbuO/rtDrKgJxquq/4/VxFqaGKYkpE6axo0bh8DAQIwYMQImJibsn30JnThxAu/evUO1atXg5uYm73DkLjY2FoMHD0ZgYCAAYObMmfj7778LTYnBMF86cuQI9PTYlCCSYi1NDPNjpE6azp07hzNnzrBZxX/Q+vXrAQATJ06s8v3Bbty4gUGDBiEmJgaamprw8vJC//795R0WUwG0bdsW1apVk3cYFYb5fy1NiRm5SM7IhbYa+1LCMNKQekRwXV1d9s3uBz148AA3btyAkpISJk2aJO9w5IaIsHbtWjg6OiImJgaNGzdGcHAwS5gYppSoC5RgqJn/Je3NZ9baxDDSkjpp+vPPP7Fw4UJkZLBr4iW1YcMGAMDAgQO/OyCooqIiunXrhm7dulWqaVRSU1MxaNAgeHh4QCgUYujQobhz5w7q168v79CYCsLc3LxSfSbKCuvXxDAlJ/XludWrVyMyMhJGRkawsLAo1OeETVnwbXFxcdi/fz8AYPr06d8tr6KiUuFHPP/as2fP0K9fP4SHh0NZWRlr167F5MmTWf84RipRUVHyDqFCMtdXR/DrRLyJZy1NDCMtqZMmNmXBj/n333+Rk5MDW1tb2NnZyTucMnfgwAGMHz8e6enpqFGjBg4fPozWrVvLOyyGqTIKWpqiWGdwhpGa1EmTp6dnacRRJeTm5mLz5s0Aqt5gljk5OZg5cyZ3abJz5844cOAADA0N5RwZw1QtBXfQvWYtTQwjNan7NDEld/ToUXz48AHGxsYYOHCgRNukp6dDXV0d6urqSE+vmCe5d+/ewdHRkUuY5s+fjwsXLrCEiWHkoKFJ/rQzj98nIy41S87RMEzFInXSJBQKsWrVKtja2sLY2Bh6enpiD6Z4BfPMTZo0CXy+5DOMZ2RkVNiO91euXEGLFi0QFBQEHR0dnD59GkuWLGEdeJlSkZXFkoDvqVtNEy3MdJArJPjefSvvcBimQpE6aVq8eDHWrFkDNzc3JCcnw8PDA/369YOCggIWLVpUCiFWDsHBwQgKCoKysnKVGGZAJBJh2bJl6NKlCz59+gRra2vcv38fPXr0kHdoTCUjEonw559/onr16tDQ0OBmJViwYAF27twp5+jKJ/c2FgAAnztvkCsUfbswwzAcqZMmHx8f/Pvvv/j111+hpKSEIUOGYMeOHVi4cCFu375dGjFWCgWXpgYPHgwjIyM5R1O6kpKS0LdvX8ybNw8ikQijR4/GrVu3ULt2bXmHxlRCS5Ysgbe3N1asWCHWgtukSRPs2LFDjpGVX65NTGCgIcDHlGxceBor73AYpsKQOmmKjY1F06ZNAeRP2pucnAwA6NGjR6W7NV5WYmNj4evrCwD4+eef5RxN6QoJCUHLli1x6tQpCAQC7NixA7t27YKqqqq8Q2MqqT179mD79u0YNmyY2GVfKysrPH/+XI6RlV98JQUMtTMDAOy59UbO0TBMxSF10lSjRg3ExMQAAOrUqYOLFy8CyL/8VNWnAynOtm3bkJubC3t7e7Rq1Ure4ZQab29v2Nvb49WrV7CwsMDNmzcxduxYeYfFVHLv379H3bp1Cy0XiUTIzc2VQ0QVwzA7Mygp8HD39Wc8+5Ai73AYpkKQOmnq27cv/P39AeS3mixYsACWlpYYOXIkxowZI/MAK7qcnBxs2bIFQOUdZiArKwsTJkzA6NGjkZWVhW7duuH+/fto2bKlvENjqoBGjRrh+vXrhZYfOXIEzZs3l0NEFYORlgqcmxgDAPbefi3fYBimgpB6nKZly5Zxv7u5ucHMzAxBQUGwtLREz549ZRpcZXD48GF8/PgRpqamJZpTTUFBAQ4ODtzv5c3r168xYMAA3L9/HzweD4sXL8b8+fPLZaxM5bRw4UK4u7vj/fv3EIlEOHbsGMLDw7Fnzx74+fnJO7xyzd3eAmcexeD4w/eY69KQTeDLMN8hddL0NXt7e9jb28silkqpYJiByZMnF5pyRhKqqqoICAiQcVSyce7cOQwbNgyJiYnQ19fH/v370bVrV3mHxVQxvXv3xunTp/HHH39AXV0dCxcuRIsWLXD69Gl06dJF3uGVa60sdNHAWBPPY1Nx+P5bjGvPbtZgmG8pUdL04cMH3LhxA3FxcRCJxG9XrayXoErizp07uHv3LgQCASZMmCDvcGRGKBTizz//xB9//AEiQqtWrXDkyBGYmZnJOzSmimrfvj0uXbok7zAqHB6Ph1FtLDD32GPsCXqDMW1rQUGBzQHJMMWROmny9vbGxIkTwefzoa+vLzbJKo/HY0nTF9avXw8AGDJkSKUZ/TohIQHDhg3DhQsXAAA//fQT1q5dy24CYJgKqrd1dfx9NgzRnzMQ8CIOnRpU7iFRGOZHSJ00LViwAAsXLsS8efNYv5Vv+PDhAw4fPgzgx4YZSE9Ph4WFBYD8/kPq6uqyCK9EgoODMWDAAERHR0NVVRXbtm3DiBEj5BYPU3Xp6uqKfWH7ls+fP5dyNBWbKl8Rbq1q4t/rUdh96w1LmhjmG6ROmjIyMjB48GCWMH3H1q1bkZeXh3bt2qFFixY/VFd8fLyMoioZIsL27dsxbdo05OTkoG7dujh27Bg3XhfDlLV169ZxvyckJGDJkiVwdnbm+lcGBQXhwoULWLBggZwirFiGtzbHjhtRCHzxCVHx6ahlIL8vZwxTnvGIiKTZYPbs2dDT08PcuXNLK6ZSlZKSAm1tbSQnJ0NLS6tU9pGdnY2aNWvi06dPOHz4MAYMGFDiutLT06GhoQEASEtLK/OWpoyMDPz000/Ys2cPgPwhJ7y8vKCtrV2mcTAVS1l8zgr0798fHTt2xNSpU8WWb9y4EZcvX8aJEydKdf9loSyO5xjvYFx5HocxbWthYc9GpbIPhimvJP2MSd1ctHTpUgQGBsLR0RE///wzPDw8xB4McPDgQXz69Ak1atRAnz595B1Oib18+RL29vbYs2cPFBQUsGLFChw9epQlTEy5cuHCBbi4uBRa7uLigsuXL0td36ZNm2BhYQEVFRXY2dnh7t273yx/+PBhNGjQACoqKmjatCnOnj0rtn7UqFHg8Xhij6LilbeR9uYAgMP33yI9O0/O0TBM+VSipOnChQv4+PEjHj9+jIcPH3KPkJCQUgixYiEibpiBKVOmQEnph0d1kIuTJ0/CxsYGjx49gpGREfz9/TFr1iyJ+5EwTFnR19fHyZMnCy0/efIk9PX1parr4MGD8PDwgKenJx48eAArKys4OzsjLi6uyPK3bt3CkCFDMHbsWDx8+BB9+vRBnz598OTJE7FyLi4uiImJ4R4HDhyQKq6y0MHSELUM1JGalYcTIe/lHQ7DlE8kJR0dHfLy8pJ2s3IjOTmZAFBycnKp1H/jxg0CQCoqKvTp06cfri8tLY0AEABKS0uTQYTflpubS3PmzOH22bZtW3r//n2p75epXEr7c/YlLy8vUlRUpB49etCff/5Jf/75J/Xo0YOUlJSkPlfZ2trSlClTuOdCoZBMTU1p6dKlRZYfNGgQde/eXWyZnZ0dTZw4kXvu7u5OvXv3liqOr5XV8dx5/RWZz/GjLmsCSCQSleq+GKY8kfQzJnVLk0AgQNu2bWWYtlUuBa1Mw4YNg4GBgZyjkc7Hjx/RtWtXLF++HADwyy+/4OrVqzA1NZVzZAxTvFGjRuHmzZvQ0tLCsWPHcOzYMWhpaeHGjRsYNWqUxPXk5OTg/v37cHJy4pYpKCjAyckJQUFBRW4TFBQkVh4AnJ2dC5UPCAhAtWrVUL9+ffz0009ISEj4ZizZ2dlISUkRe5SF/i1rQI2viBcf03D7FbvrkGG+JvW1o+nTp2PDhg1ccsD837t373D06FEAshvkU0FBATY2NtzvpeXmzZsYNGgQPnz4AA0NDezatQsDBw4stf0xjCzZ2dnBx8fnh+qIj4+HUCiEkZH4LfdGRkZ4/vx5kdvExsYWWT42NpZ77uLign79+qFWrVqIjIzEb7/9BldXVwQFBUFRUbHIepcuXYrFixf/0OspCW1VZfRtXh0+d6KxJ+g17OtId3mTYSo7qZOmu3fv4sqVK/Dz80Pjxo0LTQ1y7NgxmQVX0WzZsgVCoRCOjo5o1qyZTOpUVVVFcHCwTOoqCv3XB2vmzJnIy8tDw4YNcfToUTRs2LDU9skwsiYUCnHixAmEhYUBABo3boxevXoVm5SUpcGDB3O/N23aFM2aNUOdOnUQEBCAzp07F7nNvHnzxG6sSUlJQc2aNUs9VgAYaW8BnzvRuPjsIz4kZcJUR7VM9sswFYHUSZOOjg769etXGrFUaJmZmdi2bRuAijOVTFpaGsaNG4eDBw8CyD+5//vvv9wQBwxTEURERKB79+549+4d6tevDyC/paZmzZo4c+YM6tSpI1E9BgYGUFRUxMePH8WWf/z4EcbGxkVuY2xsLFV5AKhduzYMDAwQERFRbNIkEAjkNsp+fWNN2NfWR9CrBOy/E42ZzvXlEgfDlEdSJU15eXno2LEjunbt+s2TQlV04MABJCQkwNzcHD179pR3ON8VFhaG/v37IywsDEpKSlizZg2mTp3K7o5jKpxp06ahdu3aCAoKgp6eHoD8AS+HDx+OadOm4cyZMxLVw+fz0bJlS/j7+3NDhYhEIvj7+xcaA6qAvb09/P39MWPGDG7ZpUuXvjmJ+bt375CQkAATExPJXqAcuLcxR9CrBBy4G42fO9eFQEn+LXYMUy5I28NcVVWVXr9+XdIO6nJXGnehiEQisrKyIgC0YsUKmdVLRJSenk7m5uZkbm5O6enpMqnz4MGDpK6uTgCoevXqdPPmTXJwcKDp06fLpH6GKcu759TU1OjRo0eFloeEhJC6urpUdfn6+pJAICBvb2969uwZTZgwgXR0dCg2NpaIiEaMGEFz587lyt+8eZOUlJRo1apVFBYWRp6enqSsrEyPHz8mIqLU1FSaOXMmBQUFUVRUFF2+fJlatGhBlpaWlJWVJXFcZXk8iYhy84TU+u/LZD7Hj47ef1sm+2QYeSq1u+dsbW3x8OFD2WZuFdz169cRGhoKVVVVjB07VqZ1ExHevHmDN2/egKQbvL2QnJwczJgxA25ubkhPT0fHjh3x4MEDtGnTRkbRMkzZEwgESE1NLbQ8LS0NfD5fqrrc3NywatUqLFy4ENbW1ggJCcH58+e5zt7R0dGIiYnhyrdp0wb79+/H9u3bYWVlhSNHjuDEiRNo0qQJAEBRURGPHj1Cr169UK9ePYwdOxYtW7bE9evXy/Uk10qKChjeOn+wy91Bb+QcDcOUI9JmYwcPHqTatWvThg0b6NatWxQaGir2KImNGzeSubk5CQQCsrW1pTt37hRb1svLixtDqOAhEAgk3ldpfGPr378/ARAbm0VWZDVO07t376hNmzZcXfPmzaPc3FxuPWtpYmSpLFtGRowYQY0bN6bbt2+TSCQikUhEQUFB1KRJE3J3dy/1/ZeFsm5pIiL6lJpFlr+dJfM5fnTucUyZ7Zdh5KHUWpoGDx6MqKgoTJs2DW3btoW1tTWaN2/O/ZSWtCPwAoCWlpbY6Lpv3sjvm1B0dDSOHz8OAPj555/lFse3XL16FS1atMCtW7egra2NEydO4O+//y52tPLExESMHDkSurq6UFNTg6urK16+fAkgv+XL0NAQR44c4cpbW1uL9c+4ceMGBAIBMjIySveFMQzyx0arU6cO7O3toaKiAhUVFbRt2xZ169bF+vXr5R1ehWWgIeCmVvE4FIKnH5LlHBHDyJ/Ud89FRUXJNIA1a9Zg/PjxGD16NABg69atOHPmDHbt2lXspMA8Hq/cdETftGkTRCIROnfujMaNG8s7HDFEhBUrVuC3336DSCRCs2bNcPToUdStW/eb240aNQovX77EqVOnoKWlhTlz5qBbt2549uwZlJWV0aFDBwQEBGDAgAFITExEWFgYVFVV8fz5czRo0ACBgYFo1aoV1NTUyuiVMlWZjo4OTp48iYiICG7IgYYNG373fc583xzXBgj/mIrrL+Mx1vseTk5tCyMtFXmHxTByI3XSZG5uLrOdF4zAO2/ePG7Z90bgBfL7Kpibm0MkEqFFixb4+++/i01YsrOzkZ2dzT2X5ci6GRkZ+PfffwGUv2EGkpOTMWrUKG6Gd3d3d2zevPm7iUxBsnTz5k2ur5OPjw9q1qyJEydOYODAgXB0dOSGV7h27RqaN28OY2NjBAQEoEGDBggICICDg0Opvj6G+VrdunVZoiRjyooK2Di0BfpvuYWIuDSM33MPByfYQ5XP7qZjqqYSDTEdGRmJn3/+GU5OTnBycsK0adMQGRkpdT3fGoH3yxF1v1S/fn3s2rULJ0+exL59+yASidCmTRu8e/euyPJLly6FtrY295DlAHE+Pj5ITExErVq10L17d5nV+6MePXoEGxsbnDhxAnw+H9u2bYOXl5dELT8FQxDY2dlxy/T19VG/fn3uW7yDgwOePXuGT58+ITAwEI6OjnB0dERAQAByc3Nx69YtODo6ltbLYxgx/fv356b++dKKFSvYqPYyoK2qjF3uraCrpoxH75LhcSgEItGP3ZTCMBWV1EnThQsX0KhRI9y9exfNmjVDs2bNcOfOHTRu3BiXLl0qjRjF2NvbY+TIkbC2toaDgwOOHTsGQ0NDruXja/PmzUNycjL3ePv2rUzioP9G0gaAqVOnltrIwzweD40aNUKjRo0kGkNp7969aN26NSIiImBubo6bN29iwoQJMh1/qWnTptDT00NgYKBY0hQYGIjg4GDk5uayO/KYMnPt2jV069at0HJXV1dcu3ZNDhFVPmb6atg+0gZ8RQWcexKL1ZfC5R0Sw8iF1Jfn5s6di19++QXLli0rtHzOnDno0qWLxHWVZATerykrK6N58+aIiIgocn1pjawbEBCAJ0+eQF1dHWPGjJF5/QXU1NTw9OnT75bLzs7GjBkzsHXrVgD5813t27cP+vrSzR3VsGFD5OXl4c6dO1zik5CQgPDwcDRq1AhAfiLXvn17nDx5Ek+fPkW7du2gpqaG7OxsbNu2DTY2NlBXV5fylTJMyRQ3tICysnKZTXRbFbSy0MPSfk3x6+FQbLoaiVoGGhjQsoa8w2KYMiV1S1NYWFiRYxGNGTMGz549k6quL0fgLVAwAu+3RtT9klAoxOPHj8t8dN2CViZ3d3fo6OiU6b6/9ubNG7Rr1w5bt24Fj8fD4sWLcebMGakTJgCwtLRE7969MX78eNy4cQOhoaEYPnw4qlevjt69e3PlHB0dceDAAVhbW0NDQwMKCgro0KEDfHx8WH8mpkw1bdqUmwroS76+vlyiz8hG/5Y1MLVjfr+xecce4W7UZzlHxDBlS+qWJkNDQ4SEhMDS0lJseUhICKpVqyZ1AB4eHnB3d4eNjQ1sbW2xbt06pKenc3fTjRw5EtWrV8fSpUsBAH/88Qdat26NunXrIikpCStXrsSbN28wbtw4qfddUlFRUTh58iQAFDu9Qlm5cOEChg4dis+fP0NPTw8+Pj5wcXH5oTq9vLwwffp09OjRAzk5OejQoQPOnj0rNjmzg4MDNzlxAUdHR5w8eZL1Z2LK1IIFC9CvXz9ERkaiU6dOAAB/f38cOHAAhw8flnN0lY9Hl3p4FZ+Gs49jMXHvPRyf3BYWBqxlmakipB0AavHixaSjo0PLli2ja9eu0bVr12jp0qWko6NDf/zxR4kGldqwYQOZmZkRn88nW1tbun37NrfOwcFBbIC6GTNmcGWNjIyoW7du9ODBA4n3JYtB4n799VcCQF27di1xHZJKT0+nRo0aUaNGjcSmUREKhbR48WLi8XgEgGxsbCr09DZM5VLWgzH6+flRmzZtSE1NjfT19aljx44UEBBQJvsuC/IY3PJbMrLzqNeG62Q+x486rrpKSek58g6JYX6IpJ8xHpF0c3MQEdatW4fVq1fjw4cPAABTU1PMmjUL06ZNK/cTvqakpEBbWxvJycnQ0tKSevu0tDTUqFEDycnJ8PPzK/W75tLT06GhocHtW11dHQkJCRgxYgTOnTsHAJg4cSLWr19frqdlYKqWH/2cMeLK4/GMS8lC7003EZOchXZ1DeA1uhWUFUt0QzbDyJ2knzGJ3uGnTp1Cbm4ugPxOwL/88gvevXvH3ZH27t07TJ8+vdwnTLKwb98+JCcno27dunB1dS3z/d+/fx8tW7bEuXPnoKKiAm9vb2zduhWC9++BMWOA//paMUxVkpSUhB07duC3337D58/5/WwePHiA9+/fyzmyyqualgp2uNtAja+IGxHx8Dz19Ifnx2SY8k6ipKlv375ISkoCkD8BZcEUJ5qamtDU1Cy14Mob+mKYgZ9//hkKCmX7rcrLywtt2rTBmzdvUKdOHdy+fRvuLi7Azz8DDRoAFy7k/2SYKuTRo0eoV68eli9fjpUrV3LnqmPHjokNnMvIXmNTbawf3Bw8HrD/TjSWnw9HnlAk77AYptRI9F/f0NAQt2/fBpCfOFSFFqWiXL58GWFhYdDQ0MCoUaPKfP8///wzcnJy0KtXL9y7cgVWR48CdeoA+/YBixcDISGAjQ3w4QMQFQU8fw6EhgJ37wLXrwPFDBjKMBWZh4cHN/WPisr/p/jo1q0bG6epDHRpZIT53RoCALYGRmLA1iC8SUiXc1QMUzokSpomTZqE3r17Q1FRkZv3TVFRschHZVbQyjR69Ogy61fw6tUr7ncej4dly5bhuJcXdOrWBf78E8jKApKTgd9+A6pVA/T1gerVgdq1gYYNAWtrwM4O6NABGD++2P04OjpixowZpf+CvrJo0SJYW1uXaNuAgADweDyuZaEo3t7eEg0JwePxuClnmIolODgYEydOLLS8evXqxc4swMjWuPa18c+Q5tBUUULI2yR0W38dh++9ZZfrmEpHoiEHFi1ahMGDByMiIgK9evWCl5eX3McmKmsRERE4c+YMgLIbZuD06dMYPnw499zPzy9/5GOhEHBxAS5eBPLygDZtgB49AHNzQCAo/mFmViZxl5U2bdogJiYG2traEm+zaNEinDhxAiEhIaUXGFOmBAJBkYNYvnjxAoaGhnKIqGrqZWWKlua6+OVgCO5GfcasI48QEP4Jf/VtAh21woOPMkxFJPE4TQ0aNED9+vXh7u6O/v37c3d0VRWbNm0CEcHV1RX16tUr1X3l5eVh4cKF3NhUAoEAhoaG/x//SFEROHUqv4Vp61Zg3br8lqZ+/QBPT6Bp01KNr7zg8/kSjxzPVF69evXCH3/8gUOHDgHIbzWMjo7GnDlz0L9/fzlHV7VU11HFgfGtse1aJNZcfIEzj2PwIDoRqwdZoU0dA3mHxzA/TKqezEQEHx8fxMTElFY85VJqaip27doFAJg+fXqp7isuLg7Ozs5cwjR9+nSkpKTg7du3hSfc1dYG5szJ77+0bRvw6BHwA5fY8vLyMHXqVGhra8PAwAALFizgmtf37t0LGxsbaGpqwtjYGEOHDuVuCAD+f6nM398fNjY2UFNTQ5s2bRAeLj5H1bJly2BkZARNTU2MHTsWWVlZ3LonT55AQUEBnz59AgB8/vwZCgoKGDx4MFdmyZIlaNeundg+v7w85+3tDTMzM6ipqaFv375ISEgQW7d48WKEhoaCx+OBx+PB29ubWx8fH4++fftCTU0NlpaWOHXqVImPJVN2Vq9ejbS0NFSrVg2ZmZlwcHBA3bp1oampib/++kve4VU5igo8THasi2OT26CWgTpikrMwbMcdLD0Xhpw81kmcqeCkHQCqUaNGFBQUJO1m5UZJBonbuHEjAaD69euTUCgstdhu3bpF1atXJwCkrq5Ovr6+0lUgFOY/SsDBwYE0NDRo+vTp9Pz5c9q3bx+pqanR9u3biYho586ddPbsWYqMjKSgoCCyt7cnV1dXbvurV68SALKzs6OAgAB6+vQptW/fntq0acOVOXjwIAkEAtqxYwc9f/6c5s+fT5qammRlZUVERCKRiAwMDOjw4cNERHTixAkyMDAgY2Njrg4nJyeaP3++2D4TExOJiOj27dukoKBAy5cvp/DwcFq/fj3p6OiQtrY2ERFlZGTQr7/+So0bN6aYmBiKiYmhjIwMIiICQDVq1KD9+/fTy5cvadq0aaShoUEJCQklOp5VnTwGY7x+/Tpt2rSJli9fTpcuXSqz/ZaF8ja4paTSsnJp7tFQMp/jR+Zz/Kj7P9fo5cdUeYfFMIVI+hmTOmk6deoUtWvXjh4/flzi4ORJ2pOPUCikevXqEQDauHFjqcQkEonon3/+ISUlJQJADRo0oKdPn5bKvorj4OBADRs2JJFIxC2bM2cONWzYsMjywcHBBIBSU/NPgAUJzOXLl7kyZ86cIQCUmZlJRET29vY0efJksXrs7Oy4pImIqF+/fjRlyhQiyh/9fdasWaSrq0thYWGUk5NDampqdPHiRbF9FiRNQ4YMoW7duonV7+bmxiVNRESenp5i+ysAgH7//XfueVpaGgGgc+fOFfn6mW+rqP/ky6uKfjzPPY4hq8UXyHyOH9X//SztuvGKsnLz5B0Ww3Ak/YxJPdDQyJEjcffuXVhZWUFVVRV6enpij8rm4sWLePHiBbS0tDBy5EiZ15+WloZhw4Zh2rRpyMvLw6BBg3D37l1uotHMzEy0atUKrVq1QmZmpsz3/6XWrVuLDSdhb2+Ply9fQigU4v79++jZsyfMzMygqanJTcobHR0tVkezZs243wsmUS64jBcWFgY7Ozux8l9PzOzg4ICAgAAAQGBgIDp16oQOHTogICAAwcHByM3NRdu2bYuMX5L6v+XL2NXV1aGlpSV2CZIpf0QiEXbt2oUePXqgSZMmaNq0KXr16oU9e/awO7fKEZcmxrgwowPa1TVAVq4Ii08/Q4cVV7Hj+iukZ+fJOzyGkZjUE/auW7euFMIovwqGGRgzZozMB/J8/vw5+vfvj2fPnkFJSQkrV64sNLK6SCTCvXv3uN/lISsrC87OznB2doaPjw8MDQ0RHR0NZ2dn5OTkiJX9clLfgtchTdwFQx+8fPkSz549Q7t27fD8+XMEBAQgMTGR6y9VGr6MHciPX17HnPk+IkKvXr1w9uxZWFlZoWnTpiAihIWFYdSoUTh27BgbRqIcMdJSwZ4xtvC58wabrkYiNiULS86EYePVCLjbW2BUGwvoqrO77JjyTeqkyd3dvTTiKJdevHiBc+fOgcfjyXyYgSNHjmD06NFIS0uDiYkJDh06xHVwlpc7d+6IPb99+zYsLS3x/PlzJCQkYNmyZahZsyYAcImcNBo2bIg7d+6ItdgVDJpaoGnTptDV1cWSJUtgbW0NDQ0NODo6Yvny5UhMTPz/HYTfqP/r1/AlPp8PoVAodexM+ePt7Y1r167B398fHTt2FFt35coV9OnTB3v27CmVFmKmZBQUeBhhb4FBrWri+IP32BoYidcJGVjv/xL/Xn+FobZmGNe+Noy1Vb5fGcPIQYnmAYmMjMTvv/+OIUOGcJcvzp07h6dPn8o0OHnbuHEjAKBHjx6oU6eOTOrMzc3Fr7/+ioEDByItLQ2Ojo54+PCh3BMmIP9Sm4eHB8LDw3HgwAFs2LAB06dPh5mZGfh8PjZs2IBXr17h1KlT+PPPP6Wuf/r06di1axe8vLzw4sULeHp6FnrP8Hg8dOjQAT4+PlyC1KxZM2RnZ8Pf35+7LFiUadOm4fz581i1ahVevnyJjRs34vz582JlLCwsEBUVhZCQEMTHxyM7O1vq18GUDwcOHMBvv/1WKGECgE6dOmHu3Lnw8fGRQ2TM9wiUFDHY1gz+vzpi49DmaGSihYwcIXbciEL7FVcw9+gjRMWzUcWZ8kfqpCkwMBBNmzbFnTt3cOzYMaSlpQEAQkND4enpKfMA5SU5ORleXl4A8v8Zy0JMTAw6deqENWvWAADmzJmDS5cuwcjISCb1/6iRI0ciMzMTtra2mDJlCqZPn44JEybA0NAQ3t7eOHz4MBo1aoRly5Zh1apVUtfv5uaGBQsWYPbs2WjZsiXevHmDn376qVA5BwcHCIVCLmlSUFBAhw4dwOPxiu3PBOT3yfr333+xfv16WFlZ4eLFi/j999/FyvTv3x8uLi7o2LEjDA0NceDAAalfB1M+PHr0CC4uLsWud3V1RWhoaBlGxEhLUYGHHs1McWZaO3iPbgVbCz3kCgm+wW/ReXUApux/gMAXn5DL5rNjygkeSdlb0t7eHgMHDoSHhwc0NTURGhqK2rVr4+7du+jXrx/evXtXWrHKREpKCrS1tZGcnPzNqVDWr1+PGTNmoGHDhnj69OkPz7cXGBgINzc3fPz4EVpaWti9ezf69Onz3e3S09O5gUTT0tKgrq7+Q3EwTFmQ9HP2I/h8Pt68ecPdcPC1Dx8+oFatWpWiNbEsjmd5ce/1Z2wOiMSV5/+/CUNXTRkuTYzRo5kp7GrpQUmxbCdLZyo/ST9jUvdpevz4Mfbv319oebVq1RAfHy9tdeWSSCTChg0bAOS3Mv1IwkREWLVqFebNmwehUIimTZvi6NGjsLS0lFW4DFMlCYVCKCkVfwpTVFREXh67M6uisbHQw65Renj2IQX7777BucexSEjPwYG7b3Hg7lsYaPDh0sQY3ZuawraWHhQVquYE8ox8SJ006ejoICYmBrVq1RJb/vDhQ1SvXl1mgcnTuXPnEBkZCR0dHYwYMaLE9SQnJ2P06NE4fvw4AGDEiBHYunWr1Hd/GRiw6QcY5mtEhFGjRkEgEBS5vjK0MFVljUy1sKRPUyzq2Rh3oj7D79EHnH8Si/i0HOy7HY19t6NhqClAtybG6GFlipZmulBgCRRTyqROmgYPHow5c+bg8OHD3C3ZN2/exMyZMyvNXSoFwwyMGzeuxJfDHj9+jP79++Ply5fg8/lYv349Jk6cKHWrlbq6OjetCMMw/yfJnbyV5ZxUlSkpKqBtXQO0rWuAP3o3wa3IBJz5L4H6lJqN3UFvsDvoDXTVlNG6tj5a19aHfR19WFbT+OFuFQzzNan7NOXk5GDKlCnw9vbmmseFQiGGDh0Kb29vKCoqllasMvG965ZhYWFo1KgRFBQUEBkZCQsLC6n3sW/fPkyYMAGZmZkwMzPD4cOHYWtrK4PoGaZiqEp9cMoCO56F5eSJcDMiHqcffcClpx+R+tUgmQYafNjV1of9f0lUbQN1lkQxxZL0MyZ10lTg7du3ePz4MdLS0tC8efMK00fnewdm8uTJ2LJlC/r06cNdVpNUdnY2PDw8sHnzZgBA165d4ePjwy6vMVUO+ycvW+x4fluuUIRH75IQFJmAoFcJuPc6EdlfTQ5spCXgWqKaVtdGPSNN8JVYh3Imn8yTJpFIhJUrV+LUqVPIyclB586d4enpCVVVVZkFXRa+dWCSkpJQvXp1ZGRk4MqVK0WO/1Kc6OhoDBw4EHfv3gUALFy4EAsXLvzhlrfMzEy4uroCyO9rVdGON1M1sX/yssWOp3Sy84QIfZv8XxIVjwfRScj5KoniKyqgvrEmmlTXQpPq2mhiqo36xppQUS7fV0uY0iHzu+f++usvLFq0CE5OTlBVVcX69esRFxeHXbt2ySTg8mDXrl3IyMhAkyZNvjny9NcuXbqEIUOGICEhAbq6uti3bx+6desmk5hEIhECAwO53xmGYZhvEygpwraWHmxr6WE6LJGVK8SD6ETcjkzA/ehEPHmfguTMXDx+n4zH75MBvAUAKCnwYGmkiSam+YlUPSNN1K2mAQMNPru0xwCQImnas2cPNm/ejIkTJwIALl++jO7du2PHjh1QUKj4TZxCoZAbAVzSYQZEIhH+/vtvLFy4EESEli1b4siRIyXqB8UwDMOUDhVlRbSpY4A2dfK7ShAR3iVm4sl/SdPj98l48j4ZiRm5CItJQVhMCg7f//+Yg1oqSqhTTQN1DTVQp5oG6hhqoG41DdTUVWVjRlUxEidN0dHRYq0nTk5O4PF4+PDhA2rUqFEqwZWlM2fOICoqCnp6ehg2bNh3yycmJmLEiBE4c+YMAGD8+PH4559/oKLC5kxiGIYpz3g8HmrqqaGmnhpcm+YPjkpE+JCchSf/JVBPP6QgIi4NbxMzkJKVh4fRSXgYnSRWD19RARYGaqhloI6aumr/1amKmrpqqKGrBlU+u9RX2UicNOXl5RVKCJSVlZGbmyvzoORh/fr1APKTn++No/TgwQP0798fr1+/hoqKCjZv3ozRo0eXRZgMwzBMKeDxeKiuo4rqOqpwbmzMLc/KFSIqPh2Rn9IQGZeOiE9piIxLw6v4NGTlivDiYxpefEwrsk4DDQGXRBX8NNZWgYm2Koy1VKClqsQu+1UwEidNRQ0kl5WVhUmTJomNZXTs2DHZRlgGnjx5gitXrkBBQQGTJ0/+ZtmdO3diypQpyM7ORu3atXH06FFYW1uXTaAMwzBMmVJRVkRDEy00NBHvHCwSEd4nZSLiUxqiEzLw9nMGoj9n4G1iJt59zkBqdh7i07IRn5ZdqIWqgKqyIky0VWCkpZL/U1uFe15NUwBDTQEMNASsc3o5InHSVNRAcsOHD5dpMPJSMGVK3759YWZmVmSZzMxMTJ06lev43rNnT+zevRu6urplFifDMAxTPigo/P8S39eICMmZuXj7ORNvE/MTqreJGXiXmInY5CzEpmQhKSMXmblCvIpPx6v49G/uS1OgxCVQBpp8GGgIYKghgMF/y/TUlaGrxoeeOh9aKspsZPRSJHHS5OXlVZpxyM3nz5+xd+9eAPkdwIvy6tUrDBgwAA8fPoSCggL+/PNPzJ07t8w6wEs77QrDMAwjPzweDzpqfOio8dG0hnaRZTJzhPiYkoWY5KyvfuYnVp9SsxGfloMcoQip2XlIzc77bnIFAAo8QFeND111PvTU+NBVV4aeOh+6anzoqClDW1UZ2qr8/34qc8vU+IrsUqEEpJ5GpbLZuXMnMjMzYW1tjfbt2xda7+fnhxEjRiApKQkGBgY4cOAAnJycyiw+dXV1pKd//4PCMAzDVByqfEVYGKjDwqD4qbqICCmZefj032W++LTs/5KpbMSn5uBTWjYS0rLxOSMHiem5SMvOg4iAhPQcJKTnSBWPkgIPOmrK0FJVhpaKMjRVlKClogwtVSVoqihDU6AETZX/fhf7qQR1gRI0BEoQKClU+sSrSidNeXl5xQ4zIBQK4enpib/++gsA0Lp1axw+fLhS3CnIMAzDlH88Hg/aasrQVlNG3Woa3y2fnSdEUkYuPqfnIDE9B4kZuf8lVDn4nJ6D5MxcJGXk/8x/5CE5Mwe5QkKeiBCfloP4NOmSrS8pKfC4BEpDoAR1gSI0VJShIVCEOj8/uVLjK/7/J18JagJFqPEVocZX4p6rKucvU+Urgq9YvhKxKp00nTp1CtHR0TAwMMCQIUO45Z8+fcLQoUNx+fJlAMDPP/+MVatWgc/nyytUhmEYhvkmgZIijLQUYaQl+dA3RITMXOF/CVV+MpWSmYvUrDykZuUi5b+f+c/zkPLVsvTsPGTkCAEAeSLiEjJZUeABanwlqBQkUsr5yVTB7yrcQ4FbV2iZsiLaWhpAS0X5h+Op0knTP//8AwCYMGECN5zCnTt3MGDAALx79w5qamrYsWOHWEJV1rKystC/f38AwNGjR9k4UAzDMIzM8Hg8qPGVoMZXgol2yabpEooI6Tn5CVR6dt5/yZQQadm5SMsW5i/PyUNGtlD8Z46QS7q+XJ6VK0SuMH+GNxEBadl5SPtqQmZpXfqlA0uafsTjx48RGBgIRUVF/PTTTyAibNmyBTNmzEBubi7q1auHY8eOoXHjxnKNUygU4uzZs9zvDMMwDFOeKCrw8vs/ySApKZArFCEzV4jMnPxHRo7w/89zhcj4L7nKyv1/uaw8IbJyvliWK/yvjBCaMoqtyiZN27ZtAwD0798furq6GDFiBHx8fAAAAwYMwM6dO9nEmAzDMAwjB8qKClBWVJBpIiYL5WLSnE2bNsHCwgIqKiqws7PD3bt3v1n+8OHDaNCgAVRUVNC0aVOuJUYahw4dApA/NpOdnR18fHygqKiI1atX49ChQyxhYhiGYRhGjNyTpoMHD8LDwwOenp548OABrKys4OzsjLi4uCLL37p1C0OGDMHYsWPx8OFD9OnTB3369MGTJ0+k2m/BiN4TJkzA06dPYWxsjKtXr8LDw6Nc9dRnGIZhGKZ84BERyTMAOzs7tGrVirv1XyQSoWbNmvj5558xd+7cQuXd3NyQnp4OPz8/blnr1q1hbW2NrVu3fnd/KSkp0NYWH2ysQ4cOOHjwIIyNjYvZSn7S09OhoZF/q2laWprYlDUMU14VfM6Sk5NZq60MsOPJMKVL0s+YXFuacnJycP/+fbHBIhUUFODk5ISgoKAitwkKCio0uKSzs3Ox5b9n5syZuHz5crlMmBiGYRiGKT/k2hE8Pj4eQqEQRkZGYsuNjIzw/PnzIreJjY0tsnxsbGyR5bOzs5Gdnc09T05OBgAoKSnBy8sLvXr1QmZmJjIzM3/kpZSaL0cDT0lJYXfQMRVCSkoKgPwxYJgfV3AcC44rwzCyJek5q9LfPbd06VIsXry40PK8vDyMGDFCDhGVnKmpqbxDYBippKamFroczkgvNTUVAFCzZk05R8Iwldv3zllyTZoMDAygqKiIjx8/ii3/+PFjsZfLjI2NpSo/b948eHh4cM+TkpJgbm6O6OjoCnMyT0lJQc2aNfH27dsK05+BxVw2ymvMRITU1FSW6MuIqakp3r59C01NzQp/o0p5fc+WFHs95Zukr0fSc5ZckyY+n4+WLVvC398fffr0AZDfEdzf3x9Tp04tcht7e3v4+/tjxowZ3LJLly7B3t6+yPICgQACgaDQcm1t7Qr3htDS0mIxlwEWs2xUlC8lFYGCgkKlm/eyPL5nfwR7PeWbJK9HknOW3C/PeXh4wN3dHTY2NrC1tcW6deuQnp6O0aNHAwBGjhyJ6tWrY+nSpQCA6dOnw8HBAatXr0b37t3h6+uLe/fuYfv27fJ8GQzDMAzDVHJyT5rc3Nzw6dMnLFy4ELGxsbC2tsb58+e5zt7R0dFQUPj/TX5t2rTB/v378fvvv+O3336DpaUlTpw4gSZNmsjrJTAMwzAMUwXIPWkCgKlTpxZ7OS4gIKDQsoEDB2LgwIEl2pdAIICnp2eRl+zKKxZz2WAxM0zpq2zvWfZ6yjdZvx65D27JMAzDMAxTEch9GhWGYRiGYZiKgCVNDMMwDMMwEmBJE8MwDMMwjASqXNK0adMmWFhYQEVFBXZ2drh79668Q+IsWrQIPB5P7NGgQQNufVZWFqZMmQJ9fX1oaGigf//+hQb6LG3Xrl1Dz549YWpqCh6PhxMnToitJyIsXLgQJiYmUFVVhZOTE16+fClW5vPnzxg2bBi0tLSgo6ODsWPHIi0tTS7xjho1qtAxd3FxkVu8QP4o9q1atYKmpiaqVauGPn36IDw8XKyMJO+F6OhodO/eHWpqaqhWrRpmzZqFvLy8UoubYb7le5/FikaSz2lFsmXLFjRr1owbz8je3h7nzp2Td1gysWzZMvB4PLHxHUuqSiVNBw8ehIeHBzw9PfHgwQNYWVnB2dkZcXFx8g6N07hxY8TExHCPGzducOt++eUXnD59GocPH0ZgYCA+fPiAfv36lWl86enpsLKywqZNm4pcv2LFCvzzzz/YunUr7ty5A3V1dTg7OyMrK4srM2zYMDx9+hSXLl2Cn58frl27hgkTJsglXgBwcXERO+YHDhwQW1+W8QJAYGAgpkyZgtu3b+PSpUvIzc1F165dxeYh/N57QSgUonv37sjJycGtW7ewe/dueHt7Y+HChaUWN8N8iySfxYpEks9pRVKjRg0sW7YM9+/fx71799CpUyf07t0bT58+lXdoPyQ4OBjbtm1Ds2bNZFMhVSG2trY0ZcoU7rlQKCRTU1NaunSpHKP6P09PT7KysipyXVJSEikrK9Phw4e5ZWFhYQSAgoKCyihCcQDo+PHj3HORSETGxsa0cuVKbllSUhIJBAI6cOAAERE9e/aMAFBwcDBX5ty5c8Tj8ej9+/dlGi8Rkbu7O/Xu3bvYbeQZb4G4uDgCQIGBgUQk2Xvh7NmzpKCgQLGxsVyZLVu2kJaWFmVnZ5dJ3AxTnKI+ixXd15/TykBXV5d27Ngh7zBKLDU1lSwtLenSpUvk4OBA06dP/+E6q0xLU05ODu7fvw8nJydumYKCApycnBAUFCTHyMS9fPkSpqamqF27NoYNG4bo6GgAwP3795GbmysWf4MGDWBmZlZu4o+KikJsbKxYjNra2rCzs+NiDAoKgo6ODmxsbLgyTk5OUFBQwJ07d8o8ZiB/LLBq1aqhfv36+Omnn5CQkMCtKw/xJicnAwD09PQASPZeCAoKQtOmTblBYgHA2dkZKSkpFf6bI8OUR19/TisyoVAIX19fpKenFztFWUUwZcoUdO/eXexc+aPKxeCWZSE+Ph5CoVDsnwgAGBkZ4fnz53KKSpydnR28vb1Rv359xMTEYPHixWjfvj2ePHmC2NhY8Pl86OjoiG1jZGSE2NhY+QT8lYI4ijrGBetiY2NRrVo1sfVKSkrQ09OTy+twcXFBv379UKtWLURGRuK3336Dq6srgoKCoKioKPd4RSIRZsyYgbZt23Kj3kvyXoiNjS3y71CwjmEY2Snqc1oRPX78GPb29sjKyoKGhgaOHz+ORo0ayTusEvH19cWDBw8QHBws03qrTNJUEbi6unK/N2vWDHZ2djA3N8ehQ4egqqoqx8gqr8GDB3O/N23aFM2aNUOdOnUQEBCAzp07yzGyfFOmTMGTJ0/E+rYxDFO+VJbPaf369RESEoLk5GQcOXIE7u7uCAwMrHCJ09u3bzF9+nRcunQJKioqMq27ylyeMzAwgKKiYqE7jD5+/AhjY2M5RfVtOjo6qFevHiIiImBsbIycnBwkJSWJlSlP8RfE8a1jbGxsXKjjfV5eHj5//lwuXkft2rVhYGCAiIgIAPKNd+rUqfDz88PVq1fFZriX5L1gbGxc5N+hYB3DMLJR3Oe0IuLz+ahbty5atmyJpUuXwsrKCuvXr5d3WFK7f/8+4uLi0KJFCygpKUFJSQmBgYH4559/oKSkBKFQWOK6q0zSxOfz0bJlS/j7+3PLRCIR/P39y+0127S0NERGRsLExAQtW7aEsrKyWPzh4eGIjo4uN/HXqlULxsbGYjGmpKTgzp07XIz29vZISkrC/fv3uTJXrlyBSCSCnZ1dmcf8tXfv3iEhIQEmJiYA5BMvEWHq1Kk4fvw4rly5glq1aomtl+S9YG9vj8ePH4slfJcuXYKWllaF+9bIMOXR9z6nlYFIJEJ2dra8w5Ba586d8fjxY4SEhHAPGxsbDBs2DCEhIVBUVCx55T/clbwC8fX1JYFAQN7e3vTs2TOaMGEC6ejoiN1hJE+//vorBQQEUFRUFN28eZOcnJzIwMCA4uLiiIho0qRJZGZmRleuXKF79+6Rvb092dvbl2mMqamp9PDhQ3r48CEBoDVr1tDDhw/pzZs3RES0bNky0tHRoZMnT9KjR4+od+/eVKtWLcrMzOTqcHFxoebNm9OdO3foxo0bZGlpSUOGDCnzeFNTU2nmzJkUFBREUVFRdPnyZWrRogVZWlpSVlaWXOIlIvrpp59IW1ubAgICKCYmhntkZGRwZb73XsjLy6MmTZpQ165dKSQkhM6fP0+GhoY0b968UoubYb7le+eOikaSz2lFMnfuXAoMDKSoqCh69OgRzZ07l3g8Hl28eFHeocmErO6eq1JJExHRhg0byMzMjPh8Ptna2tLt27flHRLHzc2NTExMiM/nU/Xq1cnNzY0iIiK49ZmZmTR58mTS1dUlNTU16tu3L8XExJRpjFevXiUAhR7u7u5ElD/swIIFC8jIyIgEAgF17tyZwsPDxepISEigIUOGkIaGBmlpadHo0aMpNTW1zOPNyMigrl27kqGhISkrK5O5uTmNHz++UBJdlvESUZHxAiAvLy+ujCTvhdevX5OrqyupqqqSgYEB/frrr5Sbm1tqcTPMt3zv3FHRSPI5rUjGjBlD5ubmxOfzydDQkDp37lxpEiYi2SVNPCKikrdTMQzDMAzDVA1Vpk8TwzAMwzDMj2BJE8MwDMMwjARY0sQwDMMwDCMBljQxDMMwDMNIgCVNDMMwDMMwEmBJE8MwDMMwjARY0sQwDMMwDCMBljQxDMMwDMNIgCVNP2DRokWwtrb+ZpnXr1+Dx+MhJCSkTGKShIWFBdatWyfvMEqdt7c3dHR0Sq3+ESNG4O+//y61+iUlz/dYTk4OLCwscO/evTLfN8MUx9HRETNmzCjRtpKc10eNGoU+ffp8s0xAQAB4PF6hibUlsXPnTnTt2lWq/ZUXgwcPxurVq6Xe7tq1a+jZsydMTU3B4/Fw4sQJqbbPysrCqFGj0LRpUygpKRV5vI4dO4YuXbrA0NAQWlpasLe3x4ULF6TaD0ua/hMUFARFRUV0795d4m1mzpwpNmlqeXtjF5c0BAcHY8KECSWuV9qk60dOHj/Czc0NL1684J5LcjKUVGhoKM6ePYtp06aJLX/69CkGDRoEQ0NDCAQC1KtXDwsXLkRGRoZYOQsLC/B4PPB4PKirq6NFixY4fPiwVDGMHj0av//++w+/lgIlOVHx+XzMnDkTc+bMkVkcTMU3atQo7v3N4/Ggr68PFxcXPHr0SN6hfdfX53VJ/EiS9rWsrCwsWLAAnp6eMqmvwI+e/3bv3o127dp9t9zvv/+Ov/76C8nJyVLVn56eDisrK2zatKlE8QmFQqiqqmLatGlwcnIqssy1a9fQpUsXnD17Fvfv30fHjh3Rs2dPPHz4UOL9sKTpPzt37sTPP/+Ma9eu4cOHD98sS0TIy8uDhoYG9PX1yyhC2TE0NISampq8wyh1qqqqqFatWqnUvWHDBgwcOBAaGhrcstu3b8POzg45OTk4c+YMXrx4gb/++gve3t7o0qULcnJyxOr4448/EBMTg4cPH6JVq1Zwc3PDrVu3JNq/UCiEn58fevXqJdPXVRLDhg3DjRs38PTpU3mHwpQjLi4uiImJQUxMDPz9/aGkpIQePXrIO6zvkvd5/ciRI9DS0kLbtm3lFkNRTp48KdH5pkmTJqhTpw727dsnVf2urq5YsmQJ+vbtW+T67OxszJw5E9WrV4e6ujrs7OwQEBDArVdXV8eWLVswfvx4GBsbF1nHunXrMHv2bLRq1QqWlpb4+++/YWlpidOnT0scJ0uaAKSlpeHgwYP46aef0L17d3h7e4utL2gpOXfuHFq2bAmBQIAbN26IZe6LFi3C7t27cfLkSe7b1Zd/0FevXqFjx45QU1ODlZUVgoKCuHUFLUJ+fn6oX78+1NTUMGDAAGRkZGD37t2wsLCArq4upk2bBqFQyG33rTdRQEAARo8ejeTkZC6eRYsWASjcUpSUlISJEyfCyMgIKioqaNKkCfz8/CQ+fjweDzt27EDfvn2hpqYGS0tLnDp1CkD+paOOHTsCAHR1dcHj8TBq1CgAgEgkwtKlS1GrVi2oqqrCysoKR44cKXTc/f39YWNjAzU1NbRp0wbh4eFcmdDQUHTs2BGamprQ0tJCy5YtuUtFX7a0eXt7Y/HixQgNDeWOh7e3N8aMGVPoRJ6bm4tq1aph586dRb5eoVCII0eOoGfPntwyIsLYsWPRsGFDHDt2DLa2tjA3N8fAgQNx+vRpBAUFYe3atWL1aGpqwtjYGPXq1cOmTZugqqrKfXg3b94MS0tLqKiowMjICAMGDBDb9tatW1BWVkarVq24Zc+fP0ebNm24v2FgYCAXW926dbFq1SqxOkJCQsDj8RAREQELCwsAQN++fcHj8bjnQP7JskWLFlBRUUHt2rWxePFi5OXlcet1dXXRtm1b+Pr6Fnm8mKpJIBDA2NgYxsbGsLa2xty5c/H27Vt8+vSJKzNnzhzUq1cPampqqF27NhYsWIDc3FxufcE5du/evbCwsIC2tjYGDx6M1NRUrkx6ejpGjhwJDQ0NmJiYFLo0tHHjRjRp0oR7fuLECfB4PGzdupVb5uTkxLXaft0iIxQK4eHhAR0dHejr62P27Nn4csrWUaNGITAwEOvXr+fOLa9fv+bW379/v9jzV1F8fX3Fzi1fWrx4MXdpadKkSdwXsT179kBfXx/Z2dli5fv06YMRI0YUe/4D8s//48aN4+rt1KkTQkNDxerJysrCxYsXuaTpe+ennj17yvx8MHXqVAQFBcHX1xePHj3CwIED4eLigpcvX5a4TpFIhNTUVOjp6Um+0Q9P+VsJ7Ny5k2xsbIiI6PTp01SnTh0SiUTc+oLZuZs1a0YXL16kiIgISkhIIE9PT7KysiIiotTUVBo0aBC5uLhQTEwMxcTEUHZ2NkVFRREAatCgAfn5+VF4eDgNGDCAzM3NuRnnvby8SFlZmbp06UIPHjygwMBA0tfXp65du9KgQYPo6dOndPr0aeLz+eTr68vFNW7cOGrTpg1du3aNIiIiaOXKlSQQCOjFixeUnZ1N69atIy0tLS6e1NRUIiIyNzentWvXEhGRUCik1q1bU+PGjenixYsUGRlJp0+fprNnzxZ7vL7cnih/tu8aNWrQ/v376eXLlzRt2jTS0NCghIQEysvLo6NHjxIACg8Pp5iYGEpKSiIioiVLllCDBg3o/PnzFBkZSV5eXiQQCCggIEDsuNvZ2VFAQAA9ffqU2rdvT23atOH23bhxYxo+fDiFhYXRixcv6NChQxQSEsIdV21tbSIiysjIoF9//ZUaN27MHY+MjAy6efMmKSoq0ocPH7g6jx07Rurq6tzx+tqDBw8IAMXGxhZatn///iK36dKlC/deKeoYEhFpa2uTh4cHBQcHk6KiIu3fv59ev35NDx48oPXr14uVnTlzJk2YMIGIiHuP1ahRg44cOULPnj2jcePGkaamJsXHxxMR0V9//UWNGjUSq2PatGnUoUMHIiKKi4vjZmiPiYmhuLg4IiK6du0aaWlpkbe3N0VGRtLFixfJwsKCFi1aJFbXnDlzyMHBocjXzlQ97u7u1Lt3b+55amoqTZw4kerWrUtCoZBb/ueff9LNmzcpKiqKTp06RUZGRrR8+XJuvaenJ2loaFC/fv3o8ePHdO3aNTI2NqbffvuNK/PTTz+RmZkZXb58mR49ekQ9evQgTU1Nbkb7R48eEY/H497TM2bMIAMDA3JzcyMiopycHFJTU6NLly5x+/zys7p8+XLS1dWlo0eP0rNnz2js2LGkqanJvb6kpCSyt7en8ePHc+eWvLw8ic5fRdHW1hY7zxccTw0NDXJzc6MnT56Qn58fGRoacschIyODtLW16dChQ9w2Hz9+JCUlJbpy5Uqx5z8iIicnJ+rZsycFBwfTixcv6NdffyV9fX1KSEjg6vLz86N69eoREUl0fjp37hzx+XzKysr65mstDgA6fvw49/zNmzekqKhI79+/FyvXuXNnmjdvXqHtv37/Fafgb/vx40fJY5O4ZCXWpk0bWrduHRER5ebmkoGBAV29epVbX/DmP3HihNh2X3+4ivpDFfxD27FjB7fs6dOnBIDCwsKIKP+fOwCKiIjgykycOJHU1NTE/nE7OzvTxIkTiUiyN9GXScOXvvyHfeHCBVJQUKDw8PBvHKHityfKf4P//vvv3PO0tDQCQOfOnSOi/x+/xMRErkxWVhapqanRrVu3xOoeO3YsDRkyRGy7y5cvc+vPnDlDACgzM5OIiDQ1Ncnb27vIOL9+/V//vQo0atRI7ETds2dPGjVqVLGv//jx46SoqCiWWPv6+hIAevjwYZHbTJs2jVRVVbnnXx7D7Oxs+vvvvwkA+fn50dGjR0lLS4tSUlKKjcHS0pL8/PyI6P/vsWXLlnHrc3NzqUaNGtzrev/+PSkqKtKdO3eIKP8fhYGBgdix+/pERZT/fvr777/Flu3du5dMTEzElq1fv54sLCyKjZepWtzd3UlRUZHU1dVJXV2dAJCJiQndv3//m9utXLmSWrZsyT339PQkNTU1sc/CrFmzyM7OjojykzE+ny+WLCQkJJCqqiqXNIlEItLX16fDhw8TEZG1tTUtXbqUjI2NiYjoxo0bpKysTOnp6dw+vzxPmJiY0IoVK7jnBZ+tL8/1Dg4O3P4KSHL++lpiYiIBoGvXroktd3d3Jz09PS5GIqItW7aQhoYGl4T+9NNP5Orqyq1fvXo11a5dmztPFXX+u379OmlpaRVKburUqUPbtm3jno8fP55mzpxJRCTR+Sk0NJQA0OvXr4st8y1fn4v8/PwIAPd+KngoKSnRoEGDCm0vSdLk4+MjlixLSqnE7VqVRHh4OO7evYvjx48DAJSUlODm5oadO3fC0dFRrKyNjU2J99OsWTPudxMTEwBAXFwcGjRoAABQU1NDnTp1uDJGRkawsLAQ6zNjZGSEuLg4AMDjx48hFApRr149sf1kZ2dLdT0+JCQENWrUKFSPtL58ferq6tDS0uJiLUpERAQyMjLQpUsXseU5OTlo3rx5sXV/eezMzMzg4eGBcePGYe/evXBycsLAgQPFjqMkxo0bh+3bt2P27Nn4+PEjzp07hytXrhRbPjMzEwKBADwer9A6+qLZ/nvmzJmD33//HVlZWdDQ0MCyZcvQvXt3pKamwtzcHLVr14aLiwtcXFy4S58AEBYWhg8fPqBz585i9dnb23O/KykpwcbGBmFhYQAAU1NTdO/eHbt27YKtrS1Onz6N7OxsDBw48JsxhoaG4ubNm/jrr7+4ZUKhEFlZWcjIyOBiUlVVLdTZnanaOnbsiC1btgAAEhMTsXnzZri6uuLu3bswNzcHABw8eBD//PMPIiMjkZaWhry8PGhpaYnVY2FhAU1NTe65iYkJd26JjIxETk4O7OzsuPV6enqoX78+95zH46FDhw4ICAiAk5MTnj17hsmTJ2PFihV4/vw5AgMD0apVqyL7eSYnJyMmJkas/oLPlqSf9W+dv76WmZkJAFBRUSm0zsrKSixGe3t7pKWl4e3btzA3N8f48ePRqlUrvH//HtWrV4e3tzfXIb84oaGhSEtLK/Q/IzMzE5GRkQDyz2mnT5/GoUOHAABdunT55vkJyD8fAJDZOSEtLQ2Kioq4f/8+FBUVxdZ9+T9SUr6+vhg3bhwOHz5cbKfx4lT5pGnnzp3Iy8uDqakpt4yIIBAIsHHjRmhra3PL1dXVS7wfZWVl7veCN7FIJCpyfUGZopYVbCOrN1HBm/tHfSvWoqSlpQEAzpw5g+rVq4utEwgExdb99bFbtGgRhg4dijNnzuDcuXPw9PSEr69vsZ0JizJy5EjMnTsXQUFBuHXrFmrVqoX27dsXW97AwAAZGRnIyckBn88HAC7pDAsLK5T0FSz/OjGdNWsWRo0aBQ0NDRgZGXGvTVNTEw8ePEBAQAAuXryIhQsXYtGiRQgODoaOjg5OnTqFLl26FHli/ZZx48ZhxIgRWLt2Lby8vODm5vbdGwLS0tKwePFi9OvXr9C6L/f/+fNnGBoaShUPU7mpq6ujbt263PMdO3ZAW1sb//77L5YsWYKgoCAMGzYMixcvhrOzM7S1teHr61uoT5K055aiODo6Yvv27bh+/TqaN28OLS0tLpEKDAyEg4NDyV/od3zv3P8lfX198Hg8JCYmSr2f5s2bw8rKCnv27EHXrl3x9OlTnDlz5pvbpKWlwcTERKz/bYGC/qB3795FXl4e2rRpA+D75ycg/3wAQGbnhObNm0MoFCIuLu6b52ZJHDhwAGPGjIGvr69Ud8sXqNIdwfPy8rBnzx6sXr0aISEh3CM0NBSmpqY4cOCAVPXx+Xyxjtql6cs3Ud26dcUeBXcOSBJPs2bN8O7dO7Fb82WtILH4MpZGjRpBIBAgOjq6UPw1a9aUqv569erhl19+wcWLF9GvXz94eXkVG0dRx0NfXx99+vSBl5cXvL29MXr06G/ur6CT6LNnz8SWNWjQAGvXri10QgwNDcXly5cxZMgQseUGBgbc3+vrb4NKSkpwcnLCihUr8OjRI7x+/Zpr/Tp58iR69+5dKK7bt29zv+fl5eH+/fto2LAht6xbt27cHSbnz5/HmDFjxLZXVlYudHxatGiB8PDwQn+junXrQkHh/6ePJ0+eFJksMkwBHo8HBQUFrjXl1q1bMDc3x/z582FjYwNLS0u8efNGqjrr1KkDZWVl3Llzh1uWmJhY6Hzm4OCAZ8+e4fDhw9wVBEdHR1y+fBk3b94sdFWhgLa2NkxMTMTqL/hsfUlW534+n49GjRqJnVsKhIaGcscOyP+8a2hoiJ0vx40bB29vb3h5ecHJyUlsXVExtmjRArGxsVBSUir0+TYwMACQf77p3r272Jfzb52fgPzzQY0aNbg6JJGWlsb9DwaAqKgohISEIDo6GvXq1cOwYcMwcuRIHDt2DFFRUbh79y6WLl0qlhg+e/YMISEh+Pz5M5KTk8XqA4D9+/dj5MiRWL16Nezs7BAbG4vY2Fiphkeo0kmTn58fEhMTMXbsWDRp0kTs0b9//2LvniqOhYUFHj16hPDwcMTHx4vdBSJrkryJLCwskJaWBn9/f8THxxfZVOrg4IAOHTqgf//+uHTpEqKionDu3DmcP39eZrGam5uDx+PBz88Pnz59QlpaGjQ1NTFz5kz88ssv2L17NyIjI/HgwQNs2LABu3fvlqjezMxMTJ06FQEBAXjz5g1u3ryJ4OBgsUThSxYWFtwHMT4+XuxOk3HjxmH37t0ICwuDu7v7N/draGiIFi1a4MaNG9wyHo+HnTt34tmzZ+jfvz/u3r2L6OhoHD58GD179oS9vb3E47j4+fnhn3/+QUhICN68eYM9e/ZAJBKhfv36iIuLw71794q8dXvTpk04fvw4nj9/jilTpiAxMVEsMVJUVMSoUaMwb948WFpail3OKzg+/v7+iI2N5b7pLly4EHv27MHixYvx9OlThIWFwdfXt9D4UNevXxcbjI9hsrOzuX9KYWFh+Pnnn5GWlsbdGWZpaYno6Gj4+voiMjIS//zzD9dNQlIaGhoYO3YsZs2ahStXruDJkycYNWqUWEIP5H851NXVxf79+8WSphMnTiA7O/ubt/dPnz4dy5Ytw4kTJ/D8+XNMnjy50JhzFhYWuHPnDl6/fo34+HipW8K+5OzsLHZuKZCTk4OxY8fi2bNnOHv2LDw9PTF16lSx1zp06FC8e/cO//77b6EvRUWd/5ycnGBvb48+ffrg4sWLeP36NW7duoX58+dzdyGfOnVKbKiBb52fCpTkfHDv3j00b96c+/Ll4eGB5s2bY+HChQAALy8vjBw5Er/++ivq16+PPn36IDg4WOwyZ7du3dC8eXOcPn0aAQEBYvUBwPbt25GXl4cpU6bAxMSEe0yfPl3yQKXqAVXJ9OjRg7p161bkujt37hAACg0NLbIjM1HhjnVxcXHUpUsX0tDQIAB09epVrpPulx2ECzr7FXQ2L6rDdlGd9r7u3JaTk0MLFy4kCwsLUlZWJhMTE+rbty89evSIKzNp0iTS19cnAOTp6UlEhTtyJyQk0OjRo0lfX59UVFSoSZMmXCfjohTVEfzrDsTa2trk5eXFPf/jjz/I2NiYeDweubu7E1F+B81169ZR/fr1SVlZmQwNDcnZ2ZkCAwOJqOgO5A8fPiQAFBUVRdnZ2TR48GCqWbMm8fl8MjU1palTp3KdLL8+rllZWdS/f3/S0dHh7hQrIBKJyNzcvNj3w9c2b95MrVu3LrT80aNH1L9/f9LT0yNlZWWqU6cO/f7772IdOIs6hl+6fv06OTg4kK6uLqmqqlKzZs3o4MGDRES0Y8cOatu2rVj5gvfY/v37ydbWlvh8PjVq1IiuXLlSqO7IyEgCINaxtcCpU6eobt26pKSkRObm5tzy8+fPU5s2bUhVVZW0tLTI1taWtm/fzq2/desW6ejocHfjMIy7uzsB4B6amprUqlUrOnLkiFi5WbNmkb6+Pndn2Nq1a79788batWvF3p+pqak0fPhwUlNTIyMjI1qxYkWRHbN79+5NSkpK3M01QqGQdHV1C32Ov95nbm4uTZ8+nbS0tEhHR4c8PDxo5MiRYufi8PBwat36f+3csUvjYBjH8eeGUDJZXAoawU4OUlqcbsiWvZ1CB+lUOhUHQQotVAqOQgcdHA1unQIdHKSD4NC/oFuHUp1ECJRA0KHvTVe4q717e0ZO7fezhvfNk3d4+ZG8T74r0zRn+9Pf9q9FBoOBMk1z1mX8cz0LhYI6Pj6erVelUnm1O61UKqn19fW5a4v2v8lkog4ODtTGxoYyDENtbW2p/f19NR6P1XA4VIlEQoVhOJvnT/uTUkpFUaTW1tZUv99f+Iyf2Tellji5CnxRYRjK5uamXF5evnp+53dRFMnOzo50Op25NzbvKZ/Pi23bUqvV/mn83d2dOI4j9/f3kkqlYqmpWCxKNpuVRqMRy3zAqnNdV/b29qRery891nEc2d3dlbOzszfX0W63pdfryfX1tfaYi4sL8X1fbm5u3nz/j2ilP88B0+lUHh8f5eTkRJLJpPYftk3TlKurK3l6enrnCn9l2/bc2Sgdz8/P8vDwIK1WS1zXjS0wvby8SCaTkcPDw1jmAyByenq6dFdYEATi+77c3t5KtVqNpQ7LspYOboZhyPn5eSz3/4h404SVNhqNJJ1Oi2VZ4nneXBv/V+F5npTLZcnlctLtduc6FgF8btvb2xIEgTSbTTk6Ovrf5XxZhCYAAAANfJ4DAADQQGgCAADQQGgCAADQQGgCAADQQGgCAADQQGgCAADQQGgCAADQQGgCAADQQGgCAADQ8ANppKLU32zxDQAAAABJRU5ErkJggg==", 78 | "text/plain": [ 79 | "
" 80 | ] 81 | }, 82 | "metadata": {}, 83 | "output_type": "display_data" 84 | } 85 | ], 86 | "source": [ 87 | "hardware=\"nvidia_A6000\"\n", 88 | "bandwidth = hardware_params[hardware][\"bandwidth\"]\n", 89 | "max_OPS = hardware_params[hardware][\"FP16\"]\n", 90 | "\n", 91 | "fig=plt.figure(figsize=(6, 2.5))\n", 92 | "plt.subplot(1, 2, 1)\n", 93 | "y_max = max_OPS\n", 94 | "turning_point = y_max / bandwidth\n", 95 | "\n", 96 | "plt.plot(\n", 97 | " [0, turning_point/3, turning_point * 3], [0, y_max, y_max], color=\"black\"\n", 98 | ")\n", 99 | "plt.plot(\n", 100 | " [0, turning_point, turning_point * 3], [0, y_max, y_max], color=\"black\"\n", 101 | ")\n", 102 | "plt.xlabel(\"Arithmetic Intensity (OPs/byte)\")\n", 103 | "plt.ylabel(\"Performance (OPS)\")\n", 104 | "\n", 105 | "plt.ylim(0, y_max * 1.1)\n", 106 | "plt.xlim(0, turning_point * 1.2)\n", 107 | "\n", 108 | "plt.vlines(50, 0, max_OPS*2, color=\"black\", linestyle=\"--\")\n", 109 | "plt.annotate(\n", 110 | " \"low\\nbandwidth\",\n", 111 | " xy=(50, max_OPS*(50/(max_OPS / bandwidth))),\n", 112 | " xytext=(75, max_OPS*(50/(max_OPS / bandwidth))*0.7),\n", 113 | " arrowprops=dict(arrowstyle=\"->\", color=\"red\"),\n", 114 | ")\n", 115 | "\n", 116 | "# plt.vlines(180, 0, max_OPS*2, color=\"black\", linestyle=\"--\")\n", 117 | "plt.annotate(\n", 118 | " \"high\\nbandwidth\",\n", 119 | " xy=(50, max_OPS*(50*3/(max_OPS / bandwidth))),\n", 120 | " xytext=(75, max_OPS*(50*3/(max_OPS / bandwidth))*1.05),\n", 121 | " arrowprops=dict(arrowstyle=\"->\", color=\"red\"),\n", 122 | ")\n", 123 | "plt.title(\"Roofline Model\")\n", 124 | "\n", 125 | "plt.subplot(1, 2, 2)\n", 126 | "plt.plot(\n", 127 | " bandwidths, performances\n", 128 | ")\n", 129 | "plt.xlabel(\"Bandwidth (byte/s)\")\n", 130 | "plt.ylabel(\"Decode Time (s)\")\n", 131 | "\n", 132 | "plt.title(\"LLaMA-2-13b\")\n", 133 | "\n", 134 | "plt.tight_layout()\n", 135 | "# save pdf\n", 136 | "plt.savefig(f\"../output/hardware_bandwidth.pdf\", bbox_inches=\"tight\")" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [] 145 | } 146 | ], 147 | "metadata": { 148 | "kernelspec": { 149 | "display_name": "houmo_llm", 150 | "language": "python", 151 | "name": "python3" 152 | }, 153 | "language_info": { 154 | "codemirror_mode": { 155 | "name": "ipython", 156 | "version": 3 157 | }, 158 | "file_extension": ".py", 159 | "mimetype": "text/x-python", 160 | "name": "python", 161 | "nbconvert_exporter": "python", 162 | "pygments_lexer": "ipython3", 163 | "version": "3.10.13" 164 | } 165 | }, 166 | "nbformat": 4, 167 | "nbformat_minor": 2 168 | } 169 | -------------------------------------------------------------------------------- /examples/plot_memory.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import sys\n", 10 | "sys.path.append('..')\n", 11 | "import numpy as np\n", 12 | "import matplotlib.pyplot as plt\n", 13 | "from hardwares.hardware_params import hardware_params\n", 14 | "from model_analyzer import ModelAnalyzer" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 2, 20 | "metadata": {}, 21 | "outputs": [ 22 | { 23 | "name": "stdout", 24 | "output_type": "stream", 25 | "text": [ 26 | "use config file configs/Llama.py for meta-llama/Llama-2-13b-hf\n" 27 | ] 28 | } 29 | ], 30 | "source": [ 31 | "model_id=\"meta-llama/Llama-2-13b-hf\"\n", 32 | "hardware=\"nvidia_A6000\"\n", 33 | "analyzer=ModelAnalyzer(model_id,hardware)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 3, 39 | "metadata": {}, 40 | "outputs": [ 41 | { 42 | "data": { 43 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAH1CAYAAAA57b10AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAACebklEQVR4nOzdeVyN+fs/8Ncp7amktHA4iYShECaGFqXik2HM2CsNIWWpQbKVMdYhsoxsKdswlrEL3xZrZMu+DEm2ihEt2pzu3x/9usfRqc6du+XU9Xw8zoNzn/u+3+9TV+c6932/7/clYBiGASGEEEJqJYWa7gAhhBBCykaJmhBCCKnFKFETQgghtRglakIIIaQWo0RNCCGE1GKUqAkhhJBajBI1IYQQUotRoiaEEEJqMUrUhBBCSC1GiZoQQgipxShRE0IIIbUY50R99+5d2NvbQ1tbG4qKilBUVISCggIUFRWron+EEEJIvSbgWpTDysoKXbp0gYeHBzQ0NCRes7Cw4LVzhBBCSH3HOVFraWnh/fv3UFCgs+aEEEJIVeOcbXv06IF79+5VRV8IIYQQ8oUGXDfo3Lkz+vXrh1GjRsHQ0FDitcmTJ/PWMUIIIYRU4tS3nZ2d9B0JBIiJieGlU4QQQggpxjlRE0IIIaT6cD71DQBZWVk4duwYXrx4AaFQCBcXF2hpafHdN0IIIaTe43xEfePGDTg7O6NRo0YwMTFBcnIy3r17h6ioKHTq1Kmq+smLoqIivHr1Cg0bNoRAIKjp7hBCCKmnGIZBVlYWjI2NK76LiuGoV69ezKpVqySWrV69mvnuu++47qraPX/+nAFAD3rQgx70oEeteDx//rzC3MX5iFpXVxdv3ryRmIlMLBZDT08PGRkZXHZV7T58+AAdHR08f/6cTtUTQgipMZmZmRAKhXj//j20tbXLXZfzNWpDQ0NcunQJPXv2ZJclJCSUulWrNio53a2lpUWJmhBCSI2T5TIs50Q9a9YsuLi4wM3NDSKRCMnJydi5cyfWrFlTqU4SQgghpGycZyYbNWoUjh49isLCQsTGxqKwsBCHDx+Gm5sbp/2cPXsWrq6uMDY2hkAgwMGDByvcJi4uDp07d4aKigpatWqFiIgIrt0nhBBC5Eqlbs/q3bs3evfu/VUN5+TkwMLCAj///DN++OGHCtd/+vQp+vfvjwkTJmDnzp2Ijo7G2LFjYWRkBCcnp6/qCyGEEFJbyZSojx8/jn79+gEADh8+XOZ6AwYMkLlhFxcXuLi4yLx+WFgYTExMsGLFCgBA27Ztcf78eaxcuZISNY9sbW1haWmJVatW1Uj7cXFxsLOzQ0ZGBnR0dGqkD0Q2FCtEVhQrX0emRD1jxgw2UU+ZMkXqOgKBgFOi5io+Ph4ODg4Sy5ycnDB16tQyt8nPz0d+fj77PDMzs6q6RwghhFQJmRL1nTt32P8/ffq0yjpTntTUVBgYGEgsMzAwQGZmJnJzc6GmplZqm8WLF2P+/PlV16ng8ofUc9/fB373R2oXPuOFYqVuo1ghn+E8mKx///5Sl1fl0XRlBQYG4sOHD+zj+fPnNd2lWiUnJwfu7u7Q1NSEkZERe1mhRH5+PqZNm4amTZtCQ0MD3bt3R1xcnMQ6Fy5cgK2tLdTV1dGoUSM4OTmx99Pn5+dj8uTJaNKkCVRVVfHdd9/hypUrEtsfP34cZmZmUFNTg52dHZKTk0v18/z58+jVqxfU1NQgFAoxefJk5OTk8PqzIOWjWCGyoljhH+dEfe7cOanLz58//9WdKY+hoSHS0tIklqWlpUFLS0vq0TQAqKiosPdM073TpU2fPh1nzpzBoUOHcOrUKcTFxeH69evs676+voiPj8fu3btx69Yt/PTTT3B2dsY///wDAEhMTESfPn3Qrl07xMfH4/z583B1dYVYLAZQfMlk//79iIyMxPXr19GqVSs4OTnh3bt3AIDnz5/jhx9+gKurKxITEzF27FjMnDlToo9PnjyBs7MzBg8ejFu3bmHPnj04f/48fH19q+mnRACKFSI7ihX+yTwz2erVqwEAAQEBWLp0qcRrT548wenTp3Hv3r3KdUIgwN9//42BAweWuU5AQACOHz+O27dvs8tGjBjBzjMui8zMTGhra+PDhw/8JG05PvWdnZ2Nxo0bY8eOHfjpp58AAO/evUOzZs0wbtw4+Pv7o2XLlkhJSYGxsTG7nYODA7p164ZFixZhxIgRSElJkfolLScnB40aNUJERARGjBgBACgsLIRIJMLUqVMxffp0zJo1C4cOHcLdu3fZ7WbOnImlS5eygz7Gjh0LRUVFbNiwgV3n/PnzsLGxQU5ODlRVVavqR8Q/OT2dSbFSAyhW6nyscMlHMt+e9ffffwMo/qGU/B8AFBQUYGBgwPme5uzsbDx+/Jh9/vTpUyQmJkJXVxfNmzdHYGAgXr58iW3btgEAJkyYgLVr12LGjBn4+eefERMTg7/++gvHjh3j1C4p9uTJExQUFKB79+7sMl1dXbRp0wYAcPv2bYjFYpiZmUlsl5+fj8aNGwMo/uZb8scobf+FhYUSM9gpKSmhW7duuH//PgDg/v37Eu0DgLW1tcTzmzdv4tatW9i5cye7jGEYFBUV4enTp2jbti3Xt044olghsqJYqRoyJ+rY2FgAwLRp07B8+fKvbvjq1auws7Njn/v7+wMAPDw8EBERgdevXyMlJYV93cTEBMeOHYOfnx9CQ0PRrFkzbN68uUZvzRLl7eJ1f8m87u3rZGdnQ1FREdeuXZOY1x0ANDU1AaDMSw5892P8+PGYPHlyqdeaN29e5e2TilGsEFlRrFQO5wlPli9fjszMTBw9ehQvX75Es2bN0K9fvwonFf+Sra0tyjvrLu0I3dbWFjdu3ODaZSKFqakplJSUcPnyZTYwMzIy8OjRI9jY2KBTp04Qi8VIT09Hr169pO6jY8eOiI6Oljqy3tTUFMrKyrhw4QJatGgBoPhszJUrV9hb6tq2bVvqvvxLly5JPO/cuTPu3buHVq1afe1bJpVEsUJkRbFSNTgPJrtw4QJMTEzw22+/ISYmBr/99htatmyJCxcuVEX/SBXR1NTEmDFjMH36dMTExODOnTsYPXo0WxfVzMwMI0eOhLu7Ow4cOICnT58iISEBixcvZi83BAYG4sqVK5g4cSJu3bqFBw8eYP369Xj79i00NDTg7e2N6dOnIyoqCvfu3YOXlxc+fvyIMWPGACi+nPHPP/9g+vTpePjwIXbt2lXqC1pAQAAuXrwIX19fJCYm4p9//sGhQ4dq7aCPuohihciKYqVqcE7UPj4+WLRoEe7du4cTJ07g7t27WLJkCSZOnFgV/SNV6Pfff0evXr3g6uoKBwcHfPfdd+jSpQv7+tatW+Hu7o5ffvkFbdq0wcCBA3HlyhX2m7KZmRlOnTqFmzdvolu3brC2tsahQ4fQoEHxiZolS5Zg8ODBcHNzQ+fOnfH48WOcPHkSjRo1AlB8imn//v04ePAgLCwsEBYWhkWLFkn0sWPHjjhz5gwePXqEXr16oVOnTpg3b57EQBRS9ShWiKwoVvjHuR51w4YN8eHDB/YbElBcj1pHRwdZWVm8d5BPfI/6Fs3kdyBb8hLp96iTOkJOR/KSGkCxUudxyUecj6gdHBxK3Q516tQpODo6ct0VIYQQQirAeTCZrq4ufvzxR9jb26NFixZ49uwZYmJiMGLECHbkNgCEhITw2lFCCCGkPuKcqIuKijB06FAAwMePH6Gvr4+hQ4dCLBazU7wRQgghhB+cE/XWrVuroh+EEEIIkYLzNWqgePBYUlISbt26JfEgVc/W1rbc0p5VITk5GQKBAImJiZXaPi4uDgKBAO/fv+e1X6RiFC9EGooL+cI5UR88eBCGhoZo1aoVLC0t2UenTp2qon+EZzUR7D169MDr1685T4rDlwMHDqBv375o3LjxV31Q1EcULxQv0lBclB0X8fHxsLe3h4aGBrS0tNC7d2/k5uZ+VducE/XkyZOxdOlS5OTkoKioiH2UVDYh5EvKysowNDSEQCCokfZzcnLw3XfflSomQ2onihcijTzERXx8PJydndG3b18kJCTgypUr8PX1lbiduTI4b52Xl4fRo0dXy3ysRLpPnz7B19cX2tra0NPTw9y5c9npWLdv3w4rKys0bNgQhoaGGDFiBNLT0wEUn3oqmV+9UaNGEAgEGD16NIDiQYLLli1Dq1atoKKigubNm2PhwoUS7SYlJcHOzg7q6uqwsLBAfHw8+9qzZ8/g6uqKRo0aQUNDA+3bt8fx48cBlP72bWtrC4FAUOpRUjP2/fv3GDt2LPT19aGlpQV7e3vcvHmz0j8vNzc3zJs3Dw4ODpXehzyjeOGmvsQLxQU3ssSFn58fJk+ejJkzZ6J9+/Zo06YNhgwZAhUVlUq3C1RyZrI//vjjqxolXycyMhINGjRAQkICQkNDERISgs2bNwMonvd2wYIFuHnzJg4ePIjk5GT2j0goFGL//v0AgIcPH+L169cIDQ0FUDxt35IlSzB37lzcu3cPu3btgoGBgUS7s2fPxrRp05CYmAgzMzMMHz4cnz59AlAcF/n5+Th79ixu376NpUuXspPsf+nAgQN4/fo1+/jhhx/Qpk0btr2ffvoJ6enpOHHiBK5du4bOnTujT58+bL3Zc+fOQVNTs9zH51Vx6juKF4oXaSgu+I2L9PR0XL58GU2aNEGPHj1gYGAAGxsbqeU6ueI86tvLyws2NjZYvHhxqV/A58XBSdURCoVYuXIlBAIB2rRpg9u3b2PlypXw8vLCzz//zK7XsmVLrF69Gl27dkV2djY0NTWhq6sLAGjSpAl0dHQAAFlZWQgNDcXatWvh4eEBoHjy+++++06i3WnTpqF//+LZ0+bPn4/27dvj8ePHMDc3R0pKCgYPHowOHTqwbZelpA8AsHLlSsTExODy5ctQU1PD+fPnkZCQgPT0dPZb6PLly3Hw4EHs27cP48aNg5WVVYXXDb+MzdqAz2pryRzWpXiRz3ipahQX/MZFUlISACA4OBjLly+HpaUltm3bhj59+uDOnTto3bq1zPv6EudE/eOPP8LExASDBg2Curp6pRsmlfftt99KXKextrbGihUrIBaLkZiYiODgYNy8eRMZGRkoKioCAKSkpKBdu3ZS93f//n3k5+ejT58+5bbbsWNH9v9GRkYAir9FmpubY/LkyfD29sapU6fg4OCAwYMHS6wvzYkTJzBz5kwcOXKErU978+ZNtvj853Jzc/HkyRMAxWXw5KXqTW1A8ULxIg3FBb9xUfIzGj9+PDw9PQEAnTp1QnR0NMLDw7F48eJK75tzor516xYyMjKgpKRU6UZJ1cjLy4OTkxOcnJywc+dO6OvrIyUlBU5OTigoKChzO1nHG3z+Oy/5Ay8JzrFjx8LJyQnHjh3DqVOnsHjxYqxYsQKTJk2Suq979+5h2LBhWLJkCfr27csuz87OhpGREeLi4kptU/LN/dy5c3BxcSm3rxs2bMDIkSNlel/1FcXLfyhe/kNx8R8ucVHypePLLzJt27ZFSkqKTPsoC+dE/e233+LJkycwNzf/qoZJ5V2+fFni+aVLl9C6dWs8ePAA//77L5YsWQKhUAgAuHr1qsS6ysrKACAxSr9169ZQU1NDdHQ0xo4dW+l+CYVCTJgwARMmTEBgYCA2bdok9Q/s7du3cHV1xeDBg+Hn5yfxWufOnZGamooGDRpAJBJJbYdOZXJD8ULxIg3FBb9xIRKJYGxsjIcPH0osf/ToUYVfCCrCOVF36dIFTk5OGDVqVKk3MXny5K/qDJFNSkoK/P39MX78eFy/fh1r1qzBihUr0Lx5cygrK2PNmjWYMGEC7ty5gwULFkhs26JFCwgEAhw9ehT9+vWDmpoaNDU1ERAQgBkzZkBZWRk9e/bEmzdvcPfuXbbGa0WmTp0KFxcXmJmZISMjA7GxsWjbtq3UdQcPHgx1dXUEBwcjNTWVXa6vrw8HBwdYW1tj4MCBWLZsGczMzPDq1SscO3YMgwYNgpWVFedTVu/evUNKSgpevXoFAOwfkqGhIQwNDWXej7yieKF4kYbigt+4EAgEmD59OoKCgmBhYQFLS0tERkbiwYMH2Ldvn8ztSMM5USckJKBly5a4ePGixHKBQECJupq4u7sjNzcX3bp1g6KiIqZMmYJx48ZBIBAgIiICs2bNwurVq9G5c2csX74cAwYMYLdt2rQp5s+fj5kzZ8LT0xPu7u6IiIjA3Llz0aBBA8ybNw+vXr2CkZERJkyYIHOfxGIxfHx88OLFC2hpacHZ2RkrV66Uuu7Zs2cBFP+xf+7p06cQiUQ4fvw4Zs+eDU9PT7x58waGhobo3bt3pY96Dh8+zF4zAoBhw4YBAIKCghAcHFypfcoTihdu6ku8UFxwI0tcTJ06FXl5efDz88O7d+9gYWGB06dPw9TUtFJtluBcj1qeUT1qUpP4jBeKlTqO6lHXeVVaj5oQQggh1YfzqW8FBYUyp3CjaUQJIYQQfnE+or5x4wauX7/OPo4ePQobGxusX7+ec+Pr1q2DSCSCqqoqunfvjoSEhHLXX7VqFdq0aQM1NTUIhUL4+fkhLy+Pc7uEEEKIvOB8RG1hYVHqeZcuXeDg4IBx48bJvJ89e/bA398fYWFh6N69O1atWgUnJyc8fPgQTZo0KbX+rl27MHPmTISHh6NHjx549OgRRo8eDYFAgJCQEK5vgxBCCJELvFyjVlVVxbNnzzhtExISAi8vL3h6eqJdu3YICwuDuro6wsPDpa5/8eJF9OzZEyNGjIBIJELfvn0xfPjwCo/CCTcikQirVq2q6W4QOUCxQrigeKk8zol69erVEo8lS5bAwcEBtra2Mu+joKAA165dk6hCoqCgAAcHB4lKKp/r0aMHrl27xibmpKQkHD9+HP369Suznfz8fGRmZko8SO23cOFC9OjRA+rq6uwsQl9KSUlB//79oa6ujiZNmmD69OnsxP5A8YT9jo6ObOUca2trnDx5ssw2lyxZAoFAgKlTp/L8bkh1yM/Ph6WlpdQ6wbdu3UKvXr2gqqoKoVCIZcuWSbxeWFiIX3/9FaamplBVVYWFhQWioqJKtfHy5UuMGjUKjRs3hpqaGjp06FBqIhBSu8nrZwvnU99///23xHNNTU24uLiUmhmmPG/fvoVYLC51P5uBgQEePHggdZsRI0bg7du3+O6778AwDD59+oQJEyZg1qxZZbazePFizJ8/X+Z+kdqhoKAAP/30E6ytrbFly5ZSr4vFYvTv3x+Ghoa4ePEiXr9+DXd3dygpKWHRokUAiu+xdHR0xKJFi6Cjo4OtW7fC1dUVly9fRqdOnST2d+XKFWzYsKHCOYVJ7TVjxgwYGxuXKmOYmZmJvn37wsHBAWFhYbh9+zZ+/vln6OjosJfq5syZgx07dmDTpk0wNzfHyZMnMWjQIFy8eJGNlYyMDPTs2RN2dnY4ceIE9PX18c8//6BRo0bV/l5J5cnrZwvnI+rY2FiJx5EjRzB//vwyv53wJS4uDosWLcIff/yB69ev48CBAzh27FipGXM+FxgYiA8fPrCP58+fV2kfq1p5tVoB4M6dO3BxcYGmpiYMDAzg5uaGt2/fsq/n5OTA3d0dmpqaMDIywooVK2Bra1vuN72KaroGBwfD0tIS27dvh0gkgra2NoYNG4asrKxKv8/58+fDz8+PraDzpVOnTuHevXvYsWMHLC0t4eLiggULFmDdunXsXMSrVq3CjBkz0LVrV7Ru3RqLFi1C69atceTIEYl9ZWdnY+TIkdi0aVOd+tCtL7ECFBdlOHXqFJYvX17qtZ07d6KgoADh4eFo3749hg0bhsmTJ0uMa9m+fTtmzZqFfv36oWXLlvD29ka/fv2wYsUKdp2lS5dCKBRi69at6NatG0xMTNC3b9+vnsiitqgv8SKvny2cE/W1a9fw8uVLAMVTqk2ZMgW//PILp9PKenp6UFRURFpamsTytLS0Mqfomzt3Ltzc3DB27Fh06NABgwYNwqJFi7B48WJ2QvcvqaioQEtLS+Ihz8qr1fr+/XvY29ujU6dOuHr1KqKiopCWloYhQ4aw20+fPh1nzpzBoUOHcOrUKcTFxVVYmrSimq4A8OTJExw8eBBHjx7F0aNHcebMGSxZsoR9fdGiRRXWfeUyaX18fDw6dOggcUbGyckJmZmZuHv3rtRtioqKkJWVJVEar+Rn2r9//3KLwcuj+hIraWlp8PLywvbt26VW84uPj0fv3r3ZuakBsINWMzIyABSfNldVVZXYrqRUYonDhw/DysoKP/30E5o0aYJOnTph06ZN5f485El9iZeK1NbPFs6nvj09PXHgwAEAxXVFnz17BlVVVXh7e8tcZFtZWRldunRBdHQ0Bg4cCKD4zUZHR8PX11fqNh8/foSCguT3CkVFRQBAfZlcrbxarWvXrkWnTp3Y0zMAEB4eDqFQiEePHsHY2BhbtmzBjh072DJ0kZGRaNasWZntyVLTFSj+3UVERKBhw4YAADc3N0RHR2PhwoUAgAkTJkj8UUtjbGws888hNTVV6mWTktekWb58ObKzsyX6sXv3bly/fh1XrlyRuW15UR9ihWEYjB49GhMmTICVlRWSk5NLrZuamgoTExOJZZ/HSqNGjeDk5ISQkBD07t0bpqamiI6OxoEDByTmhUhKSsL69evh7++PWbNm4cqVK5g8eTKUlZXZ2svyrD7Eiyxq62cL50SdkpKCVq1agWEYHD58GPfv34eamhrnU0D+/v7w8PCAlZUVunXrhlWrViEnJ4edS9Xd3R1NmzZla3i6uroiJCQEnTp1Qvfu3fH48WPMnTsXrq6ubMKu68qr1Xrz5k3Exsay34I/9+TJE+Tm5qKgoADdu3dnl+vq6qJNmzZltidLTVegeDRnyR8SUFzuLT09XaKdL79tVqddu3Zh/vz5OHToEHvr3/PnzzFlyhScPn261NFUXVAfYmXNmjXIyspCYGCgTOuXJTQ0FF5eXjA3N4dAIICpqSk8PT0l7kApKiqClZUVm6w6deqEO3fuICwsrE4k6voQL1Whuj5bOCdqJSUlZGVl4e7du2jWrBn09fUhFouRm5vLaT9Dhw7FmzdvMG/ePKSmpsLS0hJRUVHst5eUlBSJI+g5c+ZAIBBgzpw5ePnyJfT19eHq6sp+s6oPyqvVmp2dDVdXVyxdurTUdkZGRnj8+DHn9mSp6QqgVG1ygUAgcTli0aJFEt/Gpbl37x6aN28uU78MDQ1L3ZZXchnly0snu3fvxtixY7F3716JU1DXrl1Deno6OnfuzC4Ti8U4e/Ys1q5di/z8fLn+AlgfYiUmJgbx8fHsEVkJKysrjBw5EpGRkTA0NJR6iQ34L1b09fVx8OBB5OXl4d9//4WxsTFmzpwpcVRpZGQktc7w/v37y+2rvKgP8SKL2vrZwjlRu7q6wt7eHllZWRg9ejSA4oEG5Z3mKIuvr2+Zp7q//AU2aNAAQUFBCAoK4txOXVJWrdbOnTtj//79EIlEaNCg9K/V1NQUSkpKuHz5Mhu0GRkZePToEWxsbKS2JUtNV1nwfXrK2toaCxcuRHp6Ovst9vTp09DS0pL4MP3zzz/x888/Y/fu3ejfX7KIRZ8+fXD79m2JZZ6enjA3N0dAQIBcJ+kSdT1WVq9ejd9++41d/urVKzg5OWHPnj3s0Z21tTVmz56NwsJC9kP/9OnTaNOmTakBPqqqqmjatCkKCwuxf/9+iX707NlTap3hLys3ybO6Hi+yqK2fLZwTdVhYGCIjI6GkpAQ3NzcAxYPK5s2bx7lxwk15tVp9fHywadMmDB8+HDNmzICuri4eP36M3bt3Y/PmzdDU1MSYMWMwffp0NG7cGE2aNMHs2bNLXff/nCw1XWXB9fRUSkoKW/tVLBaz98W2atUKmpqa6Nu3L9q1awc3NzcsW7YMqampmDNnDnx8fNijq127dsHDwwOhoaHo3r07e31JTU0N2traaNiwIb755huJdjU0NNC4ceNSy+VRfYiVL4+SSk7NmpqasgcOI0aMwPz58zFmzBgEBATgzp07CA0NlSidePnyZbx8+RKWlpZ4+fIlgoODUVRUhBkzZrDr+Pn5oUePHli0aBGGDBmChIQEbNy4ERs3bpSpr7VdfYgXQH4/WzgnamVlZXh5eUkss7Ozq1TjhJvyarUaGxvjwoULCAgIQN++fZGfn48WLVrA2dmZ/YP5/fff2dNYDRs2xC+//IIPH8ougScQCHiv6SqLefPmITIykn1ecm9ibGwsbG1toaioiKNHj8Lb2xvW1tbQ0NCAh4cHfv31V3abjRs34tOnT/Dx8YGPjw+73MPDAxEREVXW99qivsRKRbS1tXHq1Cn4+PigS5cu0NPTw7x58ySmO87Ly8OcOXOQlJQETU1N9OvXD9u3b5c4Bdu1a1f8/fffCAwMxK+//goTExOsWrUKI0eOrIF3xb/6Ei/y+tnCuR51VlYWVq5ciWvXrpW6ny0mJobXzvGN6lGXZmtrC0tLS5rarxrIez1qipVqVAfqUVO8lI9LPuJ8RO3m5obnz59j8ODB0NDQqHQna6uSWc9kKdnZtCG/1zFrohJYkyZNoKWlVSeqkCkqKqJBgwZllmElhBB5xDlRx8XFISUlRe4nD5GmoKAAr1+/xsePH2VaP9iudJWvr/H06VNe9ycLb29vKCsr10jbVUFdXR1GRkYSE1wQQog845yohUIhCgsLq6IvNaqoqAhPnz6FoqIijI2NoaysXOGRWYEav0U+TAyr/8vPl5NByCuGYVBQUIA3b97g6dOnaN26dbmDWQh30m6lIaQsFC/84ZyofX19MXToUAQEBJS66C/PRQ0KCgpQVFQEoVAodSpCaQQN+D1dXBcn3qhOampqUFJSwrNnz1BQUEA/T0JIncA5UXt7ewMoPXBMIBDIdF23tqOjMPlGvz9CSF3DOVGXVQCDEEIIIfyr9OFHamoqrl69Wmp6PkIIIYTwh/MR9du3bzFy5EicPn0aKioqKCgogKOjI7Zv3w59ff2q6GON4/t+6YrUxD2yn4uLi4OdnR0yMjJkrjMeHByMgwcPsjP9EEII4QfnI+pJkyZBV1cXL1++RG5uLl6+fAldXd0y5+wmVSssLAwNGzbEp0+f2GXZ2dlQUlKCra2txLpxcXEQCAQS1Wmk6dGjB16/fg1tbR4nXQAqLCRPCCGkNM5H1DExMXj69Ck7MtrQ0BAbN26UqDRDqo+dnR2ys7Nx9epVfPvttwCAc+fOwdDQEJcvX0ZeXh47+jk2NhbNmzevsCSpsrJyqUoxhBBCagbnI2pVVVVkZGRILHv//n2pUnOkerRp06ZUubi4uDh8//33MDExwaVLlySW29nZoaioCIsXL4aJiQnU1NRgYWGBffv2SawnEAjw/v17dtmmTZvYW9cGDRqEkJAQqafFt2/fDpFIBG1tbQwbNoydZnb06NE4c+YMQkNDIRAIIBAIkJyczPePgxBC6hzOiXrEiBHo168f9u3bhytXrmDv3r1wdXWtM5PTyyM7OzvExsayz0smmLexsWGX5+bm4vLly7Czs8PixYuxbds2hIWF4e7du/Dz88OoUaNw5swZqfu/cOECJkyYgClTpiAxMRGOjo5S64A/efIEBw8exNGjR3H06FGcOXMGS5YsAQCEhobC2toaXl5eeP36NV6/fg2hUFgFPw1CCKlbOJ/6XrBgAdTU1DBr1iy8ePECzZo1w6hRoxAYGFgV/SMysLOzw9SpU/Hp0yfk5ubixo0bsLGxQWFhIcLCwgAA8fHxyM/Ph62tLdq1a4f/+7//g7W1NQCgZcuWOH/+PDZs2CC1fuyaNWvg4uKCadOmAQDMzMxw8eJFHD16VGK9oqIiREREoGHDhgCK54WPjo7GwoULoa2tDWVlZairq9NpdUII4YBzom7QoAHmzZtH9adrEVtbW+Tk5ODKlSvIyMiAmZkZ9PX1YWNjA09PT+Tl5SEuLg4tW7ZEdnY2Pn78CEdHR4l9FBQUsCXfvvTw4UMMGjRIYlm3bt1KJWqRSMQmaQAwMjJCeno6T++SkPpDlLeLt30l87YnUlNkTtTXr1/HwYMHJepylggKCsKgQYNgaWnJZ9+IjFq1aoVmzZohNjYWGRkZ7FGxsbExhEIhLl68iNjYWNjb2yM7OxsAcOzYMTRt2lRiP187zkBJSUniuUAgoAlyCCHkK8l8jXrJkiVo166d1Ne++eYbLFq0iLdOEe7s7OwQFxeHuLg4iduyevfujRMnTiAhIQF2dnZo164dVFRUkJKSglatWkk8yrpm3KZNG1y5ckVi2ZfPZaGsrFwnppklhJDqJPMR9aVLlxAZGSn1NVdXV/zyyy+8dYpwZ2dnBx8fHxQWFkpcZ7axsYGvry8KCgpgZ2eHhg0bYtq0afDz80NRURG+++47fPjwARcuXICWlhY8PDxK7XvSpEno3bs3QkJC4OrqipiYGJw4cYJz3WeRSITLly8jOTkZmpqa0NXVpbm5CSGkAjIn6oyMjDJr/CopKeHdu3e8daq2KWumsFsv3vPaTsdmOpXe1s7ODrm5uTA3N5eoamZjY4OsrCz2Ni6geECgvr4+Fi9ejKSkJOjo6KBz586YNWuW1H337NkTYWFhmD9/PubMmQMnJyf4+flh7dq1nPo4bdo0eHh4oF27dsjNzcXTp08hEokq/Z4JIaQ+kPlwxsjICHfu3JH62u3bt9kkwMW6desgEomgqqqK7t27IyEhodz1379/Dx8fHxgZGUFFRQVmZmY4fvw453brIpFIBIZhcP/+fYnlLVq0AMMwePDgAbtMIBBgypQpePDgAQoKCpCeno6oqCj07t0bQPHgNIZhJO6T9vLywosXL/Dx40f8/fffSE5ORqtWrdjXg4ODS00fOnXqVIl7pc3MzBAfH4+PHz+CYRhK0oQQIgOZE/WQIUMwadIkdgKLEllZWZg6dSqGDRvGqeE9e/bA398fQUFBuH79OiwsLODk5FTmKOGSOcWTk5Oxb98+PHz4EJs2bSo1IIpUjeXLl+PmzZt4/Pgx1qxZg8jISKmnyQkhhPBL5lPfc+bMgaOjI0xNTeHi4oKmTZvi5cuXiIqKgrm5OWbPns2p4ZCQEHh5ecHT0xNA8ZzVx44dQ3h4OGbOnFlq/fDwcLx79w4XL15kRxfTEVn1SUhIwLJly5CVlYWWLVti9erVGDt2bE13ixBC6jyZj6hVVVURGxuL33//HZ8+fcLVq1dRWFiIZcuWITo6mp1PWhYFBQW4du0aHBwc/uuIggIcHBwQHx8vdZvDhw/D2toaPj4+MDAwYEealzeKOD8/H5mZmRIPUjl//fUX0tPTkZubi7t372LChAk13SVCCKkXOE140qBBA3h4eHz1Kc+3b99CLBZLDHoCAAMDA4lrqZ9LSkpCTEwMRo4ciePHj+Px48eYOHEiCgsLERQUJHWbxYsXY/78+V/VV0IIIaQmyc29MUVFRWjSpAk2btyILl26YOjQoZg9ezY7RaY0gYGB+PDhA/t4/vx5NfaYEEII+XqcpxDlg56eHhQVFZGWliaxPC0trcx5oI2MjKCkpARFRUV2Wdu2bZGamoqCggKpt46pqKhQVS9CCCFyrUaOqJWVldGlSxdER0ezy4qKihAdHc0WivhSz5498fjxY4kpKR89egQjI6My7+8mhBBC5B3nRF1QUMBLw/7+/ti0aRMiIyNx//59eHt7Iycnhx0F7u7uLlGRy9vbG+/evcOUKVPw6NEjHDt2DIsWLYKPjw8v/SGEEEJqI86nvo2MjDBixAj8/PPPZVZbksXQoUPx5s0bzJs3D6mpqbC0tERUVBQ7wCwlJUViekmhUIiTJ0/Cz88PHTt2RNOmTTFlyhQEBARUug+EEEJIbcc5UR87dgwRERGwt7dHixYt4OnpiVGjRqFx48acG/f19YWvr6/U1+Li4kots7a2xqVLlzi389WCtaUu7lhl7X2oqj3XOgKBAH///TcGDhxY010hhJBaifOp72+//RZhYWFITU1FQEAATpw4gebNm+Onn37CiRMnwDBMVfSTlGP06NGlEt2+ffugqqqKFStWwNXVFc7OzlK3PXfuHAQCAW7dulUNPSWEEMJVpUd9q6io4Pvvv0dBQQFevXqFU6dO4caNGygqKsKGDRvg6OjIZz8JB5s3b4aPjw/CwsLg6ekJU1NTDB48GC9evECzZs0k1t26dSusrKzQsWOVnR8ghBDyFSo16vvs2bPw9PSEkZERtmzZAn9/f7x+/RqPHz/GokWL4O7uznc/iYyWLVuGSZMmYffu3ezAvP/973/Q19dHRESExLrZ2dnYu3cvxowZU+b+8vPzERAQAKFQCBUVFbRq1QpbtmwBAIjFYowZMwYmJiZQU1NDmzZtEBoaWmof4eHhaN++PVRUVGBkZFTqcsfbt28xaNAgqKuro3Xr1jh8+LDE63fu3IGLiws0NTVhYGAANzc3vH37tjI/HkIIkTucE7WpqSmGDRsGAwMDXLlyBWfPnsXo0aOhrq4OABg2bBgaNWrEe0dJxQICArBgwQIcPXoUgwYNYpc3aNAA7u7uiIiIkLg0sXfvXojFYgwfPrzMfbq7u+PPP//E6tWrcf/+fWzYsAGampoAim+pa9asGfbu3Yt79+5h3rx5mDVrFv766y92+/Xr18PHxwfjxo3D7du3cfjwYYmqWwAwf/58DBkyBLdu3UK/fv0wcuRItmzq+/fvYW9vj06dOuHq1auIiopCWloahgwZwsvPjBBCajsBw+GiclFREdauXQtvb2+2MIY8yczMhLa2Nj58+AAtLS2J1/Ly8vD06VOYmJiUnre8jMFkVYbjYLLRo0fjzz//REFBAaKjo2Fvb19qnQcPHqBt27aIjY2Fra0tAKB3795o0aIFtm/fLnW/jx49Qps2bXD69GmJednL4+vri9TUVOzbtw8A0LRpU3h6euK3336Tur5AIMCcOXOwYMECAEBOTg40NTVx4sQJODs747fffsO5c+dw8uRJdpsXL15AKBTi4cOHMDMzk9hfub/HGiaaeYy3fZVVI53UDRQrdV95+ehLnI6oFRQUEBgYiAYNamRCM1KOjh07QiQSISgoCNnZ2aVeNzc3R48ePRAeHg4AePz4Mc6dO1fuae/ExEQoKirCxsamzHXWrVuHLl26QF9fH5qamti4cSNSUlIAAOnp6Xj16hX69OlTYd9LaGhoQEtLiy13evPmTcTGxkJTU5N9mJubAwCePHlS7n4JIaQu4Hzqu2PHjvQBWQs1bdoUcXFxePnyJZydnUvVDQeAMWPGYP/+/cjKysLWrVthampabhJWU1Mrt83du3dj2rRpGDNmDE6dOoXExER4enqyk+JUtH2JL8/OCAQCdga67OxsuLq6IjExUeLxzz//oHfv3jLtnxBC5BnnQ+Pvv/8erq6umDhxIoRCocSkJAMGDOC1c4SbFi1a4MyZM7Czs4OzszOioqLQsGFD9vUhQ4ZgypQp2LVrF7Zt2wZvb28IBIIy99ehQwcUFRXhzJkzUk99X7hwAT169MDEiRPZZZ9/iWvYsCFEIhGio6NhZ2dXqffUuXNn7N+/HyKRiM7kEFJFOkR24G1ftz1u87YvUozzJ9+GDRsAACEhIRLLBQIBJepaQCgUIi4uDnZ2dnByckJUVBR7/UNTUxNDhw5FYGAgMjMzMXr06HL3JRKJ4OHhgZ9//hmrV6+GhYUFnj17hvT0dAwZMgStW7fGtm3bcPLkSZiYmGD79u24cuUKTExM2H0EBwdjwoQJaNKkCVxcXJCVlYULFy5g0qRJMr0fHx8fbNq0CcOHD8eMGTOgq6uLx48fY/fu3di8ebNEkRZCSOXcfppS010g5eCcqJ8+fVoV/ajdyhjcdevFe16b6dhMh5f9NGvWTCJZnzx5kk3WY8aMwZYtW9CvXz8YGxtXuK/169dj1qxZmDhxIv799180b94cs2bNAgCMHz8eN27cwNChQyEQCDB8+HBMnDgRJ06cYLf38PBAXl4eVq5ciWnTpkFPTw8//vijzO/F2NgYFy5cQEBAAPr27Yv8/Hy0aNECzs7OEmdzCCGkruI06rtEUVEREhIS2NG3Xbt2lYsPzUqP+i5DbU3U9RmN+iZ1QbXHCp93ttSjKZC/BpdR35U6onZ1dcXz589hbGyMV69eoVmzZjhy5AhatmxZ6U4TQgghpDTOh8E+Pj5wdnbGmzdvcP/+fbx58wb9+/encpOEEEJIFeB8RJ2QkICDBw9CWVkZAKCsrIwFCxbIdL2TEEIIIdxwPqLW0dHB48ePJZYlJSVBR0eHrz4RQggh5P/jfETt7e0NJycnTJ48GSKRCMnJyVi7dq3Mt9sQQgghRHacE/Uvv/wCfX197Nixgy2buGDBAqqYRQghhFSBSk315O7uTomZEEIIqQaVStRv3rzB7du3SxV/oJnJCCGEEH5xTtTr16+Hn58ftLW12RrUAE0hSgghhFQFzok6ODgYx48fl1rzuK7ic8J6WdCk9oQQQkpwvj1LWVmZygvWIgKBoNxHcHBwjfQrNzcXurq60NPTQ35+PqdtR48ejYEDB1ZNxwghRM5wTtRz587FnDlz2JrDX2vdunUQiURQVVVF9+7dkZCQINN2u3fvhkAgqPcf6K9fv2Yfq1atgpaWlsSyadOm1Ui/9u/fj/bt28Pc3BwHDx6skT4QQkhdwDlR9+zZE/v27YOGhgZ0dXUlHlzt2bMH/v7+CAoKwvXr12FhYQEnJyekp6eXu11ycjKmTZuGXr16cW6zrjE0NGQf2traEAgEEss0NTURFxcHgUCAkydPolOnTlBTU4O9vT3S09Nx4sQJtG3bFlpaWhgxYgQ+fvzI7tvW1ha+vr7w9fWFtrY29PT0MHfuXMhSx2XLli0YNWoURo0ahS1btpR6/e7du/jf//4HLS0tNGzYEL169cKTJ08QHByMyMhIHDp0iD0rEBcXx+ePjBBC5Arna9TDhg1Dr169sGbNGonBZJUREhICLy8veHp6AgDCwsJw7NgxhIeHY+bMmVK3EYvFGDlyJObPn49z587h/fv3Ze4/Pz9f4rRrZmbmV/VX3gUHB2Pt2rVQV1fHkCFDMGTIEKioqGDXrl3Izs7GoEGDsGbNGgQEBLDbREZGYsyYMUhISMDVq1cxbtw4NG/eHF5eXmW28+TJE8THx+PAgQNgGAZ+fn549uwZWrRoAQB4+fIlevfuDVtbW8TExEBLSwsXLlzAp0+fMG3aNNy/fx+ZmZnYunUrAFTqSyAhhNQVnBN1cnIybt68+dVlLQsKCnDt2jUEBgayyxQUFODg4ID4+Pgyt/v111/RpEkTjBkzBufOnSu3jcWLF2P+/Plf1c+65LfffkPPnj0BFNelDgwMxJMnT9iqZz/++CNiY2MlErVQKMTKlSshEAjQpk0b3L59GytXriw3UYeHh8PFxQWNGjUCADg5OWHr1q3s9fJ169ZBW1sbu3fvhpKSEgDAzMyM3V5NTQ35+fkwNDTk9f0TQog84pxtnZ2dZb6OXJ63b99CLBbDwMBAYrmBgQFSU1OlbnP+/Hls2bIFmzZtkqmNwMBAfPjwgX08f/78q/stzzp27Mj+38DAAOrq6hKlSQ0MDEpddvj2228hEAjY59bW1vjnn38gFoultiEWixEZGYlRo0axy0aNGoWIiAgUFRUBABITE9GrVy82SRNCCCkb5yNqbW1tODs7o3///qWSbEhICG8d+1JWVhbc3NywadMm6OnpybSNiooKVFRUqqxP8ubzxCgQCEolSoFAwCbTyjp58iRevnyJoUOHSiwXi8WIjo6Go6Mj1NTUvqoNQgipTzgnarFYjEGDBgEAMjIyKt2wnp4eFBUVkZaWJrE8LS1N6inPJ0+eIDk5Ga6uruyykqTSoEEDPHz4EKamppXuD5Hu8uXLEs8vXbqE1q1bQ1FRUer6W7ZswbBhwzB79myJ5QsXLsSWLVvg6OiIjh07IjIyEoWFhVKPqpWVlcs8YieEkPqGc6IuGeDztZSVldGlSxdER0ezt1gVFRUhOjoavr6+pdY3NzfH7duSE4HMmTMHWVlZCA0NhVAo5KVfRFJKSgr8/f0xfvx4XL9+HWvWrMGKFSukrvvmzRscOXIEhw8fxjfffCPxmru7OwYNGoR3797B19cXa9aswbBhwxAYGAhtbW1cunQJ3bp1Q5s2bSASiXDy5Ek8fPgQjRs3hra2Np0mJ4TUW5Wa6/uff/7BX3/9hdevX2Pt2rV4+PAh8vPzJa6BysLf3x8eHh6wsrJCt27dsGrVKuTk5LCjwN3d3dG0aVMsXrwYqqqqpT78S2pgf7mcb2XNFHbrxXte2+nYTIfX/fHB3d0dubm56NatGxQVFTFlyhSMGzdO6rrbtm2DhoYG+vTpU+q1Pn36QE1NDTt27MDkyZMRExOD6dOnw8bGBoqKirC0tGQHunl5eSEuLg5WVlbIzs5GbGwsbG1tq/JtEkJIrcV5MNmRI0fQtWtX3L9/H9u2bQNQfAq8MhNrDB06FMuXL8e8efNgaWmJxMREREVFsde+U1JS8Pr1a877ra9Gjx4t9XY1W1tbMAzDfrEpa93g4GAkJiZKLFNSUsL69evx4cMHvHv3DgsXLpQYXPa5X375BRkZGWWezs7IyMDkyZMBFA9sO3nyJHJycpCZmYmzZ8+yA9v09fVx6tQpZGVlgWEYStKEkHqN8xH1rFmzcPz4cfTo0YO9/aZTp06lPuBlVTKhhjQVTXQRERFRqTYJIYQQecH5iPrFixfo0aMHALBHVkpKSjT4hxBCCKkCnI+ozczMcObMGdjY2LDLzp49i7Zt2/LaMVLzaOpOQgipeZwT9cKFCzFo0CB4enoiPz8fs2bNwtatW/Hnn39WRf8IIZURrM3z/j7wuz9CiMw4n/p2cHBAXFwc8vPzYWdnh4yMDJw4caLODPiRpeAEqb3o90cIqWsqdXtWx44dsXbtWr77UqNKRip//PiRZs6SYyXVv+i+a0JIXSFzoj58+HCF6wwYMOCrOlOTFBUVoaOjw851ra6uXuZtSCWYT/zU5C6Rl5fH6/7qE4Zh8PHjR6Snp0NHR6fMmdMIIUTeyJyoBw4cCENDQ6iqqko9vSgQCOQ6UQNgpy6tqB52ifSMXF7bV86lI/mvpaOjQ1W3CCF1isyJ2sXFBQkJCfjhhx/g6emJLl26VGW/aoRAIICRkRGaNGmCwsLCCtcfeyCO1/ajf7HldX/1jZKSEh1JE0LqHJkT9bFjx5Camopt27Zh1KhRUFJSgqenJ9zc3GSuZiUvFBUVZfrAf5nF773jqqqqvO6PEEKI/OM06tvQ0BAzZszA/fv3sWHDBty+fRtCoZCX+tSEEEIIKY3z7VkA8OjRIxw6dAinTp1C165doa+vz3e/CCGEEAIOiTo7OxtbtmxBz5494ejoCCUlJcTFxeHs2bMwMTGpyj4SQggh9ZbM16iNjIwgFArh6ekJR0dHKCgo4OPHj7h16xa7Dtcyl4Qjmm2KEELqHZkTdU5ODh48eICAgADMnDmz1C1aAoGACnMQQgghPJM5URcVFVVlPwghhBAiRaUGkxFCCCGkelRqrm9ST1T3NXE+26Pr79VLnmNFlvYIv6rzb70OxAolajkiytvF6/6S63B7FbVFCCHygk59E0IIIbVYpRM1wzB4/fo1n30hhBBCyBc4J+qsrCy4u7tDVVUVrVq1AgAcPHgQ8+bN471zhBBCSH3H+Rr1pEmTIBaLcefOHXTr1g0A8O233yIgIAC//vor5w6sW7cOv//+O1JTU2FhYYE1a9aw+/3Spk2bsG3bNty5cwcA0KVLFyxatKjM9Qmpr+R5fEFtbI+QmsT5iDoqKgqbN29G69atIRAIABQX60hLS+Pc+J49e+Dv74+goCBcv34dFhYWcHJyKrMedFxcHIYPH47Y2FjEx8dDKBSib9++ePnyJee2CSGEEHnAOVGrqKjg06dPEsv+/fdf6Orqcm48JCQEXl5e8PT0RLt27RAWFgZ1dXWEh4dLXX/nzp2YOHEiLC0tYW5ujs2bN6OoqAjR0dFS18/Pz0dmZqbEgxBCCJEnnBO1q6srJk6ciA8fiu8ly8/Px4wZMzBo0CBO+ykoKMC1a9fg4ODwX2cUFODg4ID4+HiZ9vHx40cUFhaW+SVh8eLF0NbWZh9CoZBTHwkhhJCaxjlRL126FPn5+WjcuDHev38PTU1NZGZmcr4+/fbtW4jFYhgYGEgsNzAwQGpqqkz7CAgIgLGxsUSy/1xgYCA+fPjAPp4/f86pj4QQQkhN4zyYTENDA3/99RfevHmDZ8+eQSgUlkq21WHJkiXYvXs34uLioKqqKnUdFRUVqKioVHPPCCGEEP5U6oj61atX0NfXh5WVVaWTtJ6eHhQVFUsNQktLS4OhoWG52y5fvhxLlizBqVOnqLQmIYSQOo1zoj5z5gxatmwJR0dH7NixAx8/fqxUw8rKyujSpYvEQLCSgWHW1tZlbrds2TIsWLAAUVFRsLKyqlTbhBBCiLzgnKiPHz+OZ8+eoV+/fggJCYGBgQHc3d3LHHldHn9/f2zatAmRkZG4f/8+vL29kZOTA09PTwCAu7s7AgMD2fWXLl2KuXPnIjw8HCKRCKmpqUhNTUV2djbntgkhhBB5UKmiHAYGBvDz84Ofnx/u3r2LadOmoW/fvhCLxZz2M3ToULx58wbz5s1DamoqLC0tERUVxZ5OT0lJgYLCf98l1q9fj4KCAvz4448S+wkKCkJwcHBl3gohhNR71V0QpzrbqwuT41S6etbr16+xc+dObN++HcnJyfDw8KjUfnx9feHr6yv1tbi4OInnycnJlWqDEEIIkVecT33v2LEDffv2hYmJCaKjoxEQEIDU1NQyJykhhBBCSOVxPqJevnw53NzcsG3btgpHZxNCCCHk63BO1ImJiVXQDUIIIYRII1OiDgsLw4QJEwAAq1evLnO9yZMn89MrQgghhACQMVEfPnyYTdR///231HUEAgElakIIIYRnMiXq48ePs/+PjY2tss4QQgghRBLnUd8dOnSQutzS0vJr+0IIIYSQL3BO1GXdy5ySkvK1fSGEEELIF2Qe9e3v7w8AKCwsZP9fIikpCS1btuS3Z4QQQgiRPVFnZGQAKC6cUfJ/AFBQUEC7du0QEhLCf+8IIYSQek7mRL1161YAQOfOnTFp0qQq6xAhhBBC/sN5wpPPk3RWVhYYhmGfa2lp8dMrQgghhACoxGCylJQU9O3bF+rq6tDR0UGjRo3YByGEEEL4xTlRT5w4EY0aNcLFixehqamJ69evY8CAAdiwYUNV9I8QQgip1zif+o6Pj8ezZ8+gqakJgUAACwsLbN68Gb169cLYsWOroo+EEEJIvcX5iFpRUREqKioAiq9Jv3v3Dtra2nj+/DnvnSOEEELqO85H1JaWloiJiYGTkxN7FK2uro527dpVRf8IIYSQeo3zEfWWLVtgbm4OAAgNDUXjxo1RWFiIyMhI3jtHCCGE1Hecj6iFQiH7fz09PWzatInXDhFCCCHkPzKXuZTFgAEDvqozhBBCCJEkU6KeMmVKhesIBAJK1IQQQgjPZLpG/fTp0wofSUlJlerAunXrIBKJoKqqiu7duyMhIaHc9ffu3Qtzc3OoqqqiQ4cOErWyCSGEkLqG82AyAPj06RMuXLiAPXv2AABycnKQk5PDeT979uyBv78/goKCcP36dVhYWMDJyQnp6elS17948SKGDx+OMWPG4MaNGxg4cCAGDhyIO3fuVOZtEEIIIbUe58FkDx48gKurK3Jzc/H+/XsMHToU0dHR+PPPP/Hnn39y2ldISAi8vLzg6ekJAAgLC8OxY8cQHh6OmTNnllo/NDQUzs7OmD59OgBgwYIFOH36NNauXYuwsLBS6+fn5yM/P599/uHDBwBAZmYmp36WpSj/Iy/7KVFRv6g9/tqqCdX5/uT5d1cb26tu1f23UJfbq62xUrKfz+tllInhyM7OjlmzZg3DMAyjo6PDMAzDfPjwgWnWrBmn/eTn5zOKiorM33//LbHc3d2dGTBggNRthEIhs3LlSoll8+bNYzp27Ch1/aCgIAYAPehBD3rQgx618vH8+fMK8yXnI+rExET83//9H4DiAWRA8QxlWVlZnPbz9u1biMViGBgYSCw3MDDAgwcPpG6Tmpoqdf3U1FSp6wcGBsLf3599XlRUhHfv3qFx48Zs36taZmYmhEIhnj9/Xi3Vxag9+VXXf5Z1vb3qVp3vr67/7moiVhiGQVZWFoyNjStcl3OiNjAwQHJyMlq2bMkue/ToEZo1a8Z1V1VORUWFne60hI6OTo30RUtLq1o/LKg9+VXXf5Z1vb3qVp3vr67/7qq7PW1tbZnW4zyYzNvbG4MHD8aRI0cgFotx6tQpuLm5wdfXl9N+9PT0oKioiLS0NInlaWlpMDQ0lLqNoaEhp/UJIYQQecc5UU+ePBleXl4IDAyEWCzG1KlT4ebmhgkTJnDaj7KyMrp06YLo6Gh2WVFREaKjo2FtbS11G2tra4n1AeD06dNlrk8IIYTIPVkGfsnixYsXnLfZvXs3o6KiwkRERDD37t1jxo0bx+jo6DCpqakMwzCMm5sbM3PmTHb9CxcuMA0aNGCWL1/O3L9/nwkKCmKUlJSY27dv8/U2eJeXl8cEBQUxeXl51J4ctled6vrPsq63V92q8/3V9d9dbY8VAcPIMjb8P//++y8aNWoEBYXig/HU1FQsXLgQW7ZswceP3IfBr127Fr///jtSU1NhaWmJ1atXo3v37gAAW1tbiEQiREREsOvv3bsXc+bMQXJyMlq3bo1ly5ahX79+nNslhBBC5IHMifratWv44Ycf8OLFC+jp6eHAgQO4efMmAgICYGdnh1mzZuHbb7+t6v4SQggh9YrMidrOzg7dunWDh4cHNm/ejN27d6Np06bYuHEjOnXqVNX9JIQQQuolmRO1np4eUlNT0aBBA+Tm5kJTUxMvXryAkZFRVfeREEIIqbdkHvVdUFCABg2Kb7tWU1ODtrY2JWlCCCGkisk84UlBQQFWr17NPs/Pz5d4DhTfukUIIYQQ/sh86tvW1rbcaTcFAgFiYmJ46xiRDwzDVNt0rDXRHuFXdf7+KFbkG322/Ifz7VmkdrO3t8fWrVvRokWLamlPWVkZN2/eRNu2betke3VdXY4XihV+1eVYqYn2uOA81zfhJj8/Hy9evECzZs1KzTv+NQ4fPix1+dmzZ3H06FEIhUIAwIABA3hp7/PiJp8Ti8VYsmQJGjduDKC4dKk8tldbULzU7rZqE4oV+WiPD3REzaOIiAi0adMG1tbWyMvLg4+PDyIjI8EwDBQUFDBmzBiEhoby8keloKAAgUBQbi1TgUAAsVj81W2VtGdhYVGqqMmZM2dgZWUFDQ0NXi9/VHd7NYHihZ/fH8UKxUptbo8X1TUFWn1gYmLCXLp0iWEYhpk2bRojEomYAwcOMPfv32cOHjzImJmZMdOnT+elLWdnZ6Z///5MWlqaxPIGDRowd+/e5aWNzy1evJgxMTFhoqOj62R7NYHiRf7aqikUK/LbHh8oUfNIRUWFefbsGcMwDGNmZsacOHFC4vUzZ84wzZs35629kJAQRigUMkeOHGGXVWWwJSQkMGZmZswvv/zCFBQU1Ln2qhvFi3y2VRMoVuS7va/FuXoWKZuhoSGePHkCAMjJyYGenp7E6/r6+vj33395a8/Pzw+HDx9GQEAAxo8fX6m51rno2rUrrl27hjdv3sDKygp37typ0lGS1d1edaN4kc+2agLFiny399Vq+ptCXTJr1izG2tqaycjIYGbOnMm4uroyWVlZDMMwTE5ODjNkyBCmb9++vLebk5PDjB8/nmndujWjqKhYLd8K//zzT8bAwIBRUFCok+1VB4oX+W+rulCs1J32KoMSNY/y8/OZAQMGMI0aNWIcHR0ZVVVVRl1dnWndujWjoaHBNG/enHn48CEvbSUlJZVadujQIWbq1Kmlri1VlefPnzMHDx5ksrOz62R7VY3ipW60VR0oVupWe1zRqO8qEBUVhSNHjiApKQlFRUUwMjJCz549MWLECGhoaPDShoKCAlq0aAE7OzvY2dnB3t4eTZs25WXf0vz4448YO3YsnJycavwUEcMwiIqKwpYtW7Bv374a7QsfKF6qDsUKd/U1VoBaHC81+jWBVFpsbCwTFBTE2NjYMKqqqoyCggLTqlUrZty4ccyff/7JpKam8tqevb09o6CgwDRr1oyZO3cu8+TJE173L4ukpCRmzpw5TLNmzRgVFRWmf//+1d4HeVXf4oVipfLqW6wwTO2PF0rUdUBubi4THR3NzJ07l+nVqxejoqLCKCgoMO3ateO1neTkZCYoKIgxMTFhFBQUGDs7O2bnzp1MXl4er+18Li8vj9mxYwdjZ2fHKCkpMQoKCkxISAjz4cOHKmuzrqur8UKxwr+6GisMI1/xQom6GiUmJjIKCgpVtv/8/HwmJiaGmT59OqOlpVWlbUVHRzMjR45k1NXVmUaNGjETJ05krl69ytv+r169ynh7ezM6OjqMlZUVExoayqSmptbqWyj4RvEiG4oVihUu5DFeKFFXo8TEREYgEPC2v/z8fObMmTNMcHAwY2try6ipqTFmZmbM2LFjmW3btrH3XValzMxMJiwsjNHV1WUUFRV526+ioiIzdepU5sGDBxLLa/MfE98oXmRDsUKxwoU8xgvN9c2jH374odzXP3z4wNtgCXt7e1y+fBkmJiawsbHB+PHjsWvXrmqtEf706VNEREQgIiICHz58gIODA2/77tOnD7Zs2YL09HS4ubnVioEmfKN44SdeKFYoVriQx3ihRM2jI0eOwNHREQYGBlJf52tuXAA4d+4cjIyMYG9vD1tbW9jY2LCTyVelvLw87Nu3D+Hh4Th79iyEQiHGjBkDT09PdrJ+Ppw8eRLPnz/H1q1b4e3tjdzcXAwdOhQAav0flawoXviJF4oVihUu5DJeavqQvi7p0KEDs3nz5jJfv3HjBm/XdrKzs5kTJ04wAQEBTLdu3RhlZWXmm2++YXx8fJi9e/cy6enpvLRT4vLly8z48eMZHR0dRlVVlRk+fDhz+vRppqioiNd2ynLq1Clm+PDhjKqqKtO6dWsmMDCQuXbtWrW0XVUoXqoGxcrXqU+xwjDyES+UqHk0evRoZuLEiWW+fu/ePUYkElVJ25mZmczx48eZ6dOnM127dmWUlZWZ9u3b87Z/gUDAWFpaMmvWrGHevXvH2365evfuHbN69WrG0tKySge0VAeKl6pFscKP+hArDFO744UmPOFRfn4+xGIx1NXVq73toqIiXLlyBbGxsYiNjcX58+eRl5fH2ymxs2fPonfv3uWus3v3bgwbNoyX9mRx/fp1dO7cudra4xvFS/XFC8VK5dW3WAFqYbzU9DcFUjlisZi5fPkys3TpUsbZ2Zlp2LAho6CgwAiFQsbd3Z3ZunUrk5yczFt77du3ZzIyMsp8/c8//2SUlJR4a68sdnZ2vL6v+qI+xgvFSuXUx1hhmNodLzSYjEdisRjLly/H4cOHUVBQgD59+iAoKAhqamq8t6Wjo4OcnBwYGhrCzs4OK1euhK2tLUxNTXlvCyiuzuPi4oLo6OhS3+r/+usvuLm5YdGiRby1d/jwYanLz549i6NHj7KDSwYMGMBbm9WN4oWfeKFY4VddjhVATuOlpr8p1CW//voro6CgwPTt25f5/vvvGVVVVcbT07NK2goLC+NtEn5ZZGVlMV26dGEcHR3Z+q0MwzB//fUXo6yszCxZsoTX9gQCAaOgoMAIBIIyH7XtOhJXFC/8oFjhV12OFYaRz3ihRM2jVq1aMWFhYezz06dPM8rKyoxYLK7BXvEnPT2dMTc3Z3788UemqKiI2bt3L6OkpMQsXLiQ97acnZ2Z/v37l6rWU5snJeCK4oUfFCvyjz5bykeJmkfKyspMSkqKxDIVFRXm+fPnNdQj/qWkpDDNmzdn+vTpwygrKzMLFiyosrZCQkIYoVDIHDlyhF1Wm/+YuKJ44Q/Fivyjz5ay0ahvHikqKiI1NRX6+vrssoYNG+LWrVswMTGpwZ59vVu3brH/f/DgAdzd3fH9999j9uzZEut17NiR13YTExMxcuRIfPfdd1i5ciW0tbVx8+ZNtGvXjtd2agLFC7/xQrEin+izpWKUqHmkoKAAFxcXqKiosMuOHDkCe3t7iVqxBw4cqInufRUFBQUIBAIwDMP+C6DU//mcIanEx48f4e/vj5iYGCQlJeHWrVu18o+JK4oX/uOFYoVihQt5iRca9c0jd3f3UlPQjRo1qoZ6w6+nT59We3slRwrq6uoICwvD4cOHERsbCz09vWrtS1WheOGvLYoV+UWfLRWjI2oeJSUlQSQSQUFBoaa7IvcUFBTQokUL2NnZwc7ODvb29mjatGlNd4tXFC/8oFghXMhjvNBvnUetW7fG27dv2edDhw5FWlpaDfao6tnb2+PZs2e87zcmJgYeHh5ISkrCuHHj0Lx5c7Ru3Rrjx4/H7t2768TPleKFHxQrdRN9tvyHjqh5pKCggNTUVDRp0gRA8WCPmzdvomXLljXcs69X1iQBP/zwA0JDQ6t0koC8vDxcvHgRcXFxiIuLQ0JCAgoLC2Fubo67d+/y3l51oXjhP14oVuQPfbZUjBI1j+ryH9PnAz7KUlUDPkoUFBTgwoULOHHiBDZs2IDs7Owqba+qUbxUXbxQrMiPmo4VoPbHC5365pFAICg14KPW1jflyMnJCS4uLkhNTUVRURH7UFRUxJ07d1BUVMR7YBcUFODs2bOYP38+7OzsoKOjgwkTJiAjIwNr166t9kEofKN44S9eKFbkF322VIxGffOIYRiMHj2avYUiLy8PEyZMkLh9ApDPWyhOnDiBlStXwsrKCn/88Qf+97//VWl79vb2uHz5MkxMTGBjY4Px48dj165dMDIyqtJ2qxPFCz8oVv5DsVIxeYwXOvXNI09PT5nW27p1axX3pOpU1yQBSkpKMDIywsCBA2FrawsbGxs0btyY1zZqGsULPyhW/kOxUjF5jBdK1ISz6pgkICcnB+fOnUNcXBxiY2ORmJgIMzMz2NjYsH9cn8/SRGqvqo4XipW6gz5bpKNETWTy+SQBJUomCQgMDGQHuVSVrKwsnD9/HrGxsYiLi8PNmzfRunVr3Llzp0rbJZVTk/FCsSJf6LOlYpSoiUxqepKAoqIiXLlyBbGxsYiNjcX58+eRl5dXq0Zmkv/UZLxQrMgX+mypGCVqIpOS+wzj4uJw+fJlFBQUoGXLlrC3t2f/wAwMDHhrr6ioCFevXmVPT124cAE5OTlo2rQp256dnR1atGjBW5uEP9UZLxQr8o0+WypGiZpwVh2TBGhpaSEnJweGhobsH46trS1MTU152T+pPlUdLxQrdQd9tkhHiZpUWlVOErBhwwbY2dnBzMyMl/2RmldV8UKxUvfQZ4skStREZgUFBbh06RI76OLy5csQCoXo3bs3evfuDRsbGzRv3rymu0lqCYoXIiuKlfJRoiYy+XKSgF69esHGxqZWTxJAag7FC5EVxUrFKFETmcjjJAGk5lC8EFlRrFSMEjWRiTxOEkBqDsULkRXFSsUoUZNKkYdJAkjtQfFCZEWxUhpVzyKVoqGhAV1dXejq6qJRo0Zo0KAB7t+/X9PdIrUUxQuRFcVKaXRETWQij5MEkJpD8UJkRbFSMUrURCbyOEkAqTkUL0RWFCsVo0RNZCKPkwSQmkPxQmRFsVIxStSEEEJILUaDyQghhJBajBI1IYQQUotRoiaEEEJqMc6J+u7du7C3t4e2tjYUFRWhqKgIBQUFKCoqVkX/CCGEkHqN82AyKysrdOnSBR4eHtDQ0JB4zcLCgtfOEUIIIfUd50StpaWF9+/fQ0GBzpoTQgghVY1ztu3Rowfu3btXFX0hhBBCyBcacN2gc+fO6NevH0aNGgVDQ0OJ1yZPnsxbxwghhBBSiVPfdnZ20nckECAmJoaXThFCCCGkGM1MRgghhNRinE99A8X1Qo8dO4YXL15AKBTCxcUFWlpafPeNEEIIqfc4H1HfuHEDzs7OaNSoEUxMTJCcnIx3794hKioKnTp1qqp+EkIIIfUS50Tdu3dvDB48GFOmTGGXrVmzBn/99RfOnTvHewf5VFRUhFevXqFhw4YQCAQ13R1CCCH1FMMwyMrKgrGxcYW3O3NO1Lq6unjz5o3ETGRisRh6enrIyMioXI+rScmpekIIIaQ2eP78OZo1a1buOpyvURsaGuLSpUvo2bMnuywhIaHUrVq1UcOGDQEU/2DomjohhJCakpmZCaFQyOal8nBO1LNmzYKLiwvc3NwgEomQnJyMnTt3Ys2aNZXqbHUqOd2tpaVFiZoQQkiNk+UyLOeZyUaNGoWjR4+isLAQsbGxKCwsxOHDh+Hm5lapThJCCCGkbJWasLt3797YuHEjjh8/jo0bN6J3796c93H27Fm4urrC2NgYAoEABw8erHCbuLg4dO7cGSoqKmjVqhUiIiK4d54QQgiRIzIl6uPHj7P/P3z4cJkPLnJycmBhYYF169bJtP7Tp0/Rv39/2NnZITExEVOnTsXYsWNx8uRJTu2SmvHx40cMHjwYWlpaEAgEeP/+PUQiEVatWsWu8/kXtuTkZAgEAiQmJtZIf0nNoVghsqovsSLTNeoZM2agX79+ACBxW9bnBAIBBgwYIHPDLi4ucHFxkXn9sLAwmJiYYMWKFQCAtm3b4vz581i5ciWcnJxk3g+pGZGRkTh37hwuXrwIPT09aGtr48qVK6VKpRJCsUJkVV9iRaZEfefOHfb/T58+rbLOlCc+Ph4ODg4Sy5ycnDB16tQyt8nPz0d+fj77PDMzs6q6V28VFBRAWVm5wvWePHmCtm3b4ptvvmGX6evrV2XXSC1DsUJkRbEiifOo7/79++PYsWOllg8YMIDz6W8uUlNTYWBgILHMwMAAmZmZyM3NhZqaWqltFi9ejPnz51dZnzpEduB1f7c9bvO6v8qwtbVlg3779u1QUlKCt7c3fv31VwgEAohEIowZMwb//PMPDh48iB9++AERERE4f/48AgMDcfXqVejp6WHQoEFYvHgxNDQ0YGtrizNnzgAoPvNiY2ODuLg4iEQiTJ06tdwvW3UJn/FCsVK3UaxQrHyO82CysmYfO3/+/Fd3hm+BgYH48OED+3j+/Dmv+7/9NIXXR20RGRmJBg0aICEhAaGhoQgJCcHmzZvZ15cvXw4LCwvcuHEDc+fOxZMnT+Ds7IzBgwfj1q1b2LNnD86fPw9fX18AwIEDB+Dl5QVra2u8fv0aBw4cqKm3RnhGsUJkRbFSeTIfUa9evRoAUFhYyP6/xJMnT6p8whNDQ0OkpaVJLEtLS4OWlpbUo2kAUFFRgYqKSpX2qy4SCoVYuXIlBAIB2rRpg9u3b2PlypXw8vICANjb2+OXX35h1x87dixGjhzJfoNt3bo1Vq9eDRsbG6xfvx66urpQV1eHsrKyXEyMQ2RHsUJkRbFSeTIn6r///htAcaIu+T8AKCgowMDAoMpvlbK2tpYYfQ4Ap0+fhrW1dZW2Wx99++23EjfhW1tbY8WKFRCLxQAAKysrifVv3ryJW7duYefOnewyhmFQVFSEp0+fom3bttXTcVLtKFaIrChWKk/mRB0bGwsAmDZtGpYvX/7VDWdnZ+Px48fs86dPnyIxMRG6urpo3rw5AgMD8fLlS2zbtg0AMGHCBKxduxYzZszAzz//jJiYGPz1119Sr5eTqvXliMrs7GyMHz8ekydPLrVu8+bNq6tbpBaiWCGyolgpG+fBZMuXL0dmZiaOHj2Kly9folmzZujXrx+0tbU57efq1auws7Njn/v7+wMAPDw8EBERgdevXyMl5b/rtiYmJjh27Bj8/PwQGhqKZs2aYfPmzXRrVhW4fPmyxPNLly6hdevWEoVYPte5c2fcu3cPrVq1qo7ukVqEYoXIimKl8jgn6gsXLmDAgAEwMDBAixYtkJKSAl9fXxw+fFiiUEdFbG1tUV7hLmmn0m1tbXHjxg2uXa4yorxdvO4vmde9VV5KSgr8/f0xfvx4XL9+HWvWrGHvX5cmICAA3377LXx9fTF27FhoaGjg3r17OH36NNauXVuNPSfVjWKFyIpipfI4J2ofHx8sWrQI48ePZ5dt2rQJEydOxM2bN3ntHKkZ7u7uyM3NRbdu3aCoqIgpU6Zg3LhxZa7fsWNHnDlzBrNnz0avXr3AMAxMTU0xdOjQauw1qQkUK0RWFCuVx7kedcOGDfHhwweJQtdisRg6OjrIysrivYN8yszMhLa2Nj58+MBL9SzRTH6vjycv6c/r/irD1tYWlpaWElPwEX7UxXtjKVaqBsVK3cclH3G+j9rBwQFRUVESy06dOgVHR0euuyKEEEJIBTif+tbV1cWPP/4Ie3t7tGjRAs+ePUNMTAxGjBjBDggDgJCQEF47SgghhNRHnBN1UVERe43g48eP0NfXx9ChQyEWi5GRkcF7B0n1iouLq+kuEDlBsUJkRbHydTgn6q1bt1ZFPwghhBAiBedr1EDx4LGkpCTcunVL4kGqnq2tbbVPNv+1NVzj4uLYWrGkelG8EGkoLuQL50R98OBBGBoaolWrVrC0tGQfnTp1qor+EZ7VRLD36NEDr1+/5jwpDl8OHDiAvn37onHjxnJZNL4mUbxQvEhDcVF2XMTHx8Pe3h4aGhrQ0tJC7969kZub+1Vtc07UkydPxtKlS5GTk4OioiL2UTJfKyFfKpk0//N5fqtTTk4OvvvuOyxdurRG2ifcULwQaeQhLuLj4+Hs7Iy+ffsiISEBV65cga+vr8TtzJXBeeu8vDyMHj26zIpVpOp9+vQJvr6+0NbWhp6eHubOncvO8rZ9+3ZYWVmhYcOGMDQ0xIgRI5Ceng6g+NRTybStjRo1gkAgwOjRowEUDxJctmwZWrVqBRUVFTRv3hwLFy6UaDcpKQl2dnZQV1eHhYUF4uPj2deePXsGV1dXNGrUCBoaGmjfvj1bROXLb9+2trYQCASlHsnJyQCA9+/fY+zYsdDX14eWlhbs7e2/ajIdNzc3zJs3Dw4ODpXeBx9qqiQqxQs3tSVeqhrFBTeyxIWfnx8mT56MmTNnon379mjTpg2GDBny1VUcOSdqHx8f/PHHH1/VKPk65dV1LSwsxIIFC3Dz5k0cPHgQycnJ7B+RUCjE/v37AQAPHz7E69evERoaCqC4dveSJUswd+5c3Lt3D7t27YKBgYFEu7Nnz8a0adOQmJgIMzMzDB8+HJ8+fQJQHBf5+fk4e/Ysbt++jaVLl0JTU1Nq/w8cOIDXr1+zjx9++AFt2rRh2/vpp5+Qnp6OEydO4Nq1a+jcuTP69OmDd+/eASiuia6pqVnu4/OKO/UdxQvFizQUF/zGRXp6Oi5fvowmTZqgR48eMDAwgI2NDc6fPy/zPsrCedS3l5cXbGxssHjx4lK/gOvXr391h0jFyqvr+vPPP7PrtWzZEqtXr0bXrl2RnZ0NTU1N6OrqAgCaNGkCHR0dAEBWVhZCQ0Oxdu1aeHh4AABMTU3x3XffSbQ7bdo09O9fPHva/Pnz0b59ezx+/Bjm5uZISUnB4MGD0aFDB7btspT0AQBWrlyJmJgYXL58GWpqajh//jwSEhKQnp7Ofgtdvnw5Dh48iH379mHcuHGwsrKq8Lrhl7FZn1G8yF+8cD1rUhkUF/zGRVJSEgAgODgYy5cvh6WlJbZt24Y+ffrgzp07aN26tcz7+hLnRP3jjz/CxMQEgwYNgrq6eqUbJpVXXl3XxMREBAcH4+bNm8jIyEBRURGA4gnx27VrJ3V/9+/fR35+Pvr06VNuux07dmT/b2RkBKD4W6S5uTkmT54Mb29vnDp1Cg4ODhg8eLDE+tKcOHECM2fOxJEjR2BmZgaguAZtdnY2GjduLLFubm4unjx5AgBQU1OjijocULxQvEhDccFvXJT8jMaPHw9PT08AQKdOnRAdHY3w8HAsXry40vvmnKhv3bqFjIwMKCkpVbpRUjXy8vLg5OQEJycn7Ny5E/r6+khJSYGTkxMKCgrK3E7W8Qaf/85L/sBLgnPs2LFwcnLCsWPHcOrUKSxevBgrVqzApEmTpO7r3r17GDZsGJYsWYK+ffuyy7Ozs2FkZCR1goSSb+7nzp2Di4tLuX3dsGEDRo4cKdP7qq8oXv5D8fIfiov/cImLki8dX36Radu2rUTJ5srgnKi//fZbPHnyBObm5l/VMKm8suq6PnjwAP/++y+WLFkCoVAIoLju9+eUlZUBQGKUfuvWraGmpobo6GiMHTu20v0SCoWYMGECJkyYgMDAQGzatEnqH9jbt2/h6uqKwYMHw8/PT+K1zp07IzU1FQ0aNIBIJJLajjyeyqxJFC8UL9JQXPAbFyKRCMbGxnj48KHE8kePHlX4haAinBN1ly5d4OTkhFGjRpV6E5MnT/6qzhDZlFXXtXnz5lBWVsaaNWswYcIE3LlzBwsWLJDYtkWLFhAIBDh69Cj69esHNTU1aGpqIiAgADNmzICysjJ69uyJN2/e4O7duxgzZoxMfZo6dSpcXFxgZmaGjIwMxMbGom3btlLXHTx4MNTV1REcHIzU1FR2ub6+PhwcHGBtbY2BAwdi2bJlMDMzw6tXr3Ds2DEMGjQIVlZWnE9ZvXv3DikpKXj16hUAsH9IhoaGMDQ0lHk/8oriheJFGooLfuNCIBBg+vTpCAoKgoWFBSwtLREZGYkHDx5g3759MrcjDedEnZCQgJYtW+LixYsSywUCASXqalJWXVeBQICIiAjMmjULq1evRufOnbF8+XIMGDCA3bZp06aYP38+Zs6cCU9PT7i7uyMiIgJz585FgwYNMG/ePLx69QpGRkaYMGGCzH0Si8Xw8fHBixcvoKWlBWdnZ6xcuVLqumfPngVQ/Mf+uadPn0IkEuH48eOYPXs2PD098ebNGxgaGqJ3796VPuo5fPgwe80IAIYNGwYACAoKQnBwcKX2KU8oXripL/FCccGNLHExdepU5OXlwc/PD+/evYOFhQVOnz4NU1PTSrVZgnM9anlG9ahJjQrmcUal4A/87YvUPhQrdV6V1qMmhBBCSPXhfOpbQUGhzCncaBpRQgghhF+cE/WNGzcknr969Qq///47e76eEEIIIfzhfOrbwsJC4uHi4oLdu3dj7dq1nBtft24dRCIRVFVV0b17dyQkJJS7/qpVq9CmTRuoqalBKBTCz88PeXl5nNslhBBC5AUv16hVVVXx7NkzTtvs2bMH/v7+CAoKwvXr12FhYQEnJyd24vcv7dq1CzNnzkRQUBDu37+PLVu2YM+ePZg1axYfb4H8fyKRCKtWrarpbhA5QLFCuKB4qTzOiXr16tUSjyVLlsDBwQG2trac9hMSEgIvLy94enqiXbt2CAsLg7q6OsLDw6Wuf/HiRfTs2RMjRoyASCRC3759MXz48AqPwon8WbhwIXr06AF1dXV2FqEvpaSkoH///lBXV0eTJk0wffp0dmJ/oHjCfkdHR7ZyjrW1NU6ePFlmm0uWLIFAIMDUqVN5fjekOuTn58PS0lJqneBbt26hV69eUFVVhVAoxLJlyyReLywsxK+//gpTU1OoqqrCwsICUVFRpdp4+fIlRo0ahcaNG0NNTQ0dOnQoNREIqd3k9bOFc6L++++/JR4XLlyAi4sLIiMjZd5HQUEBrl27JlEuTEFBAQ4ODhIlzz7Xo0cPXLt2jU3MSUlJOH78OPr161dmO/n5+cjMzJR4kNqvoKAAP/30E7y9vaW+LhaL0b9/fxQUFODixYuIjIxEREQE5s2bx65z9uxZODo64vjx47h27Rrs7Ozg6upaaowFAFy5cgUbNmyocE5hUnvNmDEDxsbGpZZnZmaib9++aNGiBa5du4bff/8dwcHB2LhxI7vOnDlzsGHDBqxZswb37t3DhAkTMGjQIIlYycjIQM+ePaGkpIQTJ07g3r17WLFiBRo1alQt74/wQ14/Wzgn6tjYWInHkSNHMH/+/DK/nUjz9u1biMXiUjeeGxgYSMww87kRI0bg119/xXfffQclJSWYmprC1ta23FPfixcvhra2NvsomQ5PXpVXqxUA7ty5AxcXF2hqasLAwABubm54+/Yt+3pOTg7c3d2hqakJIyMjrFixAra2tuV+06uopmtwcDAsLS2xfft2iEQiaGtrY9iwYcjKyqr0+5w/fz78/PzYCjpfOnXqFO7du4cdO3bA0tISLi4uWLBgAdatW8fORbxq1SrMmDEDXbt2RevWrbFo0SK0bt0aR44ckdhXdnY2Ro4ciU2bNtWpD936EitAcVGGU6dOYfny5aVe27lzJwoKChAeHo727dtj2LBhmDx5MkJCQth1tm/fjlmzZqFfv35o2bIlvL290a9fP6xYsYJdZ+nSpRAKhdi6dSu6desGExMT9O3b96snsqgt6ku8yOtnC+dEfe3aNbx8+RJA8ZRqU6ZMwS+//FLlR6txcXFYtGgR/vjjD1y/fh0HDhzAsWPHSk1t97nAwEB8+PCBfTx//rxK+1jVyqvV+v79e9jb26NTp064evUqoqKikJaWhiFDhrDbT58+HWfOnMGhQ4dw6tQpxMXFVViatKKargDw5MkTHDx4EEePHsXRo0dx5swZLFmyhH190aJFFdZ95TJpfXx8PDp06CDxRc/JyQmZmZm4e/eu1G2KioqQlZUlURqv5Gfav3//covB80WUt4u3R0XqS6ykpaXBy8sL27dvl1rNLz4+Hr1792bnpgaKY+Xhw4fIyMgAUHzmTVVVVWK7klKJJQ4fPgwrKyv89NNPaNKkCTp16oRNmzaV+/OQJ/UlXipSWz9bON+e5enpiQMHDgAoriv67NkzqKqqwtvbW+Yi23p6elBUVERaWprE8rS0tDLn0p07dy7c3NzYyd47dOiAnJwcjBs3DrNnz4aCQunvHCoqKmwt0rqgvFqta9euRadOnbBo0SJ2WXh4OIRCIR49egRjY2Ns2bIFO3bsYMvQRUZGolmzZmW2J0tNV6A4UCMiItCwYUMAgJubG6Kjo7Fw4UIAwIQJEyT+qKWRdtqyLKmpqVLPxpS8Js3y5cuRnZ0t0Y/du3fj+vXruHLlisxty4v6ECsMw2D06NGYMGECrKyskJycXGrd1NRUmJiYSCz7PFYaNWoEJycnhISEoHfv3jA1NUV0dDQOHDggMS9EUlIS1q9fD39/f8yaNQtXrlzB5MmToayszNZelmf1IV5kUVs/Wzgn6pSUFLRq1QoMw+Dw4cO4f/8+1NTUOJ0CUlZWRpcuXRAdHY2BAwcCKP6FREdHw9fXV+o2Hz9+LJWMFRUVART/wdYH5dVqvXnzJmJjY9lvwZ978uQJcnNzUVBQgO7du7PLdXV10aZNmzLbk6WmK1A8mrPkDwkoLvf2+eh9XV3dUt82q9OuXbswf/58HDp0CE2aNAEAPH/+HFOmTMHp06dLHU3VBfUhVtasWYOsrCwEBgbKtH5ZQkND4eXlBXNzcwgEApiamsLT01NiYGtRURGsrKzYZNWpUyfcuXMHYWFhdSJR14d4qQrV9dnCOVErKSkhKysLd+/eRbNmzaCvrw+xWIzc3FxO+/H394eHhwesrKzQrVs3rFq1Cjk5Oeyk5+7u7mjatClbbNvV1RUhISHo1KkTunfvjsePH2Pu3LlwdXVlE3ZdV16t1uzsbLi6umLp0qWltjMyMsLjx485tydLTVcApWqTCwQCtr4sUHx66vNv49Lcu3cPzZs3l6lfhoaGpUb7l5yd+fKMzO7duzF27Fjs3btX4hTUtWvXkJ6ejs6dO7PLxGIxzp49i7Vr1yI/P1+u46o+xEpMTAzi4+NLnTWzsrLCyJEjERkZCUNDQ6ln7oD/YkVfXx8HDx5EXl4e/v33XxgbG2PmzJkSR5VGRkZS6wzv37+/3L7Ki/oQL7KorZ8tnBO1q6sr7O3tkZWVhdGjRwMoHmhQ3mkOaYYOHYo3b95g3rx5SE1NhaWlJaKiotjTDCkpKRJH0HPmzIFAIMCcOXPw8uVL6Ovrw9XVlT0FUl+UVau1c+fO2L9/P0QiERo0KP1rNTU1hZKSEi5fvswGbUZGBh49egQbGxupbclS01UWfJ+esra2xsKFC5Gens5+iz19+jS0tLQkPkz//PNP/Pzzz9i9ezf695cseNKnTx/cvn1bYpmnpyfMzc0REBAg10m6RF2PldWrV+O3335jl7969QpOTk7Ys2cPe3RnbW2N2bNno7CwkP3QP336NNq0aVNqgI+qqiqaNm2KwsJC7N+/X6IfPXv2lFpn+MvKTfKsrseLLGrrZwvnRB0WFobIyEgoKSnBzc0NQPGgss+Hr8vK19e3zFPdX37TatCgAYKCghAUFMS5nbqivFqtPj4+2LRpE4YPH44ZM2ZAV1cXjx8/xu7du7F582ZoampizJgxmD59Oho3bowmTZqUeW2/hCw1XWXB9fRUSkoKW/tVLBaz98W2atUKmpqa6Nu3L9q1awc3NzcsW7YMqampmDNnDnx8fNijq127dsHDwwOhoaHo3r07e31JTU0N2traaNiwIb755huJdjU0NNC4ceNSy+VRfYiVL4+SSk7NmpqasgcOI0aMwPz58zFmzBgEBATgzp07CA0NlSidePnyZbx8+RKWlpZ4+fIlgoODUVRUhBkzZrDr+Pn5oUePHli0aBGGDBmChIQEbNy4UeI2L3lWH+IFkN/PFs6JWllZGV5eXhLL7OzsKtU44aa8Wq3Gxsa4cOECAgIC0LdvX+Tn56NFixZwdnZm/2B+//139jRWw4YN8csvv+DDh7JL4AkEAt5ruspi3rx5Evfld+rUCUDxrYG2trZQVFTE0aNH4e3tDWtra2hoaMDDwwO//voru83GjRvx6dMn+Pj4wMfHh13u4eGBiIiIKut7bVFfYqUi2traOHXqFHx8fNClSxfo6elh3rx57GAlAMjLy8OcOXOQlJQETU1N9OvXD9u3b5c4Bdu1a1f8/fffCAwMxK+//goTExOsWrUKI0eOrIF3xb/6Ei/y+tnCuR51VlYWVq5ciWvXrpW6ny0mJobXzvGN6lGXZmtrC0tLS5rarxrwGS8UK3VcHahHTfFSPi75iPMRtZubG54/f47BgwdDQ0Oj0p2szcRiMQoLCytcr2lDfq9j1kSBkSZNmkBLS6tOFDdRUlKqE9eWCSHkc5wTdVxcHFJSUng5Iq1tGIZBamoq3r9/L9P6wXZNeG3/6dOnvO5PFt7e3lBWVq6RtquCjo4ODA0Ny6yZTggh8oZzohYKhTIdbcqjkiTdpEkTqKurV/hhX6DG72xsJobV/+Xny8kg5BXDMPj48SN7j6WRkVEN96jukXYrDSFloXjhD+dE7evri6FDhyIgIKDURX95LmogFovZJP3lTfhlETTg93RxXZx4ozqpqakBAHtrBZ0GJ4TUBZwTdUnVkS8HjgkEAokp9+RNyVkCafMFE/lR8vsrLCykRE0IqRM4J+rPZ4Wpi+japnyj3x8hpK7hXD2rRGpqKq5evVpqej5CCCGE8Idzon779i2cnJxgbGyMXr16wdjYGM7Oznjz5k1V9I8QQgip1zif+p40aRJ0dXXx8uVLGBkZITU1Ff7+/vD19cWePXuqoo81ju+JTSpSE5NZfC4uLg52dnbIyMiQmJ2pPMHBwTh48CA7JR8hhBB+cD6ijomJwZYtW9jbXwwNDbFx40bExsby3jlSsbCwMDRs2BCfPn1il2VnZ0NJSQm2trYS68bFxUEgEEiUkZOmR48eeP36NbS1eZwdCcUzFU2dOpXXfRJCSF3HOVGrqqoiIyNDYtn79+9LlZoj1cPOzg7Z2dm4evUqu+zcuXMwNDTE5cuXJWYci42NRfPmzSusHa6srEyThhBCSC3BOVGPGDEC/fr1w759+3DlyhXs3bsXrq6udWZyennTpk2bUnVd4+Li8P3338PExASXLl2SWG5nZ4eioiIsXrwYJiYmUFNTg4WFBfbt2yexnkAgkJihbdOmTRAKhVBXV8egQYMQEhIi9bT49u3bIRKJoK2tjWHDhrHzwY8ePRpnzpxBaGgoBAIBBAIBkpOT+f5xEEJIncM5US9YsACDBw/GrFmzYGNjg9mzZ2PQoEFYsGBBVfSPyMDOzk7i0kNJJRgbGxt2eW5uLi5fvgw7OzssXrwY27ZtQ1hYGO7evQs/Pz+MGjUKZ86ckbr/CxcuYMKECZgyZQoSExPh6OgotQ74kydPcPDgQRw9ehRHjx7FmTNnsGTJEgBAaGgorK2t4eXlhdevX+P169cQCoVV8NMghJC6hfNgsgYNGmDevHmVqj9NqoadnR2mTp2KT58+ITc3Fzdu3ICNjQ0KCwsRFhYGAIiPj0d+fj5sbW3Rrl07/N///R+sra0BAC1btsT58+exYcMGqYXe16xZAxcXF0ybNg0AYGZmhosXL+Lo0aMS6xUVFSEiIgINGzYEUFzAJTo6GgsXLoS2tjaUlZWhrq4OQ0PDqvxxEEJInSLzEfX169fLTM5BQUE02rcG2draIicnB1euXMG5c+dgZmYGfX192NjYsNep4+Li0LJlS2RnZ+Pjx49wdHSEpqYm+9i2bVuZg8wePnyIbt26SSz78jkAiEQiNkkDxfNtl8y9TQghpHJkPqJesmQJfvjhB6mvffPNN1i0aBH++usv3jpGZNeqVSs0a9YMsbGxyMjIYI+KjY2NIRQKcfHiRcTGxsLe3h7Z2dkAgGPHjqFp06YS+/naAYFKSkoSzwUCQZ2fyY4QQqqazEfUly5dwvfffy/1NVdXV4lBS6T62dnZIS4uDnFxcRK3ZfXu3RsnTpxAQkIC7Ozs0K5dO6ioqCAlJQWtWrWSeJR1zbhNmza4cuWKxLIvn8tCWVlZrueDJ4SQmiDzEXVGRgaUlZWlvqakpIR3797x1inCnZ2dHXx8fFBYWChxndnGxga+vr4oKCiAnZ0dGjZsiGnTpsHPzw9FRUX47rvv8OHDB1y4cAFaWlrw8PAote9Jkyahd+/eCAkJgaurK2JiYnDixAnOt2+JRCJcvnwZycnJ0NTUhK6uLhQUKj2LLSGE1AsyJ2ojIyPcuXMHFhYWpV67fft2na7/W9ZMYbdevOe1nY7NdCq9rZ2dHXJzc2Fubi5RftTGxgZZWVnsbVxA8ch9fX19LF68GElJSdDR0UHnzp0xa9Ysqfvu2bMnwsLCMH/+fMyZMwdOTk7w8/PD2rVrOfVx2rRp8PDwQLt27ZCbm4unT59CJBJV+j0TQkh9IPPhzJAhQzBp0iT2vtgSWVlZmDp1KoYNG8a58XXr1kEkEkFVVRXdu3dHQkJCueu/f/8ePj4+MDIygoqKCszMzHD8+HHO7dZFIpEIDMPg/v37EstbtGgBhmHw4MEDdplAIMCUKVPw4MEDFBQUID09HVFRUejduzeA4sFpDMNI3Cft5eWFFy9e4OPHj/j777+RnJyMVq1asa8HBweXGlA4depUiXulzczMEB8fj48fP4JhGErShBAiA5mPqOfMmQNHR0eYmprCxcUFTZs2xcuXLxEVFQVzc3PMnj2bU8N79uyBv78/wsLC0L17d6xatQpOTk54+PAhmjRpUmr9goICODo6okmTJti3bx+aNm2KZ8+eyTwXNfk6y5cvh6OjIzQ0NHDixAlERkbijz/+qOluEUJInSdzolZVVUVsbCx27tyJU6dO4erVq9DT08OyZcswcuRINGjA7ZbskJAQeHl5wdPTE0DxnNXHjh1DeHg4Zs6cWWr98PBwvHv3DhcvXmRHF9MRWfVJSEjAsmXLkJWVhZYtW2L16tUYO3ZsTXeLEELqPE7ZtUGDBvDw8JA64IiLgoICXLt2DYGBgewyBQUFODg4ID4+Xuo2hw8fhrW1NXx8fHDo0CHo6+tjxIgRCAgIgKKiotRt8vPzkZ+fzz7PzMz8qn7XZ3TrHSGE1IwaGXL79u1biMViiUFPAGBgYIDU1FSp2yQlJWHfvn0Qi8U4fvw45s6dixUrVuC3334rs53FixdDW1ubfdCUlYQQQuSN3NwbU1RUhCZNmmDjxo3o0qULhg4ditmzZ7NTZEoTGBiIDx8+sI/nz59XY48JIYSQr8d5rm8+6OnpQVFREWlpaRLL09LSypwH2sjICEpKShKnudu2bYvU1FQUFBRIvcdbRUWFym8SQgiRa5wTdVlJkQtlZWV06dIF0dHRGDhwIIDiI+bo6Gj4+vpK3aZnz57YtWsXioqK2EkyHj16BCMjo6/uDyGE1CaivF287SuZtz2RmsL51LeRkREmTZqEGzdufFXD/v7+2LRpEyIjI3H//n14e3sjJyeHHQXu7u4uMdjM29sb7969w5QpU/Do0SMcO3YMixYtgo+Pz1f1gxBCCKnNOB9RHzt2DBEREbC3t0eLFi3g6emJUaNGoXHjxpz2M3ToULx58wbz5s1DamoqLC0tERUVxQ4wS0lJkZheUigU4uTJk/Dz80PHjh3RtGlTTJkyBQEBAVzfAqlFBAIB/v77b/bMCiGk+nWI7MDbvm573OZtX6QY50T97bff4ttvv0VoaCgOHDiAyMhIzJo1C/369cPPP/8MZ2dnmeeA9vX1LfNUd1xcXKll1tbWNVP8I1hb6uKOVdbeB06rjx49Gu/fv8fBgwfZZfv27cOoUaOwcOFCxMXFobCwEFFRUaW2PXfuHHr37o2bN2+iY8cqe0eEEEIqqdKDyVRUVPD999+joKAAr169wqlTp3Djxg0UFRVhw4YNcHR05LOfhIPNmzfDx8cHYWFh8PT0hKmpKQYPHowXL16gWbNmEutu3boVVlZWlKQJIaSWqtTtWWfPnoWnpyeMjIywZcsW+Pv74/Xr13j8+DEWLVoEd3d3vvtJZLRs2TJMmjQJu3fvZq/3/+9//4O+vj4iIiIk1s3OzsbevXsxZsyYMveXn5+PgIAACIVCqKiooFWrVtiyZQsAQCwWY8yYMTAxMYGamhratGmD0NDQUvsIDw9H+/btoaKiAiMjo1JnUd6+fYtBgwZBXV0drVu3xuHDhyVev3PnDlxcXKCpqQkDAwO4ubnh7du3lfnxEEKI3OGcqE1NTTFs2DAYGBjgypUrOHv2LEaPHg11dXUAwLBhw9CoUSPeO0oqFhAQgAULFuDo0aMYNGgQu7xBgwZwd3dHREQEGIZhl+/duxdisRjDhw8vc5/u7u74888/sXr1aty/fx8bNmyApqYmgOKR+s2aNcPevXtx7949zJs3D7NmzZKYxWz9+vXw8fHBuHHjcPv2bRw+fFiimAcAzJ8/H0OGDMGtW7fQr18/jBw5ki2b+v79e9jb26NTp064evUqoqKikJaWhiFDhvDyMyOEkNqO06nvoqIiTJkyBd7e3ux829Lcu3fvqztGuDlx4gQOHTqE6Oho2Nvbl3r9559/xu+//44zZ87A1tYWQPFp78GDB0NbW/o1+EePHuGvv/7C6dOn4eDgAABo2bIl+7qSkhLmz5/PPjcxMUF8fDz++usvNpH+9ttv+OWXXzBlyhR2va5du0q0M3r0aPbLwqJFi7B69WokJCTA2dkZa9euRadOnbBo0SJ2/fDwcAiFQjx69AhmZmZcfkyEECJ3OB1RKygoIDAwkHMBDlL1OnbsCJFIhKCgIGRnZ5d63dzcHD169EB4eDgA4PHjxzh37ly5p70TExOhqKgIGxubMtdZt24dunTpAn19fWhqamLjxo1ISUkBAKSnp+PVq1fo06dPhX0voaGhAS0tLaSnpwMAbt68idjYWGhqarIPc3NzAMCTJ0/K3S8hhNQFnE99d+zYkT4ga6GmTZsiLi4OL1++hLOzc6m64QAwZswY7N+/H1lZWdi6dStMTU3LTcJqamrltrl7925MmzYNY8aMwalTp5CYmAhPT08UFBTItH2JL8/OCAQCFBUVASi+ju7q6orExESJxz///MPWzyaEkLqM86Hx999/D1dXV0ycOBFCoVDiXucBAwbw2jnCTYsWLXDmzBnY2dnB2dkZUVFRaNiwIfv6kCFDMGXKFOzatQvbtm2Dt7d3ubfSdejQAUVFRThz5gx76vtzFy5cQI8ePTBx4kR22edf4ho2bAiRSITo6GjY2dlV6j117twZ+/fvh0gkojM5hJB6ifMn34YNGwAU15P+nEAgoERdCwiFQsTFxcHOzg5OTk6IioqClpYWAEBTUxNDhw5FYGAgMjMzMXr06HL3JRKJ4OHhgZ9//hmrV6+GhYUFnj17hvT0dAwZMgStW7fGtm3bcPLkSZiYmGD79u24cuUKTExM2H0EBwdjwoQJaNKkCVxcXJCVlYULFy5g0qRJMr0fHx8fbNq0CcOHD8eMGTOgq6uLx48fY/fu3di8eXOZJU4JIaSu4Hzq++nTp1IfSUlJVdE/UgnNmjVDXFwc3r59CycnJ4k63GPGjEFGRgacnJxgbGxc4b7Wr1+PH3/8ERMnToS5uTm8vLyQk5MDABg/fjx++OEHDB06FN27d8e///4rcXQNAB4eHli1ahX++OMPtG/fHv/73//wzz//yPxejI2NceHCBYjFYvTt2xcdOnTA1KlToaOjI3E2hxBC6ioB8/n9OjIqKipCQkICXrx4AaFQiK5du8rFh2ZmZia0tbXx4cMH9iizRF5eHp4+fQoTExOoqqrKtL9bL97z2r+OzXR43V99VJnfY3URzTzG276Sl/TnbV+k9qnuWKEpRKtfefnoS5xPfT99+hSurq54/vw5jI2N8erVKzRr1gxHjhyRuHWHEEIIIV+P82Gwj48PnJ2d8ebNG9y/fx9v3rxB//79qYoVIYQQUgU4H1EnJCTg4MGDbA1oZWVlLFiwQKbrnYQQQgjhhvMRtY6ODh4/fiyxLCkpCTo6Onz1iRBCCCH/H+cjam9vbzg5OWHy5MkQiURITk7G2rVrZb7dhhBCCCGy45yof/nlF+jr62PHjh1s2cQFCxZQxSxCCCGkClRqqid3d3dKzIQQUkfcfppS010g5ahUon7z5g1u375dqvgDzUxGCCGE8Itzol6/fj38/Pygra3N1qAGaApRQgghpCpwTtTBwcE4fvy41JrHhBBCCOEX50StrKxc78oL8jm9niy4TMFXXvUrAAgKCkJwcPBX9oi73NxcNG3aFAoKCnj58iVUVFRk3nb06NF4//49Dh48WHUdJIQQOcH5Puq5c+dizpw5bM3hr7Vu3TqIRCKoqqqie/fuSEhIkGm73bt3QyAQYODAgbz0Q169fv2afaxatQpaWloSy6ZNm1Yj/dq/fz/at28Pc3NzSriEEPIVOCfqnj17Yt++fdDQ0ICurq7Eg6s9e/bA398fQUFBuH79OiwsLODk5IT09PRyt0tOTsa0adPQq1cvzm3WNYaGhuxDW1sbAoFAYpmmpibi4uIgEAhw8uRJdOrUCWpqarC3t0d6ejpOnDiBtm3bQktLCyNGjMDHjx/Zfdva2sLX1xe+vr7Q1taGnp4e5s6dC1nquGzZsgWjRo3CqFGjsGXLllKv3717F//73/+gpaWFhg0bolevXnjy5AmCg4MRGRmJQ4cOQSAQQCAQIC4ujs8fGSGEyBXOp76HDRuGXr16Yc2aNRKDySojJCQEXl5e8PT0BACEhYXh2LFjCA8Px8yZM6VuIxaLMXLkSMyfPx/nzp3D+/fvv6oP9UlwcDDWrl0LdXV1DBkyBEOGDIGKigp27dqF7OxsDBo0CGvWrEFAQAC7TWRkJMaMGYOEhARcvXoV48aNQ/PmzeHl5VVmO0+ePEF8fDwOHDgAhmHg5+eHZ8+eoUWLFgCAly9fonfv3rC1tUVMTAy0tLRw4cIFfPr0CdOmTcP9+/eRmZmJrVu3AkClvgQSQkhdwTlRJycn4+bNm19d1rKgoADXrl1DYGAgu0xBQQEODg6Ij48vc7tff/0VTZo0wZgxY3Du3Lly28jPz0d+fj77/PO6zPXRb7/9hp49ewIorksdGBiIJ0+esFXPfvzxR8TGxkokaqFQiJUrV0IgEKBNmza4ffs2Vq5cWW6iDg8Ph4uLCxo1agQAcHJywtatW9lr5evWrYO2tjZ2794NJSUlAICZmRm7vZqaGvLz82FoaMjr+69P+B5XQaULCak5nLOts7OzzNeRy/P27VuIxWIYGBhILDcwMEBqaqrUbc6fP48tW7Zg06ZNMrWxePFiaGtrsw+hUPjV/ZZnHTt2ZP9vYGAAdXV1idKkBgYGpS47fPvttxID1qytrfHPP/9ALBZLbUMsFiMyMhKjRo1il40aNQoREREoKioCACQmJqJXr15skiaEEFI2zkfU2tracHZ2Rv/+/Usl2ZCQEN469qWsrCy4ublh06ZN0NPTk2mbwMBA+Pv7s88zMzPrdbL+PDEKBIJSiVIgELDJtLJOnjyJly9fYujQoRLLxWIxoqOj4ejoCDU1ta9qgxBC6hPOiVosFmPQoEEAgIyMjEo3rKenB0VFRaSlpUksT0tLk3rK88mTJ0hOToarqyu7rCSpNGjQAA8fPoSpqanENioqKpxuCyKlXb58WeL5pUuX0Lp1aygqKkpdf8uWLRg2bBhmz54tsXzhwoXYsmULHB0d0bFjR0RGRqKwsFDqUbWysnKZR+yEEFLfcE7UJQN8vpaysjK6dOmC6Oho9haroqIiREdHw9fXt9T65ubmuH1b8jrZnDlzkJWVhdDQ0Hp9pFyVUlJS4O/vj/Hjx+P69etYs2YNVqxYIXXdN2/e4MiRIzh8+DC++eYbidfc3d0xaNAgvHv3Dr6+vlizZg2GDRuGwMBAaGtr49KlS+jWrRvatGkDkUiEkydP4uHDh2jcuDG0tbXpNDkhpN6q1Fzf//zzD/766y+8fv0aa9euxcOHD5Gfny9xDVQW/v7+8PDwgJWVFbp164ZVq1YhJyeHHQXu7u6Opk2bYvHixVBVVS314V9SA/vL5XwrayDNrRfveW2nYzMdXvfHB3d3d+Tm5qJbt25QVFTElClTMG7cOKnrbtu2DRoaGujTp0+p1/r06QM1NTXs2LEDkydPRkxMDKZPnw4bGxsoKirC0tKSHejm5eWFuLg4WFlZITs7G7GxsbC1ta3Kt0kIIbUW58FkR44cQdeuXXH//n1s27YNQPEp8MpMrDF06FAsX74c8+bNg6WlJRITExEVFcVe+05JScHr168577e+KpnR60u2trZgGIb9YlPWusHBwUhMTJRYpqSkhPXr1+PDhw949+4dFi5cWOZsaL/88gsyMjLKPJ2dkZGByZMnAyge2Hby5Enk5OQgMzMTZ8+eZQe26evr49SpU8jKygLDMJSkCSH1Gucj6lmzZuH48ePo0aMHe/tNp06dSn3Ay6pkQg1pKproIiIiolJtEkIIIfKC8xH1ixcv0KNHDwD/zTOtpKREg38IIYSQKsD5iNrMzAxnzpyBjY0Nu+zs2bNo27Ytrx0jNY+m7iSEkJrHOVEvXLgQgwYNgqenJ/Lz8zFr1ixs3boVf/75Z1X0jxBCCKnXOCdqBwcHxMXFYePGjbCzs0NGRgZOnDgBS0vLKuhe9ZOl4ASpvej3R0jtx+cUt/VhettK3Z7VsWNHrF27lu++1KiSkcofP36kmbPkWEn1L7rvunrR3OKEVB2ZE/Xhw4crXGfAgAFf1ZmapKioCB0dHXaua3V19TJvQyrBfOKnJneJvLw8XvdXnzAMg48fPyI9PR06OjplzpxGCCHyRuZEPXDgQBgaGkJVVVXq6UWBQCDXiRoAO3VpRfWwS6Rn5PLavnIuHcl/LR0dHaq6RQipU2RO1C4uLkhISMAPP/wAT09PdOnSpSr7VSMEAgGMjIzQpEkTFBYWVrj+2ANxvLYf/Ystr/urb5SUlOhImhBS58icqI8dO4bU1FRs27YNo0aNgpKSEjw9PeHm5iZzNSt5oaioKNMH/sssfu8dV1VV5XV/pP66/TSlprtACOEJpwlPDA0NMWPGDNy/fx8bNmzA7du3IRQKealPTQghhJDSOM9MBgCPHj3CoUOHcOrUKXTt2hX6+vp894sQQggh4HDqOzs7G3v27EF4eDhevHgBd3d3xMXFoVWrVlXZP/IZugWGEELqH5kTtZGREYRCITw9PeHo6AgFBQV8/PgRt27dYtfhWuaSEEIIIeWTOVHn5OTgwYMHCAgIwMyZM0vdoiUQCKgwRxWjAUKEEFL/yJyoi4qKqrIfhBBCCJGiUlOIkvqBrokTQkjNq9Sob0IIIYRUDzqiJmWq9mviwdo87usDf/sitQ6d7SGyqguxQolajojydvG6v+Q63F5FbRFCiLyo9KlvhmHw+vVrPvtCCCGEkC9wTtRZWVlwd3eHqqoqO9nJwYMHMW/ePN47RwghhNR3nE99T5o0CWKxGHfu3EG3bt0AAN9++y0CAgLw66+/cu7AunXr8PvvvyM1NRUWFhZYs2YNu98vbdq0Cdu2bcOdO3cAAF26dMGiRYvKXJ8QUj2qezwDzSkg36rz91cXYoXzEXVUVBQ2b96M1q1bQyAQACgu1pGWlsa58T179sDf3x9BQUG4fv06LCws4OTkVGY96Li4OAwfPhyxsbGIj4+HUChE37598fLlS85tE0IIIfKAc6JWUVHBp0+fJJb9+++/0NXV5dx4SEgIvLy84OnpiXbt2iEsLAzq6uoIDw+Xuv7OnTsxceJEWFpawtzcHJs3b0ZRURGio6M5t00IIYTIA86J2tXVFRMnTsSHD8W3v+Tn52PGjBkYNGgQp/0UFBTg2rVrcHBw+K8zCgpwcHBAfHy8TPv4+PEjCgsLy/ySkJ+fj8zMTIkHIYQQIk84X6NeunQpPD090bhxYxQVFUFTUxMDBw7E6tWrOe3n7du3EIvFMDAwkFhuYGCABw8eyLSPgIAAGBsbSyT7zy1evBjz58/n1C9C6gJ5vrWuNrZHSE3inKg1NDTw119/4c2bN3j27BmEQmGpZFsdlixZgt27dyMuLg6qqqpS1wkMDIS/vz/7PDMzE0KhsLq6SAghhHy1Sh1Ru7m5wdjYGPr6+pVuWE9PD4qKiqUGoaWlpcHQ0LDcbZcvX44lS5bg//7v/8otramiogIVFZVK95EQQgipaZyvUZ85cwYtW7aEo6MjduzYgY8fP1aqYWVlZXTp0kViIFjJwDBra+syt1u2bBkWLFiAqKgoWFlZVaptQgghRF5wTtTHjx/Hs2fP0K9fP4SEhMDAwADu7u6VGnnt7++PTZs2ITIyEvfv34e3tzdycnLg6ekJAHB3d0dgYCC7/tKlSzF37lyEh4dDJBIhNTUVqampyM7O5tw2IYQQIg8qNYWogYEB/Pz8cP36dVy6dAlv3rxB3759Oe9n6NChWL58OebNmwdLS0skJiYiKiqKveadkpIiMU3p+vXrUVBQgB9//BFGRkbsY/ny5ZV5G4QQQkitV+miHK9fv8bOnTuxfft2JCcnw8PDo1L78fX1ha+vr9TX4uLiJJ4nJydXqg1CCCFEXnE+ot6xYwf69u0LExMTREdHIyAgAKmpqWVOUkIIIYSQyuN8RL18+XK4ublh27ZtFY7OJoQQUvtVd4nZ6myvLtxzzzlRJyYmVkE3CCGEECKNTIk6LCwMEyZMAIByZyCbPHkyP70ihBBCCAAZE/Xhw4fZRP33339LXUcgEFCiJoQQQngmU6I+fvw4+//Y2Ngq6wwhhBBCJHEe9d2hQwepyy0tLb+2L4QQQgj5AudEXda9zCkpKV/bF0IIIYR8QeZR3yVVqAoLCyUqUgFAUlISWrZsyW/PCCGEECJ7os7IyABQXDij5P8AoKCggHbt2iEkJIT/3hFCCCH1nMyJeuvWrQCAzp07Y9KkSVXWIUIIIYT8h/OEJ58n6aysLDAMwz7X0tLip1eEEEIIAVCJwWQpKSno27cv1NXVoaOjg0aNGrEPQgghhPCLc6KeOHEiGjVqhIsXL0JTUxPXr1/HgAEDsGHDhqroHyGEEFKvcT71HR8fj2fPnkFTUxMCgQAWFhbYvHkzevXqhbFjx1ZFHwkhhJB6i/MRtaKiIlRUVAAUX5N+9+4dtLW18fz5c947RwghhNR3nI+oLS0tERMTAycnJ/YoWl1dHe3atauK/hFCCCH1Gucj6i1btsDc3BwAEBoaisaNG6OwsBCRkZG8d44QQgip7zgfUQuFQvb/enp62LRpE68dIoQQQsh/ZC5zKYsBAwZ8VWcIIYQQIkmmRD1lypQK1xEIBJSoCSGEEJ7JdI366dOnFT6SkpIq1YF169ZBJBJBVVUV3bt3R0JCQrnr7927F+bm5lBVVUWHDh0kamUT8v/au/eYps4/DOBPy01BBbwBshbQFZV5mxI23ASKODFs6pyZc07QMEXxssWFOVyUOfObumw4zLYoUdTMISozBswwGig6vCCgwLwuGShqBHZhiFVgtt/fH4ZuTJAW315O+X6SRjiU85zap+el7el7GGPM3ph8MBkAPHz4EKdOncL+/fsBAFqtFlqt1uT17N+/H6tWrUJKSgrOnz+PsWPHYurUqaivr+/w+qdPn8bcuXMRHx+PCxcuYObMmZg5cyYuXrzYnZvBGGOM2TyTB+qrV69i5MiRmDNnDuLj4wEA+fn53ZrsJDU1FYsWLcLChQsRFBSEbdu2wdXVFRkZGR1ePy0tDdHR0UhKSsLIkSOxYcMGjB8/Hl9//bXJ2YwxxpgUmHzUd2JiIt577z0sX77cML93REQEli1bZtJ6WltbUVZWhuTkZMMyuVyOqKgonDlzpsPfOXPmzGPnwp46dSoOHz7c4fVbWlrQ0tJi+L6xsREAcPfuXZO2tTP6lvtC1tOmq+3iPHFZ1mDJ2yfl+84W8yzN0o8Fe86z1a60reffJ7bqFJnI09OTdDqd4es27u7uJq3n9u3bBIBOnz7dbnlSUhKFhIR0+DtOTk6UmZnZbtk333xDgwcP7vD6KSkpBIAvfOELX/jCF5u83Lx5s8vx0uRn1F5eXrh+/TqGDh1qWPbLL7/gmWeeMXVVZpecnNzuGbher8eff/6JAQMGQCaTWWQb7t69C4VCgZs3b1rkNKCcJ132/n9p73mWZsnbZ+/3nTW6QkRoamrCkCFDuryuyQP10qVL8cYbb+DTTz+FTqfDsWPHsHbtWixfvtyk9QwcOBAODg6oq6trt7yurg7e3t4d/o63t7dJ13dxcTHMS97Gw8PDpO0UpV+/fhbdWXCedNn7/6W951maJW+fvd93ls5zd3c36nomH0y2cuVKLFq0CMnJydDpdHj//fcxf/58LFmyxKT1ODs7Y8KECcjPzzcs0+v1yM/PR2hoaIe/Exoa2u76AHD8+PFOr88YY4xJncnPqIFHB5QlJia2W3b79m34+vqatJ5Vq1YhLi4OwcHBCAkJwVdffQWtVouFCxcCAGJjY+Hr64uNGzcCeDTxSnh4OL788kvExMQgKysLpaWlSE9P787NYIwxxmyeyc+o//jjD+j1esP3tbW1WLFiBVQqlcnhc+bMwRdffIF169Zh3LhxKC8vx9GjR+Hl5QUAqKmpwZ07dwzXnzhxIjIzM5Geno6xY8ciOzsbhw8fxqhRo0zOthQXFxekpKQ89hI850kjz5Ls/f/S3vMszZK3z97vO1vviozImGPDgbKyMsyaNQu3bt3CwIEDcejQIVRUVGD16tVQq9VYs2YNXnzxRXNvL2OMMdajGD1Qq9VqhISEIC4uDjt27EBWVhZ8fX2Rnp6O559/3tzbyRhjjPVIRg/UAwcORG1tLRwdHfHgwQP06dMHt27dgo+Pj7m3kTHGGOuxjH6PurW1FY6Oj4496927N9zd3XmQZowxxszM6KO+W1tbsXXrVsP3LS0t7b4HHn10izHGGGPiGP3Sd0RExBNn85LJZCgoKBC2YYwxxhgzYaBmrCNEZLHpWK2Rx8Sy5P3HXZE23rf8o1vno2bGa2lpwa+//truLF7mFBkZiRs3blgkC3j0+cMrV67YbZ6lcV+kmWUN3BVp55miWzOTsY7t3r0bw4cPR2hoKJqbm7Fs2TLs2bMHRAS5XI74+HikpaUJ+VB9Tk5Oh8tPnjyJI0eOQKFQAACmT5/+1FkAHju9aBudTodNmzZhwIABAB6dY1yKedbAfRFz/3FXuCu2nCcCv/Qt0NChQ7Fv3z688MILSEpKQnZ2NlJTUzFy5Ehcu3YNH374IWbMmIHPP//8qbPkcjlkMtkTz2Uqk8mg0+meOqstb+zYsY+d1OTEiRMIDg6Gm5ub0OMULJ1nDdwXMfcfd4W7Yst5QnR5IkxmNBcXF7px4wYREQUGBlJeXl67n584cYKUSqWQrOjoaIqJiaG6urp2yx0dHenSpUtCMv5t48aNFBAQQPn5+XaZZw3cF+llWQt3Rbp5IvBALZCfnx8VFBQQEZGvry+VlJS0+/nly5fJzc1NWF5qaiopFArKzc01LDNn2c6dO0eBgYH0wQcfUGtrq93lWRr3RZpZ1sBdkXbe0+KBWqA1a9ZQaGgoNTQ00EcffUSvvfYaNTU1ERGRVqulN998k1555RWhmRcuXKCgoCBavHgxabVas5etqamJYmNjacyYMfTzzz+Tk5OTXeVZEvdFulmWxl2Rft7T4IFaoJaWFpo+fTp5enrSlClTqFevXuTq6koqlYrc3NxIqVTStWvXhOdqtVpKSEgglUpFDg4OFinbvn37yMvLi+RyuV3mWQL3RfpZlsJdsZ+87uCDyczg6NGjyM3NRVVVFfR6PXx8fPDSSy/h7bffhpubm5CM6upqBAQEtFuWk5MDjUaD5ORkDB48WEjOk9y6dQtlZWWIiooSdrtsKc9SuC/SzrIk7op95JmKB2qJksvl8PPzg1qthlqtRmRkJHx9fc2WN3v2bLz77ruYOnWq1ScFICIcPXoUO3fuRHZ2tlW3RSp6al+4K6brqV0BbLcvPOGJRBUUFCAuLg5VVVVYvHgxlEolVCoVEhISkJWVhbq6OqF5DQ0NiImJgVKpxLp161BVVSV0/caorq7G2rVroVQq8frrr6O5udni2yBVPa0v3JXu62ldASTQFyu+7N7jlJeXk1wuF77eBw8eUH5+Pq1du5YmTZpELi4uJJfLKSgoSGjO9evXKSUlhQICAkgul5Narabvv/+empubheb8W3NzM+3du5fUajU5OTmRXC6n1NRUamxsNFumreC+mIa7wl0xhZT6wgO1BZWXl5NMJjPb+ltaWqigoICSkpKoX79+ZnngtsnPz6d58+aRq6sreXp6UmJiIpWWlgpbf2lpKS1dupQ8PDwoODiY0tLSqLa21qY/QiEa98U43BXuiimk2Bd+j1qgWbNmPfHnjY2NKCwsFDajT2trK86ePQuNRoPCwkIUFxdDoVAgLCwMYWFhCA8Ph1KpFJLVmaamJmRmZmLNmjVobGzEw4cPhazX0dERK1aswJIlSzB8+HDDcicnJ1RUVCAoKEhIjjVxX8T0hbvCXTGFFPvCc30LlJubiylTpsDLy6vDn4t6EAGPJsgvLi5GQEAAwsPDkZCQgMzMTPj4+AjL6Ep1dTV2796N3bt3o7GxEVFRUcLWPXnyZOzcuRP19fWYP3++TRxoIhr3RUxfuCvcFVNIsi/WfkpvT0aPHk07duzo9OcXLlwQ9pKRo6MjKRQKWrFiBf3www/0+++/C1lvVx48eEDfffcdqdVqcnBwIH9/f1q/fj3V1NQIz6qpqaH169eTv78/eXl50cqVK8nR0ZEuX74sPMsauC/icFe4K6aQWl94oBZowYIFlJiY2OnPL1++TP7+/kKy7t27R3l5ebR69WoKCQkhZ2dnGjVqFC1btowOHjxI9fX1QnLaFBcXU0JCAnl4eFCvXr1o7ty5dPz4cdLr9UJzOnPs2DGaO3cu9erVi1QqFSUnJ1NZWZlFss2F+2Ie3JWn05O6QiSNvvB71AK1tLRAp9PB1dXV4tlNTU0oKioyvKdUUVEBlUqFixcvCll/2xln4uPjMW/ePHh6egpZr6kaGhqwd+9eZGRkoLKyUuhLfpbGfTEv7ooYPaErgG33hQdqO6HX61FSUgKNRgONRoOioiI0NzcLK9vJkycRFhb2xOtkZWXhrbfeEpJnjPPnz2P8+PEWy7MnPa0v3JXu62ldAWywL9Z9Qm9fHj58SJs2baKJEydScHAwrV69mu7fv2+WLJ1OR8XFxbR582aKjo6mvn37klwuJ4VCQbGxsbRr1y66fv26sLznnnuOGhoaOv35vn37yMnJSVheZ9RqtdDbZU3cF/P2hbvSPT2xK0S23Rc+6lugzz77DJ988gmioqLQu3dvpKWlob6+HhkZGcKzPDw8oNVq4e3tDbVajS1btiAiIgLDhg0TngUAgwYNwrRp05Cfn//Yy28HDhzA/Pnz8dlnnwnLy8nJ6XD5yZMnceTIESgUCgDA9OnThWVaGvdFTF+4K2LZc1cAifbF2n8p2JNnn32Wtm3bZvj++PHj5OzsTDqdTnjWtm3bzHK2nM40NTXRhAkTaMqUKYbztxIRHThwgJydnWnTpk1C82QyGcnlcpLJZJ1ezDnpgiVwX8Tgrohlz10hkmZfeKAWyNnZ+bGPEri4uNDNmzettEVi1dfX04gRI2j27Nmk1+vp4MGD5OTkRP/73/+EZ0VHR1NMTAzV1dW1W27LsweZivsiBndF+njf8mQ8UAskl8sf++hCnz59qKqqykpbJF5NTQ0plUqaPHkyOTs704YNG8yWlZqaSgqFgnJzcw3LbPnBZCruizjcFenjfUvn+KhvgeRyOaZNmwYXFxfDstzcXERGRrY7x+mhQ4essXlPpbKy0vD11atXERsbixkzZuDjjz9ud70xY8YIzS0vL8e8efPw8ssvY8uWLXB3d7fZaf5MxX0R2xfuCnfFFFLqCw/UAi1YsMCoqeh27dplga0RSy6XQyaTgYgM/wJ47GtzfPbw/v37WLVqFQoKClBVVYXKykqbfDCZivsivi/cFe6KKaTSFx6oBaqqqoK/vz/kcvs7zfeNGzeMup6fn5+QvOrqagQEBLRblpOTA41Gg+TkZAwePFhIjjVxX8T0hbsibbxv6RoP1AI5ODjgzp07hjt6zpw52Lp1a6cT6bPOyeVy+Pn5Qa1WQ61WIzIyEr6+vtbeLKG4L2JwV5gppNgX+/vzzIr++zfPjz/+CK1Wa6WtsYzIyEij/yI2RUFBAeLi4lBVVYXFixdDqVRCpVIhISEBWVlZqKurE55padwXMbgr9on3Lf/gCU+YUSw9SUBERAQiIiIAAM3NzTh9+jQKCwtRWFiIPXv24O+//8aIESNw6dIlIXlMLEv2hbsibbxv6Rq/9C2Qg4MDamtrMWjQIABA3759UVlZ+dj7IVL07wM+OmOuAz7atLa24tSpU8jLy8P27dtx7949m5o431TcF/P1hbsiHdbuCmD7feGBWqD/foSio49PANL8CMW0adPg4OCAjIyMdgdbODk5me0jDa2trTh79qzhrD3FxcVQKBQICwtDWFgYwsPDoVQqhedaCvdFHO7KI9wV40itL/zSt0BxcXHtvn/nnXestCXi5eXlYcuWLQgODsa3336LV1991ax5kZGRKC4uRkBAAMLDw5GQkIDMzEz4+PiYNdeSuC9icFekjfctXeNn1MwklpokwMnJCT4+Ppg5cyYiIiIQHh6OAQMGCM1g5meJvnBX7APvWzrHR30zk4wbNw4lJSWQyWQYN27cE99Xehp//fUX0tPT4erqis2bN2PIkCEYPXo0li9fjuzsbPz2229myWViWaIv3BX7wPuWzvEzamYUa08S0NTUhKKiIsN7ShUVFVCpVLh48aJZc1n3WLMv3BVp4X1L13igZkax9iQBer0eJSUl0Gg00Gg0KCoqQnNzs00dmcn+Yc2+cFekhfctXeOBmhml7XOGbUdItra2YujQoYiMjDQ8wETOkqTX61FaWorCwkJoNBqcOnUKWq0Wvr6+hjy1Wi1sWkEmliX7wl2RNt63dI0Hamay/04ScO7cOeGTBPTr1w9arRbe3t6GB05ERASGDRsmZP3McszdF+6K/eB9S8d4oGbdZs5JArZv3w61Wo3AwEAh62PWZ66+cFfsD+9b2uOBmhlNapMEMOvivjBjcVeejAdqZpT/ThIwadIkhIeH2/QkAcx6uC/MWNyVrvFAzYwixUkCmPVwX5ixuCtd44GaGUWr1eKnn34yHClZXl6OwMBAhIeHGx5cbScMYIz7wozFXekaD9SsW6QwSQCzHdwXZizuyuN4ClHWLW5ubujfvz/69+8PT09PODo64sqVK9beLGajuC/MWNyVx/EzamYUKU4SwKyH+8KMxV3pGg/UzChSnCSAWQ/3hRmLu9I1HqiZUaQ4SQCzHu4LMxZ3pWs8UDPGGGM2jA8mY4wxxmwYD9SMMcaYDeOBmjHGGLNhPFAzxhhjNowHasYYY8yG8UDNGGOM2TAeqBljjDEb9n8tYh0XJprYCgAAAABJRU5ErkJggg==", 44 | "text/plain": [ 45 | "
" 46 | ] 47 | }, 48 | "metadata": {}, 49 | "output_type": "display_data" 50 | } 51 | ], 52 | "source": [ 53 | "# Quantization\n", 54 | "\n", 55 | "fig=plt.figure(figsize=(5, 5))\n", 56 | "bar_width = 0.7\n", 57 | "\n", 58 | "for step in ['decode','prefill']:\n", 59 | " plt.subplot(2, 1, 1 if step=='decode' else 2)\n", 60 | " weights=[]\n", 61 | " kvs=[]\n", 62 | " tmp_acts=[]\n", 63 | " categories=[]\n", 64 | " annotation_xs=[]\n", 65 | " annotation_texts=[]\n", 66 | " xs=[]\n", 67 | " x_st=0\n", 68 | " for batchsize,seqlen in [(1,1024),(1,4096),(16,1024)]:\n", 69 | " FP16_sum=0\n", 70 | " annotation_xs.append(x_st)\n", 71 | " annotation_texts.append(f\"{step}\\nbatchsize={batchsize}\\n seqlen={seqlen}\")\n", 72 | " for w,a,kv,quantization in [(16,16,16,\"FP16\"),(4,16,16,\"W4\"),(4,16,4,\"W4KV4\"),(4,4,4,\"W4A4\")]:\n", 73 | " # print(f\"batchsize={batchsize}, seqlen={seqlen}, w={w}, a={a}, kv={kv}\")\n", 74 | " result=analyzer.analyze(seqlen,batchsize,w,a,kv)\n", 75 | " weights.append(result[\"total_results\"][step][\"memory_consumption_weight\"])\n", 76 | " kvs.append(result[\"total_results\"][step][\"memory_consumption_kv_cache\"])\n", 77 | " tmp_acts.append(result[\"total_results\"][step][\"memory_consumption_tmp_act\"])\n", 78 | " xs.append(x_st)\n", 79 | " x_st+=1\n", 80 | " categories.append(f\"{quantization}\")\n", 81 | " if quantization==\"FP16\":\n", 82 | " FP16_sum=weights[-1]+kvs[-1]+tmp_acts[-1]\n", 83 | " weights[-1]/=FP16_sum\n", 84 | " kvs[-1]/=FP16_sum\n", 85 | " tmp_acts[-1]/=FP16_sum\n", 86 | " x_st+=1\n", 87 | " plt.bar(xs, weights, bar_width, label='Weight')\n", 88 | " plt.bar(xs, kvs, bar_width, bottom=weights, label='KV cache')\n", 89 | " plt.bar(xs, tmp_acts, bar_width, bottom=np.array(kvs) + np.array(weights), label='Tmp Act')\n", 90 | "\n", 91 | " for ann_x,ann_text in zip(annotation_xs,annotation_texts):\n", 92 | " plt.annotate(ann_text, (ann_x+2.2, 0.75), ha='center')\n", 93 | " plt.ylabel('Relative Memory Consumption', fontsize=9)\n", 94 | " plt.xticks(xs, categories, rotation=90)\n", 95 | " plt.legend()\n", 96 | " plt.tight_layout()\n", 97 | "plt.savefig(\"../output/quantization_memory_consumption.pdf\",bbox_inches='tight')" 98 | ] 99 | } 100 | ], 101 | "metadata": { 102 | "kernelspec": { 103 | "display_name": "houmo_llm", 104 | "language": "python", 105 | "name": "python3" 106 | }, 107 | "language_info": { 108 | "codemirror_mode": { 109 | "name": "ipython", 110 | "version": 3 111 | }, 112 | "file_extension": ".py", 113 | "mimetype": "text/x-python", 114 | "name": "python", 115 | "nbconvert_exporter": "python", 116 | "pygments_lexer": "ipython3", 117 | "version": "3.10.13" 118 | } 119 | }, 120 | "nbformat": 4, 121 | "nbformat_minor": 2 122 | } 123 | --------------------------------------------------------------------------------