├── data ├── shakespeare │ ├── readme.md │ └── prepare.py └── openwebtext │ ├── readme.md │ └── prepare.py ├── config ├── eval_gpt2.py ├── eval_gpt2_xl.py ├── eval_gpt2_large.py ├── eval_gpt2_medium.py └── finetune_shakespeare.py ├── LICENSE ├── sample.py ├── bench.py ├── README.md ├── train.py ├── model.py └── scaling_laws.ipynb /data/shakespeare/readme.md: -------------------------------------------------------------------------------- 1 | 2 | # tiny shakespeare 3 | 4 | Tiny shakespeare, of the good old char-rnn fame :) 5 | 6 | After running `prepare.py`: 7 | 8 | - train.bin has 301,966 tokens 9 | - val.bin has 36,059 tokens 10 | -------------------------------------------------------------------------------- /config/eval_gpt2.py: -------------------------------------------------------------------------------- 1 | # evaluate the base gpt2 2 | # n_layer=12, n_head=12, n_embd=768 3 | # 124M parameters 4 | batch_size = 8 5 | eval_iters = 500 # use more iterations to get good estimate 6 | eval_only = True 7 | wandb_log = False 8 | init_from = 'gpt2' 9 | -------------------------------------------------------------------------------- /config/eval_gpt2_xl.py: -------------------------------------------------------------------------------- 1 | # evaluate the base gpt2 2 | # n_layer=48, n_head=25, n_embd=1600 3 | # 1558M parameters 4 | batch_size = 8 5 | eval_iters = 500 # use more iterations to get good estimate 6 | eval_only = True 7 | wandb_log = False 8 | init_from = 'gpt2-xl' 9 | -------------------------------------------------------------------------------- /config/eval_gpt2_large.py: -------------------------------------------------------------------------------- 1 | # evaluate the base gpt2 2 | # n_layer=36, n_head=20, n_embd=1280 3 | # 774M parameters 4 | batch_size = 8 5 | eval_iters = 500 # use more iterations to get good estimate 6 | eval_only = True 7 | wandb_log = False 8 | init_from = 'gpt2-large' 9 | -------------------------------------------------------------------------------- /config/eval_gpt2_medium.py: -------------------------------------------------------------------------------- 1 | # evaluate the base gpt2 2 | # n_layer=24, n_head=16, n_embd=1024 3 | # 350M parameters 4 | batch_size = 8 5 | eval_iters = 500 # use more iterations to get good estimate 6 | eval_only = True 7 | wandb_log = False 8 | init_from = 'gpt2-medium' 9 | -------------------------------------------------------------------------------- /data/openwebtext/readme.md: -------------------------------------------------------------------------------- 1 | 2 | ## openwebtext dataset 3 | 4 | after running `prepare.py` (preprocess) we get: 5 | 6 | - train.bin is ~17GB, val.bin ~8.5MB 7 | - train has ~9B tokens (9,035,582,198) 8 | - val has ~4M tokens (4,434,897) 9 | 10 | this came from 8,013,769 documents in total. 11 | 12 | references: 13 | 14 | - OpenAI's WebText dataset is discussed in [GPT-2 paper](https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf) 15 | - [OpenWebText](https://skylion007.github.io/OpenWebTextCorpus/) dataset 16 | -------------------------------------------------------------------------------- /config/finetune_shakespeare.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | out_dir = 'out-shakespeare' 4 | eval_interval = 200 5 | wandb_log = False # feel free to turn on 6 | wandb_project = 'shakespeare' 7 | wandb_run_name = 'ft-' + str(time.time()) 8 | compile = False # takes too little time to finetune, not worth it 9 | 10 | # save a nice and overfit checkpoint that 11 | # will only speak Shakespeare and forgets 12 | # everything else about the world #dark 13 | always_save_checkpoint = True 14 | 15 | dataset = 'shakespeare' 16 | init_from = 'gpt2-xl' 17 | batch_size = 1 18 | block_size = 512 19 | 20 | learning_rate = 1e-5 21 | max_iters = 1000 22 | decay_lr = False 23 | -------------------------------------------------------------------------------- /data/shakespeare/prepare.py: -------------------------------------------------------------------------------- 1 | import os 2 | import requests 3 | import tiktoken 4 | import numpy as np 5 | 6 | # download the tiny shakespeare dataset 7 | if not os.path.exists('input.txt'): 8 | data_url = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt' 9 | with open('input.txt', 'w') as f: 10 | f.write(requests.get(data_url).text) 11 | 12 | with open('input.txt', 'r') as f: 13 | data = f.read() 14 | n = len(data) 15 | train_data = data[:int(n*0.9)] 16 | val_data = data[int(n*0.9):] 17 | 18 | # encode with tiktoken gpt2 bpe 19 | enc = tiktoken.get_encoding("gpt2") 20 | train_ids = enc.encode_ordinary(train_data) 21 | val_ids = enc.encode_ordinary(val_data) 22 | print(f"train has {len(train_ids)} tokens") 23 | print(f"val has {len(val_ids)} tokens") 24 | 25 | # export to bin files 26 | train_ids = np.array(train_ids, dtype=np.uint16) 27 | val_ids = np.array(val_ids, dtype=np.uint16) 28 | train_ids.tofile('train.bin') 29 | val_ids.tofile('val.bin') 30 | 31 | # train.bin has 301,966 tokens 32 | # val.bin has 36,059 tokens 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Andrej Karpathy 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /sample.py: -------------------------------------------------------------------------------- 1 | """ 2 | Sample from a trained model 3 | """ 4 | import os 5 | import torch 6 | import tiktoken 7 | from model import GPTConfig, GPT 8 | 9 | # ----------------------------------------------------------------------------- 10 | # todo make these overridable like in train.py 11 | out_dir = 'out' 12 | device = 'cuda:2' 13 | compile = False 14 | start = "\n" # or "<|endoftext|>" or whatever you like 15 | num_samples = 10 # number of samples to draw 16 | max_new_tokens = 500 # number of tokens generated in each sample 17 | temperature = 0.8 # higher temperature (up to 1) is more random, lower (down to 0) means more greedy 18 | top_k = 200 # retain only the top_k most likely tokens, clamp others to have 0 probability 19 | seed = 1337 20 | # ----------------------------------------------------------------------------- 21 | 22 | torch.manual_seed(seed) 23 | torch.cuda.manual_seed(seed) 24 | torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul 25 | torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn 26 | 27 | # model 28 | ckpt_path = os.path.join(out_dir, 'ckpt.pt') 29 | checkpoint = torch.load(ckpt_path, map_location=device) 30 | gptconf = GPTConfig(**checkpoint['model_args']) 31 | model = GPT(gptconf) 32 | model.load_state_dict(checkpoint['model']) 33 | model.eval() 34 | model.to(device) 35 | if compile: 36 | model = torch.compile(model) # requires PyTorch 2.0 (optional) 37 | 38 | # encode the beginning of the prompt 39 | enc = tiktoken.get_encoding("gpt2") 40 | start_ids = enc.encode(start, allowed_special={"<|endoftext|>"}) 41 | x = (torch.tensor(start_ids, dtype=torch.long, device=device)[None, ...]) 42 | 43 | for k in range(num_samples): 44 | 45 | with torch.no_grad(): 46 | with torch.amp.autocast(device_type="cuda", dtype=torch.bfloat16): 47 | y = model.generate(x, max_new_tokens, temperature=temperature, top_k=top_k) 48 | 49 | print(enc.decode(y[0].tolist())) 50 | print('---------------') 51 | -------------------------------------------------------------------------------- /data/openwebtext/prepare.py: -------------------------------------------------------------------------------- 1 | # saves the openwebtext dataset to a binary file for training. following was helpful: 2 | # https://github.com/HazyResearch/flash-attention/blob/main/training/src/datamodules/language_modeling_hf.py 3 | 4 | from tqdm import tqdm 5 | import numpy as np 6 | import tiktoken 7 | from datasets import load_dataset # huggingface datasets 8 | 9 | # number of workers in .map() call 10 | # good number to use is ~order number of cpu cores // 2 11 | num_proc = 8 12 | 13 | # takes 54GB in huggingface .cache dir, about 8M documents (8,013,769) 14 | dataset = load_dataset("openwebtext") 15 | 16 | # owt by default only contains the 'train' split, so create a test split 17 | split_dataset = dataset["train"].train_test_split(test_size=0.0005, seed=2357, shuffle=True) 18 | split_dataset['val'] = split_dataset.pop('test') # rename the test split to val 19 | 20 | # this results in: 21 | # >>> split_dataset 22 | # DatasetDict({ 23 | # train: Dataset({ 24 | # features: ['text'], 25 | # num_rows: 8009762 26 | # }) 27 | # val: Dataset({ 28 | # features: ['text'], 29 | # num_rows: 4007 30 | # }) 31 | # }) 32 | 33 | # we now want to tokenize the dataset. first define the encoding function (gpt2 bpe) 34 | enc = tiktoken.get_encoding("gpt2") 35 | def process(example): 36 | ids = enc.encode_ordinary(example['text']) # encode_ordinary ignores any special tokens 37 | ids.append(enc.eot_token) # add the end of text token, e.g. 50256 for gpt2 bpe 38 | # note: I think eot should be prepended not appended... hmm. it's called "eot" though... 39 | out = {'ids': ids, 'len': len(ids)} 40 | return out 41 | 42 | # tokenize the dataset 43 | tokenized = split_dataset.map( 44 | process, 45 | remove_columns=['text'], 46 | desc="tokenizing the splits", 47 | num_proc=num_proc, 48 | ) 49 | 50 | # concatenate all the ids in each dataset into one large file we can use for training 51 | for split, dset in tokenized.items(): 52 | arr_len = np.sum(dset['len']) 53 | filename = f'{split}.bin' 54 | dtype = np.uint16 # (can do since enc.max_token_value == 50256 is < 2**16) 55 | arr = np.memmap(filename, dtype=dtype, mode='w+', shape=(arr_len,)) 56 | 57 | print(f"writing {filename}...") 58 | idx = 0 59 | for example in tqdm(dset): 60 | arr[idx : idx + example['len']] = example['ids'] 61 | idx += example['len'] 62 | arr.flush() 63 | 64 | # train.bin is ~17GB, val.bin ~8.5MB 65 | # train has ~9B tokens (9,035,582,198) 66 | # val has ~4M tokens (4,434,897) 67 | 68 | # to read the bin files later, e.g. with numpy: 69 | # m = np.memmap('train.bin', dtype=np.uint16, mode='r') 70 | -------------------------------------------------------------------------------- /bench.py: -------------------------------------------------------------------------------- 1 | """ 2 | A much shorter version of train.py for benchmarking 3 | """ 4 | import os 5 | import numpy as np 6 | import time 7 | import torch 8 | from model import GPTConfig, GPT 9 | 10 | device = 'cuda' 11 | torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul 12 | torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn 13 | torch.manual_seed(1337) 14 | 15 | batch_size = 8 16 | block_size = 1024 17 | dtype = torch.bfloat16 18 | compile = True 19 | 20 | # data loading init 21 | real_data = True 22 | if real_data: 23 | dataset = 'openwebtext' 24 | data_dir = os.path.join('data', dataset) 25 | train_data = np.memmap(os.path.join(data_dir, 'train.bin'), dtype=np.uint16, mode='r') 26 | def get_batch(split): 27 | data = train_data # note ignore split in benchmarking script 28 | ix = torch.randint(len(data) - block_size, (batch_size,)) 29 | x = torch.stack([torch.from_numpy((data[i:i+block_size]).astype(np.int64)) for i in ix]) 30 | y = torch.stack([torch.from_numpy((data[i+1:i+1+block_size]).astype(np.int64)) for i in ix]) 31 | x, y = x.to(device), y.to(device) 32 | return x, y 33 | else: 34 | # alternatively, if fixed data is desired to not care about data loading 35 | x = torch.randint(50257, (batch_size, block_size), device=device) 36 | y = torch.randint(50257, (batch_size, block_size), device=device) 37 | get_batch = lambda split: (x, y) 38 | 39 | # model init 40 | gptconf = GPTConfig( 41 | block_size = block_size, # how far back does the model look? i.e. context size 42 | n_layer = 12, n_head = 12, n_embd = 768, # size of the model 43 | dropout = 0, # for determinism 44 | ) 45 | model = GPT(gptconf) 46 | model.to(device) 47 | 48 | optimizer = model.configure_optimizers(weight_decay=1e-2, learning_rate=1e-4, betas=(0.9, 0.95)) 49 | 50 | if compile: 51 | print("Compiling model...") 52 | model = torch.compile(model) # pytorch 2.0 53 | 54 | profile = False # use pytorch profiler, or just simple benchmarking? 55 | if profile: 56 | # useful docs on pytorch profiler: 57 | # - tutorial https://pytorch.org/tutorials/intermediate/tensorboard_profiler_tutorial.html 58 | # - api https://pytorch.org/docs/stable/profiler.html#torch.profiler.profile 59 | wait, warmup, active = 5, 5, 5 60 | num_steps = wait + warmup + active 61 | with torch.profiler.profile( 62 | activities=[torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA], 63 | schedule=torch.profiler.schedule(wait=wait, warmup=warmup, active=active, repeat=1), 64 | on_trace_ready=torch.profiler.tensorboard_trace_handler('./bench_log'), 65 | record_shapes=True, 66 | profile_memory=True, 67 | with_stack=True, # incurs an additional overhead, disable if not needed 68 | with_flops=True, 69 | with_modules=False, # only for torchscript models atm 70 | ) as prof: 71 | 72 | for k in range(num_steps): 73 | X, Y = get_batch('train') 74 | with torch.autocast(device_type='cuda', dtype=dtype): 75 | logits, loss = model(X, Y) 76 | optimizer.zero_grad(set_to_none=True) 77 | loss.backward() 78 | optimizer.step() 79 | lossf = loss.item() 80 | print(f"{k}/{num_steps} loss: {lossf:.4f}") 81 | 82 | prof.step() # notify the profiler at end of each step 83 | 84 | else: 85 | 86 | # simple benchmarking 87 | torch.cuda.synchronize() 88 | for stage, num_steps in enumerate([10, 20]): # burnin, then benchmark 89 | t0 = time.time() 90 | for k in range(num_steps): 91 | X, Y = get_batch('train') 92 | with torch.autocast(device_type='cuda', dtype=dtype): 93 | logits, loss = model(X, Y) 94 | optimizer.zero_grad(set_to_none=True) 95 | loss.backward() 96 | optimizer.step() 97 | lossf = loss.item() 98 | print(f"{k}/{num_steps} loss: {lossf:.4f}") 99 | torch.cuda.synchronize() 100 | t1 = time.time() 101 | if stage == 1: 102 | print(f"time per iteration: {(t1-t0)/num_steps*1000:.4f}ms") 103 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # nanoGPT 3 | 4 | The simplest, fastest repository for training/finetuning medium-sized GPTs. It's a re-write of [minGPT](https://github.com/karpathy/minGPT), which I think became too complicated, and which I am hesitant to now touch. Still under active development, currently working to reproduce GPT-2 on OpenWebText dataset. The code itself aims by design to be plain and readable: `train.py` is a ~300-line boilerplate training loop and `model.py` a ~300-line GPT model definition, which can optionally load the GPT-2 weights from OpenAI. That's it. 5 | 6 | ## install 7 | 8 | Dependencies: 9 | 10 | - [pytorch](https://pytorch.org) <3 11 | - [numpy](https://numpy.org/install/) <3 12 | - `pip install datasets` for huggingface datasets <3 (if you want to download + preprocess OpenWebText) 13 | - `pip install tiktoken` for OpenAI's fast bpe code <3 14 | - `pip install wandb` for optional logging <3 15 | - `pip install tqdm` 16 | - `pip install networkx` 17 | 18 | ## usage 19 | 20 | To render a dataset we first tokenize some documents into one simple long 1D array of indices. E.g. for OpenWebText see: 21 | 22 | ``` 23 | $ cd data/openwebtext 24 | $ python prepare.py 25 | ``` 26 | 27 | To download and tokenize the [OpenWebText](https://huggingface.co/datasets/openwebtext) dataset. This will create a `train.bin` and `val.bin` which holds the GPT2 BPE token ids in one sequence, stored as raw uint16 bytes. Then we're ready to kick off training. The training script currently by default tries to reproduce the smallest GPT-2 released by OpenAI, i.e. the 124M version of GPT-2. We can demo train as follows on a single device, though I encourage you to read the code and see all of the settings and paths up top in the file: 28 | 29 | ``` 30 | $ python train.py 31 | ``` 32 | 33 | To train using PyTorch Distributed Data Parallel (DDP) run the script with torchrun. For example to train on a node with 4 GPUs run: 34 | 35 | ``` 36 | $ torchrun --standalone --nproc_per_node=4 train.py 37 | ``` 38 | 39 | To my knowledge, running this with the current script with the GPT-2 hyperparameters should reproduce the GPT-2 result, provided that OpenWebText ~= WebText. I'd like to make the code more efficient before attempting to go there. Once some checkpoints are written to the output directory (e.g. `./out` by default), we can sample from the model: 40 | 41 | ``` 42 | $ python sample.py 43 | ``` 44 | 45 | Training on 1 A100 40GB GPU overnight currently gets loss ~3.74, training on 4 gets ~3.60. Training on an 8 x A100 40GB node for 400,000 iters (~1 day) atm gets down to 3.1. Random chance at init is -ln(1/50257) = 10.82. Which brings us to baselines. 46 | 47 | ## baselines 48 | 49 | OpenAI GPT-2 checkpoints allow us to get some baselines in place for openwebtext. We can get the numbers as follows: 50 | 51 | ``` 52 | $ python train.py eval_gpt2 53 | $ python train.py eval_gpt2_medium 54 | $ python train.py eval_gpt2_large 55 | $ python train.py eval_gpt2_xl 56 | ``` 57 | 58 | and observe the following losses on train and val: 59 | 60 | | model | params | train loss | val loss | 61 | | ------| ------ | ---------- | -------- | 62 | | gpt2 | 124M | 3.11 | 3.12 | 63 | | gpt2-medium | 350M | 2.85 | 2.84 | 64 | | gpt2-large | 774M | 2.66 | 2.67 | 65 | | gpt2-xl | 1558M | 2.56 | 2.54 | 66 | 67 | I briefly tried finetuning gpt2 a bit more on our OWT and didn't notice dramatic improvements, suggesting that OWT is not much much different from WT in terms of the data distribution, but this needs a bit more thorough attempt once the code is in a better place. 68 | 69 | ## finetuning 70 | 71 | For an example of how to finetune a GPT on new text go to `data/shakespeare` and look at `prepare.py` to download the tiny shakespeare dataset and render it into a `train.bin` and `val.bin`. Unlike OpenWebText this will run in seconds. Finetuning takes very little time, e.g. on a single GPU just a few minutes. Run an example finetuning like: 72 | 73 | ``` 74 | $ python train.py finetune_shakespeare 75 | ``` 76 | 77 | This will load the config parameter overrides in `config/finetune_shakespeare.py` (I didn't tune them much though). Basically, we initialize from a GPT2 checkpoint with `init_from` and train as normal, except shorter and with a small learning rate. The best checkpoint (lowest validation loss) will be in the `out_dir` directory, e.g. in `out-shakespeare` by default, per the config file. You can then run the code in `sample.py` to generate infinite Shakespeare. Note that you'll have to edit it to point to the correct `out_dir`. 78 | 79 | ## benchmarking 80 | 81 | For model benchmarking `bench.py` might be useful. It's identical what happens in the meat of the training loop of `train.py`, but omits much of the other complexities. 82 | 83 | ## efficiency notes 84 | 85 | Code by default now uses [PyTorch 2.0](https://pytorch.org/get-started/pytorch-2.0/). At the time of writing (Dec 29, 2022) this makes `torch.compile()` available in the nightly release. The improvement from the one line of code is noticeable, e.g. cutting down iteration time from ~250ms / iter to 135ms / iter. Nice work PyTorch team! 86 | 87 | ## todos 88 | 89 | A few todos I'm aware of: 90 | 91 | Optimizations 92 | 93 | - Additional optimizations to the running time 94 | - Investigate need for an actual Data Loader with a dedicated worker process for data 95 | - Look into more efficient fused optimizers (e.g. apex) 96 | - Re-evaluate use of flash attention (previously I wasn't able to get the forward pass to match up so I took it out) 97 | - CUDA Graphs? 98 | - Investigate potential speedups from Lightning or huggingface Accelerate 99 | 100 | Features / APIs 101 | 102 | - Add back fp16 support? (would need to also add back gradient scaler) 103 | - Add CPU support 104 | - Finetune the finetuning script, I think the hyperparams are not great 105 | - Replace poor man's configurator, and make sample.py configurable... 106 | - Report and track other metrics e.g. perplexity, num_tokens, MFU, ... 107 | - Eval zero-shot perplexities on PTB, WikiText, other related benchmarks 108 | 109 | Suspiciousness 110 | 111 | - Current initialization (PyTorch default) departs from GPT-2. In a very quick experiment I found it to be superior to the one suggested in the papers, but that can't be right? 112 | - I don't currently seem to need gradient clipping but it is very often used (?). Nothing is exploding so far at these scales but maybe I'm laeving performance on the table. Evaluate with/without. 113 | - I am still not 100% confident that my GPT-2 small reproduction hyperparameters are good, if someone has reproduced GPT-2 I'd be eager to exchange notes ty 114 | - I keep seeing different values cited for weight decay and AdamW betas, look into 115 | - I can't exactly reproduce Chinchilla paper results, see [scaling_laws.ipynb](scaling_laws.ipynb) notebook 116 | 117 | Results 118 | 119 | - Actually reproduce GPT-2 results and have clean configs that reproduce the result. It was estimated ~3 years ago that the training cost of 1.5B model was ~$50K (?). Sounds a bit too high. 120 | 121 | # acknowledgements 122 | 123 | Thank you [Lambda labs](https://lambdalabs.com) for supporting the training costs of nanoGPT experiments. 124 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | """ 2 | This training script can be run both on a single gpu in debug mode, 3 | and also in a larger training run with distributed data parallel (ddp). 4 | 5 | To run in debug mode example: 6 | $ python train.py --batch_size=32 --other=args 7 | 8 | To run DDP on 4 gpus on one node, example: 9 | $ torchrun --standalone --nproc_per_node=4 train.py 10 | """ 11 | 12 | import os 13 | import sys 14 | import time 15 | import math 16 | from ast import literal_eval 17 | 18 | import wandb 19 | import numpy as np 20 | import torch 21 | from torch.nn.parallel import DistributedDataParallel as DDP 22 | from torch.distributed import init_process_group, destroy_process_group 23 | 24 | from model import GPTConfig, GPT 25 | 26 | # ----------------------------------------------------------------------------- 27 | # default config values 28 | # I/O 29 | out_dir = 'out' 30 | eval_interval = 2000 31 | log_interval = 1 32 | eval_iters = 200 33 | eval_only = False # if True, script exits right after the first eval 34 | always_save_checkpoint = True # if True, always save a checkpoint after each eval 35 | # wandb logging 36 | wandb_log = False # disabled by default 37 | wandb_entity = 'karpathy' 38 | wandb_project = 'owt' 39 | wandb_run_name = 'gpt2' # 'run' + str(time.time()) 40 | # data 41 | dataset = 'openwebtext' 42 | batch_size = 12 43 | block_size = 1024 44 | # model 45 | device = 'cuda:0' 46 | init_from = 'scratch' # 'scratch' or 'resume' or 'gpt2*' 47 | dropout = 0.0 # for pretraining 0 is good, for finetuning try 0.1+ 48 | n_layer = 12 49 | n_head = 12 50 | n_embd = 768 51 | # adamw optimizer 52 | learning_rate = 6e-4 # max learning rate 53 | max_iters = 400000 # total number of training iterations 54 | weight_decay = 1e-2 55 | betas = (0.9, 0.95) 56 | # learning rate decay settings 57 | decay_lr = True # whether to decay the learning rate 58 | warmup_iters = 2000 # how many steps to warm up for 59 | lr_decay_iters = 400000 # should be ~= max_iters per Chinchilla 60 | min_lr = 6e-5 # minimum learning rate, should be ~= learning_rate/10 per Chinchilla 61 | # DDP settings 62 | backend = 'nccl' # 'nccl', 'gloo', etc. 63 | compile = True # use PyTorch 2.0 to compile the model to be faster 64 | # ----------------------------------------------------------------------------- 65 | # poor man's Configurator. Potentially a bad idea. Example usage: 66 | # $ python train.py override_file --batch_size=32 67 | # this will first run config/override_file.py, then override batch_size to 32 68 | for arg in sys.argv[1:]: 69 | if '=' not in arg: 70 | # assume it's the name of a config file 71 | assert not arg.startswith('--') 72 | config_file = os.path.join('config', arg + '.py') 73 | print(f"Overriding config with {config_file}:") 74 | with open(config_file) as f: 75 | print(f.read()) 76 | exec(open(config_file).read()) 77 | else: 78 | # assume it's a --key=value argument 79 | assert arg.startswith('--') 80 | key, val = arg.split('=') 81 | key = key[2:] 82 | if key in globals(): 83 | try: 84 | # attempt to eval it it (e.g. if bool, number, or etc) 85 | attempt = literal_eval(val) 86 | except (SyntaxError, ValueError): 87 | # if that goes wrong, just use the string 88 | attempt = val 89 | # ensure the types match ok 90 | assert type(attempt) == type(globals()[key]) 91 | # cross fingers 92 | print(f"Overriding: {key} = {attempt}") 93 | globals()[key] = attempt 94 | else: 95 | raise ValueError(f"Unknown config key: {key}") 96 | # ----------------------------------------------------------------------------- 97 | ddp = int(os.environ.get('LOCAL_RANK', -1)) != -1 # is this a ddp run? 98 | if ddp: 99 | init_process_group(backend=backend) 100 | gpu_id = int(os.environ["LOCAL_RANK"]) 101 | device = f"cuda:{gpu_id}" 102 | else: 103 | gpu_id = 0 # gpu_id 0 means this is the (single) master process, basically 104 | 105 | if gpu_id == 0: 106 | os.makedirs(out_dir, exist_ok=True) 107 | torch.manual_seed(1337 + gpu_id) # note: each worker gets a different seed 108 | torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul 109 | torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn 110 | 111 | # poor man's data loader, TODO evaluate need for actual DataLoader 112 | data_dir = os.path.join('data', dataset) 113 | train_data = np.memmap(os.path.join(data_dir, 'train.bin'), dtype=np.uint16, mode='r') 114 | val_data = np.memmap(os.path.join(data_dir, 'val.bin'), dtype=np.uint16, mode='r') 115 | def get_batch(split): 116 | data = train_data if split == 'train' else val_data 117 | ix = torch.randint(len(data) - block_size, (batch_size,)) 118 | x = torch.stack([torch.from_numpy((data[i:i+block_size]).astype(np.int64)) for i in ix]) 119 | y = torch.stack([torch.from_numpy((data[i+1:i+1+block_size]).astype(np.int64)) for i in ix]) 120 | x, y = x.to(device), y.to(device) 121 | return x, y 122 | 123 | # init these up here, can override if init_from='resume' (i.e. from a checkpoint) 124 | iter_num = 0 125 | best_val_loss = 1e9 126 | 127 | # model init 128 | model_args = dict(n_layer = n_layer, n_head = n_head, n_embd = n_embd, block_size = block_size, dropout = dropout) 129 | if init_from == 'scratch': 130 | # init a new model from scratch 131 | print("Initializing a new model from scratch") 132 | gptconf = GPTConfig(**model_args) 133 | model = GPT(gptconf) 134 | elif init_from == 'resume': 135 | print(f"Resuming training from {out_dir}") 136 | # resume training from a checkpoint. 137 | ckpt_path = os.path.join(out_dir, 'ckpt.pt') 138 | checkpoint = torch.load(ckpt_path, map_location=device) 139 | checkpoint_model_args = checkpoint['model_args'] 140 | for k, v in model_args.items(): 141 | assert checkpoint_model_args[k] == v, "for now" 142 | # TODO: think through how passed in params should interact with checkpoint params 143 | gptconf = GPTConfig(**model_args) 144 | model = GPT(gptconf) 145 | state_dict = checkpoint['model'] 146 | # fix the keys of the state dictionary :( 147 | # honestly no idea how checkpoints sometimes get this prefix, have to debug more 148 | unwanted_prefix = '_orig_mod.' 149 | for k,v in list(state_dict.items()): 150 | if k.startswith(unwanted_prefix): 151 | state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k) 152 | model.load_state_dict(state_dict) 153 | iter_num = checkpoint['iter_num'] 154 | best_val_loss = checkpoint['best_val_loss'] 155 | elif init_from.startswith('gpt2'): 156 | print(f"Initializing from OpenAI GPT-2 weights: {init_from}") 157 | # initialize from OpenAI GPT-2 weights 158 | override_args = dict(dropout=dropout) 159 | model = GPT.from_pretrained(init_from, override_args) 160 | # read off and override the GPT sizing model args from the model config 161 | model_args['n_layer'] = model.config.n_layer 162 | model_args['n_head'] = model.config.n_head 163 | model_args['n_embd'] = model.config.n_embd 164 | # crop down the model block size if desired 165 | if block_size < model.config.block_size: 166 | model.crop_block_size(block_size) 167 | model.to(device) 168 | 169 | # optimizer 170 | optimizer = model.configure_optimizers(weight_decay, learning_rate, betas) 171 | if init_from == 'resume': 172 | optimizer.load_state_dict(checkpoint['optimizer']) 173 | 174 | # compile the model 175 | if compile: 176 | print("compiling the model... (takes a ~minute)") 177 | unoptimized_model = model 178 | model = torch.compile(model) # requires PyTorch 2.0 179 | 180 | # wrap model into DDP container 181 | if ddp: 182 | model = DDP(model, device_ids=[gpu_id]) 183 | 184 | @torch.no_grad() 185 | def estimate_loss(): 186 | out = {} 187 | model.eval() 188 | for split in ['train', 'val']: 189 | losses = torch.zeros(eval_iters) 190 | for k in range(eval_iters): 191 | X, Y = get_batch(split) 192 | with torch.amp.autocast(device_type="cuda", dtype=torch.bfloat16): 193 | logits, loss = model(X, Y) 194 | losses[k] = loss.item() 195 | out[split] = losses.mean() 196 | model.train() 197 | return out 198 | 199 | # learning rate decay scheduler (cosine with warmup) 200 | def get_lr(iter): 201 | # 1) linear warmup for warmup_iters steps 202 | if iter < warmup_iters: 203 | return learning_rate * iter / warmup_iters 204 | # 2) if iter > lr_decay_iters, return min learning rate 205 | if iter > lr_decay_iters: 206 | return min_lr 207 | # 3) in between, use cosine decay down to min learning rate 208 | decay_ratio = (iter - warmup_iters) / (lr_decay_iters - warmup_iters) 209 | assert 0 <= decay_ratio <= 1 210 | coeff = 0.5 * (1.0 + math.cos(math.pi * decay_ratio)) # coeff ranges 0..1 211 | return min_lr + coeff * (learning_rate - min_lr) 212 | 213 | # logging 214 | if wandb_log and gpu_id == 0: 215 | wandb.init(project=wandb_project, entity=wandb_entity, name=wandb_run_name) 216 | wandb.config = { 217 | "batch_size": batch_size, 218 | "block_size": block_size, 219 | "learning_rate": learning_rate, # TODO log everything else too 220 | } 221 | 222 | # training loop 223 | t0 = time.time() 224 | while True: 225 | 226 | # determine the learning rate for this iteration 227 | if decay_lr: 228 | lr = get_lr(iter_num) 229 | for param_group in optimizer.param_groups: 230 | param_group['lr'] = lr 231 | else: 232 | lr = learning_rate 233 | 234 | if iter_num % eval_interval == 0 and gpu_id == 0: 235 | losses = estimate_loss() 236 | print(f"step {iter_num}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}") 237 | if wandb_log: 238 | wandb.log({ 239 | "iter": iter_num, 240 | "train/loss": losses['train'], 241 | "val/loss": losses['val'], 242 | "lr": lr, 243 | }) 244 | if losses['val'] < best_val_loss or always_save_checkpoint: 245 | best_val_loss = losses['val'] 246 | raw_model = model.module if ddp else model 247 | if iter_num > 0: 248 | checkpoint = { 249 | 'model': raw_model.state_dict(), 250 | 'optimizer': optimizer.state_dict(), 251 | 'model_args': model_args, 252 | 'iter_num': iter_num, 253 | 'best_val_loss': best_val_loss, 254 | } 255 | print(f"saving checkpoint to {out_dir}") 256 | torch.save(checkpoint, os.path.join(out_dir, 'ckpt.pt')) 257 | if iter_num == 0 and eval_only: 258 | break 259 | 260 | X, Y = get_batch('train') 261 | with torch.amp.autocast(device_type="cuda", dtype=torch.bfloat16): 262 | logits, loss = model(X, Y) 263 | 264 | optimizer.zero_grad(set_to_none=True) 265 | loss.backward() 266 | # TODO: gradient clipping evaluate need for 267 | optimizer.step() 268 | 269 | t1 = time.time() 270 | dt = t1 - t0 271 | t0 = t1 272 | if iter_num % log_interval == 0 and gpu_id == 0: 273 | lossf = loss.item() # loss as float. TODO CPU-GPU sync: profile, make sure not slow af 274 | print(f"iter {iter_num}: loss {lossf:.4f}, time {dt*1000:.2f}ms") 275 | iter_num += 1 276 | 277 | # termination conditions 278 | if iter_num > max_iters: 279 | break 280 | 281 | if ddp: 282 | destroy_process_group() 283 | -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- 1 | """ 2 | Full definition of a GPT Language Model, all of it in this single file. 3 | References: 4 | 1) the official GPT-2 TensorFlow implementation released by OpenAI: 5 | https://github.com/openai/gpt-2/blob/master/src/model.py 6 | 2) huggingface/transformers PyTorch implementation: 7 | https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt2/modeling_gpt2.py 8 | """ 9 | 10 | import math 11 | from dataclasses import dataclass 12 | 13 | import torch 14 | import torch.nn as nn 15 | from torch.nn import functional as F 16 | 17 | # @torch.jit.script # good to enable when not using torch.compile, disable when using (our default) 18 | def new_gelu(x): 19 | """ 20 | Implementation of the GELU activation function currently in Google BERT repo (identical to OpenAI GPT). 21 | Reference: Gaussian Error Linear Units (GELU) paper: https://arxiv.org/abs/1606.08415 22 | """ 23 | return 0.5 * x * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3.0)))) 24 | 25 | class CausalSelfAttention(nn.Module): 26 | 27 | def __init__(self, config): 28 | super().__init__() 29 | assert config.n_embd % config.n_head == 0 30 | # key, query, value projections for all heads, but in a batch 31 | self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd) 32 | # output projection 33 | self.c_proj = nn.Linear(config.n_embd, config.n_embd) 34 | # regularization 35 | self.attn_dropout = nn.Dropout(config.dropout) 36 | self.resid_dropout = nn.Dropout(config.dropout) 37 | # causal mask to ensure that attention is only applied to the left in the input sequence 38 | self.register_buffer("bias", torch.tril(torch.ones(config.block_size, config.block_size)) 39 | .view(1, 1, config.block_size, config.block_size)) 40 | self.n_head = config.n_head 41 | self.n_embd = config.n_embd 42 | 43 | def forward(self, x): 44 | B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd) 45 | 46 | # calculate query, key, values for all heads in batch and move head forward to be the batch dim 47 | q, k ,v = self.c_attn(x).split(self.n_embd, dim=2) 48 | k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs) 49 | q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs) 50 | v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs) 51 | 52 | # causal self-attention; Self-attend: (B, nh, T, hs) x (B, nh, hs, T) -> (B, nh, T, T) 53 | att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1))) 54 | att = att.masked_fill(self.bias[:,:,:T,:T] == 0, float('-inf')) 55 | att = F.softmax(att, dim=-1) 56 | att = self.attn_dropout(att) 57 | y = att @ v # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs) 58 | y = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side 59 | 60 | # output projection 61 | y = self.resid_dropout(self.c_proj(y)) 62 | return y 63 | 64 | class MLP(nn.Module): 65 | 66 | def __init__(self, config): 67 | super().__init__() 68 | self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd) 69 | self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd) 70 | self.dropout = nn.Dropout(config.dropout) 71 | 72 | def forward(self, x): 73 | x = self.c_fc(x) 74 | x = new_gelu(x) 75 | x = self.c_proj(x) 76 | x = self.dropout(x) 77 | return x 78 | 79 | class Block(nn.Module): 80 | 81 | def __init__(self, config): 82 | super().__init__() 83 | self.ln_1 = nn.LayerNorm(config.n_embd) 84 | self.attn = CausalSelfAttention(config) 85 | self.ln_2 = nn.LayerNorm(config.n_embd) 86 | self.mlp = MLP(config) 87 | 88 | def forward(self, x): 89 | x = x + self.attn(self.ln_1(x)) 90 | x = x + self.mlp(self.ln_2(x)) 91 | return x 92 | 93 | @dataclass 94 | class GPTConfig: 95 | block_size: int = 1024 96 | vocab_size: int = 50257 97 | n_layer: int = 12 98 | n_head: int = 12 99 | n_embd: int = 768 100 | dropout: float = 0.1 101 | 102 | class GPT(nn.Module): 103 | 104 | def __init__(self, config): 105 | super().__init__() 106 | assert config.vocab_size is not None 107 | assert config.block_size is not None 108 | self.config = config 109 | 110 | self.transformer = nn.ModuleDict(dict( 111 | wte = nn.Embedding(config.vocab_size, config.n_embd), 112 | wpe = nn.Embedding(config.block_size, config.n_embd), 113 | drop = nn.Dropout(config.dropout), 114 | h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]), 115 | ln_f = nn.LayerNorm(config.n_embd), 116 | )) 117 | self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) 118 | 119 | # report number of parameters (note we don't count the decoder parameters in lm_head) 120 | n_params = sum(p.numel() for p in self.transformer.parameters()) 121 | print("number of parameters: %.2fM" % (n_params/1e6,)) 122 | 123 | def forward(self, idx, targets=None): 124 | device = idx.device 125 | b, t = idx.size() 126 | assert t <= self.config.block_size, f"Cannot forward sequence of length {t}, block size is only {self.config.block_size}" 127 | pos = torch.arange(0, t, dtype=torch.long, device=device).unsqueeze(0) # shape (1, t) 128 | 129 | # forward the GPT model itself 130 | tok_emb = self.transformer.wte(idx) # token embeddings of shape (b, t, n_embd) 131 | pos_emb = self.transformer.wpe(pos) # position embeddings of shape (1, t, n_embd) 132 | x = self.transformer.drop(tok_emb + pos_emb) 133 | for block in self.transformer.h: 134 | x = block(x) 135 | x = self.transformer.ln_f(x) 136 | logits = self.lm_head(x) 137 | 138 | # if we are given some desired targets also calculate the loss 139 | loss = None 140 | if targets is not None: 141 | loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1) 142 | 143 | return logits, loss 144 | 145 | def crop_block_size(self, block_size): 146 | # model surgery to decrease the block size if necessary 147 | # e.g. we may load the GPT2 pretrained model checkpoint (block size 1024) 148 | # but want to use a smaller block size for some smaller, simpler model 149 | assert block_size <= self.config.block_size 150 | self.config.block_size = block_size 151 | self.transformer.wpe.weight = nn.Parameter(self.transformer.wpe.weight[:block_size]) 152 | for block in self.transformer.h: 153 | block.attn.bias = block.attn.bias[:,:,:block_size,:block_size] 154 | 155 | @classmethod 156 | def from_pretrained(cls, model_type, override_args): 157 | assert model_type in {'gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl'} 158 | # only dropout can be overridden see more notes below 159 | assert all(k == 'dropout' for k in override_args) 160 | from transformers import GPT2LMHeadModel 161 | print("loading weights from pretrained gpt: %s" % model_type) 162 | 163 | # n_layer, n_head and n_embd are determined from model_type 164 | config_args = { 165 | 'gpt2': dict(n_layer=12, n_head=12, n_embd=768), # 124M params 166 | 'gpt2-medium': dict(n_layer=24, n_head=16, n_embd=1024), # 350M params 167 | 'gpt2-large': dict(n_layer=36, n_head=20, n_embd=1280), # 774M params 168 | 'gpt2-xl': dict(n_layer=48, n_head=25, n_embd=1600), # 1558M params 169 | }[model_type] 170 | # we can override the dropout rate 171 | if 'dropout' in override_args: 172 | config_args['dropout'] = override_args['dropout'] 173 | # block_size is always 1024 for GPT model checkpoints 174 | # if one wants a lower block_size it has to be done through model surgery 175 | # later, by calling crop_block_size() 176 | 177 | # create a from-scratch initialized minGPT model 178 | config = GPTConfig(block_size=1024, **config_args) 179 | model = GPT(config) 180 | sd = model.state_dict() 181 | 182 | # init a huggingface/transformers model 183 | model_hf = GPT2LMHeadModel.from_pretrained(model_type) 184 | sd_hf = model_hf.state_dict() 185 | 186 | # copy while ensuring all of the parameters are aligned and match in names and shapes 187 | keys = [k for k in sd_hf if not k.endswith('attn.masked_bias')] # ignore these 188 | transposed = ['attn.c_attn.weight', 'attn.c_proj.weight', 'mlp.c_fc.weight', 'mlp.c_proj.weight'] 189 | # basically the openai checkpoints use a "Conv1D" module, but we only want to use a vanilla Linear 190 | # this means that we have to transpose these weights when we import them 191 | assert len(keys) == len(sd) 192 | for k in keys: 193 | if any(k.endswith(w) for w in transposed): 194 | # special treatment for the Conv1D weights we need to transpose 195 | assert sd_hf[k].shape[::-1] == sd[k].shape 196 | with torch.no_grad(): 197 | sd[k].copy_(sd_hf[k].t()) 198 | else: 199 | # vanilla copy over the other parameters 200 | assert sd_hf[k].shape == sd[k].shape 201 | with torch.no_grad(): 202 | sd[k].copy_(sd_hf[k]) 203 | 204 | return model 205 | 206 | def configure_optimizers(self, weight_decay, learning_rate, betas): 207 | """ 208 | This long function is unfortunately doing something very simple and is being very defensive: 209 | We are separating out all parameters of the model into two buckets: those that will experience 210 | weight decay for regularization and those that won't (biases, and layernorm/embedding weights). 211 | We are then returning the PyTorch optimizer object. 212 | """ 213 | 214 | # separate out all parameters to those that will and won't experience regularizing weight decay 215 | decay = set() 216 | no_decay = set() 217 | whitelist_weight_modules = (torch.nn.Linear, ) 218 | blacklist_weight_modules = (torch.nn.LayerNorm, torch.nn.Embedding) 219 | for mn, m in self.named_modules(): 220 | for pn, p in m.named_parameters(): 221 | fpn = '%s.%s' % (mn, pn) if mn else pn # full param name 222 | # random note: because named_modules and named_parameters are recursive 223 | # we will see the same tensors p many many times. but doing it this way 224 | # allows us to know which parent module any tensor p belongs to... 225 | if pn.endswith('bias'): 226 | # all biases will not be decayed 227 | no_decay.add(fpn) 228 | elif pn.endswith('weight') and isinstance(m, whitelist_weight_modules): 229 | # weights of whitelist modules will be weight decayed 230 | decay.add(fpn) 231 | elif pn.endswith('weight') and isinstance(m, blacklist_weight_modules): 232 | # weights of blacklist modules will NOT be weight decayed 233 | no_decay.add(fpn) 234 | 235 | # validate that we considered every parameter 236 | param_dict = {pn: p for pn, p in self.named_parameters()} 237 | inter_params = decay & no_decay 238 | union_params = decay | no_decay 239 | assert len(inter_params) == 0, "parameters %s made it into both decay/no_decay sets!" % (str(inter_params), ) 240 | assert len(param_dict.keys() - union_params) == 0, "parameters %s were not separated into either decay/no_decay set!" \ 241 | % (str(param_dict.keys() - union_params), ) 242 | 243 | # create the pytorch optimizer object 244 | optim_groups = [ 245 | {"params": [param_dict[pn] for pn in sorted(list(decay))], "weight_decay": weight_decay}, 246 | {"params": [param_dict[pn] for pn in sorted(list(no_decay))], "weight_decay": 0.0}, 247 | ] 248 | optimizer = torch.optim.AdamW(optim_groups, lr=learning_rate, betas=betas) 249 | return optimizer 250 | 251 | @torch.no_grad() 252 | def generate(self, idx, max_new_tokens, temperature=1.0, top_k=None): 253 | """ 254 | Take a conditioning sequence of indices idx (LongTensor of shape (b,t)) and complete 255 | the sequence max_new_tokens times, feeding the predictions back into the model each time. 256 | Most likely you'll want to make sure to be in model.eval() mode of operation for this. 257 | """ 258 | for _ in range(max_new_tokens): 259 | # if the sequence context is growing too long we must crop it at block_size 260 | idx_cond = idx if idx.size(1) <= self.config.block_size else idx[:, -self.config.block_size:] 261 | # forward the model to get the logits for the index in the sequence 262 | logits, _ = self(idx_cond) 263 | # pluck the logits at the final step and scale by desired temperature 264 | logits = logits[:, -1, :] / temperature 265 | # optionally crop the logits to only the top k options 266 | if top_k is not None: 267 | v, _ = torch.topk(logits, top_k) 268 | logits[logits < v[:, [-1]]] = -float('Inf') 269 | # apply softmax to convert logits to (normalized) probabilities 270 | probs = F.softmax(logits, dim=-1) 271 | # sample from the distribution 272 | idx_next = torch.multinomial(probs, num_samples=1) 273 | # append sampled index to the running sequence and continue 274 | idx = torch.cat((idx, idx_next), dim=1) 275 | 276 | return idx 277 | -------------------------------------------------------------------------------- /scaling_laws.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "attachments": {}, 5 | "cell_type": "markdown", 6 | "metadata": {}, 7 | "source": [ 8 | "Reproducing some results from [Chinchilla](https://arxiv.org/pdf/2203.15556.pdf). I can't get the numbers to match exactly, please open an Issue if you understand why the results are off by a bit. Current running hypothesis is that this is because I am using the FLOPs = 6\\*N\\*D formula, instead of taking all the Gopher sizes and calculating their FLOPs individually, and using that to interpolate?" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 1, 14 | "metadata": {}, 15 | "outputs": [], 16 | "source": [ 17 | "import matplotlib.pyplot as plt\n", 18 | "import numpy as np\n", 19 | "%matplotlib inline" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 2, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "gpt2 = dict(\n", 29 | " seq_len = 1024,\n", 30 | " vocab_size = 50257,\n", 31 | " d_model = 768,\n", 32 | " num_heads = 12,\n", 33 | " num_layers = 12,\n", 34 | ")" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 3, 40 | "metadata": {}, 41 | "outputs": [ 42 | { 43 | "data": { 44 | "text/plain": [ 45 | "123.653376" 46 | ] 47 | }, 48 | "execution_count": 3, 49 | "metadata": {}, 50 | "output_type": "execute_result" 51 | } 52 | ], 53 | "source": [ 54 | "def params(seq_len, vocab_size, d_model, num_heads, num_layers, ffw_size=4):\n", 55 | " \"\"\" Given GPT config calculate total number of parameters\"\"\" \n", 56 | " # token and position embeddings\n", 57 | " embeddings = d_model * vocab_size + d_model * seq_len\n", 58 | " # transformer blocks\n", 59 | " attention = 3*d_model**2 + 3*d_model # weights and biases\n", 60 | " attproj = d_model**2 + d_model\n", 61 | " ffw = d_model*(ffw_size*d_model) + ffw_size*d_model\n", 62 | " ffwproj = ffw_size*d_model*d_model + d_model\n", 63 | " layernorms = 2*2*d_model\n", 64 | " # dense\n", 65 | " ln_f = 2*d_model\n", 66 | " dense = d_model*vocab_size # note: no bias here\n", 67 | " # note: embeddings are not included in the param count!\n", 68 | " total_params = num_layers*(attention + attproj + ffw + ffwproj + layernorms) + ln_f + dense\n", 69 | " return total_params\n", 70 | "\n", 71 | "params(**gpt2)/1e6 # OpenAI reports gpt2 (small) as having 124M params, good." 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 4, 77 | "metadata": {}, 78 | "outputs": [ 79 | { 80 | "data": { 81 | "text/plain": [ 82 | "766.006788096" 83 | ] 84 | }, 85 | "execution_count": 4, 86 | "metadata": {}, 87 | "output_type": "execute_result" 88 | } 89 | ], 90 | "source": [ 91 | "def flops(seq_len, vocab_size, d_model, num_heads, num_layers, ffw_size=4):\n", 92 | " \"\"\" \n", 93 | " Given GPT config calculate total number of FLOPs, see Chinchilla \n", 94 | " paper Appendix F as reference: https://arxiv.org/pdf/2203.15556.pdf\n", 95 | " \"\"\" \n", 96 | " key_size = d_model // num_heads\n", 97 | "\n", 98 | " # embeddings\n", 99 | " embeddings = 2 * seq_len * vocab_size * d_model\n", 100 | "\n", 101 | " # attention\n", 102 | " # key, query, value projections\n", 103 | " attention = 2 * 3 * seq_len * d_model * (key_size * num_heads)\n", 104 | " # key @ query logits\n", 105 | " attlogits = 2 * seq_len * seq_len * (key_size * num_heads)\n", 106 | " # softmax\n", 107 | " attsoftmax = 3 * num_heads * seq_len * seq_len # TODO why?\n", 108 | " # softmax @ value reductions\n", 109 | " attvalue = 2 * seq_len * seq_len * (key_size * num_heads)\n", 110 | " # final linear\n", 111 | " attlinear = 2 * seq_len * (key_size * num_heads) * d_model\n", 112 | " att = attention + attlogits + attsoftmax + attvalue + attlinear\n", 113 | " # feed forward\n", 114 | " dense = 2 * seq_len * (d_model * ffw_size + d_model * ffw_size)\n", 115 | "\n", 116 | " # logits\n", 117 | " logits = 2 * seq_len * d_model * vocab_size\n", 118 | "\n", 119 | " forward_flops = embeddings + num_layers * (att + dense) + logits\n", 120 | " backward_flops = 2 * forward_flops # as in Kaplan et al. 2020\n", 121 | " total_flops = forward_flops + backward_flops\n", 122 | "\n", 123 | " return total_flops\n", 124 | "\n", 125 | "flops(**gpt2)/1e9" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": 5, 131 | "metadata": {}, 132 | "outputs": [ 133 | { 134 | "name": "stdout", 135 | "output_type": "stream", 136 | "text": [ 137 | "params: 69.72M\n", 138 | "approx flops: 428.34B\n", 139 | "chinchilla flops: 434.11B\n", 140 | "ratio (chinchilla / approx): 1.01\n" 141 | ] 142 | } 143 | ], 144 | "source": [ 145 | "# Reproduce Table A4 from Chinchilla paper Appendix, \n", 146 | "# comparing accurate flops above to approximate flops F = 6*N*D\n", 147 | "# note Chinchilla uses vocab_size = 32K\n", 148 | "\n", 149 | "chin_73M = dict(seq_len = 1024, vocab_size = 32000, d_model = 640, num_heads = 10, num_layers = 10)\n", 150 | "args = chin_73M\n", 151 | "\n", 152 | "D = 1024 # dataset size, cancels anyway\n", 153 | "N = params(**args) \n", 154 | "F = flops(**args)\n", 155 | "\n", 156 | "approx_flops = 6*D*N\n", 157 | "chinch_flops = F * (float(D) / args['seq_len'])\n", 158 | "\n", 159 | "print(f\"params: {N/1e6:.2f}M\")\n", 160 | "print(f\"approx flops: {approx_flops/1e9:.2f}B\")\n", 161 | "print(f\"chinchilla flops: {chinch_flops/1e9:.2f}B\")\n", 162 | "print(f\"ratio (chinchilla / approx): {chinch_flops / approx_flops:.2f}\")" 163 | ] 164 | }, 165 | { 166 | "attachments": {}, 167 | "cell_type": "markdown", 168 | "metadata": {}, 169 | "source": [ 170 | "Well this is awkward because Chinchilla paper claims that number of params for these args are 73M (we see only 70M), and the ratio is supposed to be 1.03 but we get 1.01. TODO stare at more..." 171 | ] 172 | }, 173 | { 174 | "attachments": {}, 175 | "cell_type": "markdown", 176 | "metadata": {}, 177 | "source": [ 178 | "## scaling laws" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": 6, 184 | "metadata": {}, 185 | "outputs": [ 186 | { 187 | "data": { 188 | "text/plain": [ 189 | "" 190 | ] 191 | }, 192 | "execution_count": 6, 193 | "metadata": {}, 194 | "output_type": "execute_result" 195 | }, 196 | { 197 | "data": { 198 | "image/png": "iVBORw0KGgoAAAANSUhEUgAABDQAAAHWCAYAAACIWtlUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAACWeElEQVR4nO3deXxU1f3/8fckkAUkURAIS9gVEAHBNWAFK4JIEap14WsL7tqCirRa6a8qiBRtXXDFnbhRVwQrFgQUEFmUrQUXFGXTJuAGAYQAM/f3B2XqkPkMueFOZubO6/l43Ec7d86ce+5MyHxyvPe8A47jOAIAAAAAAEghGYkeAAAAAAAAgFtMaAAAAAAAgJTDhAYAAAAAAEg5TGgAAAAAAICUw4QGAAAAAABIOUxoAAAAAACAlMOEBgAAAAAASDlMaAAAAAAAgJTDhAYAAAAAAEg5TGgAMRQXFysQCGjdunWJHgoAAIAv+bHe+tvf/qZWrVopMzNTxx13nCSpRYsWuuSSSxI6LsBvmNAAAAAA4HsTJkzQ+eefr2bNmikQCMScXNiyZYuuuuoq1a9fX7Vr19bpp5+uZcuWVeo4b7/9tm666SZ1795dEydO1F/+8hePzgDAgWokegAAAAAAEG933XWXtm3bppNOOkklJSVmu1AopH79+ulf//qXbrzxRh155JF65JFH1LNnTy1dulRHHXVUzOO88847ysjI0FNPPaWsrCyvTwPATzChAQAAAMD35s6dG74647DDDjPbvfrqq1qwYIFeeeUV/epXv5IkXXDBBTr66KN12223adKkSTGPs3nzZuXm5jKZAVQDbjkBXHrkkUfUoUMHZWdnq3Hjxho6dKi2bNkS0ebzzz/Xeeedp4KCAuXk5Khp06a66KKLtHXr1nCbmTNn6tRTT9Xhhx+uww47TG3bttWf/vSnaj4bAACA5FSZmkuSHn74YbVq1Uq5ubk66aST9N5776lnz57q2bNnRLvmzZsrEAgc9LivvvqqGjZsqHPPPTe8r379+rrgggs0depUlZeXm68NBAKaOHGiduzYoUAgoEAgoOLiYrP9l19+qfPPP19169ZVrVq1dMopp2jatGkRbebMmaNAIKCXXnpJf/rTn1RQUKDatWvrnHPO0caNGyPaVqYGBfyEKzQAF0aNGqXRo0erV69e+u1vf6vVq1drwoQJ+vDDD/X++++rZs2a2r17t/r06aPy8nJde+21Kigo0Ndff60333xTW7ZsUX5+vj766CP94he/UKdOnXT77bcrOztba9as0fvvv5/oUwQAAEi4ytRc0r51MYYNG6af/exnuuGGG7Ru3ToNHDhQRxxxhJo2bVqlYy9fvlxdu3ZVRkbkf/s96aST9Pjjj+uzzz5Tx44do772ueee0+OPP64PPvhATz75pCSpW7duUdtu2rRJ3bp1048//qjrrrtO9erV0zPPPKNzzjlHr776qn75y19GtB87dqwCgYD++Mc/avPmzRo/frx69eqlFStWKDc3t1I1KOA7DgDTxIkTHUnO2rVrnc2bNztZWVlO7969nWAwGG7z0EMPOZKcp59+2nEcx1m+fLkjyXnllVfMfu+77z5HkvPNN9/E/RwAAACS2U/rLcdxKl1zlZeXO/Xq1XNOPPFEZ8+ePeF2xcXFjiSnR48e5jFr167tDBkyxHzusssuq7B/2rRpjiRn+vTpMc9nyJAhTu3atSvsb968ecQxhw8f7khy3nvvvfC+bdu2OS1btnRatGgRPvd3333XkeQ0adLEKSsrC7d9+eWXHUnO/fff7zhO5WpQwG+45QSopFmzZmn37t0aPnx4xIz9lVdeqby8vPDlgftnv2fMmKEff/wxal+HH364JGnq1KkKhULxHTgAAEAKqWzNtWTJEn333Xe68sorVaPG/y48v/jii3XEEUdU+fg7d+5UdnZ2hf05OTnh573w1ltv6aSTTtKpp54a3nfYYYfpqquu0rp16/Txxx9HtB88eLDq1KkTfvyrX/1KjRo10ltvvSWpcjUo4DdMaACVtH79eklS27ZtI/ZnZWWpVatW4edbtmypESNG6Mknn9SRRx6pPn366OGHH464d/HCCy9U9+7ddcUVV6hhw4a66KKL9PLLLzO5AQAA0l5la679/9umTZuIdjVq1FCLFi2qfPzc3Nyo62Ts2rUr/LwX1q9fX+EcJal9+/bh53/qwHSVQCCgNm3aaN26dZIqV4MCfsOEBhAH99xzj/7973/rT3/6k3bu3KnrrrtOHTp00FdffSVp3xfhvHnzNGvWLP3mN7/Rv//9b1144YU688wzFQwGEzx6AACA9NWoUaOosa779zVu3Li6h1RpB6tBAb9hQgOopObNm0uSVq9eHbF/9+7dWrt2bfj5/Tp27Kg///nPmjdvnt577z19/fXXevTRR8PPZ2Rk6IwzztC9996rjz/+WGPHjtU777yjd999N/4nAwAAkKQqW3Pt/981a9ZEtNu7d2/4qoWqOO6447Rs2bIKV84uXrxYtWrV0tFHH13lvn+qefPmFc5Rkj799NPw8z/1+eefRzx2HEdr1qypcDXKwWpQwE+Y0AAqqVevXsrKytIDDzwgx3HC+5966ilt3bpV/fr1kySVlZVp7969Ea/t2LGjMjIywpcvfv/99xX6P+644yQpZhQYAACA31W25jrhhBNUr149PfHEExG11wsvvKAffvihysf/1a9+pU2bNmny5Mnhfd9++61eeeUV9e/fP+r6GlVx9tln64MPPtDChQvD+3bs2KHHH39cLVq00DHHHBPR/tlnn9W2bdvCj1999VWVlJSob9++kipXgwJ+Q2wrUEn169fXyJEjNXr0aJ111lk655xztHr1aj3yyCM68cQT9etf/1qS9M4772jYsGE6//zzdfTRR2vv3r167rnnlJmZqfPOO0+SdPvtt2vevHnq16+fmjdvrs2bN+uRRx5R06ZNIxaGAgAASDeVrbmysrI0atQoXXvttfr5z3+uCy64QOvWrVNxcbFat26tQCAQ0e8//vEP/etf/5Ik7dmzR//+9791xx13SJLOOeccderUSdK+CY1TTjlFl156qT7++GMdeeSReuSRRxQMBjV69GjPzvPmm2/W3//+d/Xt21fXXXed6tatq2eeeUZr167Va6+9ViE2tm7dujr11FN16aWXatOmTRo/frzatGmjK6+8UlLlalDAb5jQAFwYNWqU6tevr4ceekg33HCD6tatq6uuukp/+ctfwnnonTt3Vp8+ffSPf/xDX3/9tWrVqqXOnTvrn//8p0455RRJ+740161bp6efflrffvutjjzySPXo0UOjR48mIxwAAKS9ytRckjRs2DA5jqN77rlHf/jDH9S5c2e98cYbuu6668KpJPu99tpreuaZZ8KPly9fruXLl0uSmjZtGp7QyMzM1FtvvaUbb7xRDzzwgHbu3KkTTzxRxcXFURfxrKqGDRtqwYIF+uMf/6gHH3xQu3btUqdOnfSPf/wjfBXKT/3pT3/Sv//9b40bN07btm3TGWecoUceeUS1atWSVLkaFPCbgPPT67gAAAAAIIWFQiHVr19f5557rp544olED+eQzZkzR6effrpeeeUV/epXv0r0cICkwhoaAAAAAFLSrl27dOB/n3322Wf1/fffq2fPnokZFIBqwy0nAAAAAFLSokWLdMMNN+j8889XvXr1tGzZMj311FM69thjdf755yd6eADijAkNAAAAACmpRYsWKiws1AMPPKDvv/9edevW1eDBg3XnnXcqKysr0cMDEGcJveVk3rx56t+/vxo3bqxAIKApU6ZEPD958mT17t1b9erVUyAQ0IoVKyrV7yuvvKJ27dopJydHHTt21FtvveX94AEAAGCizkN1aNGihd544w2VlpZq9+7dKi0t1dNPP60GDRokemie6dmzpxzHYf0MIIqETmjs2LFDnTt31sMPP2w+f+qpp+quu+6qdJ8LFizQoEGDdPnll2v58uUaOHCgBg4cqFWrVnk1bAAAABwEdR4AIN6SJuUkEAjo9ddf18CBAys8t27dOrVs2VLLly/XcccdF7OfCy+8UDt27NCbb74Z3nfKKafouOOO06OPPurxqAEAAHAw1HkAgHjw3RoaCxcu1IgRIyL29enTp8Jljj9VXl6u8vLy8ONQKKTvv/8+fAkkAFQnx3G0bds2NW7cWBkZ8b2QbteuXdq9e7dn/WVlZSknJ8ez/gDgp6jzAKQ66jxv+W5Co7S0VA0bNozY17BhQ5WWlpqvGTdunEaPHh3voQGAKxs3blTTpk3j1v+uXbvUsvlhKt0c9KzPgoICrV27Num+7AD4A3UeAL+gzvOG7yY0qmLkyJERs/1bt25Vs2bN1Oa3tyozO/LDytgTvY+MvdH3B/ZGv6PHam/vj95PwPj5DATdHtfq39gfsvp32Y+5P+TquFZ7We9PyGpvna/V3thvtY/Vl7XfuivMOrbL9uZdZ0HjzbPam/uN47p976z+jfbmzXTGz5A1Tovjth8Xd/ft1R7N11uqU6eOqzG5tXv3bpVuDmrt0ubKq3Po/4WgbFtILY9fr927dyfVFx2A9GbVeU3uu1kZudmRjTOM+sbYHwgY+zOifxdkZLrr39pfw+rf2J9p9hP9uz7TOK9Mo/8sqx+X48kyClXrfGsEjPEErH6ij7Omcb5ZRuFvjWdfX9GPkW30ZbWvaZ2by36s9yLb2F/TKJ5rmv1EH0+WeV7G+2ONx3gfair6/myrvXExVpZxlVZWIHpdVFOZRv+V31+2PaTmXddR53nEdxMaBQUF2rRpU8S+TZs2qaCgwHxNdna2srOzK+zPzM6pOKFh/CxY+wPmF5S7fswvOmsixWpv/GPOML+Q3U1EZMjlF7653+WEhvFLTeZ4vGpv7Dfbxxqryz+E3f7h7ET/AjEnNIz29sSCOYMQvRvrPXL52VjtHbO9u3FaHNf9uFiu6L9Nq+tS6Lw6GZ580QFAvHlZ52XkZisj94Ci3KqfjIkItxMameaEhjVB4W5iwfWERqY3ExrmxIjZ3vij0yiEXU9omP24mzzINr4aY09oRH9RjtGX1d6eGIn+2dQ0/zCPftwc4zO2/vC3Jn1yzEklt/1Y74PRT/TdyjGPG32/NaGRbU5oWON0t1+izvOK786sqKhIs2fPjtg3c+ZMFRUVJWhEAJDcgk7Isw0A4ok6DwDc8Xudl9ArNLZv3641a9aEH69du1YrVqxQ3bp11axZM33//ffasGGD/vOf/0iSVq9eLWnf7Pz+mfjBgwerSZMmGjdunCTp+uuvV48ePXTPPfeoX79+evHFF7VkyRI9/vjj1Xx2AJAaQnIUcnMFSYx+AGA/6jwASDy/13kJvUJjyZIl6tKli7p06SJJGjFihLp06aJbb71VkvTGG2+oS5cu6tevnyTpoosuUpcuXSJiuTZs2KCSkpLw427dumnSpEl6/PHH1blzZ7366quaMmWKjj322Go8MwAAgPRGnQcAiLeAY95An77KysqUn5+vtsP/UulFQY11bJSxJzGLglrtjVsHzXG6XczTHKe19oV5Xm4XBTV+jPda/SRwUVC3i3m6PbbLRT7NXwHWmhgh44fI7SKZRnt7PF4tFuryfK1FTT1aXDTa+Pc6ezRHU7V161bl5eW568+F/b/r/rO6qWeLRTVu+1Xcxw0Ah2L/776mj4yqsIaGtXZYqqyt4dVioTU9WlvD7WKh1vi9Wiw02+zH7doaxh8EMcZkrYmR43qxUG8WHbUWC80xFvm0x+OuH3uxUKsfbxYLtdb68G5tDReLgm4LqUHb9dR5HvHdoqAAAHeCjqOgB3PbXvQBAAAA7/i9zvPdoqAAAAAAAMD/uEIDANKc3xeLAgAASFd+r/OY0ACANBeSo6CPv+gAAADSld/rPG45AQAAAAAAKYcrNFxyjJVwjd32E8Z+q3/HWGk3YM2UGe1ltHeMqS1jQWCzf8da1dvoxuRy/CZrys5lEIVr5vglJ8P4LK2kFrfHsBbscdve7XGt99oIRUl1AeNzdELGG+E2/aQa+f1SRACIKhjYt/2E9VvM+la3frNnRA88kIzviKDdk6sj7zXa2wW/dVzjBIz0E+u8dhu9Z5njsVhnYMUDGs1DLv/0MfupGeNFRgKK25rU7X929qqflGF89q7/hLDeOOsFRnsXx91TzTWh3+s8JjQAIM35ffVrAACAdOX3Os+3c3YAgNRx5513KhAIaPjw4Wab4uJiBQKBiC0nJ6f6BgkAAICkwhUaAJDmQvLmLqyq9vHhhx/qscceU6dOnQ7aNi8vT6tXrw4/DsS4vQsAACDdJbrOizcmNAAgzQU9Wv26Kn1s375dF198sZ544gndcccdB20fCARUUFBQleEBAACknUTWedWBW04AAJ4qKyuL2MrLy822Q4cOVb9+/dSrV69K9b19+3Y1b95chYWFGjBggD766COvhg0AAIAUwxUaMTiBiqkjqX5xs+uUlnhzueKz67QXi5nUYfRjvXFG0oXcJpbE4jbxxeUl+NYl+1byjevUkoDRUUb0D9lK1nH9jmZEP24gFP0A5vnGO7Uk6vsfqMIJV13Q8eZHdn8fhYWFEftvu+02jRo1qkL7F198UcuWLdOHH35Yqf7btm2rp59+Wp06ddLWrVt19913q1u3bvroo4/UtGnTQx0+gHQT9Tps4zvR6IL0k/27ST/5X19WAgrpJ9Uj+dNP9lTzzRte13nJhgkNAEhzXt9buXHjRuXl5YX3Z2dnV2i7ceNGXX/99Zo5c2alF/YsKipSUVFR+HG3bt3Uvn17PfbYYxozZswhjR0AAMCPWEMDAAAX8vLyIiY0olm6dKk2b96srl27hvcFg0HNmzdPDz30kMrLy5WZaf2nzH1q1qypLl26aM2aNZ6MGwAAAKmFCQ0ASHMhBRT04MazkIs+zjjjDK1cuTJi36WXXqp27drpj3/840EnM6R9EyArV67U2Wef7XqsAAAA6SARdV51YkIDANJcyNm3edFPZdWpU0fHHntsxL7atWurXr164f2DBw9WkyZNNG7cOEnS7bffrlNOOUVt2rTRli1b9Le//U3r16/XFVdcceiDBwAA8KFE1HnViQkNAEBS2rBhgzJ+srjrDz/8oCuvvFKlpaU64ogjdPzxx2vBggU65phjEjhKAAAAJAoTGl4xrsCxUjlcRxhY3Xh15Y/b8Qeijz9gpH5YgRBWOonbNBazvVfvv5fM99T6EIyxuk1YMVI/FHQZW2Klljhu409cssZvpJaY71uc2f8G4pyWcgiCHl2KeKh9zJkzJ+bj++67T/fdd98hHQMAwqLF2Zn/CZL0k1j9kH5Smb5IP0ms5Ek/2V3NtV+y1HnxwoQGAKQ5v3/RAQAApCu/13m+nYMDAAAAAAD+xRUaAJDmQk5AIeu+LZf9AAAAIHn4vc5jQgMA0pzfL0UEAABIV36v87jlBAAAAAAApByu0IgloIor33o0MWWncsS5f2+694yVomKln3j3/hvHNfa7TlGJ9UG6Td9wm34S7/YWqx9zlW7riTinpRjMz97t+N2uXB01NSajWoN4gspQ0IP57cR8cgBQNYFgQIFg5O9+x0wbIf1EIv0kjPQTH6n+9JPd1ZzE5/c6jwkNAEhzjkf3VjpJem8lAABAuvJ7nefbuTYAAAAAAFC9xo0bpxNPPFF16tRRgwYNNHDgQK1evTqizdVXX63WrVsrNzdX9evX14ABA/Tpp5+6PhYTGgCQ5vYvFuXFBgAAgOSRiDpv7ty5Gjp0qBYtWqSZM2dqz5496t27t3bs2BFuc/zxx2vixIn65JNPNGPGDDmOo969eysYdHdzC7ecAECaCzoZCpqLhbjpx4PBAAAAwDOJqPOmT58e8bi4uFgNGjTQ0qVLddppp0mSrrrqqvDzLVq00B133KHOnTtr3bp1at26daWPxYQGAAAAAAA4qLKysojH2dnZys7OjvmarVu3SpLq1q0b9fkdO3Zo4sSJatmypQoLC12NhwkNjyQqtcR1monZj8u0EYNn74P5AmM8GUZ7c1XyBDLTNIzPwJoOdfseufwQXKd+eLX0sfU+GItHOxnGgELGC9y2dytqaokUyIjev5MEP6MhBRTy4A7EUHVGswDAoQqpQjBBwEotIf1EEuknYbG+Mt0moJB+kqTil36yp5rLJa/rvAMnHG677TaNGjXKfl0opOHDh6t79+469thjI5575JFHdNNNN2nHjh1q27atZs6cqawsd/+CmdAAgDTn1foXrKEBAACQXLyu8zZu3Ki8vLzw/oNdnTF06FCtWrVK8+fPr/DcxRdfrDPPPFMlJSW6++67dcEFF+j9999XTk5OpcfFhAYAAAAAADiovLy8iAmNWIYNG6Y333xT8+bNU9OmTSs8n5+fr/z8fB111FE65ZRTdMQRR+j111/XoEGDKj0eJjQAIM15t1gUt5wAAAAkk0TUeY7j6Nprr9Xrr7+uOXPmqGXLlpV6jeM4Ki8vdzUuJjQAIM3tu7fy0C9F9KIPAAAAeCcRdd7QoUM1adIkTZ06VXXq1FFpaamkfVdk5Obm6ssvv9RLL72k3r17q379+vrqq6905513Kjc3V2effbarcfl2GRcAAAAAAFC9JkyYoK1bt6pnz55q1KhReHvppZckSTk5OXrvvfd09tlnq02bNrrwwgtVp04dLViwQA0aNHB1LK7QiMEJVEztcJ0qYvEqtcRl/65Z/cS7f7O9R8keGUZ76wNw3T7GibkJcZbsc7Au+7KObR3XSv0IuowtMdI9zGWx452W4pbxPgSM9BPzyj2XqSWBKJ9XwAnYq4nHQUgZCpJyAiDdhAL7tgjRf4+RfhL7yKSf/IT51pF+4g+Hnn5SXu0pJ9Vf5zkHuT2lcePGeuuttw51SJKY0ACAtMcaGgAAAP7k9zovoXNn8+bNU//+/dW4cWMFAgFNmTIl4nnHcXTrrbeqUaNGys3NVa9evfT555/H7HPUqFEKBAIRW7t27eJ4FgAAADgQdR4AIN4SOqGxY8cOde7cWQ8//HDU5//617/qgQce0KOPPqrFixerdu3a6tOnj3bt2hWz3w4dOqikpCS8Rcu8BQDsE1KGZxsA7EedBwCJ5/c6L6G3nPTt21d9+/aN+pzjOBo/frz+/Oc/a8CAAZKkZ599Vg0bNtSUKVN00UUXmf3WqFFDBQUFcRkzAPhN0AkoaK0H47IfANiPOg8AEs/vdV5yTrNIWrt2rUpLS9WrV6/wvvz8fJ188slauHBhzNd+/vnnaty4sVq1aqWLL75YGzZsiNm+vLxcZWVlERsAAADigzoPAOCFpF0UdH9WbcOGDSP2N2zYMPxcNCeffLKKi4vVtm1blZSUaPTo0frZz36mVatWqU6dOlFfM27cOI0ePfrQBpyoNBCX/ZspKlaQhjHlFbCSKIyOnIC1arjB5YrMjnHcQDKmLlhvtlcJLtaCPR61t0fp0XttpaVkRP/wA9bPhJXeYqSWmO+DW1bKjLESu5xqjDMxBD1a/TqYjP/eACSlZKjzAk7F7xDHTVSBSD852JHTL/1EMlMwSD+pWj8po/LpJ8YnEjd+r/OSdkKjqn56aWOnTp108sknq3nz5nr55Zd1+eWXR33NyJEjNWLEiPDjsrIyFRYWxn2sAJAMQk6GQh6sfh1K0tWvAfgHdR4AuOP3Oi9p58j23xu5adOmiP2bNm1ydd/k4YcfrqOPPlpr1qwx22RnZysvLy9iAwAAQHxQ5wEAvJC0ExotW7ZUQUGBZs+eHd5XVlamxYsXq6ioqNL9bN++XV988YUaNWoUj2ECQMrbfymiFxsAVAZ1HgBUD7/XeQkd1fbt27VixQqtWLFC0r4FolasWKENGzYoEAho+PDhuuOOO/TGG29o5cqVGjx4sBo3bqyBAweG+zjjjDP00EMPhR//4Q9/0Ny5c7Vu3TotWLBAv/zlL5WZmalBgwZV89kBQGoI6X8rYB/KlvjVQAAkE+o8AEg8v9d5CV1DY8mSJTr99NPDj/ff3zhkyBAVFxfrpptu0o4dO3TVVVdpy5YtOvXUUzV9+nTl5OSEX/PFF1/o22+/DT/+6quvNGjQIH333XeqX7++Tj31VC1atEj169evvhMDAABIc9R5AIB4S+iERs+ePeXEWFwkEAjo9ttv1+233262WbduXcTjF1980avhGYPyphu3aSOe9e9N9x6+D+7SSdyel/0+W2ks8W3/31fFeC5KayM1IxB0mWbiNi3FLbfpHuY8b5wXHHKbfmK0DxjtY/1Oi95RtP6r9+K5kDIU8uCYXvQBwD+Svc4LBGOkth2A9JN9SD+pDOssSD/xtJ+UUfFz31PNa2v6vc7zXcoJAMCdoJOhoAerX3vRBwAAALzj9zovOUcFAAAAAAAQA1doAECaCymgkAf3kXnRBwAAALzj9zqPCQ0ASHN+vxQRAAAgXfm9zkvOUQEAAAAAAMTAFRpuWVfaGPtdp5m47MdtSoh1YCdgrNLt8riy+jETMKw0E7fnFX23nbwR5+WFYyWHWGMyP2SX6SRWe+u4VlqKyzQQe/13l++127SUDGM8xirarlNIXDITccxVw6t5qesogspQ0IP5bS/6AIDqEggFFAgd+DvbZRIZ6SeSSD+pHNJPYkqj9JNyxzrZ+PB7nceEBgCkuZATUMiaUHPZDwAAAJKH3+u85JxmAQAAAAAAiIErNAAgzYU8uhQxxBw5AABAUvF7nceEBgCkuZCToZAHK1d70QcAAAC84/c6LzlHBQAAAAAAEANXaMQSkL0s9AHMNBPPBmPwKnXFk8HYXKe9WMwXuEyKsPrJMPqxTsBqbyyiHZOZHOPynN2mn7j+EFxym1piLXNtNvco/cRlqovrFBg36SfVvOhSUAEFPfgt4EUfAFBtQqrwlWOlkJB+sg/pJ/tY6SdSVRJQSD+JyYfpJ3ucqvyhUHV+r/OY0ACANOf3SxEBAADSld/rvOQcFQAAAAAAQAxcoQEAaS4oby4jrN4LKAEAAHAwfq/zmNAAgDTn90sRAQAA0pXf67zkHBUAAAAAAEAMXKHhkuuUEDNtxFiFOmCsfu3yKqF4jzNgjNPt1UzxTj8xwyLM99ldP27bx3qN29QSx0gPsVI87GQXo33QGI/bdA+je8dtMk2ixDn9JBkEnQwFPZh196IPAKgugVDF70w7DYT0E4n0k/91Y3/fWQkopJ/s78flcS0pnH6yu5pLYL/XeUxoAECacxRQyIN7K+1CHQAAAIng9zovOadZAAAAAAAAYuAKDQBIc36/FBEAACBd+b3OY0IDANJcyAkoFGvhFxf9AAAAIHn4vc5LzmkWAAAAAACAGLhCIwYnUDGtwn1KiKvmMcfiSf9u+7HaG1NhrhM2jPW1zXSVDKO9u4CQxDLTRoz9VtqIR2kpCXvzrPM1Vw63UkWs5sYTGUYai9G9Y70/FvPzjT6eQJT0k+r+sQ0qQ0EP5re96AMAqku0lBML6SexWpN+EtlV9GOTfnKwflwe15IC6Sfl1Xzrht/rPCY0ACDN+f1SRAAAgHTl9zovOadZAAAAAAAAYuAKDQBIcyFlKOTB/LYXfQAAAMA7fq/zmNAAgDQXdAIKenAZoRd9AAAAwDt+r/OSc5oFAAAAAAAgBq7QiCWgiss5xzltxHW8gct+vEpdccs8rpl04TL9xDwBY51u67jWst5Guop5YlZ7SbIWxrZOwmU6iWOcW8BtWoo13Wn1Y6R4KEqKh2T/zJkLPydb+onL83WVflLNq1/7fbEoAIjKRcqJhfSTWK3TMP1EshNQSD/5L9JP9lRzveT3Oo8JDQBIc46ToZAHkyhONU/EAAAAIDa/13nJOSoAQFq58847FQgENHz48JjtXnnlFbVr1045OTnq2LGj3nrrreoZIAAAAJIOExoAkOaCCni2VcWHH36oxx57TJ06dYrZbsGCBRo0aJAuv/xyLV++XAMHDtTAgQO1atWqKh0XAADA7xJd58UbExoAkOZCzv/urzy0zf2xt2/frosvvlhPPPGEjjjiiJht77//fp111lm68cYb1b59e40ZM0Zdu3bVQw89VMUzBwAA8LdE1Hnjxo3TiSeeqDp16qhBgwYaOHCgVq9eHX7++++/17XXXqu2bdsqNzdXzZo103XXXaetW7e6Pj8mNAAAniorK4vYysvLzbZDhw5Vv3791KtXr4P2u3Dhwgrt+vTpo4ULFx7ymAEAAOCNuXPnaujQoVq0aJFmzpypPXv2qHfv3tqxY4ck6T//+Y/+85//6O6779aqVatUXFys6dOn6/LLL3d9LBYFjSVKykm800zM/l1e4WOmgVjrUxvtnYCxirY5fnfHdX2+rlNgrPOy2lvn664fq33M1xipJXYii8s3w/UPkctUFIvbNBDruPEWSKb0k+pe/dqbxaL291FYWBix/7bbbtOoUaMqtH/xxRe1bNkyffjhh5Xqv7S0VA0bNozY17BhQ5WWllZtwADSWiAYUCB44O9bb76DSD+J1drP6SeSmYBC+sl/+yH9xDjTuPG6zquM6dOnRzwuLi5WgwYNtHTpUp122mk69thj9dprr4Wfb926tcaOHatf//rX2rt3r2rUqPzPGxMaAJDmQgoo5MEkyv4+Nm7cqLy8vPD+7OzsCm03btyo66+/XjNnzlROTs4hHxsAAAAVeV3nlZWVRezPzs6OWuv91P5bSerWrRuzTV5enqvJDCnBt5zMmzdP/fv3V+PGjRUIBDRlypSI5x3H0a233qpGjRopNzdXvXr10ueff37Qfh9++GG1aNFCOTk5Ovnkk/XBBx/E6QwAAAfKy8uL2KJ9yS1dulSbN29W165dVaNGDdWoUUNz587VAw88oBo1aigYrPhfsgoKCrRp06aIfZs2bVJBQUHczgVA1VHnAYD/FBYWKj8/P7yNGzcuZvtQKKThw4ere/fuOvbYY6O2+fbbbzVmzBhdddVVrseT0AmNHTt2qHPnznr44YejPv/Xv/5VDzzwgB599FEtXrxYtWvXVp8+fbRr1y6zz5deekkjRozQbbfdpmXLlqlz587q06ePNm/eHK/TAICUFnQCnm2VdcYZZ2jlypVasWJFeDvhhBN08cUXa8WKFcrMrHjJblFRkWbPnh2xb+bMmSoqKjrk9wCA96jzACDxvK7zNm7cqK1bt4a3kSNHxjz+0KFDtWrVKr344otRny8rK1O/fv10zDHHRL1F+WASestJ37591bdv36jPOY6j8ePH689//rMGDBggSXr22WfVsGFDTZkyRRdddFHU191777268sordemll0qSHn30UU2bNk1PP/20br755vicCACksETcW1mnTp0Ks/S1a9dWvXr1wvsHDx6sJk2ahGf+r7/+evXo0UP33HOP+vXrpxdffFFLlizR448/fshjB+A96jwASDyv67z9V+BWxrBhw/Tmm29q3rx5atq0aYXnt23bprPOOkt16tTR66+/rpo1rTVWbEmbcrJ27VqVlpZGrGifn5+vk08+2VzRfvfu3Vq6dGnEazIyMtSrV6+Yq+CXl5dXWJUfAJBYGzZsUElJSfhxt27dNGnSJD3++OPq3LmzXn31VU2ZMsW8fBFA8qLOAwD/chxHw4YN0+uvv6533nlHLVu2rNCmrKxMvXv3VlZWlt54440qr6mWtIuC7l+13s2K9t9++62CwWDU13z66afmscaNG6fRo0cf2oDjnWaSZP2YaSZGSojbdWhcp8kYU3NeBXWYrCnBWJfeZxiD8iq1xDhpx0hLsdI6zP7drjbtMrXETJQxj+tyQGZzlyvbu3w/XaefVKOQ9uWLe9HPoZgzZ07Mx5J0/vnn6/zzzz+k4wBIvGSo8wJOtN/Z3qSQWEg/idXaD+kndl+knxysn/RJPyn3oOZyIxF13tChQzVp0iRNnTpVderUCf9ez8/PV25ubngy48cff9Tzzz8fMdlcv379qLceW5L2Co3qNHLkyIj7gDZu3JjoIQFAtXH+u/r1oW52IQ0AiUOdByCdJaLOmzBhgrZu3aqePXuqUaNG4e2ll16SJC1btkyLFy/WypUr1aZNm4g2bn9HJ+0VGvtXrd+0aZMaNWoU3r9p0yYdd9xxUV9z5JFHKjMz0/Uq+JWJmgEAAIA3qPMAwL+cg1yh3bNnz4O2qaykvUKjZcuWKigoiFjRvqysTIsXLzZXtM/KytLxxx8f8ZpQKKTZs2ezCj4AGEJOwLMNACqDOg8Aqoff67yEXqGxfft2rVmzJvx47dq1WrFiherWratmzZpp+PDhuuOOO3TUUUepZcuWuuWWW9S4cWMNHDgw/JozzjhDv/zlLzVs2DBJ0ogRIzRkyBCdcMIJOumkkzR+/Hjt2LEjvBo2ACBSIlJOAPgfdR4AJJ7f67yETmgsWbJEp59+evjxiBEjJElDhgxRcXGxbrrpJu3YsUNXXXWVtmzZolNPPVXTp0+PWAH1iy++0Lfffht+fOGFF+qbb77RrbfeqtLSUh133HGaPn16hQWkAAAAED/UeQCAeAs4Xt284iNlZWXKz89X65v/oszsyPiYDGuhXWOB4oC1ULDVz15jFWqj/wyrf2N/IBi9f3P8XrV3eV4BYxXtgNWP1d4YTyAYfelit/1YCRVme8lO/bDGaqVgmP24a2+9F2b/Hh3X3O/yfM1fYdZK7I7Lcbrtx2L0E238e53demfbC9q6dWulc76rYv/vugFvX6aatd2vj36gPTt2a2rvp+M+bgA4FPt/97W5sWKdZ6amWfszrWQxl/1Y/+HTbG8c1wgGsPs3vuNcnq+slDvruEY/AXM8Rn1p9WOMJ5AR/bs70+gnw2ifYY1TUg3zNcaxjb5qGuknmca5ZRr9ZxlFu9XeGn+W8ceF1b6GEfuWbfYTfZw1zX6i/zFljaem8UdHjtGP1d7ab40nWvud2/dqxAkLqPM8krSLggIAqsf+1au96AcAAADJw+91XnLeCAMAAAAAABADV2gAQJrzauXqZF39GgAAIF35vc5jQgMA0pzfv+gAAADSld/rPG45AQAAAAAAKYcrNA7mwIkoY2LKmrAy57HMfqI/EZCxmrXb8Xg1fovL8zJXxXY5ftcDNToyJx6t1bLd9qMYQ7VWzHY7G2q+SUb3GcbPnMsQDxn9yHU/xjyrlShjdGOu6B4ynzDG47Yfg7HqdrT3OVDNiy75feYeAKIJhCr+DrayK+zfbua3UJXGVNle7O8Jd8d1XPZjHddKVzGTwqx+XLW2S4wMczzRv7uDdk8ujyztNV5j/9Fl9WWchJF+Yp3bbqN395kX1hlYkYtG85DLPz/NfmoaT1hRklY/Lo9rcdHPnmoOGfV7nccVGgAAAAAAIOVwhQYApDm/z9wDAACkK7/XeUxoAECac+RNtnj1XkAJAACAg/F7ncctJwAAAAAAIOVwhQYApDm/X4oIAACQrvxe5zGhEUtAFZZVdpse4joVxW2Kh+vjuktRsdNArNWv3Y3H7dVPdgqJcV5Gcoi1uLD5/lsJHta1V1ZiiSQZi1O7TV6x01Jcpo1YxzWu3woEXb557hcIN/pJlfQT6/0xXhAt/aSavzD8/kUHANFESzmxkH4Sux/STw5+dNJP/ov0E+0OUed5iVtOAAAAAABAyuEKDQBIc36fuQcAAEhXfq/zmNAAgDTn9y86AACAdOX3Oo9bTgAAAAAAQMrhCg0ASHOOE5Djway7F30AAADAO36v85jQiMEJVAwbsIIcrM83UakoiTuulXRhrIrtMnnDvKbIiC0xU108eh9kpb2YB6hKaomxtrfLtBRzjfAYY/WkvSXd0k8sUfuv3ovnQgoo5DZyyOgHAFKFm5QTC+knsftJv/QTyW2BQ/rJf6VR+ske5xB/8bjk9zqPW04AAAAAAEDK4QoNAEhzfl8sCgAAIF35vc5jQgMA0pzf760EAABIV36v87jlBAAAAAAApByu0ACANOf3SxEBAADSld/rPCY0Ygk4FVMsXCdCuNsf7zQTr9JYvEtRsTqyEjnc9W+Ox7g2yQhLidGPdWJGe8lMLXFCViKLsbK367QU4xmXCy07Rj+HulJ8mF/TTyzRujF+FuLF75ciAkA0geC+LR5IP4ndj1/TT6RYCSikn+xD+kl1p5z4vc7jlhMAAAAAAJByuEIDANKc49GliMk6cw8AAJCu/F7nMaEBAGnOkX27ldt+AAAAkDz8XudxywkAAAAAAEg5XKEBAGkupECMheHc9QMAAIDk4fc675AmNMrLy5Wdne3VWJJPoGKShHfpHu76sdJAAkYaSPzHaSVvuIsJscZvp5kYT1ipJdYiwnF+f8yUFtnvnZV+Yh7EbVqKlfphLctsvnfGZ290T/rJwUQ5rts0pUPk99WvAVSN3+u8QMjD76hKIv0kdj8pn34imXUA6ScHkz7pJ+XV/XvH53Weq8r7n//8p4YMGaJWrVqpZs2aqlWrlvLy8tSjRw+NHTtW//nPf+I1TgAAAMQRdR4AINVUakLj9ddf19FHH63LLrtMNWrU0B//+EdNnjxZM2bM0JNPPqkePXpo1qxZatWqla655hp988038R43AMAjof+ufu3FBiD1UOcBgH/5vc6r1DU5f/3rX3Xfffepb9++yohyufYFF1wgSfr666/14IMP6vnnn9cNN9zg7UgBAHHhOB6tfp2sy18DiIk6DwD8y+91XqUmNBYuXFipzpo0aaI777zzkAYEAACA6kOdBwBIVVWObd29e7dWr16tvXuNhVoAAClh/2JRXmwA/IE6DwD8we91nuuUkx9//FHXXnutnnnmGUnSZ599platWunaa69VkyZNdPPNN3s+yIQJqOKyxy7TLtynmSSmvZNhJGMYq0ebP89W0oUV4GFNqZkDdZeK4ja1xExLcRfeIhnvpxRjhW1jxWPzM3aZWmKmn7hMjjGfMFJCSD85iGirhgeqPNdcJX5f/RpA5aVTnZeIlBML6Sex+0mV9BMpRgIK6SeSSD+RpD1mDGN8+L3Oc101jxw5Uv/61780Z84c5eTkhPf36tVLL730kqeDAwAAQPWhzgMApBLXExpTpkzRQw89pFNPPTXiv+x26NBBX3zxhaeDk6Rt27Zp+PDhat68uXJzc9WtWzd9+OGHZvs5c+YoEAhU2EpLSz0fGwD4gd9XvwZQedR5AOAvfq/zXN9y8s0336hBgwYV9u/YscO+dP0QXHHFFVq1apWee+45NW7cWM8//7x69eqljz/+WE2aNDFft3r1auXl5YUfRxszAMD/q18DqDzqPADwF7/Xea6v0DjhhBM0bdq08OP9X25PPvmkioqKvBuZpJ07d+q1117TX//6V5122mlq06aNRo0apTZt2mjChAkxX9ugQQMVFBSEt2gxZAAAAPgf6jwAQCpxfYXGX/7yF/Xt21cff/yx9u7dq/vvv18ff/yxFixYoLlz53o6uL179yoYDEbcwylJubm5mj9/fszXHnfccSovL9exxx6rUaNGqXv37mbb8vJylZeXhx+XlZUd2sABIIXsm7n3YrEoDwYDIKGo8wDAX/xe57mezj711FO1YsUK7d27Vx07dtTbb7+tBg0aaOHChTr++OM9HVydOnVUVFSkMWPG6D//+Y+CwaCef/55LVy4UCUlJVFf06hRIz366KN67bXX9Nprr6mwsFA9e/bUsmXLzOOMGzdO+fn54a2wsHDfE4FD35wMYwtE39z248UYZYzFiTV+czyBqJvb83W/GcfNNLaAu83q3z7fGH1lGJvVX4aibq7HavTj9txcj996jzKib67HE+dxKiMj+ma0D2RkRN8CFe/5DsQaZzXye5wXgMpLpzovEJICwQO2kMvtwNdXtR9jk7HZ4wlE3+I+nkD0zeX7o1DA2IzjGudrFp7WOQQDUTcnFH2zxxlQKBh9M787QxlRt2AwEHULhTKMLRB12xvKiLpZ/QRDgajbnmBm1C3oBKJvoYyo2+5QZtTNam+Nf3eoRtTNar/Xib6Vh2pE3faGMqNue5yMqFt5qGbUzRpPdUpEnTdu3DideOKJqlOnjho0aKCBAwdq9erVEW0ef/xx9ezZU3l5eQoEAtqyZUuVzi/gOMk617LPF198ocsuu0zz5s1TZmamunbtqqOPPlpLly7VJ598Uqk+evTooWbNmum5556L+ny0mfvCwkK1unWsMg74rwYZe6J/kAEjOShgJBxlWMk+bvvZG/3jc9uP2X/QiM8y21v9G+N02b/VjxW7ZsXOBoz3zWxvHtf452PGhUkB45+cdQwrNtRK2wpY7d0e1/rVYO0PujuuOR7rvXM7Hmu/V/1b77PF6Cfar+C9od2avflJbd26NeIeca+VlZUpPz9fbZ4bqcxaOQd/wUEEf9ylNb8ZF/dxA/CPRNZ5Ha76izKzIn/3WTGgViS3ud+rfqy/e1yPx/gOivt4jOO6fH9k9GMe1zhfBdz1I6OfgDUeyRxrhtWXMaZARvQ6I9PoJ8Non2GMp4bZ3jiu0U9NI8410zivTKP/LOOPCKu9Nf4s448gq30N44+IbLOf6OOsafZT8Y++8u17NOG0131d55111lm66KKLdOKJJ2rv3r3605/+pFWrVunjjz9W7dq1JUnjx4/Xrl27JO1L2Prhhx90+OGHux6X6+mhn//85xo9enSF/T/88IN+/vOfux7AwbRu3Vpz587V9u3btXHjRn3wwQfas2ePWrVqVek+TjrpJK1Zs8Z8Pjs7W3l5eREbAKQLx8MNQGqjzgMAf0lEnTd9+nRdcskl6tChgzp37qzi4mJt2LBBS5cuDbcZPny4br75Zp1yyimHdH6u19CYM2eOVq5cqeXLl+uFF14Iz7Ds3r3b83srf6p27dqqXbu2fvjhB82YMUN//etfK/3aFStWqFGjRnEbGwCkMq9uF+GWEyD1UecBgL94XecduA5Rdna2srOzY75269atkqS6dese8jgOVKUbeGbNmqXS0lKdcsopWrduncdDijRjxgxNnz5da9eu1cyZM3X66aerXbt2uvTSSyXtuzxl8ODB4fbjx4/X1KlTtWbNGq1atUrDhw/XO++8o6FDh8Z1nAAAAH5AnQcAsBQWFkasSzRu3LiY7UOhkIYPH67u3bvr2GOP9Xw8rq/QkPYtyDR37lxdeumlOvHEE/XKK6+offv2Xo9N0r7ZnJEjR+qrr75S3bp1dd5552ns2LGqWbOmJKmkpEQbNmwIt9+9e7d+//vf6+uvv1atWrXUqVMnzZo1S6effnpcxgcAKc+r+0W45wTwBeo8APARj+u8jRs3Rty6d7CrM4YOHapVq1YdNL2qqlxPaOzPI8/OztakSZN0xx136KyzztIf//hHzwcnSRdccIEuuOAC8/ni4uKIxzfddJNuuukmT44dTuL46T5jQZz970uluVxsyVr00jGOa43TXEjSGL7Vv7WAkXU1k9v+rQWV7Pch+n5Z+6332frHbn1exgFi/TQ41lqS5qJQ1oVUbt8Mo7mxMJe1MKsp0xintVioNR6X/wZcc/l2mjKtFc3cLSIa7dNy/fvkUHmVUMItJ0DKS6c6LxByoixEnRq/x6xyxR699Yw3M9H2eLw5rlVvWf1Yx7VqHnsxd6Mfo7X9CrvMyDDHFL1gCdo9uTryXqO9/cehdVzjBIzFQq3z2m30nmWOx2KdgZGYYL5tLv9MNvupWWHXbq9q2sryuM5zsxbRsGHD9Oabb2revHlq2rTpoY8hCtcTGgeuyP/nP/9Z7du315AhQzwbFAAAAKofdR4A4FA5jqNrr71Wr7/+uubMmaOWLVvG7ViuJzTWrl2rI488MmLfeeedp7Zt20asWgoASA2OY19Y4rYfAKmNOg8A/CURdd7QoUM1adIkTZ06VXXq1FFpaakkKT8/X7m5uZKk0tJSlZaWhlOqVq5cqTp16qhZs2auFg91PaHRvHnzqPuPPfbYuCzyAQCIL1JOAOxHnQcA/pKIOm/ChAmSpJ49e0bsnzhxoi655BJJ0qOPPhoRE37aaadVaFMZlZrQOPfcc1VcXKy8vDyde+65MdtOnjy50gcHAABAYlHnAQC8dODti9GMGjVKo0aNOuRjVWpCIz8/P7xIVH5+/iEfFACQRKKtgFzVfgCkHOo8APAxn9d5lZrQmDhxYtT/73sZqrBirfU5uk/3oL0k12kjToaRKmLNApopKkb/Lu8vCxjrXMfqxwywMM7NMVbetpIwvEotMT9jd93Y6SfWiuLGm+cYq2u7Tj+xPgDzh85KLXH5w2Km1UQbi4u2HmANDSC9pWudFwhF+w5xnx+STEg/id1PvNNP7COTfvK/3aSf7KnmiQG/13muq+adO3fqxx9/DD9ev369xo8fr7ffftvTgQEAAKB6UecBAFKJ6wmNAQMG6Nlnn5UkbdmyRSeddJLuueceDRgwILz4BwAghTgebpU0YcIEderUKZxlXlRUpH/+859m++LiYgUCgYgtJyfH9akCiI06DwB8JgF1XnVyPaGxbNky/exnP5MkvfrqqyooKND69ev17LPP6oEHHvB8gACA+Nq/+rUXW2U1bdpUd955p5YuXaolS5bo5z//uQYMGKCPPvrIfE1eXp5KSkrC2/r16704fQA/QZ0HAP6SiDqvOrmObf3xxx9Vp04dSdLbb7+tc889VxkZGTrllFMoLgEAldK/f/+Ix2PHjtWECRO0aNEidejQIeprAoGACgoKqmN4QNqizgMApBLXV2i0adNGU6ZM0caNGzVjxgz17t1bkrR582bl5eV5PkAAQDXw8DLEsrKyiK28vDzmoYPBoF588UXt2LFDRUVFZrvt27erefPmKiwsPOjVHACqhjoPAHzIp7ebSFW4QuPWW2/V//3f/+mGG27QGWecES4+3377bXXp0sXzASZStM/OTqgw+oh3eoiVEmIsOGy1t35IHaMf92km7tJJHDOdxBqoNSAr/ST6bnu/u8gP8+ckxmvMz9LoxlxF2/rs3aafGO3Nt9RliorLRbfNlBDHZTpJwG06icXt+M1/NFHGE/MHyHteXUa4v4/CwsKI/bfddlvUnPGVK1eqqKhIu3bt0mGHHabXX39dxxxzTNS+27Ztq6efflqdOnXS1q1bdffdd6tbt2766KOP1LRp00MeO4B90qnOCwTdfHeRfhK7J3f8m36yr7fKH5n0k//tTp/0k70h60OMD6/rvGTjekLjV7/6lU499VSVlJSoc+fO4f1nnHGGfvnLX3o6OABA6tm4cWPEf8nNzs6O2q5t27ZasWKFtm7dqldffVVDhgzR3Llzo05qFBUVRVy90a1bN7Vv316PPfaYxowZ4/1JAGmKOg8AkEpcT2hIUkFBQYX7mE866SRPBgQAqGZeXUr43z72J5ccTFZWltq0aSNJOv744/Xhhx/q/vvv12OPPXbQ19asWVNdunTRmjVrDmnIACqizgMAH/G4zks2rtfQAAD4TcDDrepCodBB19vYLxgMauXKlWrUqNEhHRMAAMDfkqPOi5cqXaEBAMChGDlypPr27atmzZpp27ZtmjRpkubMmaMZM2ZIkgYPHqwmTZpo3LhxkqTbb79dp5xyitq0aaMtW7bob3/7m9avX68rrrgikacBAACABGJCAwDSXQIuRdy8ebMGDx6skpIS5efnq1OnTpoxY4bOPPNMSdKGDRuU8ZOFYH/44QddeeWVKi0t1RFHHKHjjz9eCxYsMBcRBQAAgHx/ywkTGrFEubLGSqIw00ms9lZ6iNXedZqJkSpircrsMnnDXOXWSpZwGU5i3Qxl9WNeAWWN33iB2b+VxmK8z1XhNpnGPLLLXzau00/i3I/r9BCX3P5bMpnJN+5SV6KeVzWnnCTii+6pp56K+fycOXMiHt9333267777qjAoAIguEKrC7/4KSD+J3ZM7KZ9+IsVIQCH9JFY/6ZR+ssf6QzBemNCQ3njjjUp3eM4551R5MAAAAKhe1HkAgFRVqQmNgQMHVqqzQCCgYNDtf44FACSUE4hxuZTLfgCkHOo8APAxn9d5lZrQCIWq+bIYAEC1cRz7jhi3/QBIPdR5AOBffq/zDim2ddeuXV6NAwAAAEmEOg8AkOxcT2gEg0GNGTNGTZo00WGHHaYvv/xSknTLLbccdJE3AEAScjzcAKQ06jwA8Bmf13muU07Gjh2rZ555Rn/961915ZVXhvcfe+yxGj9+vC6//HJPB5hITkbFRAQ7icJIwTDSCqwQAzMtxWUKievEDOtq0ziP0+oo4DpVxF2MSsBIonCbWhIIukuNkeQ+ecX6LK1jJFlqibkSeMjdZ+8+PcRde8dl/2ZikMVNKooVdxQvPr+3EkDlpVOd503KiYX0k9g9uZMq6SdSrLqH9BOJ9BNJ2mv+YRQnPq/zXL+bzz77rB5//HFdfPHFysz83w9Y586d9emnn3o6OAAAAFQf6jwAQCpxfYXG119/rTZt2lTYHwqFtGfPHk8GBQCoPgHHvgDGbT8AUht1HgD4i9/rPNdXaBxzzDF67733Kux/9dVX1aVLF08GBQCoRj6/txJA5VHnAYDP+LzOc32Fxq233qohQ4bo66+/VigU0uTJk7V69Wo9++yzevPNN+MxRgAAAFQD6jwAQCpxfYXGgAED9I9//EOzZs1S7dq1deutt+qTTz7RP/7xD5155pnxGCMAIJ72LxblxQYgpVHnAYDP+LzOc32FhiT97Gc/08yZM70eS/KJcsOR29QSM9HCTP1w195tmomVLGGltJiXFpmBEFbMibFKtJmK4q4fs6NMl2tEm/0b3WR69w87YLypZvqJ0Y/jcpFoa1Fp82fLbWpJyOjISgay+jEOa3K3GLdrjvXZe5GKYv5CiROvLiNM0ksRAbiTLnVeIOg+uevQkX4Suyd3ki39JNaxST/Zh/QTaa8xlrjxeZ1Xze8mAAAAAADAoavUFRpHHHGEApX8L4bff//9IQ0IAFDNfD5zDyA26jwA8DGf13mVmtAYP358+P9/9913uuOOO9SnTx8VFRVJkhYuXKgZM2bolltuicsgAQBx5PMvOgCxUecBgI/5vM6r1ITGkCFDwv//vPPO0+23365hw4aF91133XV66KGHNGvWLN1www3ejxIAAABxQZ0HAEhVrtfQmDFjhs4666wK+8866yzNmjXLk0EBAKqRz1e/BlB51HkA4DM+r/Ncp5zUq1dPU6dO1e9///uI/VOnTlW9evU8G1gycDIqpoXYqRzGfqO96xADl8c1kyuscVrhHtbCwm7HY7wRjnVgi8vmAbdLQVvvj/UCa2XqWJ+vleBi8Cz9xOVQzbfO/GUW/YfFMd7UgDkg43ytAVmpKC77Nw9gfcYuf3bdpKJY71m8RAl0qnI/AFJbOtV5gZATJYEqUQU76Sexe3Incekndl+kn8Q+cjqln+yxEgDjxO91nusJjdGjR+uKK67QnDlzdPLJJ0uSFi9erOnTp+uJJ57wfIAAAACoHtR5AIBU4vo/A15yySV6//33lZeXp8mTJ2vy5MnKy8vT/Pnzdckll3g+wG3btmn48OFq3ry5cnNz1a1bN3344YcxXzNnzhx17dpV2dnZatOmjYqLiz0fFwD4huPhBiClUecBgM/4vM5zfYWGJJ188sl64YUXvB5LVFdccYVWrVql5557To0bN9bzzz+vXr166eOPP1aTJk0qtF+7dq369euna665Ri+88IJmz56tK664Qo0aNVKfPn2qZcwAAACpijoPAJAqqjShEQwGNWXKFH3yySeSpA4dOuicc85RZqZ101XV7Ny5U6+99pqmTp2q0047TZI0atQo/eMf/9CECRN0xx13VHjNo48+qpYtW+qee+6RJLVv317z58/XfffdxxcdAADAQVDnAQBShesJjTVr1qhfv3766quv1LZtW0nSuHHjVFhYqGnTpql169aeDW7v3r0KBoPKycmJ2J+bm6v58+dHfc3ChQvVq1eviH19+vTR8OHDzeOUl5ervLw8/LisrKzqgwaAFBOQR4tFHXoXABKMOg8A/MXvdZ7rNTSuu+46tWrVShs3btSyZcu0bNkybdiwQS1bttR1113n6eDq1KmjoqIijRkzRv/5z38UDAb1/PPPa+HChSopKYn6mtLSUjVs2DBiX8OGDVVWVqadO3dGfc24ceOUn58f3goLCz09DwBIaj6P8wJQedR5AOAzPq/zXF+hMXfuXC1atEh169YN76tXr57uvPNOde/e3dPBSdJzzz2nyy67TE2aNFFmZqa6du2qQYMGaenSpZ4dY+TIkRoxYkT4cVlZ2b4vuwxVmPKxYknNJMgMI4LSiu50GavqVTyrFX1pjd9cFMZKynQd82rsN943K6LTfJ9dhlu5jU6NKWgcO95xrsbPivkZWB9aMHpHgQx3/zg8i2E142KtqN0ki3mN9j6bHwoAxFc61XkBJ9p3Y7LFpybbeNxJtzjXfa9w1xdxrrGP7Mc416BRi6JqXE9oZGdna9u2bRX2b9++XVlZ7pN5D6Z169aaO3euduzYobKyMjVq1EgXXnihWrVqFbV9QUGBNm3aFLFv06ZNysvLU25ubtTXZGdnKzs72/OxA0BK8Grl6iRd/RpA5VHnAYDP+LzOc/2fAX/xi1/oqquu0uLFi+U4jhzH0aJFi3TNNdfonHPOiccYJUm1a9dWo0aN9MMPP2jGjBkaMGBA1HZFRUWaPXt2xL6ZM2eqqKgobmMDgJTm8zgvAJVHnQcAPuPzOs/1hMYDDzyg1q1bq6ioSDk5OcrJyVH37t3Vpk0b3X///Z4PcMaMGZo+fbrWrl2rmTNn6vTTT1e7du106aWXStp3GeHgwYPD7a+55hp9+eWXuummm/Tpp5/qkUce0csvv6wbbrjB87EBAAD4CXUeACCVuL7l5PDDD9fUqVP1+eef69NPP5W0LzKrTZs2ng9OkrZu3aqRI0fqq6++Ut26dXXeeedp7NixqlmzpiSppKREGzZsCLdv2bKlpk2bphtuuEH333+/mjZtqieffJIoLwAwBByPVr9O0pl7AJVHnQcA/uL3Os/1hMZ+Rx11lI466igvxxLVBRdcoAsuuMB8vri4uMK+nj17avny5XEcFQD4iM/vrQTgHnUeAPiEz+s81xMajuPo1Vdf1bvvvqvNmzcrFIpcSXby5MmeDS7RHFVMpwlYqSVBIznBShuxAiGs9p6lmXjUj8v9Zj/WGsv2C4zm3qSZ+CH9xPWhrfANYxVtJ9PlD4v1M2cufm0c10onsdJVrPG47N9Ke4lrKoqVMAMAcZZOdV4gKB34FWJ9NSVf2kiyjccd0k8O3hfpJ7GPnMrpJ0GjD1SN63dz+PDh+s1vfqO1a9fqsMMOi8j1zs/Pj8cYAQDx5PPFogBUHnUeAPhMAuq8cePG6cQTT1SdOnXUoEEDDRw4UKtXr45os2vXLg0dOlT16tXTYYcdpvPOO69CilVluL5C47nnntPkyZN19tlnuz4YACD5+P3eSgCVR50HAP6SiDpv7ty5Gjp0qE488UTt3btXf/rTn9S7d299/PHHql27tiTphhtu0LRp0/TKK68oPz9fw4YN07nnnqv333/f1bhcT2jk5+eb2eAAAABIXdR5AIBDNX369IjHxcXFatCggZYuXarTTjtNW7du1VNPPaVJkybp5z//uSRp4sSJat++vRYtWqRTTjml0sdyfcvJqFGjNHr0aO3cudPtSwEAycgJeLcBSGnUeQDgMx7XeWVlZRFbeXn5QYewdetWSVLdunUlSUuXLtWePXvUq1evcJt27dqpWbNmWrhwoavTc32FxgUXXKC///3vatCggVq0aBGO1dpv2bJlbrsEACSSz1e/BlB51HkA4DMe13mFhYURu2+77TaNGjXKfFkoFNLw4cPVvXt3HXvssZKk0tJSZWVl6fDDD49o27BhQ5WWlroalusJjSFDhmjp0qX69a9/rYYNGyrg59X4M5wKiQiOkTjhOiXESlqwEi08SiFxn2biLoXEbcCD+Y/Lep+NjqzUGNerYlvna3xebtNP9h3Z5b8Zl7+A7HQS6z11d1i3n7HrPBnrMzAObCf6xDedxPwcPTkuq18DSIx0qvMCIafCd2aG8bud9JPqkfrpJ+6PTfrJPumUfrI3ZL05qWHjxo3Ky8sLP87Ozo7ZfujQoVq1apXmz58fl/G4ntCYNm2aZsyYoVNPPTUe4wEAVDMWBQWwH3UeAPiL13VeXl5exIRGLMOGDdObb76pefPmqWnTpuH9BQUF2r17t7Zs2RJxlcamTZtUUFDgalyu/zNgYWFhpU8AAJACiG0F8F/UeQDgMwmo8xzH0bBhw/T666/rnXfeUcuWLSOeP/7441WzZk3Nnj07vG/16tXasGGDioqKXJ2e6wmNe+65RzfddJPWrVvn9qUAAABIYtR5AIBDNXToUD3//POaNGmS6tSpo9LSUpWWloYXnM7Pz9fll1+uESNG6N1339XSpUt16aWXqqioyFXCiVSFW05+/etf68cff1Tr1q1Vq1atCotFff/99267BAAkkkeXInKFBpD6qPMAwGcSUOdNmDBBktSzZ8+I/RMnTtQll1wiSbrvvvuUkZGh8847T+Xl5erTp48eeeQR18NyPaExfvx41wcBACQxUk4A/Bd1HgD4TALqPMdctf9/cnJy9PDDD+vhhx8+hEFVMeUkbWSowk05VpqGY6VgWIkHZj/R95vtXS4IbKaBuE0hMZIozB9et/1bzT37g8ldeovZ3pzujLUet9vUDOMYRmqJyW0ijtXc+plz+9lYK+dbn4Fx3ISlolg8OK5jx/YAQFylU50XCO3bIkX/ZU36SWKlSvrJviN7c2zST/bxY/pJ0DhXVE2l3s0dO3a46tRtewBAArEoKJDWqPMAwMd8XudVakKjTZs2uvPOO1VSUmK2cRxHM2fOVN++ffXAAw94NkAAQHztj/PyYgOQeqjzAMC//F7nVeqWkzlz5uhPf/qTRo0apc6dO+uEE05Q48aNlZOTox9++EEff/yxFi5cqBo1amjkyJG6+uqr4z1uAAAAeIA6DwCQqio1odG2bVu99tpr2rBhg1555RW99957WrBggXbu3KkjjzxSXbp00RNPPKG+ffsqM9O6MQoAAADJhjoPAJCqXC0K2qxZM/3+97/X73//+3iNBwBQ3Ug5ASDqPADwJZ/Xea5TTtKJk+FUSC+x00lcpplYPxBGWorVf+VWQflpP9YT7vZb91CZ5+VZykl8V612nWZiJGMcbP3r6HuNz95KA7GXVjfaWx+ayxXOXb5F1vgDxnjsoJkEpaK4TcRxm5YS9bipseo8AKSyQNBRoELd5e57nfSTxEq29JNYPZF+8t/ujf3plH4SdFt7IyZXfw5//PHH+t3vfqcuXbqoUaNGatSokbp06aLf/e53+vjjj+M1RgBAHPl9sSgAlUOdBwD+4/c6r9JXaPzzn//UwIED1bVrVw0YMEANGzaUJG3atEkzZ85U165dNXXqVPXp0ydugwUAxEmSfkkBqB7UeQDgYz6u8yo9oXHzzTfrj3/8o26//fYKz40aNUqjRo3SjTfeyBcdAABAiqHOAwCkokrfcvLZZ5/p4osvNp8fNGiQPv/8c08GBQCoRo6HG4CURJ0HAD7l8zqv0hMaLVq00LRp08znp02bpubNm3syKABA9fH7vZUADo46DwD8ye91XqVvObn99tv1f//3f5ozZ4569eoVcW/l7NmzNX36dE2aNCluA02IDKdi6ojLtJEDU1LCrCQEs5/o+6321gya25QT92kmLhMhXLLXM05Q+kmVVgd3+RrzFKI/YaaKePUWmZ+9sd86rvWzWHEx6H3duP1ZdJuKYg3UWO3bdVqK+T5X7Mcx/2EDQHykY50XCEaro9x+mUVvT/pJYrkfPekn4dakn0iKb/pJMGSdFKqi0hMa559/vpo0aaIHHnhA99xzj0pLSyVJBQUFKioq0pw5c1RUVBS3gQIA4sSrywiTdOYewMFR5wGAT/m8zqv0hIYkdevWTd26dYvXWAAACeDVZYTJeikigMqhzgMA//F7ncd1zQAAAAAAIOV4NqHxySefqFWrVl51BwCoLj5f/RrAoaPOA4AU5fM6z9UtJ7Hs3r1b69ev96o7AEB18fm9lQAOHXUeAKQon9d5lZ7QGDFiRMznv/nmm0MeTNLJUIVrWMzUEnMFXpdpJtYPinFcJ9NlMoZnaSYu97tdedkaj9FL6qSf2K8xQzNcriodMD40c/Vr6722xhN0+QLXqSgu00mMf2Nme7dJPK7HE323/XlF6zs1VpEH4B/pWOcFQk6U71ivUs1IP0lGpJ8cvB/ST2If2Yv0k6BVu6JKKj2hcf/99+u4445TXl5e1Oe3b9/u2aAAANUnEYtFTZgwQRMmTNC6deskSR06dNCtt96qvn37mq955ZVXdMstt2jdunU66qijdNddd+nss88+xFEDkKjzAMCv/L4oaKUnNNq0aaMbbrhBv/71r6M+v2LFCh1//PGeDQwAUE0ScCli06ZNdeedd+qoo46S4zh65plnNGDAAC1fvlwdOnSo0H7BggUaNGiQxo0bp1/84heaNGmSBg4cqGXLlunYY4/1YPBAeqPOAwCf8vktJ5VeFPSEE07Q0qVLzecDgYAc8/4EAAD+p3///jr77LN11FFH6eijj9bYsWN12GGHadGiRVHb33///TrrrLN04403qn379hozZoy6du2qhx56qJpHDvgTdR4AIBVV+gqNe+65R+Xl5ebznTt3Vihk3wEIAEhSHs/cl5WVRezOzs5Wdna2+bJgMKhXXnlFO3bsUFFRUdQ2CxcurHCPf58+fTRlypRDGjKAfajzAMCnfH6FRqUnNAoKCuI5DgBAgnh9b2VhYWHE/ttuu02jRo2q0H7lypUqKirSrl27dNhhh+n111/XMcccE7Xv0tJSNWzYMGJfw4YNVVpaeugDB0CdBwA+xRoa6SzDqZguYt2kk+FuRWDzsk0r3cBtP8Z4rJk1t+v+esY435C5OrjRjbE/UeknjpWMISlg/DYIuEzHMH+rBK32RvfW6tRmzInxs278LMY/FcXYbR3Wep/dJgBZB7aSkMzzqviEk+IpJxs3boxYWNC6OqNt27ZasWKFtm7dqldffVVDhgzR3LlzzUkNAPBSIOTY31EVWxv7ST/ZJ9nG44536SexenOH9JNYrVM7/SRkjAVV4/rdPOKII1S3bt0KW7169dSkSRP16NFDEydO9GRwwWBQt9xyi1q2bKnc3Fy1bt1aY8aMiXkP55w5cxQIBCps/Fc8ADA4Hm6S8vLyIjZrQiMrK0tt2rTR8ccfr3Hjxqlz5866//77o7YtKCjQpk2bIvZt2rSJ/6oMeKw66zyJWg8A4s7jOi/ZuL5C49Zbb9XYsWPVt29fnXTSSZKkDz74QNOnT9fQoUO1du1a/fa3v9XevXt15ZVXHtLg7rrrLk2YMEHPPPOMOnTooCVLlujSSy9Vfn6+rrvuupivXb16dcR/IWzQoMEhjQUA/CpZLkUMhULmPfxFRUWaPXu2hg8fHt43c+ZMc80NAFVTnXWeRK0HAPGWLHVevLie0Jg/f77uuOMOXXPNNRH7H3vsMb399tt67bXX1KlTJz3wwAOH/EW3YMECDRgwQP369ZMktWjRQn//+9/1wQcfHPS1DRo00OGHH35IxwcAxMfIkSPVt29fNWvWTNu2bdOkSZM0Z84czZgxQ5I0ePBgNWnSROPGjZMkXX/99erRo4fuuece9evXTy+++KKWLFmixx9/PJGnAfhOddZ5ErUeAODQuL7lZMaMGerVq1eF/WeccUa4ED377LP15ZdfHvLgunXrptmzZ+uzzz6TJP3rX//S/Pnz1bdv34O+9rjjjlOjRo105pln6v3334/Ztry8XGVlZREbAKSNBFyKuHnzZg0ePFht27bVGWecoQ8//FAzZszQmWeeKUnasGGDSkpKwu27deumSZMm6fHHH1fnzp316quvasqUKTr22GMP7dwBRKjOOk+qnlqPOg9AWuOWk0h169bVP/7xD91www0R+//xj3+obt26kqQdO3aoTp06hzy4m2++WWVlZWrXrp0yMzMVDAY1duxYXXzxxeZrGjVqpEcffVQnnHCCysvL9eSTT6pnz55avHixunbtGvU148aN0+jRow95vACQkhIQ5/XUU0/FfH7OnDkV9p1//vk6//zzXQ4KgBvVWedJ1VPrUecBSGsJqPOqk+sJjVtuuUW//e1v9e6774bvrfzwww/11ltv6dFHH5W0777mHj16HPLgXn75Zb3wwguaNGmSOnTooBUrVmj48OFq3LixhgwZEvU1bdu2Vdu2bcOPu3Xrpi+++EL33XefnnvuuaivGTlypEaMGBF+XFZWpsLCQgUyHQUyIz85J2St8OsuucKxUkiMa2bMpAVrJV+XP3Bm/25jRTz7QU9M+knASPCwQkvcJ5ZIdiyH1dw4C+tn0ereWm3aOjnzzXOZWuI2FcVlOonZ3hhPwEwhsd4H6/2Pvtv1v4Eox3XMf3gAEF/VWedJ1VPrmXVe0DG/xyuP9JPYkm087lRt9HFO0jOPSvqJlBrpJ0FSTjzlekLjyiuv1DHHHKOHHnpIkydPlrTvi2Xu3Lnq1q2bJOn3v/+9J4O78cYbdfPNN+uiiy6SJHXs2FHr16/XuHHjzC+5aE466STNnz/ffD47O9tchR8A/C4gb0rL1ChPAcRSnXWeVD21HnUegHTm9zrP9YSGJHXv3l3du3f3eiwV/Pjjj8rIiJzByszMVCgU8z9/V7BixQo1atTIy6EBgH/4/FJEAO5UV50nUesBQNz5vM6r0oRGMBjUlClT9Mknn0iSOnTooHPOOUeZmda1PFXTv39/jR07Vs2aNVOHDh20fPly3XvvvbrsssvCbUaOHKmvv/5azz77rCRp/PjxatmypTp06KBdu3bpySef1DvvvKO3337b07EBAAD4UXXVeRK1HgDg0Lie0FizZo3OPvtsff311+H7F8eNG6fCwkJNmzZNrVu39mxwDz74oG655Rb97ne/0+bNm9W4cWNdffXVuvXWW8NtSkpKtGHDhvDj3bt36/e//72+/vpr1apVS506ddKsWbN0+umnezYuAPATv+eTA6i86qzzJGo9AIg3v9d5Acdamc9w9tlny3EcvfDCC+HVrr/77jv9+te/VkZGhqZNmxaXgVansrIy5efnq/DR25SRmxPxnLPHWMQlaCxAuMfYv9fd/oxg9MMGrP1W/0b7jL1GP9ZCjG7H41k/xqJZRntrPR9zPK73u18U1OzLWrPT7TES1I/Z3lxA1t2ioNYiUnb/1njcjt/toqDufttHG//evbs0d9Ed2rp1q/Ly8lz158b+33Udrv6LMrNzDv6CgwiW79JHj/0p7uMGED/pVOd1P/021ahxQJ2XaS1waOw3Fnm3FkS0+zHaG/2HrP5d9mOO0+zH2J/y43G331wXMuaYotcH8R6TYyyG7vY9krWousvzNQs367hW8II5HuNvBasfa4H/jOiFXqbRT4bRPiPKeII/lmv1/91FnecR11dozJ07V4sWLQp/yUlSvXr1dOedd1bb/ZbVJsOp+I/C+sdppZm4TSFxmdhg9mP9ozX6MccZb65n+tyln1gBHnaiiNv9VrJHjBNzuSyzYyavuJygsNJGrGQdr1JRzAkBl/24XbzbbRqLOc/hTVqKNTESLb3FTEECgDhLpzovEIr2nenVf4Ik/SS2ZBuPO7F+SuwzIP1EIv1EkkJWDYwqcT2hkZ2drW3btlXYv337dmVlZXkyKABANUvSywgBVC/qPADwIR/Xea5DcH/xi1/oqquu0uLFi+U4jhzH0aJFi3TNNdfonHPOiccYAQBxtP/eSi82AKmNOg8A/MXvdZ7rCY0HHnhArVu3VlFRkXJycpSTk6Pu3burTZs2uv/+++MxRgAAAFQD6jwAQCpxfcvJ4YcfrqlTp+rzzz/Xp59+Kklq37692rRp4/ngAADVwOf55AAqjzoPAHzG53We6wmN/Y466igdddRRXo4FAJAAfo/zAuAedR4A+IPf67xKTWiMGDGi0h3ee++9VR5MssnIcCpE/ISiRRJIcoyVcM1EBSsqyUw58SaxIWRFDSXbqtJuh1OVtJE4spJJpBhD9Si1xHrvrJ8hz1JRrJ9RI9LYXizb3c+61TxgRcJ5FBdr/tszWElF0dJS7JXEAcB7aVvnBUPKOOBLMGTejU36SfVItvG45/4MSD+R0iv9hJQTb1VqQmP58uWV6iwQ4484AECS8vmliABio84DAB/zeZ1XqQmNd999N97jAAAkiN8vRQQQG3UeAPiX3+s81yknAAAAAAAAlnnz5ql///5q3LixAoGApkyZEvH8pk2bdMkll6hx48aqVauWzjrrLH3++eeuj8OEBgCkO8fDDQAAAMkjQXXejh071LlzZz388MMVh+Q4GjhwoL788ktNnTpVy5cvV/PmzdWrVy/t2LHD1XGqnHICAPAJn99bCQAAkLYSVOf17dtXffv2jfrc559/rkWLFmnVqlXq0KGDJGnChAkqKCjQ3//+d11xxRWVPg4TGjFkZISUkRG5Yq2TEf2iFsdID7EiGOxkBqObTDO6wtjvbuGukMvVsuPOdcqJtd/lCs5mP0YvVoCHvZy4+7EaN6yZP0NmNx6lophpIMZq0+Z4rPMy0kCs99QaT4LSUizW+xYtOCnk8t8vAKAKghVvLM8wkgpIPznYceMt2cbjHuknsftJp/STkJUAmCLKysoiHmdnZys7O9tVH+Xl5ZKknJyc8L6MjAxlZ2dr/vz5riY0uOUEANLc/sWivNgAAACQPLyu8woLC5Wfnx/exo0b53pM7dq1U7NmzTRy5Ej98MMP2r17t+666y599dVXKikpcdUXV2gAQLrjlhMAAAB/8rjO27hxo/Ly8sK73V6dIUk1a9bU5MmTdfnll6tu3brKzMxUr1691LdvX9dXQTOhAQAAAAAADiovLy9iQqOqjj/+eK1YsUJbt27V7t27Vb9+fZ188sk64YQTXPXDhAYApLmA45hrmrjtBwAAAMkj2eu8/Px8SfsWCl2yZInGjBnj6vVMaABAuuOWEwAAAH9KUJ23fft2rVmzJvx47dq1WrFiherWratmzZrplVdeUf369dWsWTOtXLlS119/vQYOHKjevXu7Og4TGjEEMkPKyIxcmdZKH7ATG4xP3lg611zJ12KlIXj0h0XC0k88Sjmx3h5r/GYih7HaoRmiEmP8rtNJjCfMBRjjnIpi9+MuFcVMM7HaG0sYJ1taisVNiopjnSwAwDOBkKPAAUkJ1lcB6SdVPW68Jdt43CP9JHY/fkw/cYxEFL9ZsmSJTj/99PDjESNGSJKGDBmi4uJilZSUaMSIEdq0aZMaNWqkwYMH65ZbbnF9HCY0ACDNeZVQQsoJAABAcklUndezZ8+YC3xed911uu666w5xVExoAAC45QQAAMCffF7npcf1LgAAAAAAwFe4QgMA0hy3nAAAAPiT3+s8JjQAIN35/FJEAACAtOXzOo8JjRgyMx1lHpByYiUnWIkHISf6WrhuUwzMnx/XP1jerPoc7/QTO53EXXsrwSNktbeWLg4aKyNbyRvmCssxjmH9aJnHMPa7TEVxew52P+6Oa43fWjwoYWkp5gdj9ONBiorxawMA4KHA3pACB2YW1Ij+JUH6yT6kn1Qf0k9i95PK6Sfm35OoEiY0ACDN+f1SRAAAgHTl9zqPCQ0ASHc+vxQRAAAgbfm8ziPlBAAAAAAApByu0AAAJO1lhAAAADg0fq7zmNAAgHTnOFEXJ61SPwAAAEgePq/zmNCIoUZmqNIpJ9bn61hL7VqJCmYSgnHcGvFeLdsdK/3ETN6w0kaCVv9Ge2v4Hu03U1RcD0jmFKnbNBPPUlHcppaY/bj7mQ64jKwxZ5at/l2kiuxr7zLNJI4pKqEUWoUdAFJVwHEqJnrtNb7kSD+RRPpJMiD9JHY/qZB+ErLiFlElTGgAQJrz++rXAAAA6crvdR4TGgCQ7ny++jUAAEDa8nmdR8oJAAAAAABIOVyhAQBpLhCKsVaLy34AAACQPPxe5zGhAQDpzueXIgIAAKQtn9d5TGjEUCMzqMzMyLgNOzjBSkiIPpVlhHjIugvIsdaP3mu09yr9xFqEN2jFfljtXa4JbPbjrhvXiR9G/25TVKzj7ju2u/QQz1JRjPZmConRj53i4e68zOO6TUsxE4as9u6Sisz+o+/2JEWF1a8BoBoEpQN/CZu/fUk/idme9JPEI/0kdj9JlX5CnecpJjQAIM35ffVrAACAdOX3Oi+pFwUNBoO65ZZb1LJlS+Xm5qp169YaM2aMHOs/pf7XnDlz1LVrV2VnZ6tNmzYqLi6ungEDQCpyHO82AHCBWg8A4szndV5SX6Fx1113acKECXrmmWfUoUMHLVmyRJdeeqny8/N13XXXRX3N2rVr1a9fP11zzTV64YUXNHv2bF1xxRVq1KiR+vTpU81nAAAAAAu1HgDgUCT1hMaCBQs0YMAA9evXT5LUokUL/f3vf9cHH3xgvubRRx9Vy5Ytdc8990iS2rdvr/nz5+u+++4zv+TKy8tVXl4eflxWVubhWQBAcvP7pYgAkld11HrUeQDSmd/rvKS+5aRbt26aPXu2PvvsM0nSv/71L82fP199+/Y1X7Nw4UL16tUrYl+fPn20cOFC8zXjxo1Tfn5+eCssLPTmBAAgFTgebgDgQnXUetR5ANKaz+u8pL5C4+abb1ZZWZnatWunzMxMBYNBjR07VhdffLH5mtLSUjVs2DBiX8OGDVVWVqadO3cqNze3wmtGjhypESNGhB+XlZWpsLBQWZlBZdY4IOXESkioET0ew7rVyOonlOkuosKz9BNzsV2XKxFbcSBmSoiRaGGcVobRv9XeSi0xWeO0kjqs48bKabbSRsy+jPQQ62fL6sdK33CdZuLyuC4XbneTBhJrPHZ7d7Elbvu3fojc9BOy3gMA8JnqqPWsOi8QDCqgA9PsokcekH5StfaknyQe6Sex+0lE+olDyomnknpC4+WXX9YLL7ygSZMmqUOHDlqxYoWGDx+uxo0ba8iQIZ4dJzs7W9nZ2Z71BwCpxO+XIgJIXtVR61HnAUhnfq/zknpC48Ybb9TNN9+siy66SJLUsWNHrV+/XuPGjTO/5AoKCrRp06aIfZs2bVJeXl7UqzMAIO15tXJ1kq5+DSB5UesBQJz5vM5L6jU0fvzxR2VkRA4xMzNToZB9oVpRUZFmz54dsW/mzJkqKiqKyxgBAABQNdR6AIBDkdQTGv3799fYsWM1bdo0rVu3Tq+//rruvfde/fKXvwy3GTlypAYPHhx+fM011+jLL7/UTTfdpE8//VSPPPKIXn75Zd1www2JOAUASHr7L0X0YgMAN6j1ACC+/F7nJfUtJw8++KBuueUW/e53v9PmzZvVuHFjXX311br11lvDbUpKSrRhw4bw45YtW2ratGm64YYbdP/996tp06Z68sknySUHAItXK1cn6RcdgORFrQcAcebzOi+pJzTq1Kmj8ePHa/z48Wab4uLiCvt69uyp5cuXH/LxszKDqpFZuZQTi9v2MlbXDpnrQbtMPwkaK/C6HaaVZhI0UkuM9mb4ifFEyJgaNFNFzI6i7zaTOqzwFusapxgpJ67TQKyxWqkcZnvjM7DGavZvfAZWeovL9BDz9jzPUk4Sk6JifgdEecL6GQEAv0lkrRcIOgpU+CVspNaRfnKQ/kk/+Z/USLAg/SR2P3FNPzH+HkPVJPWEBgAg/vy++jUAAEC68nudx4QGAKS7kBPjvyy47AcAAADJw+d1XlIvCgoAAAAAABANV2gAQLrz+WJRAAAAacvndR5XaAAAAAAAgJTDFRoxZGfuVY3MyKVsQ67jQCzu3vqAESFhL5JrrKJttY++qLc9EWcFRRhTZAGjf3PB4ZBX6SRWKorLxA+PUlEkWUE25rHdp5AY+61UEbftvUpLseJMPEpFSVSKivVD7aafkPXzHycBebRY1KF3AQDVJxSq8OUVMP9bH+kn+5B+cnDJOKbKI/0kdj+epJ9Uc5qd3+s8rtAAgHTnON5tlTRu3DideOKJqlOnjho0aKCBAwdq9erVMV9TXFysQCAQseXk5Bzq2QMAAPhXAuq86sSEBgCg2s2dO1dDhw7VokWLNHPmTO3Zs0e9e/fWjh07Yr4uLy9PJSUl4W39+vXVNGIAAAAkG245AYA0l4h88unTp0c8Li4uVoMGDbR06VKddtpp9jECARUUFFR1iAAAAGklEXVedeIKDQBId46Hm6SysrKIrby8/KBD2Lp1qySpbt26Mdtt375dzZs3V2FhoQYMGKCPPvrI5ckCAACkEY/rvGTDhAYAwFOFhYXKz88Pb+PGjYvZPhQKafjw4erevbuOPfZYs13btm319NNPa+rUqXr++ecVCoXUrVs3ffXVV16fAgAAAFIAt5zEkFUjqJo19lbrMQOB6Evk7nW5rKwZEmLEbzhGioqZZmKmkxj7rfbWeKKtCCwpYMS6mKkixmnZ/Udvb6aiuEwO2deX9YTRl/XRWLOkViKL23QP1+kn7vox01I8SyeJb4qKxYvxh6p79WvHsd8vl/1I0saNG5WXlxfen52dHfN1Q4cO1apVqzR//vyY7YqKilRUVBR+3K1bN7Vv316PPfaYxowZcwgjB5CW9gYl54AvfqMyJv3kYEg/ObhkHFPlkX4Sux9X6SeepWZWjtd1XrJhQgMA0l1I3kSI/bePvLy8iAmNWIYNG6Y333xT8+bNU9OmTV0drmbNmurSpYvWrFnjdqQAAADpweM6L9lwywkAoNo5jqNhw4bp9ddf1zvvvKOWLVu67iMYDGrlypVq1KhRHEYIAACAZMcVGgCQ5hJxKeLQoUM1adIkTZ06VXXq1FFpaakkKT8/X7m5uZKkwYMHq0mTJuE1OG6//XadcsopatOmjbZs2aK//e1vWr9+va644opDHjsAAIAfccsJAMDfvFq52kUfEyZMkCT17NkzYv/EiRN1ySWXSJI2bNigjIz/XUj4ww8/6Morr1RpaamOOOIIHX/88VqwYIGOOeaYQx05AACAPyWgzqtO3HICAKh2juNE3fZPZkjSnDlzVFxcHH583333af369SovL1dpaammTZumLl26VP/gAQAAENO8efPUv39/NW7cWIFAQFOmTIl4fvv27Ro2bJiaNm2q3NxcHXPMMXr00UddH4crNGKoVWO3aib5OxQwIhX27rWiHKLPYVmpKI4VH2LtN1JIrKQOl4uGx0gtib4/7qko1nFjLZpjvNmuU0KsY7tcyNyr1BK3/XvV3u7H+qGO93FdTl9HSzmp7hlwx3Ef52L1AwApIhAKKXDAKnfmbzHST6qI9JODS8YxVR7pJ7H7iXZc6++TuElQnbdjxw517txZl112mc4999wKz48YMULvvPOOnn/+ebVo0UJvv/22fve736lx48Y655xzKn2cJP9zHQAQbwEnxoSOy34AAACQPBJV5/Xt21d9+/Y1n1+wYIGGDBkSvv34qquu0mOPPaYPPvjA1YQGt5wAAAAAAICDKisri9jKy8ur1E+3bt30xhtv6Ouvv5bjOHr33Xf12WefqXfv3q76YUIDANLd/ksRvdgAAACQPDyu8woLC5Wfnx/e9qfRufXggw/qmGOOUdOmTZWVlaWzzjpLDz/8sE477TRX/XDLCQCkuUDoIOu+uOgHAAAAycPrOm/jxo3Ky8sL78/Ozq5Sfw8++KAWLVqkN954Q82bN9e8efM0dOhQNW7cWL169ap0P0xoAAAAAACAg8rLy4uY0KiKnTt36k9/+pNef/119evXT5LUqVMnrVixQnfffTcTGl6plbFbWdEXoz5kGcaqKlZqibV/T8BYLdtMP4k+Hmut3VCGlVpi7TfuYjJSSxxrdRmjf3PZaq9SUcz9xvla7WNceR+wIixc9mWOyW1aitv2Lm9U8yzlxOX74/buh0Slq0QTsmKH4oWUEwDpKBTSgV8urnMQSD+pItJPDi4Zx1R5pJ/E6Ke6r2hNwjpvz5492rNnjzIO+NsxMzNToZC7N4gJDQBId468qReYzwAAAEguCarztm/frjVr1oQfr127VitWrFDdunXVrFkz9ejRQzfeeKNyc3PVvHlzzZ07V88++6zuvfdeV8dhQgMAAAAAAHhmyZIlOv3008OPR4wYIUkaMmSIiouL9eKLL2rkyJG6+OKL9f3336t58+YaO3asrrnmGlfHYUIDANJcwHEU8OAyQi/6AAAAgHcSVef17NlTTozXFBQUaOLEiYc6LCY0ACDtJeG9lQAAAPCAz+s8l8v7AQAAAAAAJB5XaMRQu8ZuZdU4tJmoDGP1FLdpJlYqihUGstfsP3r7YEb09sGgsUq0kWbiGP1Y6SeOleZgpaJYCSFuU1GsBA9jPFYyiZkEEiOlwkxScZnWYaaluEwtkZHKYbc3+nebNuI25cSj9hbvUk4OfTwuF3c+dI68WXE7OSfuASC6vUEp44CCw0wtiY70k4P15BbpJweXjGOqPNJPJFV7mp18XecxoQEAaY41NAAAAPzJ73Uet5wAAAAAAICUwxUaAJDuHHm0WNShdwEAAAAP+bzOY0IDANKdz1e/BgAASFs+r/O45QQAAAAAAKQcrtCIoXZmubIzK7ckrJlCYq2ybERI2Gkm0fdnGvt3Z0RfFTsjI/px95jto895BYPRjxsKRG8fyjTST4wUFdepKEbih9W/lexh9WOtRuxYiSUxfmzM11iTnmbyinUAl/0bs632eNz2b4w/QakoCUtLsURpX+0pJyF5szh6dY8bAA5FMCQ5RqzagUg/kUT6ycGPG+vY8Ub6Seye3Iln+onrWvFQ+bzOY0IDANKc31e/BgAASFd+r/OS/paTFi1aKBAIVNiGDh0atX1xcXGFtjk5OdU8agAAABwMdR4A4FAk/RUaH374oYLB/11ut2rVKp155pk6//zzzdfk5eVp9erV4ceBQGpcagUACeHzxaIAJC/qPACIM5/XeUk/oVG/fv2Ix3feeadat26tHj16mK8JBAIqKCiI99AAwB98/kUHIHlR5wFAnPm8zkv6W05+avfu3Xr++ed12WWXxZyN3759u5o3b67CwkINGDBAH330Ucx+y8vLVVZWFrEBAACg+lDnAQDcSvorNH5qypQp2rJliy655BKzTdu2bfX000+rU6dO2rp1q+6++25169ZNH330kZo2bRr1NePGjdPo0aMr7K+dWa6cA1JO3KaQWGkmNTJqGvut9tH3lweif4SZRvtMI80kMyP6+PcaaSN7M630E3f7PUtFsVJIMq00E2O/2X/05hbHSkuRFHCZmGKmcrhMWLHTQzxKUTGamykqLvt3n07iUbqKJY6pK6FKLrrvGZ/P3ANIDdVd5zmhkJwDlu13ffMK6SeSSD85tGPHW7KNxx0/pp8EqPM8lVJXaDz11FPq27evGjdubLYpKirS4MGDddxxx6lHjx6aPHmy6tevr8cee8x8zciRI7V169bwtnHjxngMHwCSU8jDDQCqiDoPAOLA53VeylyhsX79es2aNUuTJ0929bqaNWuqS5cuWrNmjdkmOztb2dnZhzpEAAAAVAF1HgCgKlLmCo2JEyeqQYMG6tevn6vXBYNBrVy5Uo0aNYrTyAAgte3PJ/diA4CqoM4DgPjwe52XEldohEIhTZw4UUOGDFGNGpFDHjx4sJo0aaJx48ZJkm6//XadcsopatOmjbZs2aK//e1vWr9+va644opEDB0Akp/P760EkNyo8wAgjnxe56XEhMasWbO0YcMGXXbZZRWe27BhgzJ+snDlDz/8oCuvvFKlpaU64ogjdPzxx2vBggU65phjqnPIAAAAqATqPABAVaXEhEbv3r3lGDNCc+bMiXh833336b777vPkuHk1diq3xp6IfZlBIz3EiGyoaSxjm2lEIVj7a1hpKVb/QXcpKnus/UaayZ5g9FW0vUpFCYWM9hlWQoixGreRdBEy0kxkpKJYKSrm4jhWe8n8WbZSOWQlphjtvUpLsVNR3PXjNm3EdcqJ0dw8gTimk8TiJo0lVN2LLoWcGAN02Q8AuJSoOk9790oH1C1OjeilMekn+8dD+olkp59IsdZNTLa0kWQbjzupnH5iJR7Gjc/rvJSY0AAAxJHPL0UEAABIWz6v81JmUVAAAAAAAID9uEIDANKeRzP3nl0qDAAAAG/4u85jQgMA0p3PL0UEAABIWz6v87jlBAAAAAAApByu0Ijh8MwflZsZ+RZZqSU1g8Z+q72xPysjK+r+nRlWaom7NJPdwegf+e6M6Ktf7wkZ++OcihK0Uk6M/kNGLETIWEU4ZBw3ZCaHuEw/iTGBaSemuNtvpqW4XDnZMVYsDlipK1Z4iHXOxnLfnqWcuO7Ho9QViwdpLNZnEjchR55cRpikq18DQFShoOREr8cORPrJf5F+cpD2dgIK6SfVIyXSTxKRZufjOo8JDQBId04oRt6vy34AAACQPHxe53HLCQAAAAAASDlcoQEA6c7ni0UBAACkLZ/XeUxoAEC68/m9lQAAAGnL53Uet5wAAAAAAICUwxUaMdTN2KFamZGrP9cM7I3aNtNYrtZtyknNkNu0lOjHzc6I3t5KS9ltpJnEOxVlr5FastdKOTH27w0a7Y1Ei5DRj5mK4nK/Y/QvSY6ZpOKuvZ1+YhzYZSKL6wQXa5zGC6xxBtyO03U6idWRR+knbtNYoghV9wy4zy9FBIBonGBIzgF1lNscB9JP/ov0k4O+hvSTxEqm9BPjz7f48Xmdx4QGAKQ7Rx590R16FwAAAPCQz+s8bjkBAAAAAAAphys0ACDd+fxSRAAAgLTl8zqPCQ0ASHehkGLdxeuuHwAAACQNn9d53HICAAAAAABSDldoxJCfsVO1MyLnfKyUkywjhSQntCfq/uxQtrHfaG8cNzsj+v7yUPSPNjsjy1X78kwj/cRovztotM+M3n6P0X6vE32uzWofrBF9JWIrFSXeaSmOE/3nIdZr7MQUK83E7X5jQK5TTuKcrmKlpbhOG3G56rZxAm6vrgt48P6HgqScAEDc7dkjBSJ/ZzuKngZH+sn+/kk/OXj/7l5D+kliJSL9hJQTbzGhAQDpzudfdAAAAGnL53Uet5wAAAAAAICUwxUaAJDuQo48uTw3lJwz9wAAAGnL53UeExoAkOYcJyTHOfQbOr3oAwAAAN7xe53HLScAAAAAACDlcIVGDEdk7tJhmZFzPjmh6KkiNa2Uk0D01BJ7f/QUErO9kZayKxR9lW63qShu91tpJuVBq72RchKy0lKMlBMrtcRIS7Ha73GZihIyEi2CVrKH7JQTK0nFSv2wUlHs5JXo4zFTUdymq0TvPkZairvxuE0zsVNFPEpRsZp70L8TrOYZcMfx5jLCJF0sCgCicYKOnAPiBgKKXm+RfrIf6Sf7xPrE3OZmkH6SjOKafpKIlBMf13lcoQEA6W7/6tdebAAAAEgeCarz5s2bp/79+6tx48YKBAKaMmVKxPOBQCDq9re//c3VcZjQAAAAAAAAntmxY4c6d+6shx9+OOrzJSUlEdvTTz+tQCCg8847z9VxuOUEANJdKCQFPLj+MUkXiwIAAEhbCarz+vbtq759+5rPFxQURDyeOnWqTj/9dLVq1crVcZjQAIB053gU58UtJwAAAMnF4zqvrKwsYnd2drays6Ov61hZmzZt0rRp0/TMM8+4fi23nAAAAAAAgIMqLCxUfn5+eBs3btwh9/nMM8+oTp06Ovfcc12/lis0YjgyI6A6GZEr1m4z0kayQ9FXfbbam6klGUbKSSh6+1qh8qj7fzTST2o50Vfp/jEYvX25E/1HxEpRcZt+ssdIMyk39pvpJy7bW+kne80EEpftjf37+jJWrTYTU6xjuEshsfoxwzdcpquY6ScuU0jctrdTTtylsbhPSzF2u005iSLgRP99Ei9OKFRhpf8q9cMtJwBSiBMMyglU7r/tkX6yvx/STw7O7btN+sk+6ZN+EghW77l6Xedt3LhReXl54f2HenWGJD399NO6+OKLlZOT4/q1XKEBAOkuAatfjxs3TieeeKLq1KmjBg0aaODAgVq9evVBX/fKK6+oXbt2ysnJUceOHfXWW28dypkDAAD4m8d1Xl5eXsR2qBMa7733nlavXq0rrriiSq9nQgMAUO3mzp2roUOHatGiRZo5c6b27Nmj3r17a8eOHeZrFixYoEGDBunyyy/X8uXLNXDgQA0cOFCrVq2qxpEDAADAK0899ZSOP/54de7cuUqv55YTAEh3IUcKVO+ioNOnT494XFxcrAYNGmjp0qU67bTTor7m/vvv11lnnaUbb7xRkjRmzBjNnDlTDz30kB599NGqjxsAAMCvElDnSdL27du1Zs2a8OO1a9dqxYoVqlu3rpo1ayZp3wKjr7zyiu65554qD4sJDQBId46jWHffuuunaqtfb926VZJUt25ds83ChQs1YsSIiH19+vTRlClTqjBYAACANOBxnVdZS5Ys0emnnx5+vL+GGzJkiIqLiyVJL774ohzH0aBBg6o8LG45AQB4yu3q16FQSMOHD1f37t117LHHmu1KS0vVsGHDiH0NGzZUaWmpJ+MGAACAN3r27CnHcSps+yczJOmqq67Sjz/+qPz8/Cofhys0Yjg8s5byMiPnfLJDu6O2NVNLrBSSwN6o+2s50dvXDkQ/7g4jFaW2Mc4doejta2VEb19upJnsMtJPzPbG/j2OlXISvX8rtcRsb6STWOkqbtNP3KaoxOrLSlJxm35itvcoXcVMP3GZ+uFV6op3KSruVpyOZ0pLaG91p5w4cjy4FNH570m6Xf166NChWrVqlebPn3/IYwCASgsFpQNSTtz+JiT9ZH8/pJ8cHOknsSXbeNxxM3ov7v5ww+s6L9kwoQEA6c4JyZtLEff1sX/V68oYNmyY3nzzTc2bN09NmzaN2bagoECbNm2K2Ldp0yYVFBRUbbwAAAB+53Gdl2yS/paTFi1aKBAIVNiGDh1qvoZYPwBIbo7jaNiwYXr99df1zjvvqGXLlgd9TVFRkWbPnh2xb+bMmSoqKorXMAHEGXUeAOBQJP2ExocffqiSkpLwNnPmTEnS+eefH7U9sX4A4I4TcjzbKmvo0KF6/vnnNWnSJNWpU0elpaUqLS3Vzp07w20GDx6skSNHhh9ff/31mj59uu655x59+umnGjVqlJYsWaJhw4Z5+n4AqD7UeQAQX4mo86pT0k9o1K9fXwUFBeHtzTffVOvWrdWjR4+o7X8a69e+fXuNGTNGXbt21UMPPVTNIweAFOGEvNsqacKECdq6dat69uypRo0ahbeXXnop3GbDhg0qKSkJP+7WrZsmTZqkxx9/XJ07d9arr76qKVOmxFxIFEByo84DgDhLQJ1XnVJqDY3du3fr+eef14gRIxQIRF8gpiqxfuXl5Sov/99inPvjA8u2V/zQfgxF/yB3GB/wDmMma6excOCPRj8/hqIvqvSjE33/LqP9Tif6YqS7jAUpd4eij3OXMUFXbvycW/v3GOe723jf9hrv/25jv7nIp7EYaXUsCho03tNELQpqLtppjCf1FwV117/FzSKfbtuHdpb/9zXVMxO+V3s8Wfdsr7E4XjSVObc5c+ZU2Hf++eeb/+UWQGqr7jov6u8+oy4JmPuN7w7jd1zAqg+MekJGfWMuwhlw2b/Rj2P2b/Rj1QzGgo7WoqAWx6idrHE6GdbnYtUwVj9GNzFqBnOsxpisvxHtfoz2bvu33iKrH3cfvb3fdT9u34d4j8fd/mg/oqHyXfte4+M6rzql1ITGlClTtGXLFl1yySVmm6rE+o0bN06jR4+usL9513VVHSoAHLLvvvvukGKsDiYrK0sFBQWaX+rd/ecFBQXKyoqepgQAsVR3nTdfUX73Wf8B0tpv1fe7zOEAgCTqPK+k1ITGU089pb59+6px48ae9jty5MiI2f4tW7aoefPm2rBhQ1x/yOKlrKxMhYWFFaITU0Wqj19K/XNg/Im1detWNWvWTHXr1o3rcXJycrR27Vrt3h09trkqsrKylJOT41l/ANIHdV7lpPp3XKqPX0r9c2D8iUWd562UmdBYv369Zs2apcmTJ8dsV5VYv+zsbGVnZ1fYn5+fn5L/SPZzE52YjFJ9/FLqnwPjT6yMjPgvc5STk5N0X0wA0g91nnup/h2X6uOXUv8cGH9iUed5I+kXBd1v4sSJatCggfr16xezHbF+AAAAqYU6DwBQFSkxoREKhTRx4kQNGTJENWpEXlRCrB8AAEDqos4DAFRVSkxozJo1Sxs2bNBll11W4bl4xPplZ2frtttui3p5Yipg/ImX6ufA+BMr1ccPAG5Q57nD+BMv1c+B8SdWqo8/2QSc6sqLAQAAAAAA8EhKXKEBAAAAAADwU0xoAAAAAACAlMOEBgAAAAAASDlMaAAAAAAAgJSTlhMa27Zt0/Dhw9W8eXPl5uaqW7du+vDDD2O+Zs6cOeratauys7PVpk0bFRcXV89go3A7/jlz5igQCFTYSktLq2W88+bNU//+/dW4cWMFAgFNmTIl4nnHcXTrrbeqUaNGys3NVa9evfT5558ftN+HH35YLVq0UE5Ojk4++WR98MEHKTP+UaNGVfg82rVrl5DxT548Wb1791a9evUUCAS0YsWKSvX7yiuvqF27dsrJyVHHjh311ltveT94xWf8xcXFFd7/nJycah//nj179Mc//lEdO3ZU7dq11bhxYw0ePFj/+c9/Dtpvdf38A0Cqoc6jzkv0+KnzKo86LzrqvMpLywmNK664QjNnztRzzz2nlStXqnfv3urVq5e+/vrrqO3Xrl2rfv366fTTT9eKFSs0fPhwXXHFFZoxY0Y1j3wft+Pfb/Xq1SopKQlvDRo0qJbx7tixQ507d9bDDz8c9fm//vWveuCBB/Too49q8eLFql27tvr06aNdu3aZfb700ksaMWKEbrvtNi1btkydO3dWnz59tHnz5pQYvyR16NAh4vOYP3++52OXDj7+HTt26NRTT9Vdd91V6T4XLFigQYMG6fLLL9fy5cs1cOBADRw4UKtWrfJq2BHj83r8kpSXlxfx/q9fv96L4UYdnzX+H3/8UcuWLdMtt9yiZcuWafLkyVq9erXOOeecmH1W588/AKQa6jzqvESPX6LOqyzqvIqo81xy0syPP/7oZGZmOm+++WbE/q5duzr/7//9v6ivuemmm5wOHTpE7LvwwgudPn36xG2clqqM/91333UkOT/88EM1jDA2Sc7rr78efhwKhZyCggLnb3/7W3jfli1bnOzsbOfvf/+72c9JJ53kDB06NPw4GAw6jRs3dsaNGxeXce/n1fhvu+02p3PnznEcaXQHjv+n1q5d60hyli9fftB+LrjgAqdfv34R+04++WTn6quv9mCUNq/GP3HiRCc/P9/TsVVGrPHv98EHHziSnPXr15ttEvXzDwDJjjovsajz9qHOqxrqvH2o89xJuys09u7dq2AwWOGyo9zcXHPmdOHCherVq1fEvj59+mjhwoVxG6elKuPf77jjjlOjRo105pln6v3334/nMCtt7dq1Ki0tjXh/8/PzdfLJJ5vv7+7du7V06dKI12RkZKhXr17V/plUZfz7ff7552rcuLFatWqliy++WBs2bIj3cD2TTP8mqmr79u1q3ry5CgsLNWDAAH300UeJHpIkaevWrQoEAjr88MOjPp9MP/8AkGyo86jzvESd9z/Ued6gzvNe2k1o1KlTR0VFRRozZoz+85//KBgM6vnnn9fChQtVUlIS9TWlpaVq2LBhxL6GDRuqrKxMO3furI5hh1Vl/I0aNdKjjz6q1157Ta+99poKCwvVs2dPLVu2rFrHHs3++zujvb/WvZ/ffvutgsGgq9fES1XGL0knn3yyiouLNX36dE2YMEFr167Vz372M23bti2u4/WK9W+iut//qmrbtq2efvppTZ06Vc8//7xCoZC6deumr776KqHj2rVrl/74xz9q0KBBysvLi9ommX7+ASDZUOdR53mJOu9/UqnOoM5LLzUSPYBEeO6553TZZZepSZMmyszMVNeuXTVo0CAtXbo00UOrFLfjb9u2rdq2bRt+3K1bN33xxRe677779Nxzz1XXsPETffv2Df//Tp066eSTT1bz5s318ssv6/LLL0/gyNJDUVGRioqKwo+7deum9u3b67HHHtOYMWMSMqY9e/boggsukOM4mjBhQkLGAAB+QJ1HnZdo1HmJRZ2XXtLuCg1Jat26tebOnavt27dr48aN+uCDD7Rnzx61atUqavuCggJt2rQpYt+mTZuUl5en3Nzc6hhyBLfjj+akk07SmjVr4jjKyikoKJCkqO/v/ucOdOSRRyozM9PVa+KlKuOP5vDDD9fRRx+dFJ9JZVj/Jqr7/fdKzZo11aVLl4S9//u/5NavX6+ZM2eas/ZScv38A0Ayos6jzvMKdd7/pHKdQZ3nb2k5obFf7dq11ahRI/3www+aMWOGBgwYELVdUVGRZs+eHbFv5syZETN/iVDZ8UezYsUKNWrUKI6jq5yWLVuqoKAg4v0tKyvT4sWLzfc3KytLxx9/fMRrQqGQZs+eXe2fSVXGH8327dv1xRdfJMVnUhnJ+m+iqoLBoFauXJmQ93//l9znn3+uWbNmqV69ejHbJ9PPPwAkM+q8xNcU1Hn7UOclFnWezyV6VdJEmD59uvPPf/7T+fLLL523337b6dy5s3PyySc7u3fvdhzHcW6++WbnN7/5Tbj9l19+6dSqVcu58cYbnU8++cR5+OGHnczMTGf69OkpMf777rvPmTJlivP55587K1eudK6//nonIyPDmTVrVrWMd9u2bc7y5cud5cuXO5Kce++911m+fHl4dd8777zTOfzww52pU6c6//73v50BAwY4LVu2dHbu3Bnu4+c//7nz4IMPhh+/+OKLTnZ2tlNcXOx8/PHHzlVXXeUcfvjhTmlpaUqM//e//70zZ84cZ+3atc7777/v9OrVyznyyCOdzZs3V/v4v/vuO2f58uXOtGnTHEnOiy++6CxfvtwpKSkJ9/Gb3/zGufnmm8OP33//fadGjRrO3Xff7XzyySfObbfd5tSsWdNZuXJlSox/9OjRzowZM5wvvvjCWbp0qXPRRRc5OTk5zkcffVSt49+9e7dzzjnnOE2bNnVWrFjhlJSUhLfy8vJwH4n8+QeAVEOdR52X6PFT5yV2/NR56SUtJzReeuklp1WrVk5WVpZTUFDgDB061NmyZUv4+SFDhjg9evSIeM27777rHHfccU5WVpbTqlUrZ+LEidU76J9wO/677rrLad26tZOTk+PUrVvX6dmzp/POO+9U23j3x4kduA0ZMsRxnH2RWLfccovTsGFDJzs72znjjDOc1atXR/TRvHlz57bbbovY9+CDDzrNmjVzsrKynJNOOslZtGhRyoz/wgsvdBo1auRkZWU5TZo0cS688EJnzZo1CRn/xIkToz7/0/H26NEj3H6/l19+2Tn66KOdrKwsp0OHDs60adNSZvzDhw8P/+w0bNjQOfvss51ly5ZV+/j3R5BF2959991wH4n8+QeAVEOdR52X6PFT5yV2/NR56SXgOI5z8Os4AAAAAAAAkkdar6EBAAAAAABSExMaAAAAAAAg5TChAQAAAAAAUg4TGgAAAAAAIOUwoQEAAAAAAFIOExoAAAAAACDlMKEBAAAAAABSDhMaAAAAAAAg5TChAc/17NlTw4cPT/QwTKtXr1ZBQYG2bdtmtikuLtbhhx9efYNKoHXr1ikQCGjFihWe9Dd9+nQdd9xxCoVCnvQHAACSB3VeaqHOg98xoYGUUFJSov/7v//T0UcfrYyMDPOL9JVXXlG7du2Uk5Ojjh076q233qrQZuTIkbr22mtVp04dT8fYokULjR8/3tM+D8aLL6nCwkKVlJTo2GOP9WRMZ511lmrWrKkXXnjBk/4AAIC/UedFR50HHBwTGkgJ5eXlql+/vv785z+rc+fOUdssWLBAgwYN0uWXX67ly5dr4MCBGjhwoFatWhVus2HDBr355pu65JJLqmnkyS8zM1MFBQWqUaOGZ31ecskleuCBBzzrDwAA+Bd1XvxQ58HvmNBA3P3www8aPHiwjjjiCNWqVUt9+/bV559/HtHmiSeeUGFhoWrVqqVf/vKXuvfeeyMuBWzRooXuv/9+DR48WPn5+VGPc//99+uss87SjTfeqPbt22vMmDHq2rWrHnrooXCbl19+WZ07d1aTJk0iXltcXKxmzZqFj//dd99FPP/FF19owIABatiwoQ477DCdeOKJmjVrVvj5nj17av369brhhhsUCAQUCAQkSd99950GDRqkJk2aqFatWurYsaP+/ve/R/T96quvqmPHjsrNzVW9evXUq1cv7dixI/z8k08+qfbt2ysnJ0ft2rXTI488En6uZcuWkqQuXbooEAioZ8+e5mdw8cUXq379+srNzdVRRx2liRMnSqo4+3/JJZeEz+Gn25w5cyTtKzr+8Ic/qEmTJqpdu7ZOPvnk8HP79e/fX0uWLNEXX3wRdTwAAMAfqPOo84BEYkIDcXfJJZdoyZIleuONN7Rw4UI5jqOzzz5be/bskSS9//77uuaaa3T99ddrxYoVOvPMMzV27FjXx1m4cKF69eoVsa9Pnz5auHBh+PF7772nE044IaLN4sWLdfnll2vYsGFasWKFTj/9dN1xxx0RbbZv366zzz5bs2fP1vLly3XWWWepf//+2rBhgyRp8uTJatq0qW6//XaVlJSopKREkrRr1y4df/zxmjZtmlatWqWrrrpKv/nNb/TBBx9I2neJ5aBBg3TZZZfpk08+0Zw5c3TuuefKcRxJ0gsvvKBbb71VY8eO1SeffKK//OUvuuWWW/TMM89IUrifWbNmqaSkRJMnT4763txyyy36+OOP9c9//lOffPKJJkyYoCOPPDJq2/vvvz98DiUlJbr++uvVoEEDtWvXTpI0bNgwLVy4UC+++KL+/e9/6/zzz9dZZ50VUbw0a9ZMDRs21HvvvWd9XAAAwAeo86jzgIRyAI/16NHDuf766x3HcZzPPvvMkeS8//774ee//fZbJzc313n55Zcdx3GcCy+80OnXr19EHxdffLGTn59/0P5/qmbNms6kSZMi9j388MNOgwYNwo87d+7s3H777RFtBg0a5Jx99tkR+y688ELz+Pt16NDBefDBB8OPmzdv7tx3330xX+M4jtOvXz/n97//veM4jrN06VJHkrNu3bqobVu3bl3hnMaMGeMUFRU5juM4a9eudSQ5y5cvj3nM/v37O5deemnU52L18dprrzk5OTnO/PnzHcdxnPXr1zuZmZnO119/HdHujDPOcEaOHBmxr0uXLs6oUaNijgsAAKQW6rzYqPOA6sUVGoirTz75RDVq1NDJJ58c3levXj21bdtWn3zyiaR9q1GfdNJJEa878LFXdu7cqZycnApj/On4JKmoqCji8fbt2/WHP/xB7du31+GHH67DDjtMn3zySXjm3hIMBjVmzBh17NhRdevW1WGHHaYZM2aEX9e5c2edccYZ6tixo84//3w98cQT+uGHHyRJO3bs0BdffKHLL79chx12WHi74447XF/i99vf/lYvvviijjvuON10001asGDBQV+zfPly/eY3v9FDDz2k7t27S5JWrlypYDCoo48+OmJMc+fOrTCm3Nxc/fjjj67GCQAAUgd1HnUekGjerQ4DJFhBQYE2bdoUsW/Tpk0qKCgIPz7yyCPDXyRu/OEPf9DMmTN19913q02bNsrNzdWvfvUr7d69O+br/va3v+n+++/X+PHj1bFjR9WuXVvDhw8Pvy4zM1MzZ87UggUL9Pbbb+vBBx/U//t//0+LFy9WrVq1JO277/TAL+LMzExX4+/bt6/Wr1+vt956SzNnztQZZ5yhoUOH6u67747avrS0VOecc46uuOIKXX755eH927dvV2ZmppYuXVphDIcddljE4++//17169d3NU4AAIBoqPNs1HlIZ1yhgbhq37699u7dq8WLF4f3fffdd1q9erWOOeYYSVLbtm314YcfRrzuwMeVUVRUpNmzZ0fsmzlzZsQsfJcuXfTxxx9XGONPxydJixYtinj8/vvv65JLLtEvf/lLdezYUQUFBVq3bl1Em6ysLAWDwQqvGzBggH7961+rc+fOatWqlT777LOINoFAQN27d9fo0aO1fPlyZWVl6fXXX1fDhg3VuHFjffnll2rTpk3Etn+RqKysLEmqcNxo6tevryFDhuj555/X+PHj9fjjj0dtt2vXLg0YMEDt2rXTvffeG/Fcly5dFAwGtXnz5gpj+mlBsWvXLn3xxRfq0qXLQccFAABSE3UedR6QaFyhgbg66qijNGDAAF155ZV67LHHVKdOHd18881q0qSJBgwYIEm69tprddppp+nee+9V//799c477+if//xneAXp/favzrx9+3Z98803WrFihbKyssJfmNdff7169Oihe+65R/369dOLL76oJUuWRPxC79Onj6644goFg8HwzPN1112n7t276+6779aAAQM0Y8YMTZ8+vcJ5TJ48Wf3791cgENAtt9yiUCgU0aZFixaaN2+eLrroImVnZ+vII4/UUUcdpVdffVULFizQEUccoXvvvVebNm0Kj3nx4sWaPXu2evfurQYNGmjx4sX65ptv1L59e0nS6NGjdd111yk/P19nnXWWysvLtWTJEv3www8aMWKEGjRooNzcXE2fPl1NmzZVTk5O1NXBb731Vh1//PHq0KGDysvL9eabb4aPcaCrr75aGzdu1OzZs/XNN9+E99etW1dHH320Lr74Yg0ePFj33HOPunTpom+++UazZ89Wp06d1K9fP0n7CoXs7OwKl3QCAAD/oM6jzgMSLtGLeMB/DlzM6fvvv3d+85vfOPn5+U5ubq7Tp08f57PPPot4zeOPP+40adLEyc3NdQYOHOjccccdTkFBQUQbSRW25s2bR7R5+eWXnaOPPtrJyspyOnTo4EybNi3i+T179jiNGzd2pk+fHrH/qaeecpo2berk5uY6/fv3d+6+++6IxaLWrl3rnH766U5ubq5TWFjoPPTQQxXOc+HChU6nTp2c7OxsZ/8/re+++84ZMGCAc9hhhzkNGjRw/vznPzuDBw92BgwY4DiO43z88cdOnz59nPr16zvZ2dnO0UcfHbEAleM4zgsvvOAcd9xxTlZWlnPEEUc4p512mjN58uTw80888YRTWFjoZGRkOD169Ij6mYwZM8Zp3769k5ub69StW9cZMGCA8+WXX4bPTT9ZLKp58+ZR3+t3333XcRzH2b17t3Prrbc6LVq0cGrWrOk0atTI+eUvf+n8+9//Dh/vqquucq6++uqoYwEAAKmLOo86jzoPySTgOP/NDQKSyJVXXqlPP/00LnFQDz/8sN544w3NmDHD874hffvtt2rbtq2WLFkSvmQSAABgP+q81EWdh2TDLSdICnfffbfOPPNM1a5dW//85z/1zDPP6JFHHonLsa6++mpt2bJF27ZtU506deJyjHS2bt06PfLII3zJAQAASdR5fkKdh2TDFRpIChdccIHmzJmjbdu2qVWrVrr22mt1zTXXJHpYAAAAOETUeQDihQkNAAAAAACQcohtBQAAAAAAKYcJDQAAAAAAkHKY0AAAAAAAACmHCQ0AAAAAAJBymNAAAAAAAAAphwkNAAAAAACQcpjQAAAAAAAAKYcJDQAAAAAAkHL+P3Wov0utV/0VAAAAAElFTkSuQmCC", 199 | "text/plain": [ 200 | "
" 201 | ] 202 | }, 203 | "metadata": {}, 204 | "output_type": "display_data" 205 | } 206 | ], 207 | "source": [ 208 | "def L(N, D):\n", 209 | " \"\"\" \n", 210 | " Approximates loss given N parameters and D dataset size (in tokens),\n", 211 | " per Chinchilla paper.\n", 212 | " \"\"\"\n", 213 | " E = 1.69 # entropy of natural language, limit of infinite model on infinite data\n", 214 | " A = 406.4\n", 215 | " B = 410.7\n", 216 | " alpha = 0.34\n", 217 | " beta = 0.28\n", 218 | " return A / (N ** alpha) + B / (D ** beta) + E\n", 219 | "\n", 220 | "ns = 10 ** np.arange(7, 11, step=2**-4) # model sizes from 10M to 100B\n", 221 | "ds = 10 ** np.arange(9, 12, step=2**-4) # dataset sizes from 1B to 1T\n", 222 | "plt.figure(figsize=(15, 5))\n", 223 | "# plot a heatmap of loss as a function of model size and dataset size\n", 224 | "plt.subplot(121)\n", 225 | "plt.imshow(np.array([[L(n, d) for d in ds] for n in ns]), extent=[9, 12, 7, 11], origin='lower')\n", 226 | "plt.xlabel('log10(dataset size)')\n", 227 | "plt.ylabel('log10(model size)')\n", 228 | "plt.title('loss')\n", 229 | "plt.colorbar()\n", 230 | "# plot the compute for each point, which is a deterministic function: flops = 6*N*D\n", 231 | "plt.subplot(122)\n", 232 | "plt.imshow(np.log10(np.array([[6*n*d for d in ds] for n in ns])), extent=[9, 12, 7, 11], origin='lower')\n", 233 | "plt.xlabel('log10(dataset size)')\n", 234 | "plt.ylabel('log10(model size)')\n", 235 | "plt.title('log10 flops')\n", 236 | "plt.colorbar()" 237 | ] 238 | }, 239 | { 240 | "attachments": {}, 241 | "cell_type": "markdown", 242 | "metadata": {}, 243 | "source": [ 244 | "Ok so given any N,D we can estimate both: 1) the loss, and 2) the total flops. Now we want to solve the following problem: Given a specific budget of flops C, find: N_opt, D_opt = argmin_{FLOPs(N,D) = C} L(N, D). i.e. how big of a model should we train and for how many tokens?" 245 | ] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "execution_count": 7, 250 | "metadata": {}, 251 | "outputs": [ 252 | { 253 | "name": "stdout", 254 | "output_type": "stream", 255 | "text": [ 256 | "best model size: 316.23M\n", 257 | "best dataset size: 10.12B\n" 258 | ] 259 | }, 260 | { 261 | "data": { 262 | "text/plain": [ 263 | "Text(0, 0.5, 'loss')" 264 | ] 265 | }, 266 | "execution_count": 7, 267 | "metadata": {}, 268 | "output_type": "execute_result" 269 | }, 270 | { 271 | "data": { 272 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEqCAYAAABEE9ZrAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAuW0lEQVR4nO3deViU9f7/8ecMy7Ajm4AspqIoKoqaCpor5m7W6ZfHLKy0Tp0sy7IT1Tl1Wg6ezJZz8tt6ytZjqannWC7pSS3FDTTRVMIFEFkUkWWQbeb+/QFOoiAIw9wzzPtxXXNdcnPPzHuY5tXn/sxn0SiKoiCEEDZGq3YBQgjREhJeQgibJOElhLBJEl5CCJsk4SWEsEkSXkIImyThJYSwSY5qF2BpRqORM2fO4OnpiUajUbscIcQVFEWhtLSUTp06odU23r6yu/A6c+YMYWFhapchhGhCdnY2oaGhjf7e7sLL09MTqP3DeHl5qVxNO6bXQ6dOtf8+cwbc3dWtR9iMkpISwsLCTJ/VxthdeF26VPTy8pLwaksODr/928tLwktct6a6daTDXghhkyS8hBA2ScJLCGGTJLyEEDZJwksIYZMkvIQQNknCSwjR5j7ecZK1B3Ioqag222NKeAkh2lS1wciSTenMX36ArMJysz2uhJcQok0dyL5AWWUNvu7ORAWbb2C4hJcQok39mH4WgLhufmi15lsMQcJLCNGmfsw4B8CI7gFmfVwJLyFEmykur+bn7AsADO/ub9bHlvASQrSZ5BPnMCrQLcCdTh1czfrYEl5CiDaz/dfaS8abzHzJCBJeQog29JMpvMx7yQgSXkKINpJZqCfrfDmOWg1DuvqZ/fElvIQQbeLHulbXgM4+eOjMv+6phJcQok2YLhkjzH/JCBJeQog2UGMwsuN4XXj1MH9nPUh4CSHawMGcYkoravB2daJviHebPIeElxDC7LYdq50SNCzCDwczTgm6nISXEMLsttbNZxzVo2ObPYeElxDCrM7rqzh4+gIAI9qovwskvIQQZvbjr2dRFOgZ5EmQt0ubPY+ElxDCrC71d42MbLtWF0h4CSHMyGhU2GaB/i6Q8BJCmNHhMyUU6qtwd3ZgYGefNn0uCS8hhNlsPVYAwLAIf5wd2zZeJLyEEGZjumSMbNtLRpDwEkKYSXF5NalZRUDbd9aDhJcQwkx+yqhdNbV7Rw9CzLxqakMkvIQQZnGpv2tkGw5MvZyElxCi1YxGhR+OWa6/CyS8hBBmkJZTzLmySjx0jgzu4muR55TwEkK02pajtZeMI3q0/RCJSyS8hBCt9r+j+QCMttAlI0h4CSFaKb+kgkM5JWg0luvvAgkvIUQr/VB3ydgvtAMBnjqLPa+ElxCiVS71d43tablWF0h4CSFaoaLaYNolaEwvCS8hhI3YdaKQi9UGgrxciAr2suhzS3gJIVrsUn/X6J4d0WjaZqONxkh4CSFaRFEU1fq7QMJLCNFCx/JLOV10EZ2jlrgIP4s/v9WE16JFi9BoNDz22GONnrNs2TI0Gk29m4tL2y3wL4Ro3KbDtQNTb+ruj5uzo8Wf3/LP2IC9e/fy3nvvER0d3eS5Xl5eHDt2zPSzpa+zhRC1vv+lNrzGRQWq8vyqt7zKysqYNWsWH3zwAT4+Ta95rdFoCAoKMt0CA9X5wwlhz3KLL5KWU4xGA2N62ml4Pfzww0yePJn4+PhmnV9WVkbnzp0JCwvjlltu4fDhw9c8v7KykpKSkno3IUTrbK5rdQ0I97HoqPrLqRpey5cvJzU1laSkpGadHxkZyUcffcTatWv5/PPPMRqNxMXFcfr06Ubvk5SUhLe3t+kWFhZmrvKFsFub6sLrZpUuGUHF8MrOzmb+/Pl88cUXze50j42NJSEhgf79+zNy5Ei++eYbAgICeO+99xq9T2JiIsXFxaZbdna2uV6CEHappKKaXScKAfX6u0DFDvuUlBQKCgoYMGCA6ZjBYGD79u28/fbbVFZW4uDgcM3HcHJyIiYmhoyMjEbP0el06HTqNGuFaI+2HjtLtUGhW4A7XQM8VKtDtfAaO3YsaWlp9Y7de++99OzZkz/96U9NBhfUhl1aWhqTJk1qqzKFEFf47VvGIFXrUC28PD096dOnT71j7u7u+Pn5mY4nJCQQEhJi6hN78cUXGTp0KBEREVy4cIHFixeTmZnJ3LlzLV6/EPaoqsbI1rpR9WpeMoKVjPNqTFZWFlrtb91yRUVF3H///eTl5eHj48PAgQPZuXMnUVFRKlYphP3YfbKQ0soa/D10xIR1ULUWjaIoiqoVWFhJSQne3t4UFxfj5WXZWfB2Ra8Hj7r+kLIycHdXtx5hFs+uTuOL3VnMHBxG0m1NDypvieZ+RlUf5yWEsA0Go8LGuilBE/oEq1yNhJcQoplSMos4V1aJl4sjsV0tPxH7ShJeQohm2XAoD4D4qECLbW92LepXIISweoqisPFwbXhN6K3uEIlLJLyEEE06eLqYnAsXcXN2YESPALXLASS8hBDNsL7uknF0z464ODU9gNwSJLyEENekKAobDuUCMLGPdVwygoSXEKIJx/JLOVVYjrOj1qI7YjdFwksIcU3r02ovGUd0D8BDZz2TciS8hBDXtN4KLxlBwksIcQ3p+aWk55fh7KAlXuWJ2FeS8BJCNGrdwdpW14ge/ni7OqlcTX0SXkKIBimKwrqDZwCYEt1J5WquJuElhGjQkdxSTpzV4+yoZWwv6/mW8RIJLyFEg75Nq211jY4MwNPFui4ZQcJLCNGA2kvG2v4ua7xkBAkvIUQDDp8pIbOwHBcnLWN6Wt8lI0h4CSEa8N+6jvqxPQNxt6KBqZeT8BJC1KMoCt/WXTJOjlZ/xdTGSHgJIerZn32B00W1y9+MtqK5jFeS8BJC1LN2fw4A43sH4epsHcvfNETCSwhhUmMwmr5lnNbfOr9lvETCSwhh8lPGOQr1Vfi5OzM8wl/tcq5JwksIYfKfA7XfMk6ODsbJwbrjwbqrE0JYzMUqg2mTjVv6h6hcTdMkvIQQAGw+ko++ykCYrysDwjuoXU6TJLyEEACsrbtkvKVfCBqNRuVqmibhJYTgQnkV29ILALjFyr9lvETCSwjBt2m5VBsUooK96B7oqXY5zSLhJYRgVcppAKbH2EarCyS8hLB7J8/pSc26gFYD023gW8ZLJLyEsHPfpNa2ukb0CKCjl4vK1TSfhJcQdsxoVPgmtXYu4+8GhKpczfWR8BLCju06WUjOhYt4ujgyzsq2NmuKhJcQdmxVSm2ra0p0MC5O1ruCREMkvISwU+VVNabdsG3tkhEkvISwWxsO5VFeZeAGPzcGdvZRu5zrJuElhJ1aVfct420DQm1iOtCVJLyEsEPZ58vZkVGIRgO3xtjO2K7LSXgJYYdW7MsGYHiEP2G+bipX0zISXkLYGYNRYUXddKA7BoWpXE3LWU14LVq0CI1Gw2OPPXbN81asWEHPnj1xcXGhb9++fPfdd5YpUIh2YvuvZ8ktrqCDmxM397atsV2Xs4rw2rt3L++99x7R0dHXPG/nzp3MnDmTOXPmsH//fqZPn8706dM5dOiQhSoVwvZ9vbf2knF6/xB0jrY1tutyqodXWVkZs2bN4oMPPsDH59pf17711ltMmDCBhQsX0qtXL1566SUGDBjA22+/baFqhbBthWWVbD6SD8CMG233khGsILwefvhhJk+eTHx8fJPnJicnX3Xe+PHjSU5ObvQ+lZWVlJSU1LsJYa9W78+h2qAQHepNr2AvtctpFUc1n3z58uWkpqayd+/eZp2fl5dHYGD9a/TAwEDy8vIavU9SUhJ//etfW1WnEO2Boigsr7tktPVWF6jY8srOzmb+/Pl88cUXuLi03TIciYmJFBcXm27Z2dlt9lxCWLPUrCIyCspwcdIytZ/tLDrYGNVaXikpKRQUFDBgwADTMYPBwPbt23n77beprKzEwaF+Z2JQUBD5+fn1juXn5xMUFNTo8+h0OnQ6nXmLF8IGfbErC4Cp0Z3wcnFSuZrWU63lNXbsWNLS0jhw4IDpNmjQIGbNmsWBAweuCi6A2NhYtmzZUu/Y999/T2xsrKXKFsImFemrWJdWOwl71tDOKldjHi0Kr08++YRvv/3W9PNTTz1Fhw4diIuLIzMzs1mP4enpSZ8+ferd3N3d8fPzo0+fPgAkJCSQmJhous/8+fPZsGEDS5Ys4ejRo7zwwgvs27ePefPmteRlCGE3VqWepqrGSO9OXvQL9Va7HLNoUXj97W9/w9XVFaj9BnDp0qW8+uqr+Pv78/jjj5utuKysLHJzc00/x8XF8eWXX/L+++/Tr18/Vq5cyZo1a0xhJ4S4mqIofLG79pJx1pDONjkJuyEaRVGU672Tm5sbR48eJTw8nD/96U/k5uby6aefcvjwYUaNGsXZs2fbolazKCkpwdvbm+LiYry8bPurYqum14OHR+2/y8rA3V3deuzYzoxz3Pnhbjx0jux+ZizuOlUHGTSpuZ/RFrW8PDw8KCwsBGDTpk2MGzcOABcXFy5evNiShxRCtJFLra5bY0KsPriuR4teybhx45g7dy4xMTGkp6czadIkAA4fPswNN9xgzvqEEK1QUFrBxsO14yDvHBKucjXm1aKW19KlS4mNjeXs2bOsWrUKPz8/oHb4w8yZM81aoBCi5b7em02NUWFgZx+bH1F/pRa1vDp06NDgfEIZyS6E9ag2GPm8bmzXXUPbV6sLWtjy2rBhAz/99JPp56VLl9K/f3/uvPNOioqKzFacEKLlNh3OJ6+kAn8PZyb1DVa7HLNrUXgtXLjQNME5LS2NJ554gkmTJnHy5EkWLFhg1gKFEC3zyc5TANw5ONyml75pTIsuG0+ePElUVBQAq1atYsqUKfztb38jNTXV1HkvhFDPL2dK2HPqPI5aTbsZUX+lFrW8nJ2dKS8vB2Dz5s3cfPPNAPj6+sqSM0JYgUutrgl9ggj0aruFD9TUopbX8OHDWbBgAcOGDWPPnj189dVXAKSnpxMaanubVwrRnhTpq1hzoHYn7HviblC3mDbUopbX22+/jaOjIytXruSdd94hJKR266T169czYcIEsxYohLg+X+3LprJuHqMtbibbXC1qeYWHh7Nu3bqrjr/xxhutLkgI0XI1BiOfJdcujjA77oZ2M4+xIS2eK2AwGFizZg1HjhwBoHfv3kybNq3BpWyEEJax/lAeORcu4ufuzLR2sODgtbQovDIyMpg0aRI5OTlERkYCtcsth4WF8e2339KtWzezFimEaJqiKHz44wkA7o7tjItT+25ItKjP69FHH6Vbt25kZ2eTmppKamoqWVlZdOnShUcffdTcNQohmmFfZhE/ny7G2VHL3e10eMTlWtTy2rZtG7t27cLX19d0zM/Pj0WLFjFs2DCzFSeEaL5Lra7fDQjBz6P9L33eopaXTqejtLT0quNlZWU4Ozu3uighxPXJLNSz6Zfa/R3mDO+icjWW0aLwmjJlCg888AC7d+9GURQURWHXrl08+OCDTJs2zdw1CiGa8NFPJ1EUGB0ZQERHT7XLsYgWhdc//vEPunXrRmxsLC4uLri4uBAXF0dERARvvvmmmUsUQlzLhfIqvt53GoC5N3VVuRrLafGSOGvXriUjI8M0VKJXr15ERESYtTghRNM+S87kYrWBXsFexHXzU7sci2l2eDW1WsQPP/xg+vfrr7/e8oqEEM12scrAx3XzGB8c2bVdD0q9UrPDa//+/c06z57+eEKo7au9WZzXVxHm68rkdrhm17U0O7wub1kJIdRXbTDywY8nAXhgRDccHVTbQ1oV9vVqhWhH/vvzGXIuXMTfw5n/N9D+VnOR8BLCBhmNCu9uOw7AvcO6tPupQA2R8BLCBm05WkB6fhkeOkfusoOpQA2R8BLCxiiKwtv/+xWAWUPD8XZ1UrkidUh4CWFjtqWf5efTxbg4aZk73H4GpV5JwksIG6IoCm9tqW113TWkMwGe7X8CdmMkvISwITsyCtmfdQGdo5YHRtpvqwskvK6prLKGc2WVapchBHCp1ZUOwJ1Dwuno2T53BWouCa9GrNmfQ2zSFhZvOKZ2KUIAkHyikL2ninB21PLgSFmtWMKrEWG+rpRW1LB6fw4FJRVqlyPsnKIovLW5tq/r9zeGtdu9GK+HhFcjBnb2ZWBnH6oMRtPEVyHUsiOjkN0nz+PsIK2uSyS8ruGBEbUdop/vyqSsskblaoS9UhSFxZtquy/uHBJOpw6uKldkHSS8rmFcr0C6+rtTWlHD8j1Zapcj7NSWIwX8nH0BVycHHh4ta+ZdIuF1DVqthvvrWl8f/XSSaoNR5YqEvTEaFV6ra3XdM+wGux7XdSUJrybcGhOCv4eOM8UVrDt4Ru1yhJ35Ni2Xo3mleOoc+cMI+x7XdSUJrya4ODlwT1ztxNf3tp1AURSVKxL2osZg5I3va8d1zb2pKx3cZGeuy0l4NcNdQzvj5uzA0bxSfjhWoHY5wk6sTDnNiXN6fNycuG/4DWqXY3UkvJqhg5szs4aEA/CPLRnS+hJtrryqhjc217a6Hh4dgaeLfa4ccS0SXs10/4iu6By1HMi+wI6MQrXLEe3cRz+dJL+kklAfV+6Otc/1upqiani98847REdH4+XlhZeXF7Gxsaxfv77R85ctW4ZGo6l3c3GxzEjjjp4uzBxc1/qqW0tJiLZwrqySd7edAGDh+Eh0jva3SmpzqBpeoaGhLFq0iJSUFPbt28eYMWO45ZZbOHz4cKP38fLyIjc313TLzMy0WL1/GNkVZwcte06eZ/cJaX2JtvHPLb9SVllD3xBvpkZ3Urscq6VqeE2dOpVJkybRvXt3evTowSuvvIKHhwe7du1q9D4ajYagoCDTLTAw0GL1Bnu7cvug2o0O/vm/DIs9r7AfJ8/p+WJ37YDoxEk90WplK8HGWE2fl8FgYPny5ej1emJjYxs9r6ysjM6dOxMWFtZkKw2gsrKSkpKSerfWeGhkNxy0Gn7KOEdqVlGrHkuIKy1af4Qao8LoyADiuvmrXY5VUz280tLS8PDwQKfT8eCDD7J69WqioqIaPDcyMpKPPvqItWvX8vnnn2M0GomLi+P06dONPn5SUhLe3t6mW1hYWKvqDfN147aYEADTGBwhzGFnxjk2Hs7HQavh6Ym91C7H6mkUlb/3r6qqIisri+LiYlauXMmHH37Itm3bGg2wy1VXV9OrVy9mzpzJSy+91OA5lZWVVFb+tqBgSUkJYWFhFBcX4+Xl1aKaswrLGbNkKzVGheUPDGVoV78WPU67pteDh0ftv8vKwN1d3XqsXI3ByJR//sTRvFISYjvz4i191C5JNSUlJXh7ezf5GVW95eXs7ExERAQDBw4kKSmJfv368dZbbzXrvk5OTsTExJCR0Xj/k06nM32beenWWuF+bsy4sbYF99rGYzLuS7Ta8r3ZHM0rxdvVicfje6hdjk1QPbyuZDQa67WUrsVgMJCWlkZwcHAbV3W1R8Z0R+eoZV9mEVvTz1r8+UX7UVxezZK6ydePx3fHx12mATWHquGVmJjI9u3bOXXqFGlpaSQmJrJ161ZmzZoFQEJCAomJiabzX3zxRTZt2sSJEydITU3lrrvuIjMzk7lz51q89iBvF+6u2+xzySZpfYmWe2vLrxSVV9O9owez7HQD2ZZwVPPJCwoKSEhIIDc3F29vb6Kjo9m4cSPjxo0DICsrC632t3wtKiri/vvvJy8vDx8fHwYOHMjOnTub1T/WFh4a1Y1/78niUE4JGw7lMbGv5VuAwrYdyyvlk+RTAPxlahRODlZ3MWS1VO+wt7TmdgY215JNx/jn/zLoFuDOxsdG4Cj/8dWSDvsmKYrCjPd2sefUeW6OCuT9hEFql2QVbKbD3tbdP6IrPm5OHD+rZ/nebLXLETbkm9Qc9pw6j6uTA89P6612OTZHwquVvFyceKzu26E3N6dTWlGtckXCFhSXV/O3744AMD++OyGyLv11k/AygzuHhNPF351zZVW8VzehVohrWbzpKIX6Krp39OC+YV3ULscmSXiZgZODlqcn9gTggx9PkFt8UeWKhDU7kH3BNH/xxVv64OwoH8OWkL+amdwcFcjgG3yprDHy2kaZNiQaVlVj5OlVB1GU2v0RYrvJ7IyWkvAyE41GwzOTa+ejfbP/NGmni1WuSFij97Yd52heKb7uzvx5ijpDfNoLCS8z6h/WgVtjQlAU+Mt/DmE02tUoFNGEjIJS01JKz0+NwldG0reKhJeZJU7sibuzA/uzLrAqtfHVLoR9MRoVnl6VRpXByOjIAKb1k0UGW0vCy8w6erkwP747AH/fcJTiizJ0QsAXuzPZl1mEu7MDL9/aF41GFhlsLQmvNnBPXBe6BdQOnXhzs3Te27vMQj1J648C8NSEnjKmy0wkvNqAs6OWF+pGTH+anMnRvNat3ipsl8Go8MTXP1NeZSC2q59pMr9oPQmvNnJT9wAm9gnCYFR4drV03turD388wb7MIjx0jrx6e7SsSW9GEl5t6C9To3B3diAls4gvdltulyNhHY7llbJkU223wZ+n9CLM103litoXCa82FOztylMTakfe/33DMRl5b0eqaows+PoAVQYjY3p25I5Brds7QVxNwquN3TW0MzHhHSirrOH5tdfe6Ui0H69tOsbhMyV0cHNi0W3y7WJbkPBqYw5aDYtui8ZRq2HTL/lsOJSrdkmijW1PP8v722sn6L/6u2g6ellmV3d7I+FlAZFBnjw4shsAf157mCJ9lcoVibZyrqySBV//DMDdQztzc+8glStqvyS8LGTemAgiOnpwtrSS59YeUrsc0QaMRoUnV/zMubJKIgM9eXay7L3YliS8LMTFyYHX7+iHg1bDtwdz+c/PZ9QuSZjZhz+dYOuxs+gctfxjZgwuTg5ql9SuSXhZUHRoB+aNjgDgz2sOkV9SoXJFwlx2nyjk7xtqty/7y9QoIoM8Va6o/ZPwsrB5YyLoG+JN8cVq/rTqoGyZ1g4UlFQw79/7MRgVbo0J4c7B4WqXZBckvCzMyUHL63f0w9lRy9ZjZ/k0WQav2rIag5F5/97P2dJKegR68MqtfWRYhIVIeKmge6AniXXLRr/y7REOn5GFC23V3zccZc/J83joHHnnroG4Oau6FapdkfBSyT1xNxDfK5Aqg5FHvtyPvrJG7ZLEdVqVcpoPfjwJwKu3R9MtwEPliuyLhJdKNBoNi2+PJtjbhRPn9PxZhk/YlNSsIhK/SQNg3ugIJslu6RYn4aUiH3dn3vp9DFpN7QakK/bJprW2ILf4In/4LIUqg5FxUYEsGNdD7ZLskoSXygZ38eXxuk1rn1tzSDbusHLlVTU88GkKZ0trB6K+MaO/LHOjEgkvK/Dw6AjG9uxIZY2RBz9PobCsUu2SRANqDEYe/fd+0nKK8XFz4sPZg/DQSQe9WiS8rIBWq+H1Gf3p4u9OzoWLzPtyPzUGo9plicsoisIL/z3M5iMFODtq+SBhkKzPpTIJLyvh7erE+3cPxN3ZgeQThfztu6NqlyQu8+62E3y+KwuNBt6a0Z9BN/iqXZLdk/CyIt0DPVlyRz8APtpxUlZftRLfpJ7m7xtq/2fy58lRTJRvFq2ChJeVmdAn2NSB/5e1h9l6rEDliuzbhkN5LFx5EIA5w7tw3/AuKlckLpHwskKPjo3gdwNCMRgVHv4ilV/OyO5DatiefpZH6+Ys3j4wlGcnyRI31kTCywppNBqSbutLbFc/9FUG7lu2lzMXZP17S9p76jwPfLaPKoORSX2DWHRbXxkSYWUkvKyUs6OWd+8eSERHD/JKKrjrw92ckyEUFrHn5Hnu+WgPFdVGRkUG8OaMGBwd5KNibeQdsWLerk58et9gQjq4cuKcnoR/7aH4YrXaZbVryccLmf3RHvRVBoZF+PHOrIE4O8rHxBrJu2LlOnVw5fO5Q/D30PFLbgn3LdtLeZVM4m4LOzLOce+yPVysNnBTd3/+NftGXJ1lNVRrJeFlA7r4u/PZnMF4uTiSklnEfcv2yioUZvb9L/nct2wvFdVGRkcG8EHCIFnG2cpJeNmIXsFefHLfYDx0juw6cZ6Ej/ZQUiGXkOawfE8Wf/hsH5U1RuJ7BfLu3QMluGyAhJcNiQn34fO5Q0wtsLs/3E1xuQRYSymKwj+3/MrT36RhVOCOQaG8e9cAdI4SXLZAwsvG9A/rwJf3D8XHzYmfTxfz+w92yUYeLVBVY+SZ1Wks+T4dgIdHd+Pvv4uWbxVtiKrv1DvvvEN0dDReXl54eXkRGxvL+vXrr3mfFStW0LNnT1xcXOjbty/fffedhaq1Hn1CvFn+QCz+HjqO5JZw2//tJD2/VO2ybEaRvoq7/7Wbf+/JRqOB56dGsXB8T1l73saoGl6hoaEsWrSIlJQU9u3bx5gxY7jllls4fPhwg+fv3LmTmTNnMmfOHPbv38/06dOZPn06hw7Z3yqkkUGefPNQHF3rVqL43Ts7ST5eqHZZVi+joJTp/7eD3XXrzv9r9iDuHSZTfmyRRrGyvbd8fX1ZvHgxc+bMuep3M2bMQK/Xs27dOtOxoUOH0r9/f959990GH6+yspLKyt8Gd5aUlBAWFkZxcTFeXl7mfwEWVqSvYu6n+0jJLMLZQcvL0/twx41hapcFej141K3pXlYG7u7q1gP85+czJK46iL7KQJivK/+afSM9AmV/RWtTUlKCt7d3k59Rq7nANxgMLF++HL1eT2xsbIPnJCcnEx8fX+/Y+PHjSU5ObvRxk5KS8Pb2Nt3Cwqzgg21GPu7OfDF3CJP6BlFlMPLUqoM8uzqNqhpZD+ySyhoDz61J49F/70dfZWBoV1/WPjxcgsvGqR5eaWlpeHh4oNPpePDBB1m9ejVRUVENnpuXl0dgYGC9Y4GBgeTl5TX6+ImJiRQXF5tu2dntb514FycH3p45gCfG9UCjgS92Z/H795PJK5aO/ONny7j9nWQ+35UFwCNjIvh8zhB83Z1Vrky0lupr2EZGRnLgwAGKi4tZuXIls2fPZtu2bY0G2PXS6XTodDqzPJY102o1PDK2O31CvJm/fD+pWReY+NZ2Fv0umvG9g9Quz+KMRoVPkk+xaP1RKmuM+Lg58caM/oyK7Kh2acJMVG95OTs7ExERwcCBA0lKSqJfv3689dZbDZ4bFBREfn5+vWP5+fkEBdnfh7Mxo3t25D/zhtO7kxdF5dX84bMUnl510K5G5GefL+euf+3mr//9hcoaIzd19+e7+TdJcLUzqofXlYxGY70O9svFxsayZcuWese+//77RvvI7NUN/u6s/uMw/jCyKxoNLN+bzaR//MiOjHNql9amqmqMvLP1OOPe2MbO44W4Ojnw0vQ+fHrfYIK9XdUuT5iZqpeNiYmJTJw4kfDwcEpLS/nyyy/ZunUrGzduBCAhIYGQkBCSkpIAmD9/PiNHjmTJkiVMnjyZ5cuXs2/fPt5//301X4ZVcnbUkjixFyN7BPDE1z+TWVjOrA93c1tMCM9O7oWfR/u6lE4+Xshf1h7i14IyAIZ08WXR76Lp4q/+t5yibagaXgUFBSQkJJCbm4u3tzfR0dFs3LiRcePGAZCVlYVW+1vjMC4uji+//JLnnnuOZ555hu7du7NmzRr69Omj1kuwenHd/Nn4+AiWbDzGp7sy+WZ/Dv87VsD8sd2ZNaSzzS/3kp5fyt/XH2XL0drlsv3cnXl2ci9ujQmRQaftnNWN82przR1D0h7tr9ui/mhe7Wj8cF83Fo6PZEp0sPk/6G08ziuzUM///XCcFSnZGBVw0Gq4c3A4T94cibebk1mfS1hWcz+jEl52psZg5Kt92bzx/a+mlVl7BXvx4MiuTO4bbL65fW0UXkdyS3hn63HWHTyDse6/3Il9glg4PpKuAR5meQ6hLgmvRth7eF2ir6zhwx9P8v724+irDACE+boyZ1gXbo0JbX3rxYzhVVVj5Ptf8vlyTyY7Mn6bAjUqMoBHx3ZnQLhP62oVVkXCqxESXvVdKK/i0+RMlu08xXl9FQA6Ry0T+wRxx41hDO3i17KNJ1oZXoqikJZTzLcHc1mVeppzZbW1aTQwqW8wD43sRp8Q7+uvS1g9Ca9GSHg17GKVgZUp2Xy+K4tjl61Q4e+hY1xUR26OCiK2m1/zF+lrQXhVVBvYd6qI7b+eZf2hXLLP/7ZjUoCnjhmDwphxYxhhvm7X9dqEbZHwaoSE17UpisLPp4v5am826w6eobTit8GtTg4aokM7MOgGHwaG+9Aj0JMwXzccGmqZNRFeiqKQff4ih88Uc+hMMamZF0jJKqo3J9PVyYExvToyNboTY3t1xEnW2rILEl6NkPBqvqoaI7tPFvL9L/l8/0s+uQ3MlXR21NLV350gbxf8PXQEeOrw0DniUnmROROjAVj6nwMUOzhTWlHNmQsV5Fy4SE7RRS5WG656vCAvF+Ii/IjvFcjoyI6yAYYdkvBqhIRXyyiKQtb5cvaeKmLvyfOk5RRz/GwZlY2sXuFaVcGRN24HoNfjK7no7HLVOc4OWiKDPOndyYs+Id7EdvOjq7+7jM+yc839jKo+MVvYBo1GQ2c/dzr7uXP7wFAADEaF00XlnDirp6C0gnNlVZwtreRilQFNud503/83KARnLy88XBwJ9nYhpIMboT6udOrgavODZIV6pOUl2oYVLkYobIPNLUYohBDXQ8JLCGGTJLyEEDZJwksIYZMkvIQQNknCSwhhkyS8hBA2ye4GqV4a1lZSUqJyJe2c/rdBqpSUgOHqqUBCNOTSZ7OpIah2F16lpbUrJrS3zWetWqdOalcgbFBpaSne3o0ve2R3I+yNRiNnzpzB09PTLufQlZSUEBYWRnZ2tswwsAH2+H4pikJpaSmdOnWqt4fFleyu5aXVagkNDVW7DNV5eXnZzYehPbC39+taLa5LpMNeCGGTJLyEEDZJwsvO6HQ6nn/+eXS69rXpbHsl71fj7K7DXgjRPkjLSwhhkyS8hBA2ScJLCGGTJLyEEDZJwksIYZMkvEQ9b7zxBr179yYqKopHH320ycmxwrJuvfVWfHx8uP322+sdX7duHZGRkXTv3p0PP/xQpeosS4ZKCJOzZ88ydOhQDh8+jJOTEyNGjOC1114jNjZW7dJEna1bt1JaWsonn3zCypUrAaipqSEqKooffvgBb29vBg4cyM6dO/Hz81O52rYlLS9RT01NDRUVFVRXV1NdXU3Hjh3VLklcZtSoUXh6etY7tmfPHnr37k1ISAgeHh5MnDiRTZs2qVSh5Uh4tSPbt29n6tSpdOrUCY1Gw5o1a646Z+nSpdxwww24uLgwZMgQ9uzZY/pdQEAATz75JOHh4XTq1In4+Hi6detmwVfQvrX2/WnMmTNnCAkJMf0cEhJCTk6OOUu3ShJe7Yher6dfv34sXbq0wd9/9dVXLFiwgOeff57U1FT69evH+PHjKSgoAKCoqIh169Zx6tQpcnJy2LlzJ9u3b7fkS2jXWvv+iPokvNqRiRMn8vLLL3Prrbc2+PvXX3+d+++/n3vvvZeoqCjeffdd3Nzc+OijjwDYvHkzERER+Pr64urqyuTJk9m1a5clX0K71tr3pzGdOnWq19LKycmhkx0sACnhZSeqqqpISUkhPj7edEyr1RIfH09ycjJQu7rszp07qaiowGAwsHXrViIjI9Uq2a405/1pzODBgzl06BA5OTmUlZWxfv16xo8f39Ylq87uFiO0V+fOncNgMBAYGFjveGBgIEePHgVg6NChTJo0iZiYGLRaLWPHjmXatGlqlGt3mvP+AMTHx/Pzzz+j1+sJDQ1lxYoVxMbGsmTJEkaPHo3RaOSpp55q9980goSXuMIrr7zCK6+8onYZohGbN29u8Pi0adPs7n80ctloJ/z9/XFwcCA/P7/e8fz8fIKCglSqSlwi78/1k/CyE87OzgwcOJAtW7aYjhmNRrZs2SKDUK2AvD/XTy4b25GysjIyMjJMP588eZIDBw7g6+tLeHg4CxYsYPbs2QwaNIjBgwfz5ptvotfruffee1Ws2n7I+2Nmimg3fvjhBwW46jZ79mzTOf/85z+V8PBwxdnZWRk8eLCya9cu9Qq2M/L+mJfMbRRC2CTp8xJC2CQJLyGETZLwEkLYJAkvIYRNkvASQtgkCS8hhE2S8BJC2CQJLyGETZLwEkLYJAkvYdVGjRrFY4891uzzly1bRocOHVr8fI2tLS+sj0zMFuIyubm5+Pj4qF2GaAYJLyEuI2tn2Q65bBTXbdSoUTzyyCM89thj+Pj4EBgYyAcffGBavsXT05OIiAjWr19f737btm1j8ODB6HQ6goODefrpp6mpqTH9Xq/Xk5CQgIeHB8HBwSxZsuSq566srOTJJ58kJCQEd3d3hgwZwtatW5tde1VVFfPmzSM4OBgXFxc6d+5MUlKS6feXXza+8MILaDSaq27Lli0DatfbSkpKokuXLri6utKvXz/TRrDCAtRe1kLYnpEjRyqenp7KSy+9pKSnpysvvfSS4uDgoEycOFF5//33lfT0dOWhhx5S/Pz8FL1eryiKopw+fVpxc3NT/vjHPypHjhxRVq9erfj7+yvPP/+86XEfeughJTw8XNm8ebNy8OBBZcqUKYqnp6cyf/580zlz585V4uLilO3btysZGRnK4sWLFZ1Op6SnpyuKoigff/yx4u3t3WjtixcvVsLCwpTt27crp06dUn788Uflyy+/NP0eUFavXq0oiqKUlpYqubm5pttrr72muLm5KWlpaYqiKMrLL7+s9OzZU9mwYYNy/Phx5eOPP1Z0Op2ydetW8/yhxTVJeInrNnLkSGX48OGmn2tqahR3d3fl7rvvNh3Lzc1VACU5OVlRFEV55plnlMjISMVoNJrOWbp0qeLh4aEYDAaltLRUcXZ2Vr7++mvT7wsLCxVXV1dTeGVmZioODg5KTk5OvXrGjh2rJCYmKorSdHg98sgjypgxY+rVcbnLw+tyycnJiouLi/LVV18piqIoFRUVipubm7Jz5856582ZM0eZOXNmo88vzEf6vESLREdHm/7t4OCAn58fffv2NR27tAvOpQ1Tjxw5QmxsLBqNxnTOsGHDKCsr4/Tp0xQVFVFVVcWQIUNMv/f19a239VpaWhoGg4EePXrUq6WysrLZu+Xcc889jBs3jsjISCZMmMCUKVO4+eabr3mfrKwspk+fzpNPPskdd9wBQEZGBuXl5YwbN67euVVVVcTExDSrFtE6El6iRZycnOr9rNFo6h27FFJGo9Fsz1lWVoaDgwMpKSk4ODjU+52Hh0ezHmPAgAGcPHmS9evXs3nzZu644w7i4+Mb7avS6/VMmzaN2NhYXnzxxXq1AHz77beEhITUu49Op7uelyVaSMJLWESvXr1YtWoViqKYgm3Hjh14enoSGhqKr68vTk5O7N69m/DwcACKiopIT09n5MiRAMTExGAwGCgoKOCmm25qcS1eXl7MmDGDGTNmcPvttzNhwgTOnz+Pr69vvfMUReGuu+7CaDTy2Wef1Ws1RkVFodPpyMrKMtUnLEvCS1jEH//4R958800eeeQR5s2bx7Fjx3j++edZsGABWq0WDw8P5syZw8KFC/Hz86Njx448++yzaLW/fSHeo0cPZs2aRUJCAkuWLCEmJoazZ8+yZcsWoqOjmTx5cpN1vP766wQHB5s21l2xYgVBQUENDmx94YUX2Lx5M5s2baKsrMzU2vL29sbT05Mnn3ySxx9/HKPRyPDhwykuLmbHjh14eXkxe/Zss/3tRMMkvIRFhISE8N1337Fw4UL69euHr68vc+bM4bnnnjOds3jxYsrKypg6dSqenp488cQTFBcX13ucjz/+mJdffpknnniCnJwc/P39GTp0KFOmTGlWHZ6enrz66qv8+uuvODg4cOONN/Ldd9/VC8lLtm3bRllZGXFxcVfVcM899/DSSy8REBBAUlISJ06coEOHDgwYMIBnnnmmBX8hcb1kAw4hhE2SQapCCJsk4SWEsEkSXkIImyThJYSwSRJeQgibJOElhLBJEl5CCJsk4SWEsEkSXkIImyThJYSwSRJeQgib9P8BG5GIwiV70WYAAAAASUVORK5CYII=", 273 | "text/plain": [ 274 | "
" 275 | ] 276 | }, 277 | "metadata": {}, 278 | "output_type": "display_data" 279 | } 280 | ], 281 | "source": [ 282 | "c = 1.92e19 # target compute budget (usually know this because we know how many GPU for how long go brrr)\n", 283 | "# sweep model sizes from 10M to 100B\n", 284 | "ns = 10 ** np.arange(7, 11, step=2**-4)\n", 285 | "# using C = 6*N*D, solve for D that maintains the compute budget c\n", 286 | "ds = c / (6 * ns)\n", 287 | "# evaluate the loss in each case\n", 288 | "losses = L(ns, ds)\n", 289 | "# find the argmin\n", 290 | "best = np.argmin(losses)\n", 291 | "print(f\"best model size: {ns[best]/1e6:.2f}M\")\n", 292 | "print(f\"best dataset size: {ds[best]/1e9:.2f}B\")\n", 293 | "# plot the loss\n", 294 | "plt.figure(figsize=(3,3))\n", 295 | "plt.plot(ns, losses)\n", 296 | "plt.xscale('log')\n", 297 | "# plot a vertical bar at the best model size\n", 298 | "plt.axvline(ns[best], color='red')\n", 299 | "plt.xlabel('model size')\n", 300 | "plt.ylabel('loss')" 301 | ] 302 | }, 303 | { 304 | "attachments": {}, 305 | "cell_type": "markdown", 306 | "metadata": {}, 307 | "source": [ 308 | "In the plot above, basically the models on the left of best are too small and trained for too long. The models on the right of best are way too large and trained for too little. The model at the red line is just right.\n", 309 | "\n", 310 | "Now, the Chinchilla paper says that best model size is 400M params and 8B tokens, so this disagrees and there is some calculations problem. TODO figure out and fix..." 311 | ] 312 | }, 313 | { 314 | "cell_type": "code", 315 | "execution_count": 8, 316 | "metadata": {}, 317 | "outputs": [ 318 | { 319 | "name": "stdout", 320 | "output_type": "stream", 321 | "text": [ 322 | "compute budget 1.000000e+17: best model size: 29.43M, best dataset size: 0.57B\n", 323 | "compute budget 1.778279e+17: best model size: 36.52M, best dataset size: 0.81B\n", 324 | "compute budget 3.162278e+17: best model size: 48.70M, best dataset size: 1.08B\n", 325 | "compute budget 5.623413e+17: best model size: 60.43M, best dataset size: 1.55B\n", 326 | "compute budget 1.000000e+18: best model size: 80.58M, best dataset size: 2.07B\n", 327 | "compute budget 1.778279e+18: best model size: 107.46M, best dataset size: 2.76B\n", 328 | "compute budget 3.162278e+18: best model size: 133.35M, best dataset size: 3.95B\n", 329 | "compute budget 5.623413e+18: best model size: 177.83M, best dataset size: 5.27B\n", 330 | "compute budget 1.000000e+19: best model size: 220.67M, best dataset size: 7.55B\n", 331 | "compute budget 1.778279e+19: best model size: 294.27M, best dataset size: 10.07B\n", 332 | "compute budget 3.162278e+19: best model size: 392.42M, best dataset size: 13.43B\n", 333 | "compute budget 5.623413e+19: best model size: 486.97M, best dataset size: 19.25B\n", 334 | "compute budget 1.000000e+20: best model size: 649.38M, best dataset size: 25.67B\n", 335 | "compute budget 1.778279e+20: best model size: 865.96M, best dataset size: 34.23B\n", 336 | "compute budget 3.162278e+20: best model size: 1074.61M, best dataset size: 49.05B\n", 337 | "compute budget 5.623413e+20: best model size: 1433.01M, best dataset size: 65.40B\n", 338 | "compute budget 1.000000e+21: best model size: 1778.28M, best dataset size: 93.72B\n", 339 | "compute budget 1.778279e+21: best model size: 2371.37M, best dataset size: 124.98B\n", 340 | "compute budget 3.162278e+21: best model size: 3162.28M, best dataset size: 166.67B\n", 341 | "compute budget 5.623413e+21: best model size: 3924.19M, best dataset size: 238.84B\n", 342 | "compute budget 1.000000e+22: best model size: 5232.99M, best dataset size: 318.49B\n", 343 | "compute budget 1.778279e+22: best model size: 6493.82M, best dataset size: 456.40B\n", 344 | "compute budget 3.162278e+22: best model size: 8659.64M, best dataset size: 608.62B\n", 345 | "compute budget 5.623413e+22: best model size: 11547.82M, best dataset size: 811.61B\n", 346 | "compute budget 1.000000e+23: best model size: 14330.13M, best dataset size: 1163.05B\n", 347 | "compute budget 1.778279e+23: best model size: 19109.53M, best dataset size: 1550.95B\n", 348 | "compute budget 3.162278e+23: best model size: 23713.74M, best dataset size: 2222.54B\n", 349 | "compute budget 5.623413e+23: best model size: 31622.78M, best dataset size: 2963.80B\n", 350 | "compute budget 1.000000e+24: best model size: 42169.65M, best dataset size: 3952.29B\n", 351 | "compute budget 1.778279e+24: best model size: 52329.91M, best dataset size: 5663.68B\n", 352 | "compute budget 3.162278e+24: best model size: 69783.06M, best dataset size: 7552.64B\n", 353 | "compute budget 5.623413e+24: best model size: 93057.20M, best dataset size: 10071.61B\n", 354 | "compute budget 1.000000e+25: best model size: 115478.20M, best dataset size: 14432.74B\n", 355 | "compute budget 1.778279e+25: best model size: 153992.65M, best dataset size: 19246.37B\n", 356 | "compute budget 3.162278e+25: best model size: 191095.30M, best dataset size: 27580.28B\n", 357 | "compute budget 5.623413e+25: best model size: 254829.67M, best dataset size: 36778.90B\n" 358 | ] 359 | }, 360 | { 361 | "data": { 362 | "text/plain": [ 363 | "" 364 | ] 365 | }, 366 | "execution_count": 8, 367 | "metadata": {}, 368 | "output_type": "execute_result" 369 | }, 370 | { 371 | "data": { 372 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1gAAAEqCAYAAAAI6vFLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAACGMklEQVR4nO3deXxU1fn48c+dyWSdzCQh+waBsEsSCKK4AUJVbLVg9autC2JLa8XlJ2ILtS60KlaqpVX6xdoq2trWrdpWW79WSsBWrCWQgMiWECD7npnMZJ2Z+/tjkoFhAmS/M8nzfr3mleTOmTtnMg+XPHPOeY6iqqqKEEIIIYQQQogB02ndASGEEEIIIYQYKSTBEkIIIYQQQohBIgmWEEIIIYQQQgwSSbCEEEIIIYQQYpBIgiWEEEIIIYQQg0QSLCGEEEIIIYQYJJJgCSGEEEIIIcQgCdK6A/7K5XJRUVFBZGQkiqJo3R0hhBBCCCGERlRVpbm5meTkZHS6s49RSYJ1BhUVFaSlpWndDSGEEEIIIYSfKC0tJTU19axtJME6zaZNm9i0aRMOhwOA999/n4iICI17JQbC6XRSXFzMhAkT0Ov1WndHjFISh0JrEoNCaxKDQmsDiUG73c6Xv/xlIiMjz9lWUVVV7W8nRzKr1YrZbKa2thaTyaR1d8QAOJ1OCgoKyMnJkQu60IzEodCaxKDQmsSg0NpAYtBqtRIXF4fFYjlnbiAjWKfpHsFyOp0A7N+/H6PRqHGvxEB0f1qh1+vlgi40I3EotCYxKLQmMSi0NpAYtNlsvW4rCdZpVq5cycqVKz0jWFlZWTKCFeC6k2X5xExoSeJQaE1iUGhNYlBobSAxaLVae91WyrQLIYQQQgghxCCREazTnD5FcO/evTJFMMA5nU6OHTsGIJ+YCc04nU4OHz4MSBwKbUgMCq1JDAqtDSQG+zJFUIpcnEH3FMGGhoYepwh2dHRw/PhxXC6XBr0TfdXR0UFwcPCQPofZbCYhIUH2TRM9ksXdQmsSg0JrEoNCawMtchETEyNFLoaKqqpUV1cTFBREUlLSOTcbE9pSVZW2tjZCQ0OHJPlRVZWWlhZqa2sBSExMHPTnEEIIIYQQgUESrNP0doqgXq8nKSkJQEaxAoCqqp7bUAgLC8NsNlNdXU1VVdWQPIcIbDI1RmhNYlBoTWJQ9JbibCfccoSIxgPYo6dijzlvUM47XFMEJcE6TW+qCLa1tXH8+HEiIyMJCwvTqKeit7qTqrCwsCGdvqcoCk1NTYwdO5bQ0NAhex4RmKR6ltCaxKDQmsSg6JHqgvoilPJ8KM93f63Zj+JyAOCa8x3UnFsG5amGq4qgJFjn0FOdfL1ej6Io6HQ6WW8TQBRFGdL3qzseZH8PcSbdsSHxIbQiMSi0JjEosNVAeT6U7YLyXVC+B9otvu0i4iF1NrqUWTCI8dLfGOxLe0mwhBBCCCGEEIOvowUqC90JVfkuKMsHywnfdkFhkJwDKbmQOhtSZoM5FQJ0IEMSrHNwOp2e4cRTjw31mh4xeLrfo6F+r7rjoaeYEaI7LiQ2hFYkBoXWJAZHONUFdYfdU/wqdru/Vu9HUb3fbxUFYiehpuRCymz31/ipoDstLRmCGgcDicG+PEYSrNP0tshFUFAQra2tUuDiDK666iqysrJ4+umne9X+d7/7Hd///vcpLy/v1/MZjUb+8Ic/cM011/R4f1tbW7/O2xft7e10dnZy8ODBIX8uEXhkcbfQmsSg0JrE4MhjaK3BVJuPqS6fyNo9GDoafdp0hsRgj5rqLlYRPQW7eTIuQ8TJBlVOqPp8WPorRS400pciF2FhYVLM4Az0ej1BQUGEh4f3qn33HlW9bX+6iooKoqOjCQkJ8blvuIpc6HQ6DAYDmZmZEhfChyzuFlqTGBRakxgcAdqscPxfKEfzUEq2o9Qf8bpb7Zrqp6bkoibnQkouOlMKkYpCpEZdPpUUufATZytyMdRFEwJdX34/3e36+/vsLpk/WP3pj+7zy+JdcSayuFtoTWJQaE1iMMA4OqDsv3A0z30rz4dTp/wpOkieBRMWwPj5KKlzICgYf/7reDiKXMgOuYNAVVVaOhya3Pqyrmj+/Pncc889/L//9/+Ijo4mISGBF198EbvdzvLly4mMjCQzM5O///3vXo/bvn07c+bMISQkhKSkJNasWYPD4fDcb7fbue222zAajSQlJfHMM8/4PHd7ezurV68mJSWFiIgILrjgAvLy8nrd946ODu6++26SkpIIDQ1l7NixrF+/3nO/oii8++67ADz22GNeCbBOp8NoNLJlyxbAvW/Z+vXrycjIICwsjOzsbN56661e90UIIYQQYkRSVaj+Anb+El67AX4yDrZcDTuehrLP3MnVmEw4/1tw42vwvRJYsRUu/yGMuwSCgrV+BX5BRrAGQWunk2mP/J8mz/3Fj64kPLj3b+Mrr7zC9773PT777DNef/11vvvd7/LOO++wdOlSfvCDH/Czn/2MW2+9lRMnThAeHk55eTlXX301t99+O6+++ioHDx5kxYoVhIaG8thjjwHw4IMPsn37dv785z8THx/PD37wA3bv3k1OTo7nee+++26++OIL/vjHP5KcnMw777zDVVddxb59+5g4ceI5+/2LX/yCv/zlL7zxxhukp6dTWlpKaWlpj21Xr17NnXfe6fn5d7/7HY8++iizZ88GYP369fzud79j8+bNTJw4kR07dnDLLbcQFxfHvHnzev27FEIIIYQIeNYKOLodjm5zj1LZqr3vD4+F8fNgvHuUiqg0LXoZUCTBOofeVBHUspJgX58/Ozubhx56CIA1a9bw1FNPERsby7e+9S0AHn74Yf73f/+XwsJCLrzwQjZt2kRaWhrPPfcciqIwefJkysvLWbNmDQ8//DAtLS385je/4be//S2XX345AFu2bCEtLc3TtxMnTvDyyy9z/PhxkpOTAXjggQf44IMPeOmll3jyySfPWenv+PHjTJw4kYsvvhhFUUhPT/dp3/18ERERRES4F09++umnPPzww7zwwgtMnz6dtrY2nnzySf7xj38wd+5cADIyMvj444954YUXuOyyy3r9uzydVBEUZyPVs4TWJAaF1iQG/YDqgtpDKBW73Zv6ntiJUnfIu0lQGIydi5oxDzVjPiRMd08F7BbA759UEdRIf6oIqqrKf79/6XB3FQC1s50WR0ev2jqdTqZOnUpLS4vnWExMDJMnT/Yci4x0L0EsLS0lKyuLzz//nPPPP5/W1lbPY3Jzc7HZbBw5coSmpiY6OjrIysrynCM0NJSJEyficDhoaWlh165dOJ1OJk+e7NWf9vZ2zGYzLS0tdHS4X8OpfTvVTTfdxLXXXsvkyZNZtGgRixcvZuHChT7nO/XxpaWlLFmyhHvvvZcvf/nLtLa28sUXX9DS0sIVV1zh9diOjg6ys7PP+Py9IVUExdlI9SyhNYlBoTWJweEX1FZPROMBIpoOENF0kIimQ+gd3n/rqCi0RE3CGpuLNS4Xe/R0VH3XVL8qJ1Tt1aDnQ0OqCGqkv1UEI3o6mZ/R6/WEhYV5VerT6XSEh4f7VO8zGAyEh4cTFBTkUw2w+zWHhYV5yp/3dN7ux3V2dqLX69m1a5dPMBuNRsLDw89ZRfCiiy7i6NGj/P3vf+ejjz7itttuY9GiRbz55pueNiEhIZ7H2+12brrpJi666CKefPJJ2traCAsL8yTO7733HikpKV7Pcerj+0OqCIqzkepZQmsSg0JrEoNDrMMOlYUo5bu69qLKR7FW+DRTDeGQ1FXpL2U2jLuE0LBoQoH44e/1sJIqgn5ipFUR7KnPZzs2depU3n77bc8xgE8++YTIyEjS0tIYM2YMBoOBzz77jLFjxwLQ2NjI4cOHmTdvHoqiMGvWLJxOJ7W1tVx6ac8jfb2pImg2m7npppu46aabuOGGG7jqqqtobGwkJibGq8+qqnLrrbficrn47W9/i06n89w/ffp0QkJCKC0tZf78+X387Z2dVBEU5yLVs4TWJAaF1iQGB4nLCbUH3VX9yna5v9Z84Z4CeCpFB3FTIWUWpM6GlNkocVNAH+TXlf6G0nBUEZQES5zVXXfdxcaNG7nnnnu4++67OXToEI8++iirVq3yVOf75je/yYMPPsiYMWOIj4/noYce8iQ1AJMmTeLmm2/mtttu45lnnmHmzJnU1taydetWsrKy+PKXv3zOfjz77LMkJSUxc+ZMdDodb775JomJiURFRfm0feyxx/joo4/48MMPsdlsNDc309raSmJiIpGRkaxevZr7778fl8vFJZdcgsVi4d///jcmk4lly5YN5q9PCCGEEGLg2pvh2L/hxE53MlWxBzp6mLIWmQyp7v2nSJkNyTkQ4g87UI0ukmCJs0pJSeFvf/sbDz74INnZ2cTExPDNb36TH/7wh542GzZswGazcc011xAZGckDDzyAxWLxOs/LL7/M448/zgMPPEB5eTmxsbFceOGFfOUrX+lVPyIjI3n66ac5cuQIer2e888/n7/97W9eiVy37du3Y7PZuOiii7yOv/TSSyxfvpwf//jHxMXFsX79eo4ePUpUVBSzZs3iBz/4QT9+Q0IIIYQQg8zZCeW7T1b2K/svuBzebQwR7pGplK6EKnU2mJI16a7wpqhalsAbJEuXLiUvL4+FCxd69jNqampi0aJFOBwOHA4H9913HytWrOj1ObvXYFkslh7XYJWUlJCRkSFrbQKAqqq0tLQQHh4+pFM6JS7E2TidTgoKCmTtgdCMxKDQmsTgWagq1B0+uaFvycfQ0ezdJnocZFwGqee7E6q4KaCT32NfDCQGz5YbnG5EjGDdd9993HHHHbzyyiueY5GRkezYsYPw8HDsdjvnnXce1113HWPGjNGwp0IIIYQQQgDN1VCyHYq7RqmaTytIERYNGfPce0+Nnw8xGRp0UvTHiEiw5s+fT15entcxvV7vqQjX3t6u+X5VQgghhBBiFOuww/FPTiZUNfu979eHQPqF7mRqwgJIzJIRqgCleYK1Y8cONmzYQH5+PpWVlbzzzjssWbLEq82mTZvYsGEDVVVVZGdn89xzzzFnzpxznrupqYl58+Zx5MgRNmzYQGxs7BC9CiGEEEIIIU7hdEBlwcmEqvQ/4Or0bpOY5U6mxs+H9LlgCNOgo2KwaZ5g2e12srOzueOOO7juuut87n/99ddZtWoVmzdv5oILLmDjxo1ceeWVHDp0iPj4s1frj4qKorCwkOrqaq677jquv/56EhISemzb3t5Oe3u75+fuWvc97fbsdDo9I2IyKub/ut+joX6vuuNBdqkXPRnI7vFCDAaJQaG1ER+D1nIo341Sno9SkQ8VBSiddq8mqjkNdfx8yJiHOu4yiDjtw/+R+rvxEwOJwb48RvMEa/HixSxevPiM9z/77LOsWLGC5cuXA7B582bef/99XnrpJdasWdOr50hISCA7O5uPP/6Y66+/vsc269evZ926dT7H9+7di9Fo9DkeFBREa2srLpfL5z7hf7o3RB5K7e3tdHZ2cvDgwSF/LhF4BrJ7vBCDQWJQaG0kxaDO0UJ40yEimg4S0XiAiKYDBLfV+7RzGIw0j5mJNW4WzXGzaQ9PBkWBTuBIGVA27H0fzQYSgzZbD2Xxz2BACVZHRwclJSVMmDCBoKDBz9U6OjrIz89n7dq1nmM6nY5Fixaxc+fOsz62urqa8PBwIiMjsVgs7Nixg+9+97tnbL927VpWrVrl+dlqtZKWlsb06dN7rCJYVlZGaGioVIsLEKqqEhY2tMPuiqJgMBikiqDoUfcnZjNmzAj4PyxEYJIYFFoL2Bh0OVBqD6JU7EGpyEep2I1SdwjltE19VUWHGjcVNTkXNXmm+2vsJCJ0eiKAJG16L04xkBjsnt3WG/3KilpaWrjnnns8VfsOHz7M+PHjueeee0hJSen1yNK51NXV4XQ6fab1JSQkeI0SLFq0iMLCQux2O6mpqbz55pvo9Xq+/e1ve6Zt3XPPPcyYMeOMzxUSEkJISAibNm1i06ZNnmHAM41gBQcH09bWJlMEA4Cqqp4RrKEs097e3k5HR4eMYIkeuVwuDh8+jKIoPe7fJsRQkxgUWguIGFRVgttqiWg6iNFyiIimA0RYjqB3+s6EaQ+Nwx41BVvUVGxRU2gxZeIKOuXD3Ip2qNg3jJ0X5zKQGBzyEay1a9dSWFhIXl4eV111lef4okWLeOyxxwYtweqtjz76qMfjBQUFw9oPIYQ4E1VVURRFPpQRmpEYFFrzxxjUddqJsBzG2HSQCMshjE0HCG5v8GnnDArHZp7kSajs5sl0hsrWP4FmuGKwXwnWu+++y+uvv86FF17oNSIwffp0iouLB61zsbGx6PV6qqurvY5XV1eTmJg4aM9zqpUrV7Jy5UrPZmJZWVkjeorg5ZdfTnZ2Nj/72c+07sqQ27BhA3/+85/ZvXv3kJxfURSCg4MZP358wMeFGHzdo+KywabQisSg0JrmMehyoNQeQCnf3TXVb497qh/ef2yrih41fppnqp8rORdiJxKm6AgDpCZ14BpIDA75FMHa2toeK/jZ7fZBnYIVHBxMbm4uW7du9ZRud7lcbN26lbvvvnvQnudUp08R3L9//xmLXIyEKYJOpxOHw0Fra2uv2u/YsYOrr76asrIyoqKihrZzp3jiiSd47733zrn27kza29u56667+OY3v9nr19qf5+js7OTIkSNDcn4R2JxOJ8XFxej1evnjVmhCYlBobVhjUFUJbq0m3FOE4iARTYfRudp9mraHJWCPnoo9air2qCm0mCeiBp3yQWlVB1Tt93mcCDwDicEhnyI4e/Zs3n//fe655x7g5LqWX//618ydO7dP57LZbBQVFXl+LikpoaCggJiYGNLT01m1ahXLli1j9uzZzJkzh40bN2K32z1VBQdbb0ewjh8/TlhYWMCPVOj1eoKCgjybMp9L9+sNDw/v9WMGg8FgQKfT9es5u5PgsLCwIV2DpdPpMBgMZGZmBnxciMGn+Se3YtSTGBRaG9IYbLNCxW53AYryfCjPR7HX+DRTQyIheRZqymzUlFxInkWQMR4zYB7cHgk/NFwjWP1aYfjkk0/ygx/8gO9+97s4HA5+/vOfc8UVV/Dyyy/zxBNP9Olcu3btYubMmcycOROAVatWMXPmTB555BEAbrzxRn7605/yyCOPkJOTQ0FBAR988MEZ97MSZ2a321m2bBmRkZEkJyfzzDPP+LT57W9/y/nnn4/JZCIpKYmbb76Zmhr3BerYsWNcfvnlAMTExKDT6TyJ7gcffMCll15KdHQ0sbGxXHPNNV7TRTs6Orj77rtJTk4mLCyMcePGsX79es/9TU1NfOtb3yI+Ph6z2czChQspLCwEYMuWLfzoRz+isLAQnU6HTqdjy5YtPb7GvLw8LrjgAoxGI9HR0VxyySUcP34cgHXr1nniDPCc69RbRkaG5/7PP/+cq6++msjISBITE7ntttuoq6vrz69eCCGEEINFVaHhKEr+FpS/rET3vxei25CB/rXr0G17HOXw31HsNai6INTEbFy5d+C6dhPO736K68ESXLe8g7rgIZh0FRjPvqeqEP3RrxGsSy65hIKCAp566ilmzJjBhx9+yKxZs9i5c+dZK/X1ZP78+eecZnf33XcP2ZTA0/W2iqDXPliqCp1DM+3snAxh7v0UeuH+++8nLy+PP/7xj8TFxbFu3Tp2797N9OnTaWlpAdxJ2EMPPcTEiROpra1l7dq13HbbbfzpT39izJgxvPbaa9x8883s2bMHk8lEaGgoLS0tNDQ0cNddd3Heeedht9t5/PHHWbJkCTt37kSn0/Hzn/+cv/zlL7zyyiukpaVRVlZGWVmZ53m/9rWvERYWxp/+9CdMJhMvvfQSCxcupKCggGuuuYZ7772Xf/zjH7z33nsAmEwmz2O7ORwOli5dyu23385vfvMbT5n/9vZ22tra6OzsxOVyeR53agJot9tZunQpc+bMoaWlhaamJi6//HJuv/12nnjiCdra2nj44Ye5/vrr+dvf/tbj71f2wRJnM5L2fxGBSWJQaG0gMahvt2Cq201kXT6m2nxCWqt92rSHJXZN9ZuCPXqqe6qfPsR9pwqUtUDZ3oG+DBHAhmsfLEUN9EVEQ6R7imBtbe0Zi1yMGzfOPRWsw47uqVRN+ulaUwbBEedsZ7PZiIuL49VXX+WGG24AoKGhgfT0dFasWHHGIhe7du3iggsuwGKxYDQaycvLY+HChdTX1591DVZdXR0JCQkUFhZy3nnncd999/HFF1/w4Ycf+kzT+9e//sU111xDVVUVISEhnuOTJk1i9erVfPvb32bdunXnLFDR0NBAXFwc//znP5k3b57XfS0tLWcscqGqKtdffz2lpaVs376dsLAwnnjiCT7++GM++OADT7uysjLGjh3LgQMHmDRpks/zt7W1cezYMVJTU2WKoPDhdDopKCiQ6VlCMxKDQmt9isHOVpSy/6Ar2YFSkoeu2rvcuaozuKf5pc/F1VWMgoi4Iey9GAkGch20Wq3ExcVhsVh8coPT9WsE6/LLL2fevHk8+uijXscbGxv52te+xj//+c/+nNYv9KvIRUcb505xhkZraxs4zz3T84svvqCjo4Ps7GxPkYewsDAmTpzoVeRiz549PPnkk+zbt4+mpib3CB3uvc6mTp1Ke3t71/O2eiVDRUVFPP744+zatYv6+nrP44qKipgwYQI33XQT1157LVOmTGHRokUsXryYhQsXAu4krjsB9H5trRw+fJjW1lbP6NPZClSEhYVxyy23sHjxYi6//HLmz5/P1772NRITEz2jSz2d49FHH2Xnzp18/PHHnufdvXs3eXl5Pf4DOnDgAGlpaT7HpciFOBspMCC0JjEotHbWGFSdhFuKiazdhaluN8aGfehcnV5NWiLH0xyXizV2FrYxWSf3nGoDjlYBVcPyOkTg8usiF3l5eezbt489e/bw2muvERHhTi86OjrYvn17f07pN/pV5CIsDHVtuSb9DTeE92qKYPeISlhYmFehCJ1O5ylyYbfbWbJkCVdeeSWvvfYacXFxnDhxgquuugq9Xk94ePgZi1zceOONjB07lhdffJHk5GRcLhczZsxAURTCw8O56KKLOHr0KH//+9/56KOPuO2221i0aBFvvvkmHR0dJCUlsW3bNp9+R0VFER4e3usiF6+++ir3338/H3zwAe+++y4//vGP+b//+z+ys7N7PMfvfvc7nn/+ebZt20ZmZqbneFtbG9dccw1PPfWUz3MkJSX12A8pciHORgoMCK1JDAqt+cRg43GUkm1wdDvKsR0orY1e7dXIJNTx8yFjPmrGZYQYEwhByqSL/vPrMu3g3tz3O9/5DhdeeCF//etfGTduXH9P5dd6ynD1ej2KonhuKAqE+I5y+ZPMzEwMBgOfffYZY8eOBdwjjocPH2bevHkoisKhQ4eor6/nqaee8ozQ5OfnA3hea/eolcvl8kz1q6+v59ChQ7z44otceumlgHva36mPAzCbzdx0003cdNNN3HDDDVx11VU0NjaSm5tLVVUVBoPhjHEUEhKC0+nsVRXAWbNmMWvWLH7wgx8wd+5c/vCHP5Cdne25v/scO3fuZMWKFbzwwgs+1S9nzZrF22+/TUZGBkFBvftn0v1a5dNhcSbdsSHxIbQiMSg01drImOp/EfTBa+hK8qDxmPf9wZGQcSmMnw/jF6DEThzS6r9idOrvdbAv7fudYCUlJbF9+3aWL1/O+eefz5tvvsnUqVP7ezq/5XQ6PdnuqcdUVfXcAkFERAR33HEHDz74IDExMcTHx/PDH/4QnU7neR1paWkEBwfzi1/8gjvvvJPPP/+cH//4xwCeNunp6SiKwl//+leuvvpqwsLCiIqKYsyYMfzqV78iMTGREydOsHbtWq/HPfvssyQlJTFz5kx0Oh1vvPEGiYmJnoqBc+fOZcmSJfzkJz9h0qRJVFRU8P7777N06VJmz57N2LFjKSkpYc+ePaSmphIZGek1RRHcJf5/9atfce2115KcnMyhQ4c4cuQIt9xyi1c7VVWpqqpi6dKl3HTTTVxxxRVUVlYC7n88cXFx3HXXXbz44ot8/etf9/zOioqKeP3113nxxRd7/EfW/Vp7ihkhuuNCYkNoRWJQDCtnJ9TsP1kyvTwfff0RJpzSRNUFQcps1Ix57pGq5FmgN5xs0LXcQIjBMpDrYF8e068Eq/vThJCQEH7/+9/z+OOPc9VVV/H973+/P6fzK/2qIhgg1q1bh8Vi4dprr8VoNHLvvffS0NCAw+GgpaWFiIgINm/ezLp163juuefIycnh8ccf53/+539oa2ujpaWF6OhoHnroIdasWcMdd9zBN77xDV544QW2bNnC6tWrmTFjBhMnTmTDhg0sXryY9vZ2WlpaCAkJ4Sc/+Yln3mv3CFFbWxsAb775JuvWrWP58uWeAhkXX3yxp1rg4sWLWbRoEZdffjlNTU1s3rzZJ3EC95q5V155hYaGBhITE1mxYgW33XabTxXBgoICqqureeWVV3jllVc8j09PT+eLL74gKiqKf/zjHzz88MNceeWVtLe3k5aWxpe+9CXa2tp6/ERNqgiKs5EKbkJrEoNiyKgqwa1V7g19uzb1DbccQefq8GnaFJxIW/JcbPGzaR6TjSuoa8p9PVAvm/mKoeXXVQR1Oh1VVVXEx5/cO+Dtt99m2bJltLa2johPx7rXYDU0NJxxDVZGRoastQkAqqrS2to65BsNt7W1UVJSwtixYyUuhA+p4Ca0JjEoBk2bxTMqpVTkQ/lulBbffSLVUHPXpr65qCmzcSZkUXCkXGJQaGagVQRjYmKGropgSUkJsbHeSwy/9rWvMXnyZM+anZGiV2uwREAY6vdL1mCJc5H1L0JrEoOizxwdUP05lOdD2S731/oequXqDJA4A1JyIXU2pMxGGTMBFAXP/7xOJ3p9lcSg0JTfrsHqLpJwuvPOO4/zzjuvP6f0WyNlDdZo1v0eDfV7JWuwxNnI+hehNYlBcU6qCk3Hu9ZN7XJ/rdqH4mz3bRqdgZqSCym5qMmz3MlV0GmzN05bRiExKLTmd2uwrrvuOrZs2YLJZOK66647a9s//elPve6AvxnJa7BGs+61XkNJ1mCJs5H1L0JrEoPidPoOKxFNB923xgOENx3E0GHxaecwmLBHTcEePQV71FTs0VNwBptPNqgD6s79f5/EoNDacK3B6nWCZTabvcptj1T92gdL+LXukauhXoMl+2CJs5E9iITWJAZHOUc7VH/uHpWq2O1eP9VQ7NNM1QdD4gzU5K7RqZRclOgMjIrCQDekkRgUWvO7fbBefvnlHr8f6c62BguQNVgBZDjWzMkaLHE2sv5FaE1icJRQVWg46r1uqmovOH2r+hEzoWvNVK573VTieRAUwlD9bykxKLTmt2uwWltbUVWV8HB3ac3jx4/zzjvvMG3aNK644or+nDKgdP+COzo6CAsL07g3wl+0tLQAYDAYztFSCCGEGESOdij9Dxz7N5R3JVStjb7twmK8kilSZkF4zPD3V4gRrl8J1le/+lWuu+467rzzTpqampgzZw7BwcHU1dXx7LPP8t3vfnew++lXgoKCCA8Pp7a2FoPBgE6n07pL4ixUVaW9vR2dTjckI1iqqtLS0kJNTQ1RUVHyqZwQQoih5XJBzX44mgfF2+D4J+Bo9W6jD4GkrK5EKhdScyE6A2TmjRBDrl8J1u7du/nZz34GwFtvvUViYiJ79uzh7bff5pFHHhlRCdaZKo3Ex8dz/Phxjh07NvydEn3W0dFBcHDwkD6H2WwmLi5OqiOJHkn1LKE1icEAZylDKcmDo9tRju1Asdd63a1GxKNmXAap57ur+yWcB/rT/t/TuDCXxKDQmt9VETxVS0sLkZGRAHz44Ydcd9116HQ6LrzwQo4fP96fU/qN3lYRFIHD6XRy7Ngxxo0bN6SjS62trVRVVQ3Z+UVgk+pZQmsSg4FF12kjsq4AU10+ptp8Qu1lXvc79aHYxmRjjZ2FNW42bZHjTo5O1QA1Xwx7n89FYlBoze+qCJ4qMzOTd999l6VLl/J///d/3H///QDU1NScc2djf9ebKoIisEjVIuEPJA6F1iQG/ZyzA8p2oZTkoRzNc1f6U0+OOKmKDpJnoWbMQx2/AFJnY9QHYwSSNet030gMCq35XRXBUz3yyCN84xvf4P7772fhwoXMnTsXcI9mzZw5sz+n9FtS6WZkkKpFwh9IHAqtSQz6EVWFmgNwdJt7LdWxf0On3bvNmEwYvwDGz0cZdwmERQ1Zdb/hIjEotOa3VQSvv/56LrnkEiorK8nOzvYcX7hwIUuXLu3PKYUQQgghRi57vbu6X/muk6XT25q824THwvj5J29RacPfTyHEgPUrwQJITEwkMTHR69icOXMG3CEhhBBCiIDW2ebed8qzD9UuaDzm2y4oDMZe5E6mJiyA+OkglYmFCHj9TrD8ydKlS8nLy2PhwoW89dZbAJSWlnLrrbdSU1NDUFAQDz/8MDfccIPGPRVCCCHEiOJyQUPxyVGp8l1Q9Tm4On3bjpl4yj5UXZX+goa2wq0QYviNiATrvvvu44477uCVV17xHAsKCmLjxo3k5ORQVVVFbm4uV199NRERERr2VAghhBABzV53clSqPL9rqp/Ft114bFcy1bWhb8osCIse/v4KIYbdiEiw5s+fT15entexpKQkkpKSAPd0xtjYWBoaGiTBEkIIIUTvWSvg6HZ3IYoTO6Gph+1ogkIhKdudTKV2jU5FjZVNfYUYpfqVYO3YsYOLLrqIoCDvhzscDj755BMuu+yyPp1rw4YN5OfnU1lZyTvvvMOSJUu82mzatIkNGzZQVVVFdnY2zz33XJ/We+Xn5+N0OklLk8WiQgghhDiL9mZ3Rb/u6n61B33bxE46JZmaDQnTQW8Y9q4KIfxTvxKsBQsWUFlZSXx8vNdxi8XCggUL+rTTsd1uJzs7mzvuuIPrrrvO5/7XX3+dVatWsXnzZi644AI2btzIlVdeyaFDh3yevycNDQ3cdtttvPjii73ukxBCCCFGCWene5rf0Two3uae+udynNJAgeQcd7n0jEvdo1OhZo06K4QIBP1KsFRVRelh2Lu+vr7PU/AWL17M4sWLz3j/s88+y4oVK1i+fDkAmzdv5v333+ell15izZo1Zz13e3s7S5YsYc2aNVx00UXnbNve3u75uXszMafT2aeEUfif7vdQ3kehJYlDoTWJwS6qCnWHUUq2uzf0Pf4vlA6bd5PojK4NfefDuEt9106N9t9hP0kMCq0NJAb78pg+JVjdI0yKonD77bcTEhLi9aR79+49ZyLTFx0dHeTn57N27VrPMZ1Ox6JFi9i5c+dZH6uqKrfffjuXX345t9566zmfa/369axbt87n+J49ezAajX3vvPAbTqeTI0eO4HQ6ZWNDoRmJQ6G10RyDhvYGTHV7MNXlY67fQ3Bbndf9DkMkljEzscbmYomdSUe4ew03LcAXR4e/wyPUaI5B4R8GEoM2m+3cjbr0KcEym91D4qqqEhkZSVhYmOe+4OBgLrzwQlasWNGXU55VXV0dTqeThIQEr+MJCQkcPHhyTvSiRYsoLCzEbreTmprKm2++idPp5PXXXycrK4t3330XgN/+9rfMmDGjx+dau3Ytq1at4sUXX+TFF1/E6XRSVFQ0aK9FaEdRlDOOugoxXCQOhdZGUwwGtTdibDqIqb4QU/1uwptLvO536Qw0R5+HNXYWlthZtJgyQZH9p4baaIpB4Z+GKwb7lGC9/PLLAIwbN47Vq1f7TUW+jz76qMfjLper1+cICQkhJCSEBx54gAceeACr1YrZbCYrKwuTyTRYXRUacDqdqKpKdna2fGImNCNxKLQ2YmOwsxWlai9KxW6Uinx0FXtQLCd8mrkSs1DHzcOVMQ81dQ5hhjDCgATfM4ohMmJjUASMgcRg9/Kh3ujXGqxHH30Uh8PBRx99RHFxMd/4xjeIjIykoqICk8k0aFPqYmNj0ev1VFdXex2vrq4mMTFxUJ7jdJs2bWLTpk2eeZb79++XKYIBzul0UlxcjF6vlwu60IzEodDaiIhB1UWorZSIpgNENLpvYc0lKKr32ggVhTZjOraY6TTHzcY6ZibOkK7CFDbgoMxQ0cKIiEER0AYSg0M2RbDb8ePHueqqqzhx4gTt7e186UtfIjIykp/85Ce0t7ezefPm/pzWR3BwMLm5uWzdutVTut3lcrF161buvvvuQXmO061cuZKVK1d6RrCmT58uI1gBrnsx44wZM+SCLjQjcSi0FpAxaKtBqdiNriLfPUJVuQelvdmnmRoRh5qciyt5FmpyLmpSDvpQE2ZA6v35j4CMQTGiDCQGh3wE67777mP27NkUFhYyZswYz/GlS5f2eQ2WzWbzWutUUlJCQUEBMTExpKens2rVKpYtW8bs2bOZM2cOGzduxG63e6oKDjYZwRp55BMz4Q8kDoXW/D0GFUcb4ZYjJ0enmg4Q0lrj086lC8EeNQl71BTs0VOxR02lMyz+5Ka+NuBID5sBC835ewyKkc+vR7A+/vhjPvnkE4KDg72Ojxs3jvLy8j6da9euXSxYsMDz86pVqwBYtmwZW7Zs4cYbb6S2tpZHHnmEqqoqcnJy+OCDD3wKXwwWGcEaeeQTM+EPJA6F1vwqBlUX1B05OTJVkY9Sc6DHqX5q7GTU5JnukamUWahxUwnRBRECxGjTe9FPfhWDYlTy6xEsl8vVYy34srIyIiMj+3Su+fPno6rqWdvcfffdQzYl8HQygjXyyCdmwh9IHAqtaRmDQW0NXiNTEU2H0DtafNp1hsRgj5rqHpmKnoLdPBmX4ZSCWjVAzYHh67gYVHIdFFrz6xGsK664go0bN/KrX/0KcJc8tNlsPProo1x99dX9OaXfOH0ES6oIBr7uZDknJ0cu6EIzEodCa8MWgx12qNqLUr4LpTwfyvNRrL6zW1RDOCTluEelknMhJRedKYVIRaFvH9WKQCHXQaG1gcTgkI9gPfPMM1x55ZVMmzaNtrY2vvGNb3DkyBFiY2P5wx/+0J9TCiGEECLQuJxQdxiloiuRKs+HM0z1I24Kaoo7kVKTcyF+Cuj69WeIEEL4tX5d2VJTUyksLOT111+nsLAQm83GN7/5TW6++WavzYcD0elTBPfu3StTBAOc0+nk8OHDAPKJmdCMxKHQ2qDEoOok3HIEU20+kXV7iGg62ONUv47QWK8iFC1Rk3AFhZ9sUOWAqs/71wcRsOQ6KLQ2kBjsyxRBRT3XAqhRqnuKYENDg0wRDHBOp5OCggKZkiA0JXEotNavGFRVaCxBKdmOcnQbHPsYpc3i3cQQAckzUVNyUZNnQUoumJKH4BWIQCfXQaG1gcSg1WolJiYGi8VyztygXyNYr7zyCrGxsXz5y18G4Hvf+x6/+tWvmDZtGn/4wx8YO3Zsf04rhBBCCK211KOUbIeS7ShH81AspV53qyEmGHcZ6vh5qGlzIW4y6OSPZSGE6NavBOvJJ5/kf//3fwHYuXMnzz//PBs3buS9997j/vvv509/+tOgdnI4yRTBkUemJAh/IHEotHamGFSc7Rgb9mGqzcdUm0+4tcjrcS4lCHvMdKyxuVjjcmkxTzqZUFV2QOW+YXsNIrDJdVBobbimCPYrwSotLSUzMxOAd999l+uvv55vf/vbXHzxxcyfP78/p/QbUkVw5JGqRcIfSBwKrXliMGsG+rov3KNTJdvhxKcoznavtmr8dPcIVcY8SL+I8OAIwoFEDfotRg65Dgqt+XUVQaPRSH19Penp6Xz44YeezYFDQ0NpbW3tzyn9luzVMDJ0v4/yXgotSRwKTbicUHsQ5finTNzzLoZt+1BaG73bRCbDhAUwfgFkXIYSmYCiTW/FCCfXQaG1/sZgX9r3K8H60pe+xLe+9S1mzpzJ4cOHPXtf7d+/n3HjxvXnlEIIIYQYDNYKKNsF5bugLB8q9kCnHR0Q3d0mOBIyLnUnVOPnQ+xEUCSlEkKIwdCvBGvTpk388Ic/pLS0lLfffpsxY8YAkJ+fz9e//vVB7aDWnE6nZzhRBKbu91DeR6EliUMxJDpsUFGA0r0HVUU+SnOlTzM12IialENF8HgS5t6ILu187z2oXK5h7LQYreQ6KLQ2kBjsy2P6lWBFRUXx/PPP+xxft25df07nV6TIxcgji2qFP5A4FAOmOglrPkZE4wEimg4S0XiA0ObjKHgnRyo6Wk0Z2KOmdu1DNYW2yHScLjh8+DCT6oLRN8oeVGL4yXVQ9FVzhwtUiAzRDcr5/LrIRbeWlhZOnDhBR0eH1/GsrKyBnFZTUuRi5JFFtcIfSByKPrOWQ9fIlFKeD5WFKJ12n2aqKQVSct37UKXMhsQsQoIjCAFiTmknMSi0JjEozqbd4eJgpZWCMgt7yywUlDZxrL6F+xdN5O4LJgzKc/h1kYva2lpuv/12Pvjggx7vH0lDv7IQc2SQRbXCH0gcirNqaYBjH8PRPCjeBo0lvm2CIyFlJqTMhtTZkJKLEumu7debFVQSg0JrEoMCQFVVjte3UFDa5Ll9UWGlw+k7XbnS0jao8eK3RS7+3//7f1gsFv7zn/8wf/583nnnHaqrq3n88cd55pln+nNKIYQQYnRxtEPpf9zJ1NE8dzEK1JP3K3pImHZKMjUbYieBbnCmygghxHBptHdQUNZEYVcyVVjaRGNLp0+7qHADOWlR5KRFkZ0WRU5qFNERwRr0eGD6lWD985//5M9//jOzZ89Gp9MxduxYvvSlL2EymVi/fj1f/vKXB7ufQgghRGBzuaD6c3cydXQbHN8JjtO2Nomb4q7qN34BjLsYQiK16KkQQvRbu8PJFxVWTyLVPdXvdMF6HdOSTeSkRTEzPYrs1CjGjglHGQEVTfuVYNntduLj4wGIjo6mtraWSZMmMWPGDHbv3j2oHdSaVLsJfFK1SPgDicNRylKGcnQblGxHKdmB0lLndbdqTEDNmA8Z81DHz4PIJO/HD2K8SAwKrUkMjiwul0p5UytFtXaKa20U19g5VN3MgUorHU7Vp31GbDjZqVFkp5rJSYtiSmIkwUG60845tBVN/bqK4OTJkzl06BDjxo0jOzubF154gXHjxrF582aSkpLOfQI/JlUERx6pWiT8gcTh6KDvtBFZt4fIut2YavMJtZd53e/Uh9I8JofmuFlY43JpM45z7z+lAsXVQPWQ9U1iUGhNYjAwdbpUKpsdlDc7KbM6KLM6KG923zrOkHOYghUmjgkmM8bAxBgDmTEGIoN1gAtoxFXXyBd1PT92KPl1FcH77ruPykr3PhuPPvooV111Fa+99hrBwcFs2bKlP6f0G1JFcOSRqkXCH0gcjkCq2lXpbxdKxW6UEzuhYg+KevITWFXRQ8os1Ix5qOMXQEoukfpgIoHkYe6uxKDQmsSgf7O1Ozhaa6eo1kZxjc0zMnWioRWny3dECiBYrzAuNoLMOCMT4iLIjDeSlWomLTrML6f6+XUVwVtuucXzfW5uLsePH+fgwYOkp6cTGxvbn1P6Lal0MzJI1SLhDyQOA1ybFSp2Q9kuKN8N5bvA1sOI05iJMGEBjJ+PMu4SCDX3qsLfcJAYFFqTGNSey6VytM5GQamF/RUWimpsFNXYqLS0nfExxpAgJsQbyYwzkhl/8pYWHUaQPrAK7/htFcEf/ehHrF69mvDwcADCw8OZNWsWra2t/OhHP+KRRx7pz2mFEEII/+DshJovupKpfPet9hBeVf6gq9LfdHeVv9TzIeMyMKdq0mUhhOhJbXO7V8GJwrImmtscPbaNNQYz4ZQkamJ8JJnxRhJMIX45IuWv+pVgrVu3jjvvvNOTYHVraWlh3bp1kmAJIYQIHKoKltKTyVTZLqgs9K3wB2BOh9Tck6XTE7MgONy3nRBCaKC1w8nnFRYKS5vYU9pEwYkmypt8r2WhBh0zUszMSIliUsLJhCoqPPBKovujfiVYqqr2mMUWFhYSExPTwyOG1tKlS8nLy2PhwoW89dZb5zwuhBBiFGuznJziV9Y1OmWv8W0XYoKUWV6b+mKMH/7+CiFED1wuleJam9dmvQermn3WSykKZMYZ3ftLdZVDn5wYiSHApvYFkj4lWNHR0SiKgqIoTJo0ySvJcjqd2Gw27rzzzkHv5Lncd9993HHHHbzyyiu9Oi6EEGKUcHZC9f5TkqldUHfYt50uyD3V79RNfcdkyqa+Qgi/oKoqFZa2rv2lGikobWJvqYXmdt+pfnGRIZ7NememRTEj1UxkqEGDXo9efUqwNm7ciKqq3HHHHaxbtw6z2ey5Lzg4mHHjxjF37txB7+S5zJ8/n7y8vF4fF0IIMQKpKjQd75rm15VMVRaCo4eF21Fj3SNS3clUUhYYwoa/z0IIcYpOp4vj9S0U1dgorrV5ClAU19po6aEmephBz4wUMznpUZ6kKskcKuulNNanBGvZsmUAZGRkcNFFF2EwDDwb3rFjBxs2bCA/P5/KykreeecdlixZ4tVm06ZNbNiwgaqqKrKzs3nuueeYM2fOgJ9bCCFEAGtt6ipA0TXdrzwf7LW+7ULN7mQqpWuaX0ouGOOGvbtCCNGttcPp3py3K4k6Um2jqNbG8Xo7nT1s0gsQpFOYEGckO81MTlo0OWnu9VOBVsVvNOjXGqx58+Z5vm9ra6Ojo8Pr/r7sG2W328nOzuaOO+7guuuu87n/9ddfZ9WqVWzevJkLLriAjRs3cuWVV3Lo0CHi4wdvLnx7ezvt7e2en7tr3cuO44FPdo4X/kDicICcnVD9OUrFbijPRynPR6k/4tNM1Rkg4TzUrkRKTZ4FYyaActofIKPwfZAYFFobjTHY1NLh3k+qxkZx9x5TtXbKm1pRe86jCA/WMz7WvafUhLiIrqp+EaTHhPewbkodVb/PgRpIDPblMf1KsFpaWvje977HG2+8QX19/YA6sHjxYhYvXnzG+5999llWrFjB8uXLAdi8eTPvv/8+L730EmvWrOl7589g/fr1rFu3zuf43r17MRqNg/Y8YvjJzvHCH0gc9pGqEmo7hql2N5G1+UTWF6J3+lbCagtPpiV6Cvaoqdijp9JiykTVd1XBcgFldijbO7x991MSg0JrIz0GWztdFDV2UtTQyZGuW0Or64ztI4MVUk1BpEQGkWo6eRsTpkOnKLgvYs3Q2UxzOewvH7aXMmINJAZtNluv2/YrwXrwwQfZtm0b//u//8utt97Kpk2bKC8v54UXXuCpp57qzyl71NHRQX5+PmvXrvUc0+l0LFq0iJ07dw7a8wCsXbuWVatWeX62Wq2kpaUxffr0Po3ICf/T/UnFjBkzRuQFXQQGicNeaK5Cd2w7SskO99fTNvFVQ6NQk2ehJs/C1fVVFz4GIyAfg52bxKDQ2kiKQYfTxZFaO3vLLBSWWdhbZqWo1oarh1GpJHMoE2IjukajTt5iIqQk+nAbSAx2z27rjX4lWH/961959dVXmT9/PsuXL+fSSy8lMzOTsWPH8tprr3HzzTf357Q+6urqcDqdJCQkeB1PSEjg4MGDnp8XLVpEYWEhdrud1NRU3nzzTebOnXvG4z0JCQkhJCSETZs2sWnTJs8onIxgBT6Xy8Xhw4dRFAWdVAQTGpE49KVztGCq34upfjemut2E24573e/UhdAcMwNr7CwssbNojcw4OdXPBhwuBUqHvd+BSmJQaC1QY1BVVepbXRQ1OtyjU40OjjZ20t7DhK0xYTomxgSRGW0gMyaIDHMQYYbu19rqvjXVcaIJTgzjaxBuA4nBIR/BamhoYPz48YB7vVVDQwMAl1xyCd/97nf7c8oB+eijj/p0XIwu3fu2qWea7CzEMJA4BMXlIMJyCFPdbsx1u4loOoBOPfkXioqC3TwJa+xMLLG52KKmnZzuJwZMYlBoLVBisLXTRXGjgyNdCVVRo4PGNt+pfmFBChOig8iMDiIzxsDE6CCiwwJ7ZG6kG64Y7FeCNX78eEpKSkhPT2fKlCm88cYbzJkzh7/+9a9ERUUNWudiY2PR6/VUV3tPE6muriYxMXHQnudUK1euZOXKlVitVsxmM1lZWTJFMMB1j0bm5OQE/JQEEbhGZRyqKtQXoSvJQ3dsB8rxf6F0eH8CqEaPwzVuPq6My1DHXkJwWDSxQKw2PR7RRmUMCr/ijzHocLo4XOOe6re33D3dr6jW7lOAQq9TmJxgJCvFRHaqmexUMxmxEeh1Ug49kAwkBod8iuDy5cspLCxk3rx5rFmzhmuuuYbnn3+ezs5Onn322f6cskfBwcHk5uaydetWT+l2l8vF1q1bufvuuwfteU51+hTB/fv3yxTBAOd0OikuLkav1/vNBV2MPqMhDhVXJ2HWo0Q0HiCi6QCRdQUEt3mXTXcYTFjjZtEcOwtrXC4d4UnuOzqBojKgbNj7PVqMhhgU/k3rGFRVlbpWF0fqOznS0MGRhk6ONjpo76Esely4jokxwUyMMTBxjIHxUQZCghTACTTQWt3AF9U+DxN+biAxOORTBO+//37P94sWLeLgwYPk5+eTmZlJVlZWn85ls9koKiry/FxSUkJBQQExMTGkp6ezatUqli1bxuzZs5kzZw4bN27Ebrd7qgoONhnBGnn88RMzMfqMuDhUVWg81lU2fRdKeT5U7UNxtns304dA+oWoGfNRx89HSZyBWdFh7vmsYgiNuBgUAWe4Y7C5zeEelSptorDMQmGphVpbu087Y0iQe1QqzUx2ahQ5aWZijSFD3j8x/Px6BOt0Y8eOZezYsf167K5du1iwYIHn5+5KfsuWLWPLli3ceOON1NbW8sgjj1BVVUVOTg4ffPCBT+ELIYQQQ6i10b3/VMVudzJVsRulxXebDjUs2rP/lJp2AaRdCIYwDToshBhNHE4Xh6ptFJY2UVBmYW+ZhaJam89UvyCdwuTESHJSzWSnRZGdamZ8bAQ6meonBpGi9nKV1y9+8Yten/Tee+/td4e0duoUwcOHD5OXlydTBANc93s5adIk+dRWaCaQ4lBxdhBmLSai6QARjQeJaDpIqN136p5LZ6DFlOnefypqCvboqbSHJ4Mif6j4o0CKQTEyDVYMdjhVKpodlFkdnn2nihs76eihql98uJ6JYwzuqX4xBjKiDYTo5Ro1Wg0kBm02G/Pnz8disZxzdluvE6yMjAyvn2tra2lpafEUtWhqaiI8PJz4+HiOHj3apw77o+4pgrW1tTJFMMA5nU4KCgpkWozQlF/Hoa0G3fF/oZT/F6ViD0r1PhRnh08zNWY8ruRcz15Uavx0CJJpNIHCr2NQjAp9jcHmtk6Ka+3etzo7ZY2tPe43FRka5FWEIivFzBijVCIVJw3kOmi1WomLi+tVgtXrKYIlJSWe73//+9/zy1/+kt/85jdMnjwZgEOHDrFixQq+853v9Kmz/kaKXIw8Wi+qFQL8Kw51jlaM9Xsx1e0msi6fcKvvh2IOgwl79FT3LWoq9qjJOINP+Q+lHqg/PHydFgPmTzEoRqeeYlBVVZraXZRZHZRbnZR1jUyVWXsujd4twqCQagoiI8rgKUSRZNSjUxSgFTpaqSipomKYXpsIDMNV5KLXI1inmjBhAm+99RYzZ870Op6fn8/111/vlYwFqu4RrIaGBhnBCnDyqa3wB5rGocsBlQUoR7ejlORB6Wcork6vJmpiFmr6XEiZjZqSC1FjZarfCCPXQqElVVU5UW/nw0/3okYmcLS+haJaO8U1NqxtjjM+LiEyhAnxRibERZAZZ2RCvPtrrDEYRa5Roo8GOoIVExMzuCNYp6qsrMTh8P3H4HQ6ffasCnTySd/I0P0+ynsptDRscaiq0HAUiv8JR/Og5GNot3i3MafDhPkwfj5kzEOJiEX+VBn55FoohktTSwcFpU2eW2FpE40t3R/seFdj0ymQFhNOZpyRzHgjE+LdXzPjjZhCDcPfeTGi9fc62Jf2/UqwFi5cyHe+8x1+/etfM2vWLMA9evXd736XRYsW9eeUfsvpdHqmC4rA1P0eyvsotDTkcWivQynZDiXuUSrF4l2UQg01w7hLUccvQM2YB9EZ3iNU8u9jxJNroRgq7Q4XByutFJSdLIl+rL7Fp51Br5AYoeO89FgmxhuZ0JVQZYwJJ8TQ8x+vEq9iMA3kOtiXx/QrwXrppZc8e1MZDO5PFhwOB1deeSW//vWv+3NKv3H6Gqy9e/fKGqwA110xBvr26YMQg2mw41BxtGFs2IepLh9T7W7CrUVe97uUIOwx52GNy8UaO4uWqEmgdD3vCSucKBxwH0RgkWuhGAyqqlJld3Zt1uu+lTR14uhhuVSSUe+p3jdxjIE0o46S4iNMmhSDXm8Hl522qmoOVA3/6xCj00Cug0O+BqvbkSNHOHDgAABTpkxh0qRJ/T2V35E1WCOHrDsQ/mBAcaiq0FB8cv+p7k19T19HlXAeasY89whV+lwIjhjEVyACnVwLRX80tnRQWGqhsKyJglL3/lJNrZ0+7aLDDWSnRp2yYa+ZqHDvCn4Sg0Jrfr0Gq9vEiROZOHHiQE7h92Su+sgg6w6EP+h1HNrroDwfynZB+S4o3w1tTb7tTCkwfgFMWAAZl6EY42UdlTgruRaKs2l3OPmiwuq1dup4D1P9goN0TE82kZMW5bmlx4T3quiExKDQmt+uwRpNZL564JN1B8IfnDEOHW1Qtdc9KlWej1KxG6XxmM/jVX0IJGW5K/wl56KmzIaodFlHJXpNroXiVKqqcqy+hcIyS1cRCgsHqqx0On0nNmXEhpOdGkVOmpns1CimJEYSHKTzauNynbmkejeJQaE1v16DNZLJGqyRR9YdCH/gdDo5fOggobZSIq2HiWg6QETjAcKtxSiq70W71ZhOS9QUzz5UraYMVF1XNS0HcLzRfROil+RaOLpZ211da6Y6OFLfSVFDJ7ZO32TKFKwwcUywZ+1UZowBY7AOcAGNuOoa+aKuf32QGBRaC4g1WCOZrMEaOWTOt9BM11Q/pXwXlOfjKt1FkMP3Aq2Gx0JKLmrXjeRZEGrWoMNiJJNr4ejR3ulkf6WVwjKLZ/3UiYZWn3YhXVP93OumoshJNZMaHTZk+0tJDAqt+eUarJdeeolrr72W2NjYPnUokMk84ZFB5nyLIdfZCpV73WumutdONZ3waqID1KBQlKRsSJkNqbmQMhula6qfrJ8SQ02uhSNPva2dohobRbU2DlU1U1DaxIHKnqf6TYiLICctmpw0Mzlp0UxJisSg1/Vw1qEjMSi05ndrsH73u99x1113MWvWLL761a9y7bXXMnXq1D51TgghAp7LBfVF3slU9X5w+W7ATuxkSMnFlTyLgzYjky9dij44dPj7LIQIWKqqUmFp40h1M0U1Noprbe6kqsZ2yua93mKNwZ4CFNlpUWSlRmEOk017hRgOfUqw/vnPf9LY2Mj777/PX/7yF5544gkSEhK49tpr+epXv8oll1yCTje8n4QIIcSQs9WcVtVvD7RbfNtFxEPqbEiZ5R6hSjk51U91OmktKAC9/IEjhOhZp9PF8foWnySquNZGS8eZF9inRoeRGW9kYryRrFR3UjWUU/2EEGfX5yIX0dHR3HLLLdxyyy10dHTwz3/+k7/85S/cfPPNtLa2cvXVV3PttdeyePFiIiICfw8WqXYT+KRqkegT1eXeY+r4v7rWT+WjWEp9mwWFQVI2asqsk1X9zKneVf3AU9lP4lBoTWLQf7hcKkW1Ng5UNlNUa6O41k5xjY1j9S04XD0vjQ/SKWTERjAhLoIJcUYy491fx8dGEBbsO3WpN1X9hpvEoNBaQFQRDA4O5qqrruKqq67il7/8Jbt27eIvf/kLP/7xjzlw4AAPP/zwQE6vCakiOPJI1SJxLsEtlZhq84mszcdUt4egTqvX/SoKbcb0rop+7sp+rZEZoOu6hHYCx+qB+jM+h8Sh0JrEoHYaW51dFfzct6KGTlodPSdSoXqFFJOelMggUk3uW0pkEIlGPUG67g9w7KDa6ayp4VDN8L2OgZIYFFoL+CqCnZ2dGAyBOxVGqgiOHFK1SPhobYRjH6MczUMp2Y7SWOJ1txpshLEXo6bOcY9QJc2E0IFdByQOhdYkBodHS4eD/RVWCrqq9xWUWqi0tPm0CzPomZYcycR4I5lxRibER5AZZyTRFIpONzKn9kkMCq35ZRXBvgjk5OpUUulmZJCqRaOcox1K/wPF2+BoHlTsAU75bEnRQ+r5MGEBjJ+PkpILesOgV/WTOBRakxgcXE6XSnGtjYITTewpbaKgtInD1c04T5vmpygwKT7SXXQiPYrs1CgmJRgJGuYKfv5AYlBoze+qCAohREBwuaD6c3cydXQbHN8JjtP2gImbAuPnw/gFMPaiAY9QCSFGvmprGwVdiVTBiSb2lVuwtftWD00whXRV8IsmJy2KGalmjCHyJ5cQo4X8axdCjAxNpScTqqPboaXO+35jgjuZGj8fxs8DU7IWvRRC+DlVVam2du0tVeMuQuH+3k6drd2nfXiwnhkpZnLSo5jZlVQlmmUrBiFGsxGRYC1dupS8vDwWLlzIW2+95Tn+3nvv8cADD+Byufj+97/Pt771LQ17KYQYNB0tUFl4yj5U+XB6pT9DBIy7xJ1QTVjgHrGSksVCiC4Op4vSxlZPKfTuzXqP1tho7mFUCkCnwKSESM/+UjnpUUyMj0Q/QtdMCSH6p98J1scff8wLL7xAcXExb731FikpKfz2t78lIyODSy65ZDD7eE733Xcfd9xxB6+88ornmMPhYNWqVWzbtg2z2Uxubi5Lly5lzJgxw9o3IcQAuZxQd/hkIlW+C6q/APW0cqmKHlJyPeuoSJkNQcGadFkI4T/aOp0crbV7RqKKu5Kpkjo7Hc6eS5nrdQpjY8KZEG/07C+VGW9kQpyRCJnqJ4Q4h35dJd5++21uvfVWbr75Zvbs2UN7u3vI3GKx8OSTT/K3v/1tUDt5LvPnzycvL8/r2Geffcb06dNJSUkBYPHixXz44Yd8/etfH9a+CSH6qLnq5Ia+ZbugogA6mn3bRSa5E6qUXPfmvskzISRy2LsrhPAfzW2d7CuzsKe0icLSJg5WNVPa2MKZ6iWHBOm69pTyvo0bE0Fw0OgrQCGEGBz9SrAef/xxNm/ezG233cYf//hHz/GLL76Yxx9/vE/n2rFjBxs2bCA/P5/KykreeecdlixZ4tVm06ZNbNiwgaqqKrKzs3nuueeYM2fOWc9bUVHhSa4AUlJSKC8v71PfhBBDrN0GlQXukanuESprD/9ODRHuBCplljuZSpkN5hTfdkKIUcPhdHGwqtldCv2Eu/BEUa2tx2TKFBrklUBNjI8kM95ISlTYiC2JLoTQTr8SrEOHDnHZZZf5HDebzTQ1NfXpXHa7nezsbO644w6uu+46n/tff/11Vq1axebNm7ngggvYuHEjV155JYcOHSI+Pr4/3e9Re3u7ZyQO3LXuAdlxfASQneP9hMsJtYdQKnZB+W6U8nyoPYCiek/RURUdxE1BTcmF5FmoKbMhbvLJTX27Bdj7KXEotBbIMaiqKhVNbRSUNVFYZqGw1MLnFRbaOn2n+KVEhZKdGkV2mpnpySYy44zEGoNReliDqaquQLuUBLRAjkExMgwkBvvymH4lWImJiRQVFTFu3Div4//6178YP358n861ePFiFi9efMb7n332WVasWMHy5csB2Lx5M++//z4vvfQSa9asOePjkpOTvUasysvLzzrqtX79etatW+dzfO/evRiNxt68FOGnZOd4bRhaa4loOkBE40Eimg4S3nQIvbPVp11HaCz2qKnYo6dij5pCS9RkXEFhJxtUdkLl58PY86EhcSi0FkgxaO90UdTQSVFDJ0caOjlS30lTu28yFR6kkBljYOIYAxNjDGTGGIgO7X5tFrBaKLeCzF/xD4EUg2JkGkgM2my2XrftV4K1YsUK7rvvPl566SUURaGiooKdO3eyevVqHn744f6cskcdHR3k5+ezdu1azzGdTseiRYvYuXPnWR87Z84cPv/8c8rLyzGbzfz9738/a9/Wrl3LqlWrPD9brVbS0tKYPn36OXdrFv6t+5OKGTNmyAV9qHTYUCoLUSry0VXsRqnYjdJc6dNMNYSjJs1ETZmFK2mWe5QqMokIIGL4ez2sJA6F1vw1BjudLg5X29hbbqGwzEphmYWjdXafqX5BOoXJCUayU81kpZrJSjUxfkyETPELIP4ag2L0GEgMds9u641+JVhr1qzB5XKxcOFCWlpauOyyywgJCWH16tXcc889/Tllj+rq6nA6nSQkJHgdT0hI4ODBg56fFy1aRGFhIXa7ndTUVN58803mzp3LM888w4IFC3C5XHzve987awXBkJAQQkJC2LRpE5s2bfIMA+7fv19GsAKc0+mkuLhYdo4fLKqTsOZjRDQecI9MNR4krPkYCqdN9UNHq2mc1+hUW+RYd7U/gE7gWB1Q5/MUI5HEodCaP8SgqqrUtjg50tDJ4Xr3CNXRxk46eijmFx+hZ2KMwXPLiDYQolcAB1BPa1U9+6uG+xWIgfCHGBSj20BicMhHsBRF4aGHHuLBBx+kqKgIm83GtGnTNEtEPvroox6PX3vttVx77bV9OtfKlStZuXIlVqsVs9ksI1gjgHxiNkBtVpQT/0ZX9pl7ZKqyAKWzxaeZGpmMmjwLV0ouavIs1MQsgoKNmAHz8Pfa70gcCq1pEYPW1k72lbtHpbpHqOrtHT7tTKFBZKW4R6WyU81kpZgZY5RtFkYauQ4Krfn1CNarr77K+eefz9SpU5k2bZrneFtbG2+88Qa33XZbf07rIzY2Fr1eT3V1tdfx6upqEhMTB+U5TicjWCOPfGLWN4qrk4jGA0TW5mOqyyei6aBPIQqnPgx71OSukamptERPoTM09mQDC2ApGd6O+zmJQ6G1oY5Bh0vluMXBkfpOjjR0cKShk/Jm30XhQQqMjQpiYkywZ+1UklGPTlGAVuhopaKkiopB76HQmlwHhdb8egTr9ttvJyIigi1btvC1r33Nc9xisbB8+fJBS7CCg4PJzc1l69atntLtLpeLrVu3cvfddw/Kc5xORrBGHvnE7BxUFaX2IMqx7ehKtqOc+MRnhEqNGY8r/WLUlFzU5FzUMRMJ1ekJBWTr7t6ROBRaG8wYdLlUyppa2VtuZW+Zhb1lFvZXNtPu8J3rlxYdRnaquWtkysS0pEhCDPJvYDSS66DQml+PYAGsW7eOW2+9lX379vHYY4/19zTYbDaKioo8P5eUlFBQUEBMTAzp6emsWrWKZcuWMXv2bObMmcPGjRux2+2eqoKDTUawRh75xMyXoa2OyNrdmGp3YarbjaG9wev+zmAzzbGzsMbl0hw7i47wU0aMqzqh6oth7nHgkzgUWutPDHa6VCqbHZQ3OymzOiizOihvdt86eqhYbDScWtUvmMwYA+aQ7g17LWCxcNgyeK9JBBa5DgqtDdcIlqKqZ9rf/Mx0Oh1VVVUcPXqUpUuXcvHFF/Pb3/4Wq9VKcnJyn+rE5+XlsWDBAp/jy5YtY8uWLQA8//zzno2Gc3Jy+MUvfsEFF1zQ1273SfcIVm1trYxgBTin00lBQQE5OTmj94LebkM58Qm6kjyUYzvQ1R3yulsNCkVNuxBXxnzUjMtQ46eDouv5XKJfJA6F1s4Wg7Z2B0fr7BTXnrwdrbVzorEVp6vnPxMMeoUpiZFeo1PjxoT3uN+UECDXQaG9gcSg1WolLi4Oi8VyztygXwmWXq+nsrKS+Ph4Tpw4wbXXXouiKGzevJmLLroooDeQO3UE6/Dhw+Tl5ckIVoDrfi8nTZo0ei7oLgcRTQcx1eYTWbcbY+MXKOrJf5cqCi3mSe4RqrhZ2KLPQ9XLgvKhNCrjUPgVh8PBngNFhMaNpcKuekajyqwO6lt7KOPXJSxIIdUUREqknlRTkOcWH6FHL8mU6AO5DgqtDSQGbTYb8+fPH7oEq3sEKz4+HoCWlhZuvvlmtm7dit1uD+gEq1v3CFZDQ4OMYAW4Ef+JmaqCtRzK81Eq8lHKd0MPlf7U6HGoGfNRM+ZBxmUQFq1Nf0epER+Hwu802DsoLLNQWNpEYZmFfeUWGls6z9g+1hjMhDgjmXERTIjv+hpnJMEUIqNSYlDIdVBobaAjWDExMb1KsPq1BuvRRx/1GtUJDw/nnXfe4dFHH2XHjh39OaUQorfarVBRgFKej1KeDxW7UWy+m8GoYdGo4y6D8fNQM+ZD9Ljh7qkQYpi0dzr5orKZgrIm9pZZKCi1cKLBdzsFBUiJCu1KoIxkxhuZEBfBhLgIosJlFFsIIQZDv0awRjKZIjjyBPSUBJeTsOaSrk19DxDReIBQ2wkUvP/ZqoqO1sjxng197dFTaDOOlXVUfiSg41D4FVVVqbQ5OdzQyZGuzXqPNXXi6OF/82Sj3lMKfbxZT1tNCedNkRgU2pDroNDacE0R7PUI1l/+8hcWL16MwWDgL3/5yxnbKYrCNddc0/ve+pnTy7RnZWXJFMEA1z1l1e+nJKgqWMqgIv/k6FRlIYqj1bepOQ01eRak5KKmzIakLEIM4YQAMcPfc9ELAROHwu/U2zvYW9pEQZnFvWFvmQVLq+9Uv5hwA9lpUWSnmslJiyIr1Yw5zOC53z01xiAxKDQj10GhtYHE4JCUaV+yZIln3VX3nlQ9URRlRKzBEmLIqSrUH0Ep2Y5SsgPK/otir/FtFhIJybPc+0+l5ELKLDAmaNBhIcRQ6nS6ONHQQlGNneJaG4eqm9lbZuFEg++HLCFBOqYnm9wV/NKiyEk1kxodJmulhBDCD/Q6wXK5XD1+P9Kcvg/W3r17ZYpggOseDgY0/8QsqL2ByNo9mOp2YardTXBbrdf9qqKnxTQee9RUWqKnYI+aSpsx7eRUv1agqBKoHPa+i4HxpzgU2mpzuKjo3leqq4pfudVBlc3Z4zQ/gJRIPRNj3HtLTRxjYKw5iCCdArSDq5r6E9XUnzj780oMCq1JDAqtDSQG+7IPVr83Gj5dU1MTUVFRg3U6zcgUwZFH0ykJHXY48Yl7lOpoHkqN9wa9qj4Y0i5EHT8fNf1CSMwm1BBGKDBmeHsqhphMjRl9Gls6KK6xU1Rro7jW5hmZKm9qO+NjwoP1jI+NIDPeSGZ8BDNSzGSlmDGdMtWvvyQGhdYkBoXW/G6K4Kl+8pOfMG7cOG688UYAbrjhBt5++22SkpL429/+RnZ2dn9O65dkt/GRoft9HPL30uWEij1wdBsU50Hpf8B12lqJxCwYPx8mLEBJuxCCw5FJPaPDsMWhGFatHU72lVvYX2GhqMbmudXbO874mJiIYDLjjO5qfqfckkyh6HRDd0WQGBRakxgUWutvDPalfb8SrM2bN/Paa68B8I9//IOPPvqIDz74gDfeeIMHH3yQDz/8sD+n9UtOp1PWlAW47vdwSN5HVYWGo54RKo5/jNJm8W5iTnWXSR8/3102PSL29A4Ofr+E3xnSOBTDxuVSKaq1UVjqLjhRWNbEoWobTlfPc/uSzKGefaUmxEV4kqoxET2XRFdV15BdEiQGhdYkBoXWBhKDfXlMvxKsqqoq0tLSAHjvvff4n//5H6644grGjRvHBRdc0J9T+g1ZgzXyDOqcb1UluLWaiMb9mOr2EFm7i5BW78IUDoOR5jEzscbNojk2l/aIFFAU6ASOlAFlA+uDCEiy9iAwNbZ2lUNvcJdDL2ropLWHhVLRoToyYwykmYJIMQWRGhlEiklPWFD3Vgnt7pulgVILlA7rq3CTGBRakxgUWvPrNVjR0dGUlpaSlpbGBx98wOOPPw649+YI9E8lZA3WyDOgOd9tFqjY472p72mV/lSdAdIuQM2Yhzp+PkpSDiadHokacSpZe+D/WjocfF5upaDMwt6yJgpKLVRafNdLhRn0zEgxear3ZaWaSTKH+n0FP4lBoTWJQaE1v16Ddd111/GNb3yDiRMnUl9fz+LFiwHYs2cPmZmZ/Tml35J5wiNDr+bbOjuh+nMoz4eyfCjfBXWHfdvpgiDhPBh3CYxfgDJ2LgRHyDoqcU6y9sB/OF0qRTU2CkobKSh1J1OHq5t9pvrpFJiUEEl2ahQ56VHkpEUxMd5IkD4wN/GWGBRakxgUWvPbNVg/+9nPGDduHKWlpTz99NOeKXSVlZXcdddd/TmlEMNLVaHpuHcyVVkIjh6qe0WNhdTZkDIbUnIhKQsMYcPfZyFEv1Vb29hzoqkrmWpkX5kFe4fvjItEUyg5aVHu0am0KGakmjGGDFrBXSGEEKNAv/7XMBgMrF692uf4/fffP+AOCTEU9J02KP4nVBa4k6myXdBS59sw1OxOolJmdyVVub5FKYQQfs3e7mBfucWdTJ1oorCsqcepfhHBemakmslJiyYnzf010RyqQY+FEEKMJPKxnBh5HB0np/qV56Mr20VO/RHfdjoDJM5wJ1HdI1RjJrgLUgghAoLTpXKkppkCz+hUE4ermzm9qF/3VL+ZXdP8ctKiyYw3oh/CkuhCCCFGJ0mwzkHKifq5rql+Srl7mp9Sng9V+1Cc7Z4m3X8+qdHjUFNmQ/Is1JRcd3IVdNqn1S7X8PVdjCpSnnhwVFnaKOwqQFFY1sS+cistPU31M4eSk2r2FKKYnmwi4vSpfkNYEt0fSQwKrUkMCq35dZn2kUzKtPs3fYeViKaD7lvjAcKbDmLosPi0cxhM2KOmYI+eQrNpEnvrDYydOuvkAsU6oO7g8HZejGpSnrhv7J0uyq0OyqwOypqdlFkdlDR20tDm+yFIaJBCZrSBiWMMTIxx32LCun/HFrBYOOJ7mRh1JAaF1iQGhdaGq0y7oqpqz7sjjnLdZdobGhqkTLtWHO1Q/fnJ8ujl+SgNxT7NVH0wJJznHp1KyUVNngUx4z1T/ZxOJwUFBVIWVmhK4tCXqqrUNrdTVGunuNZGcY2dolobxbV2aprbe3yMXqcwKd5IdpqZnNQostPMTIiTqX69ITEotCYxKLQ2kBi0Wq3ExMRgsVjOmRvICNY5SCnRYdTZCid2wtE8OPZvqNoLzg7fdjHjTylCMRsl8TwICjlrmXQpCyv8wWiNQ6dLpbyxlSM1zRTV2Ny3WhvFNTasbY4zPi7BFEJmvJHMOCOZ8UYmJ5o4L8VEeLD819VfozUGhf+QGBRa86sy7dHR0b3eRLGhoaHXHRCjmMsFVYXuhKp4G5z4FJynfWodFuNdhCJlFoTHaNJdIcS5Wds62du1PupglTuhOlpro93R8/pGnQJpMeFMjDcy4ZRkakK8EVOoYZh7L4QQQgxcrxOsjRs3DmE3xKjReOxkQlWyA1pPS8Yjk2D8Ahg/D9LmQHSGVPUTwk91Ol0cqmpmT1c59ILSRopr7T22DQ7SMT42wiuJyow3khEbQahBPskWQggxcvQ6wVq2bNlQ9mNI/PSnP+Xll19GURTWrFnDLbfconWXRp+WBncidTTPfWss8b4/OBLGXQITFsD4+RA7SRIqIfyQqqqUNbZ6SqEXlDbxebmlx5Gp1OgwctKimJ5sZmJXIpUWEy7rpIQQQowK/Z7IXlxczMsvv0xxcTE///nPiY+P5+9//zvp6elMnz59MPvYL/v27eP3v/89+fn5qKrKggUL+MpXvkJUVJTWXRvZHO3uqX5H8+DoNqgoAE6po6LoIfX8kwlVSi7oZRqQEP7G0tpJYVci1f213u67JjIyNKhrXyn3LTstilhjiAY9FkIIIfxDvxKs7du3s3jxYi6++GJ27NjBE088QXx8PIWFhfzmN7/hrbfeGux+9tmBAweYO3cuoaHufY6ys7P54IMPuOmmmzTu2QjjaIeqfXD8E3dCdXwnOFq928RNcSdT4xfA2IsgVKoyCuFPbO0OjlQ3s6/c4p7qV9bE0R6m+hn0ClOTTO5EKjWKnPQoMsZEoJORKSGEEMKjXwnWmjVrePzxx1m1ahWRkZGe45dffjnPP//8oHRsx44dbNiwgfz8fCorK3nnnXdYsmSJV5tNmzaxYcMGqqqqyM7O5rnnnmPOnDkAnHfeeaxbt46mpiZUVSUvL49JkyYNSt9GLVWFhqNQtgvKd0HXpr4+lf6MCScTqvHzwJSsSXeFECepqkq9veNkFb8aG8W1No5U26iytvX4mPSYcK+RqenJJlkvJYQQQpxDvxKs7ul3p4uPj6eurm7AnQKw2+1kZ2dzxx13cN111/nc//rrr7Nq1So2b97MBRdcwMaNG7nyyis5dOgQ8fHxTJs2jXvvvZfLL78cs9nMhRdeeNbyiu3t7bS3n6xgZ7VaAUb3juMt9VC+G6V8F0pFvvv7tiafZmpYDKTORs2Yh5oxD+Kmeq+j0vj3JzvHC38wXHHocqlUWFopqnHvLXXqHlNNrZ1nfFysMdg9OpVqJjstiqxUM2Mign3ayb+jwCXXQqE1iUGhtYHEYF8e068EKyoqisrKSjIyMryO79mzh5SUlP6c0sfixYtZvHjxGe9/9tlnWbFiBcuXLwdg8+bNvP/++7z00kusWbMGgO985zt85zvfAeBb3/oWEydOPOP51q9fz7p163yO79mzB6PROJCXEhAUZwfh1iKMTQcxNh0kwnKQ0JZKn3YunYEWUya2qCnYoqZgj5pKe1jiyYSqtBVKdw9z78/O6XRy5MgRnE6n7LshNDPYcehwqVTZnJQ3OylvdlDW7P6+otlB+xn+D1CAuHAdKZFBpJj0pEbq3d9H6jEG67paWcFm5dhBODbgXgp/ItdCoTWJQaG1gcSgzWbrddt+JVg33XQT3//+93nzzTdRFAWXy8W///1vVq9ezW233dafU/ZJR0cH+fn5rF271nNMp9OxaNEidu7c6TlWU1NDfHw8hw4d4rPPPmPz5s1nPOfatWtZtWoVL774Ii+++CJOp5OioqIhfR2aUVVC7WVEWLqSqaaDhFuPolN9N/xsjUjDHjUZm3kK9qgptJjGo+oCqyiFoiioqtrrfdyEGAoDiUNVVamyOznS4KCooZOiRgfHLA7OsLUUegWSjHpSuhKoVJP7+2RjECFB8u9gtJJrodCaxKDQ2nDFYL8SrCeffJKVK1eSlpaG0+lk2rRpOJ1OvvGNb/DDH/5wsPvoo66uDqfTSUJCgtfxhIQEDh486Pn5q1/9KhaLhYiICF5++WWCgs78ckNCQggJCeGBBx7ggQcewGq1YjabycrKwmQaAUUZrOXoSvJQSnagO7YDpcV3KqcaHouaPAtX8izU5FmoSTPRh0VhAgL5N+B0OlFVlezsbPnETGimL3HYYO9gb7mFwjIre8ss7C23YGn1/QAkPFjv3lsqzvuWFh2GQa/r4cxiNJNrodCaxKDQ2kBisHv5UG/0K8EKDg7mxRdf5OGHH+bzzz/HZrMxc+bMs07B08Kpo1m9tWnTJjZt2uSZZ7l///6AnCKo67QRWVeAqS4fU20+ofYyr/tdumBazBOxR0/FHjUVe/QUOk6d6mcHikqB0mHv+2BzOp0UFxej1+vlgi40c6Y47HCqHGvq5HBDJ0fqOznS0Em13XeOn0EHGVEGJo4xMDHGfYuP0KNTFNxbIdjAYcNeCQd9Z/cKIddCoTmJQaG1gcTgkE8R7Jaenk56evpATtEvsbGx6PV6qqurvY5XV1eTmJg4oHOvXLmSlStXekawpk+fHhgjWM4OlPJd6Ep2oBzbjlKxG0U9OX9IVXTuUalx83BlXIaaMptgfTDBQLR2vR4W3YsZZ8yYIRd0oRmn00mnw4ExeQKfVzRTWGalsMzCoepmOp2qT/uM2HCyU81kpZjJTjUxOSGS4CAZlRL9J9dCoTWJQaG1gcTgkIxgrVq1qtcnffbZZ3vdtj+Cg4PJzc1l69atntLtLpeLrVu3cvfddw/o3AEzgqWqhDYfw1S7C1Pdboz1heid3qWW2yJSscblYo3NxRabg9PQ9TosgOXQ8PdZI/KJmdCKpd1FUUMHh+s7OVLfwaG6dtpcn/q0M4XomBhjYFKMgcyum7vohAo0QUMThxqGu/dipJFrodCaxKDQmt+NYO3Zs8fr5927d+NwOJg8eTIAhw8fRq/Xk5ub2+snPxubzeZVZKKkpISCggJiYmJIT09n1apVLFu2jNmzZzNnzhw2btyI3W73VBXsr9NHsPxqDZa1AqVkO5RsRynZjmLzHsFTw2PdZdLHz0PNmI/BnMoYYIwmnfUf3clyTk6OXNDFkGnvdLK/0kphqYWCsiYKSy2UNp626TY6QoJ0TE82kZNmJjs1iuxUM6nRYbLoWww5uRYKrUkMCq0NJAaHZARr27Ztnu+fffZZIiMjeeWVV4iOdk8wa2xsZPny5Vx66aV96OqZ7dq1iwULFnh+7h5BW7ZsGVu2bOHGG2+ktraWRx55hKqqKnJycvjggw98Cl8EvOZKlE9+jnI0D6XusNddalAYjJ2LmjEfNWM+JEwDRaYQCTHUXC6Vkno7hWUWd0JV2sTBqmYcLt+pfplxEWSlmslKMRFqr+Kay3IJDQ6sSpxCCCGE6D1FVVXfvwjOISUlhQ8//JDp06d7Hf/888+54oorqKioGLQODrdTpwgePnyYvLw8TacI6tst5Hy4FAAVhZaoSVhjZ2ONm4U9ejqq3ncjUOGt+72cNGmSfGIm+sXS5uRIQ6fnVtTQib3T99Jp7prq112IIjPGQITB/aGHxKHQmsSg0JrEoNDaQGLQZrMxf/58LBbLOWe39avIhdVqpba21ud4bW0tzc3N/Tml3/DHIheOtjWosZNQx16CISxapv31kSyqFb2lqirV1naKa+0crrFRWGZhb5mFsqY2n7ahBh3Tk0xkpZrITjWTnWom2Rx6xql+EodCaxKDQmsSg0Jrflfk4lRLly5l+fLlPPPMM8yZMweA//znPzz44INcd911/Tml3/DLIhfmK6ATKCoDys7VWpxGFtWK0zldKtV2J+XNDsqs3Tf3z60O35EpBUgxBXnKo0+MMZBuDiJIpwAdoNbSUFpLw1l2NZA4FFqTGBRakxgUWhuuIhf9miLY0tLC6tWreemll+js7AQgKCiIb37zm2zYsIGIiIi+ntLvdI9gNTQ0aD6CJQbG6XRSUFAgi2pHobZOJyV1dopq7RTX2CiutVNUa+NYnZ2OHkqjA+h1CmNjwsmMNzIjxUR2WhRZKSYiQwe2bkriUGhNYlBoTWJQaG0gMWi1WomJiRm6KYLh4eH88pe/ZMOGDRQXFwMwYcKEEZFYnU4+ZRkZut9HeS9HJnu7g4NVzRTX2CiqtVFU476VNrZwpo+QQg06xscayYw3MjHe/TUz3sjYMRFDtt+UxKHQmsSg0JrEoNBaf2OwL+0HtNFwREQEMTExnu9Hou65miJwdb+H8j6ODE6XSlGNjYLSJncVvzILh6ub6aGAHwDmMAMT4iLIjDMyIT6CCXFGMuMiSIkKQ6frab2UOiSxInEotCYxKLQmMSi0NpAY7Mtj+pVguVwuHn/8cZ555hnPfMTIyEgeeOABHnroIXS6wC0VfvoarL1792q/BksMSHfFGOjbpw/CP9S3OjlSf7KCX3FjJ209rJOKCtWRZgoizRREiimI1Eg9qaYgzCG6rsITDsACrRbqT0D9ieF9HRKHQmsSg0JrEoNCawOJwSHZaPhUDz30EL/5zW946qmnuPjiiwH417/+xWOPPUZbWxtPPPFEf07rF/x6o2HRL7KxYeCwtzvYV27xjEwVljZRZW33aRcRrOe8FLPXZr2J5lANetx7EodCaxKDQmsSg0JrfrfR8KleeeUVfv3rX3Pttdd6jmVlZZGSksJdd90V0AnW6WSe8Mggc779j9Olcri6mYLSJgpONFFY1tTjVD+dApMSIpmZHkVOWhQ5adFkxhvR9zi9z79JHAqtSQwKrUkMCq357RqshoYGpkyZ4nN8ypQpNDQ09OeUfkvmCgc+mfPtHyotbRSWNlFQZmFvWRP7yq20dPi+J0nmUHLSzGSlRpGTaua8FBPhwaddqlQXgfZ2ShwKrUkMCq1JDAqt+fUarOzsbJ5//nl+8YtfeB1//vnnyc7O7s8p/YaswRp5ZM738GvtdFHU2LVuqr6TooZOGtpcPu3CghQyu/aVyowxMCnGQHRY93tkAYuFw5bh7ftQkTgUWpMYFFqTGBRa8+s1WE8//TRf/vKX+eijj5g7dy4AO3fupLS0lL/97W/9OaXfkDVYI4/M+R5aDqeLIzU2CkotFJY1UVBqoajW5lMeXa9TmJxgJDs1qmuEysyEuMCc6tcfEodCaxKDQmsSg0Jrfr0Ga968eRw+fJhNmzZx8OBBAK677jruuusukpOT+3NKvyXzhEcGmfM9OFRVpdLS5l431bV2al+5hdZO32HzlKiwrjVTUeSkR3Fespmw4NH9+5c4FFqTGBRakxgUWvPbNVgAycnJI6qYhRDCW4fDxbF6O0U1No5U2/i8wkJBaRO1zb5V/SJDgshKM3uKUGSnmYmP9O+qfkIIIYQQQ6HfCVZbWxt79+6lpqYGl8t7bcWp1QWFEP7N1u6guMZGUY2Nolr31+IaG8cbWnD2sHuvXqcwJTHSMzo1Mz2K8bHGM2zaK4QQQggxuvQrwfrggw+47bbbqKur87lPUZQRVR1Gqt0EPqla5J7aV2/voLjW7k6mau0U17q/Vlnazvg4Y0gQmfERTIgzutdPpUUxPcnkM9VPDcCqfsNN4lBoTWJQaE1iUGjNr6sI3nPPPdxwww088sgjJCQk9OcUfkuqCI48o61qka3DRXFjJ8ctDsqtDsqsDsqaHdg6fEejukWF6EgxBZFq0pNqCiI1MohUUxDRoToURQGcgAUaLRxqHLaXMqKMtjgU/kdiUGhNYlBobbiqCCqqenqtr3MzmUzs2bOHCRMm9PWhAaO7imBDQ4NUEQxwTqeTgoKCEVm1qMPh4mBVs6d6X2FZEyV1LT22VRRIjQpjQpyRCfERZMYZmRAXQWa8EXOYYZh7PvqM5DgUgUFiUGhNYlBobSAxaLVaiYmJwWKxnDM36NcI1vXXX09eXt6ITrC6SaWbkWEkVC1SVZUTDS0nK/iVNrG/wkqHw3d/qfSYcKYnm8iMN3pu42ONo76Kn9ZGQhyKwCYxKLQmMSi05rdVBJ9//nluuOEGPv74Y2bMmIHB4P3p97333tuf0wohTtHU0uFJpApLmygss9Bg7/BpZw4zkN1dcCItiuy0KGIigjXosRBCCCGE6FeC9Yc//IEPP/yQ0NBQ8vLyutZouCmKIgmWEH3U7nByoLKZghON7oSqzEJJnd2nnUGvMC3ZTE6qmZx0d0n0cWPCvf4NCiGEEEII7fQrwXrooYdYt24da9asQafTDXafBs3PfvYzfv3rX6OqKosWLeLnP/+5/CEqNNXS4aC4xk5RbbO7LHrX7Xh9C44eSqKPGxPuKYeenRbFtGQTIUEyrUIIIYQQwl/1K8Hq6Ojgxhtv9Ovkqra2lueff579+/djMBi47LLL+PTTT5k7d67WXROjQKO9w7On1JFq9/5SxTU2yptaz/iY6PCTU/1y0qLITo0iWqb6CSGEEEIElH4lWMuWLeP111/nBz/4wWD3Z1A5HA7a2tx7/HR2dhIfH69xj8RIoqoqlZa2kyNRp2zSW9/DWqluMRHB7gp+XcUnJnZ9TTKHygirEEIIIUSA61eC5XQ6efrpp/m///s/srKyfIpcPPvsswPu2I4dO9iwYQP5+flUVlbyzjvvsGTJEq82mzZtYsOGDVRVVZGdnc1zzz3HnDlzAIiLi2P16tWkp6cTFBTEnXfeOSqqHoqh4XKplNTbKTjhLjqxt9xCUXUz9o4zbzqXEhXmTqLijF7V/KQAhRBCCCHEyNWvBGvfvn3MnDkTgM8//9zrvsH6BN5ut5Odnc0dd9zBdddd53P/66+/zqpVq9i8eTMXXHABGzdu5Morr+TQoUPEx8fT2NjIe++9x7FjxwgLC2Px4sXs2LGDyy67rMfna29vp7293fOz1WoFkB3HR4D+7Npdb++gsPTk3lJ7yyxY2xw+7fQ6hbEx4Z79pLq/jo+NICKk539eEk+j00B2jxdiMEgMCq1JDAqtDSQG+/KYfiVY27Zt68/D+mTx4sUsXrz4jPc/++yzrFixguXLlwOwefNm3n//fV566SXWrFnDRx99RGZmJjExMQB8+ctf5tNPPz1jgrV+/XrWrVvnc3zv3r0YjcZBeEVCK+fatbvdqVLS2MmRhpO3GrvvP6JgHWREG5gY476NjTKQaNRj0HV/qNACtOCsreVI7RC+IBGQBrJ7vBCDQWJQaE1iUGhtIDFos9l63bZfCZbWOjo6yM/PZ+3atZ5jOp2ORYsWsXPnTgDS0tL45JNPaGtrw2AwkJeXx7e//e0znnPt2rWsWrXK87PVaiUtLY3p06efc7dm4d+6P6mYMWMGiqKjpL6FwjILe8ssFJZZOFRt67GC34S4CLJSTGSnmslONTMpwYhB77+FXYR/OzUO5Q8LoQWJQaE1iUGhtYHEYPfstt4IyASrrq4Op9NJQkKC1/GEhAQOHjwIwIUXXsjVV1/NzJkz0el0LFy4kGuvvfaM5wwJCSEkJIRNmzaxadMmzzDg/v37ZQQrgFnanBysa+c/RdU07GymuNFBS6dvMhUVomPiGAOZXaNTmTEGIgw6QAWacNU3cbB+2LsvRhCn00lxcXG/do8XYjBIDAqtSQwKrQ0kBkf8CFZvPfHEEzzxxBN9eszKlStZuXIlVqsVs9ksI1gBpLXDyReVzRSWWSgsd49QlTe1dd0bBnQCEGrQMT3p5MhUVqqJZKngJ4aYfHIrtCYxKLQmMSi0JiNYZxEbG4ter6e6utrreHV1NYmJiQM6t4xgBQaXqlLe7ORIQwdH6t3rpo5bHJw+008BUk164nQt5I4bw+S4ENJMQQTpFKAd1BoaSmtoKNXiVYjRRD65FVqTGBRakxgUWpMRrLMIDg4mNzeXrVu3ekq3u1wutm7dyt133z2gc8sIln+qbW5nb7mFwjIrhWUW9pVbsbX7VvWLMwaT1TUylZ1qZkayiTCDQkFBATk5OXJBF5qRT26F1iQGhdYkBoXWRv0Ils1mo6ioyPNzSUkJBQUFxMTEkJ6ezqpVq1i2bBmzZ89mzpw5bNy4Ebvd7qkq2F8ygqU9e6eLExYHR+o7OdzQyZGGDupaXD7tQvQKE6KDyIwxMGlMMBNjDIwJ03VN9bOB3UbJkXL5xEz4BYlDoTWJQaE1iUGhtVE/grVr1y4WLFjg+bm7wt+yZcvYsmULN954I7W1tTzyyCNUVVWRk5PDBx984FP4oq9kBGt4qKpKra2D4lq71+1onY2a5g6f9ooCE+OMZKWaPCNUE+MiCOpFVT/5xEz4A4lDoTWJQaE1iUGhtVE/gjV//nxU1bfa26nuvvvuAU8JPJ2MYA0up6pSa3dSZnVQ1uyk3Oro+r7nan7dYsJ0ZEYbmDimq6pftIEwgw5wAY04ahs50Mu9puQTM+EPJA6F1iQGhdYkBoXWRv0IllZOH8HKysqSEaxeaHe4OFZnp6jWRnFN19daOyV1dtodvtP7AHQKpMWEkxkXwYQ4IxPiI8iMMzIhLoLIUMOg9a07WZY1WEJLEodCaxKDQmsSg0JrA4nBETGCJfyXw+niSI2NglJ3KfSCsiaKamw+Ffy6BQfpGB8bwYS4rgSqK5EaNyacEINcYIUQQgghxMghCdZpTp8iuHfv3lE9RVBVVepbXRxp6ORwfQdFDZ0UNzpod/pmU+FBCimmIFJNQaRG6j3fx0fo0Xv2mLKB00ZrVTUHqobnNTidTg4fPgwgn5gJzUgcCq1JDAqtSQwKrQ0kBmWK4ACM9imCzW0O9pVbKCxr8oxQ1TS3+7QzhuiZkWImJy2KrFQzM1LMJJpC/HKzXpmSIPyBxKHQmsSg0JrEoNCaTBEUQ87hdHG42kZBWROFpRYKyywU1do4vbaIXqcwOcFIdmoUOWlmslLNTIgzotf5XzIlhBBCCCGEliTBOs1InCKoqiqWdpdXJb+Spk6KGzvpcPq2jw3XMTEmmEkx7ip+46MMhAQpQCdQR0tlHfsqh/tV9J9MSRD+QOJQaE1iUGhNYlBoTaYIaiSQpwi6XCoVllaKauwU19ooqu36WmPH0trZ42OMIUGefaVy0qLITjUTFxkyzD0fWjIlQfgDiUOhNYlBoTWJQaE1mSLoJ/xxr4YOh4vj9XaKamzuW63769FaO62dPQxJ4d6oNzU6jMw4I5nxRiYlRDIzPYrxsUZ0o2CqX/f76G/vpRhdJA6F1iQGhdYkBoXW+huDfWkvCdY5dO/4rJXmNgdbD9ZQVOPeV6qoxsaJhhYcZ6iJbtArjBsTQWZ8195SXaXRM2IjCAv2DQxVdaHhyxsW3e+hlu+jEBKHQmsSg0JrEoNCawOJwb48RhKs0/jbGqzGVicPvFfrczw0SCElUt9VEj2IFFMQaaYgEiL0pxSfsINqp6OmhkM1w9tvfyJzvoU/kDgUWpMYFFqTGBRakzVYGvG3NViqqjLvcD7J5jDPBr0T4iJIMof6ZUl0fyRzvoU/kDgUWpMYFFqTGBRakzVYfsIf5gm/cscFmj7/SCBzvoU/kDgUWpMYFFqTGBRaG441WLq+dkoIIYQQQgghRM8kwRJCCCGEEEKIQSJTBM9Bqt0EPqlaJPyBxKHQmsSg0JrEoNCaVBHUiL9VERQDJ1WLhD+QOBRakxgUWpMYFFqTKoIa8bcqgmLgpGqR8AcSh0JrEoNCaxKDQmtSRdBPSKWbkUGqFgl/IHEotCYxKLQmMSi0JlUEhRBCCCGEECKAyAjWGaiqCvRtOFD4J6fTic1mw2q1yidmQjMSh0JrEoNCaxKDQmsDicHunKA7RzgbSbDOoLm5GYC0tDSNeyKEEEIIIYTwB83NzZjN5rO2UdTepGGjkMvloqKigsjISBRF0bo7YgCsVitpaWmUlpZKwRKhGYlDoTWJQaE1iUGhtYHEoKqqNDc3k5ycjE539lVWMoJ1BjqdjtTUVK27IQaRyWSSC7rQnMSh0JrEoNCaxKDQWn9j8FwjV92kyIUQQgghhBBCDBJJsIQQQgghhBBikEiCJUa8kJAQHn30UUJCQrTuihjFJA6F1iQGhdYkBoXWhisGpciFEEIIIYQQQgwSGcESQgghhBBCiEEiCZYQQgghhBBCDBJJsIQQQgghhBBikEiCJYQQQgghhBCDRBIsIYQQQgghhBgkkmCJUWnp0qVER0dz/fXXex3/2c9+xvTp05k2bRr33nsvUmRTDIXS0lLmz5/PtGnTyMrK4s033/Tc99577zF58mQmTpzIr3/9aw17KUayM8Xg2WJTiMF0rlhraWlh7NixrF69WqMeipHubDFYUlLCggULmDZtGjNmzMBut/fp3FKmXYxKeXl5NDc388orr/DWW28BUFtby4UXXsj+/fsxGAxcdtll/PSnP2Xu3Lka91aMNJWVlVRXV5OTk0NVVRW5ubkcPnyYkJAQpk2bxrZt2zCbzeTm5vLJJ58wZswYrbssRpgzxaDVau3xeEREhNZdFiPMmWKwO9YeeughioqKSEtL46c//anGvRUj0dlicN68eTz++ONceumlNDQ0YDKZCAoK6vW5ZQRLjErz588nMjLS57jD4aCtrY3Ozk46OzuJj4/XoHdipEtKSiInJweAxMREYmNjaWho4LPPPmP69OmkpKRgNBpZvHgxH374obadFSPSmWLwTMeFGGxni7UjR45w8OBBFi9erGEPxUh3phjs/qD90ksvBSAmJqZPyRVIgiUC0I4dO7jmmmtITk5GURTeffddnzabNm1i3LhxhIaGcsEFF/DZZ5+d87xxcXGsXr2a9PR0kpOTWbRoERMmTBiCVyAC3WDGYH5+Pk6nk7S0NCoqKkhJSfHcl5KSQnl5+VC9DBHAhioGe3NcCBjaGFy9ejXr168fyu6LEWCoYvDIkSMYjUauueYaZs2axZNPPtnnvkmCJQKO3W4nOzubTZs29Xj/66+/zqpVq3j00UfZvXs32dnZXHnlldTU1Jz1vI2Njbz33nscO3aM8vJyPvnkE3bs2DEUL0EEuMGKwYaGBm677TZ+9atfDUe3xQgy1DEosSnOZahi8M9//jOTJk1i0qRJQ/4aRGAbqhh0OBx8/PHH/PKXv2Tnzp384x//4B//+EffOqcKEcAA9Z133vE6NmfOHHXlypWen51Op5qcnKyuX7/eq922bdvUr33ta56f33jjDfWuu+7y/Pz000+rP/nJT4am42LE6G8MtrW1qZdeeqn66quveo79+9//VpcsWeL5+b777lNfe+21oeu8GBEGMwbPdlyIMxnMGFyzZo2ampqqjh07Vh0zZoxqMpnUdevWDflrEIFtMGPwk08+Ua+44grPz08//bT69NNP96k/MoIlRpSOjg7y8/NZtGiR55hOp2PRokXs3LnzrI9NS0vjk08+oa2tDafTSV5eHpMnTx7qLosRpjcxqKoqt99+O5dffjm33nqrp92cOXP4/PPPKS8vx2az8fe//50rr7xy2F+DCGwDicEzHReiLwYSg+vXr6e0tJRjx47x05/+lBUrVvDII48M+2sQgW0gMXj++edTU1NDY2MjLpeLHTt2MHXq1D49vyRYYkSpq6vD6XSSkJDgdTwhIYGqqirPz4sWLeKGG27gb3/7G6mpqezcuZMLL7yQq6++mpkzZ5KVlcWECRO49tprh/sliADXmxj897//zeuvv867775LTk4OOTk57Nu3j6CgIJ555hkWLFhATk4ODzzwgFQQFH02kBg803Eh+mIgMSjEYBjo/8VPPvkkl112GVlZWUycOJGvfOUrfXr+vpXEEGKE+Oijj3o8/sQTT/DEE08Mc2/EaHPJJZfgcrl6vO/aa6+VxF4MubPF4JmOCzGYzhaD3W6//fbh6YwYlc4Wg4sXLx5QFUsZwRIjSmxsLHq9nurqaq/j1dXVJCYmatQrMZpIDAqtSQwKrUkMCq1pHYOSYIkRJTg4mNzcXLZu3eo55nK52Lp1q2wYLIaFxKDQmsSg0JrEoNCa1jEoUwRFwLHZbBQVFXl+LikpoaCggJiYGNLT01m1ahXLli1j9uzZzJkzh40bN2K321m+fLmGvRYjicSg0JrEoNCaxKDQml/HYJ9qDgrhB7Zt26YCPrdly5Z52jz33HNqenq6GhwcrM6ZM0f99NNPteuwGHEkBoXWJAaF1iQGhdb8OQYVVVXVoU/jhBBCCCGEEGLkkzVYQgghhBBCCDFIJMESQgghhBBCiEEiCZYQQgghhBBCDBJJsIQQQgghhBBikEiCJYQQQgghhBCDRBIsIYQQQgghhBgkkmAJIYQQQgghxCCRBEsIIYQQQgghBokkWEIIIYQQQggxSCTBEkIIIQbo2LFjKIpCQUHBoJ/7scceIycnZ9DPK4QQYmhIgiWEEGLUGq3JS15eHoqi0NTUpHVXhBBixJEESwghhBBCCCEGiSRYQgghBsTlcvH000+TmZlJSEgI6enpPPHEE5779+3bx+WXX05YWBhjxozh29/+NjabzXP/7bffzpIlS3jyySdJSEggKiqKH/3oRzgcDh588EFiYmJITU3l5Zdf9jyme0reH//4Ry666CJCQ0M577zz2L59u6fNli1biIqK8urru+++i6IonvvXrVtHYWEhiqKgKApbtmwBoKmpiW9961vExcVhMpm4/PLLKSwsPOfv4uDBg/3uT7ennnqKhIQEIiMj+eY3v0lbW5vX/Q6Hg3vvvZeoqCjGjBnD97//fZYtW8aSJUu83pP169eTkZFBWFgY2dnZvPXWW57f3YIFCwCIjo5GURRuv/32c742IYQQvSMJlhBCiAFZu3YtTz31FA8//DBffPEFv//970lISADAbrdz5ZVXEh0dzX//+1/efPNNPvroI+6++26vc/zzn/+koqKCHTt28Oyzz/Loo4/yla98hejoaP7zn/9w55138p3vfIeysjKvxz344IM88MAD7Nmzh7lz53LNNddQX1/fq37feOONPPDAA0yfPp3KykoqKyu58cYbAbjhhhuoqanh73//O/n5+cyaNYuFCxfS0NBw1nMOpD8Ab7zxBo899hhPPvkku3btIikpiV/+8pdebX7yk5/w2muv8fLLL/Pvf/8bq9XKu+++69Vm/fr1vPrqq2zevJn9+/dz//33c8stt7B9+3bS0tJ4++23ATh06BCVlZX8/Oc/73UfhRBCnIMqhBBC9JPValVDQkLUF198scf7f/WrX6nR0dGqzWbzHHv//fdVnU6nVlVVqaqqqsuWLVPHjh2rOp1OT5vJkyerl156qednh8OhRkREqH/4wx9UVVXVkpISFVCfeuopT5vOzk41NTVV/clPfqKqqqq+/PLLqtls9urPO++8o576X9+jjz6qZmdne7X5+OOPVZPJpLa1tXkdnzBhgvrCCy/0+DoHqz9z585V77rrLq82F1xwgVcfExIS1A0bNnh+djgcanp6uvrVr35VVVVVbWtrU8PDw9VPPvnE6zzf/OY31a9//euqqqrqtm3bVEBtbGzs8fUIIYTovyBNszshhBAB7cCBA7S3t7Nw4cIz3p+dnU1ERITn2MUXX4zL5eLQoUOeka7p06ej052cVJGQkMB5553n+Vmv1zNmzBhqamq8zj937lzP90FBQcyePZsDBw4M6DUVFhZis9kYM2aM1/HW1laKi4vP+tiB9ufAgQPceeedPufctm0bABaLherqaubMmeO5X6/Xk5ubi8vlAqCoqIiWlha+9KUveZ2no6ODmTNn9rovQggh+kcSLCGEEP0WFhY2KOcxGAxePyuK0uOx7iSiN3Q6Haqqeh3r7Ow85+NsNhtJSUnk5eX53Hf6Gqq+6G9/+qp7fdv7779PSkqK130hISGD/nxCCCG8yRosIYQQ/TZx4kTCwsLYunVrj/dPnTqVwsJC7Ha759i///1vdDodkydPHvDzf/rpp57vHQ4H+fn5TJ06FYC4uDiam5u9nvv0faqCg4NxOp1ex2bNmkVVVRVBQUFkZmZ63WJjY4e0P1OnTuU///nPGc9pNptJSEjgv//9r+eY0+lk9+7dnp+nTZtGSEgIJ06c8Ol/Wlqa53V3P1YIIcTgkgRLCCFEv4WGhvL973+f733ve7z66qsUFxfz6aef8pvf/AaAm2++mdDQUJYtW8bnn3/Otm3buOeee7j11ls90wMHYtOmTbzzzjscPHiQlStX0tjYyB133AHABRdcQHh4OD/4wQ8oLi7m97//vadKYLdx48ZRUlJCQUEBdXV1tLe3s2jRIubOncuSJUv48MMPOXbsGJ988gkPPfQQu3btGtL+3Hfffbz00ku8/PLLHD58mEcffZT9+/d7tbnnnntYv349f/7znzl06BD33XcfjY2NnmqEkZGRrF69mvvvv59XXnmF4uJidu/ezXPPPccrr7wCwNixY1EUhffee4/a2lqvqo5CCCEGSOtFYEIIIQKb0+lUH3/8cXXs2LGqwWBQ09PT1SeffNJz/969e9UFCxaooaGhakxMjLpixQq1ubnZc/+yZcs8BRq6zZs3T73vvvu8jo0dO1b92c9+pqrqyaISv//979U5c+aowcHB6rRp09R//vOfXo9555131MzMTDUsLEz9yle+ov7qV7/yKirR1tamfu1rX1OjoqJUQH355ZdVVXUX77jnnnvU5ORk1WAwqGlpaerNN9+snjhxosffwWD1R1VV9YknnlBjY2NVo9GoLlu2TP3e977nVeSis7NTvfvuu1WTyaRGR0er3//+99UbbrhBvemmmzxtXC6XunHjRnXy5MmqwWBQ4+Li1CuvvFLdvn27p82PfvQjNTExUVUURV22bFmPr0sIIUTfKap62oRwIYQQws8dO3aMjIwM9uzZQ05Ojtbd0ZTL5WLq1Kn8z//8Dz/+8Y+17o4QQox6UuRCCCGECCDHjx/nww8/ZN68ebS3t/P8889TUlLCN77xDa27JoQQAlmDJYQQQgQUnU7Hli1bOP/887n44ovZt28fH330kaeYhhBCCG3JFEEhhBBCCCGEGCQygiWEEEIIIYQQg0QSLCGEEEIIIYQYJJJgCSGEEEIIIcQgkQRLCCGEEEIIIQaJJFhCCCGEEEIIMUgkwRJCCCGEEEKIQSIJlhBCCCGEEEIMEkmwhBBCCCGEEGKQ/H/tyHwPe7NNZQAAAABJRU5ErkJggg==", 373 | "text/plain": [ 374 | "
" 375 | ] 376 | }, 377 | "metadata": {}, 378 | "output_type": "display_data" 379 | } 380 | ], 381 | "source": [ 382 | "# sweep over compute budgets from 1e17 to 1e26\n", 383 | "cs = 10 ** np.arange(17, 26, step=2**-2)\n", 384 | "best_ns = []\n", 385 | "best_ds = []\n", 386 | "for c in cs:\n", 387 | " ns = 10 ** np.arange(7, 14, step=2**-5)\n", 388 | " ds = c / (6 * ns)\n", 389 | " losses = L(ns, ds)\n", 390 | " best = np.argmin(losses)\n", 391 | " best_ns.append(ns[best])\n", 392 | " best_ds.append(ds[best])\n", 393 | " print(f\"compute budget {c:e}: best model size: {ns[best]/1e6:.2f}M, best dataset size: {ds[best]/1e9:.2f}B\")\n", 394 | "\n", 395 | "# plot both the model size and dataset size as a function of compute budget, on one plot\n", 396 | "plt.figure(figsize=(10,3))\n", 397 | "plt.plot(cs, best_ns, label='model size')\n", 398 | "plt.plot(cs, best_ds, label='dataset size')\n", 399 | "plt.xscale('log')\n", 400 | "plt.yscale('log')\n", 401 | "plt.xlabel('compute budget')\n", 402 | "plt.ylabel('model size / dataset size')\n", 403 | "plt.grid(True, which=\"both\", ls=\"-\", color='k', alpha=0.2)\n", 404 | "plt.legend()\n" 405 | ] 406 | }, 407 | { 408 | "cell_type": "code", 409 | "execution_count": null, 410 | "metadata": {}, 411 | "outputs": [], 412 | "source": [] 413 | } 414 | ], 415 | "metadata": { 416 | "kernelspec": { 417 | "display_name": "pytorch2", 418 | "language": "python", 419 | "name": "python3" 420 | }, 421 | "language_info": { 422 | "codemirror_mode": { 423 | "name": "ipython", 424 | "version": 3 425 | }, 426 | "file_extension": ".py", 427 | "mimetype": "text/x-python", 428 | "name": "python", 429 | "nbconvert_exporter": "python", 430 | "pygments_lexer": "ipython3", 431 | "version": "3.10.8" 432 | }, 433 | "orig_nbformat": 4, 434 | "vscode": { 435 | "interpreter": { 436 | "hash": "7f5833218766b48e6e35e4452ee875aac0e2188d05bbe5298f2c62b79f08b222" 437 | } 438 | } 439 | }, 440 | "nbformat": 4, 441 | "nbformat_minor": 2 442 | } 443 | --------------------------------------------------------------------------------