├── drake_meme.png ├── gen_data └── cloned_repos.dat ├── experiments ├── all_models_loss_plot.png ├── all_models_no_opt_loss_plot.png ├── non_builtins_gpt3_loss_plot.png └── all_models_no_opt_no_flan_t5_loss_plot.png ├── testfunc.py ├── eval_chat_llms ├── chat_llms_classsification_plot.png ├── eval_chat_llms_results.json └── eval_chat_llms.ipynb ├── .gitmodules ├── filter_functions_with_docstrings.py ├── filter_functions_with_docstrings_and_shuffle.py ├── generate_examples.py ├── generate_examples_no_builtins.py └── README.md /drake_meme.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Avmb/inverse_scaling_prize_code_identifier_swap/HEAD/drake_meme.png -------------------------------------------------------------------------------- /gen_data/cloned_repos.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Avmb/inverse_scaling_prize_code_identifier_swap/HEAD/gen_data/cloned_repos.dat -------------------------------------------------------------------------------- /experiments/all_models_loss_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Avmb/inverse_scaling_prize_code_identifier_swap/HEAD/experiments/all_models_loss_plot.png -------------------------------------------------------------------------------- /experiments/all_models_no_opt_loss_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Avmb/inverse_scaling_prize_code_identifier_swap/HEAD/experiments/all_models_no_opt_loss_plot.png -------------------------------------------------------------------------------- /experiments/non_builtins_gpt3_loss_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Avmb/inverse_scaling_prize_code_identifier_swap/HEAD/experiments/non_builtins_gpt3_loss_plot.png -------------------------------------------------------------------------------- /testfunc.py: -------------------------------------------------------------------------------- 1 | def foo(a, b=1, c="hello"): 2 | """ 3 | Foo 4 | """ 5 | 6 | x = a + b 7 | print(x) 8 | print(2 * len(c) + abs(-1)) 9 | 10 | -------------------------------------------------------------------------------- /eval_chat_llms/chat_llms_classsification_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Avmb/inverse_scaling_prize_code_identifier_swap/HEAD/eval_chat_llms/chat_llms_classsification_plot.png -------------------------------------------------------------------------------- /experiments/all_models_no_opt_no_flan_t5_loss_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Avmb/inverse_scaling_prize_code_identifier_swap/HEAD/experiments/all_models_no_opt_no_flan_t5_loss_plot.png -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "inverse-scaling-eval-pipeline"] 2 | path = inverse-scaling-eval-pipeline 3 | url = git@github.com:Avmb/inverse-scaling-eval-pipeline.git 4 | [submodule "gen_data/pycodesuggest"] 5 | path = gen_data/pycodesuggest 6 | url = git@github.com:uclnlp/pycodesuggest.git 7 | -------------------------------------------------------------------------------- /eval_chat_llms/eval_chat_llms_results.json: -------------------------------------------------------------------------------- 1 | {"claude-instant-v1.1": {"raw_total_accuracy": 206, "total_accuracy": 0.103, "total_accuracy_on_valid": 0.103, "num_valid": 2000, "num_examples": 2000}, "claude-v1.3": {"raw_total_accuracy": 350, "total_accuracy": 0.175, "total_accuracy_on_valid": 0.175, "num_valid": 2000, "num_examples": 2000}, "gpt-3.5-turbo-0301": {"raw_total_accuracy": 67, "total_accuracy": 0.0335, "total_accuracy_on_valid": 0.0393885949441505, "num_valid": 1701, "num_examples": 2000}, "gpt-4-0314": {"raw_total_accuracy": 37, "total_accuracy": 0.0185, "total_accuracy_on_valid": 0.018518518518518517, "num_valid": 1998, "num_examples": 2000}} -------------------------------------------------------------------------------- /filter_functions_with_docstrings.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import sys 4 | import ast 5 | import astunparse 6 | 7 | def usage(): 8 | print("Usage:", file=sys.stderr) 9 | print("%s < file_list > functions_with_docstrings" % sys.argv[0], file=sys.stderr) 10 | sys.exit(-1) 11 | 12 | def process_file(filename): 13 | try: 14 | with open(filename) as in_fs: 15 | file_str = in_fs.read() 16 | file_ast = ast.parse(file_str) 17 | for node in file_ast.body: 18 | if isinstance(node, ast.FunctionDef): 19 | docstr = ast.get_docstring(node) 20 | if docstr == None: 21 | continue 22 | func_str = astunparse.unparse(node).strip() 23 | print(func_str) 24 | print("###") 25 | 26 | except Exception: 27 | print("Can't process file: %s , skipping" % filename) 28 | 29 | def main(): 30 | if (len(sys.argv) != 1): 31 | usage() 32 | 33 | for filename in sys.stdin: 34 | process_file(filename.strip()) 35 | 36 | if __name__ == "__main__": 37 | main() 38 | 39 | -------------------------------------------------------------------------------- /filter_functions_with_docstrings_and_shuffle.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import sys 4 | import random 5 | import ast 6 | import astunparse 7 | 8 | def usage(): 9 | print("Usage:", file=sys.stderr) 10 | print("%s < file_list > functions_with_docstrings" % sys.argv[0], file=sys.stderr) 11 | sys.exit(-1) 12 | 13 | def process_file(filename, acc): 14 | try: 15 | with open(filename) as in_fs: 16 | file_str = in_fs.read() 17 | file_ast = ast.parse(file_str) 18 | for node in file_ast.body: 19 | if isinstance(node, ast.FunctionDef): 20 | docstr = ast.get_docstring(node) 21 | if docstr == None: 22 | continue 23 | func_str = astunparse.unparse(node).strip() 24 | acc.append(func_str) 25 | #print(func_str) 26 | #print("###") 27 | 28 | except Exception: 29 | print("Can't process file: %s , skipping" % filename, file=sys.stderr) 30 | 31 | def main(): 32 | if (len(sys.argv) != 1): 33 | usage() 34 | 35 | acc = [] 36 | for filename in sys.stdin: 37 | process_file(filename.strip(), acc) 38 | 39 | random.shuffle(acc) 40 | print("\n###\n".join(acc)) 41 | 42 | 43 | if __name__ == "__main__": 44 | main() 45 | 46 | -------------------------------------------------------------------------------- /generate_examples.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import sys 4 | import ast 5 | import astunparse 6 | import builtins 7 | import random 8 | random.seed(42) 9 | import copy 10 | 11 | def usage(): 12 | print("Usage:", file=sys.stderr) 13 | print("%s max_num_examples < functions_with_docstrings > examples" % sys.argv[0], file=sys.stderr) 14 | sys.exit(-1) 15 | 16 | def output_csv(prompt, bad_class, good_class): 17 | prompt_str = '"' + prompt.replace('"', '""') + '"' 18 | #classes_str = '"' + repr([bad_class, good_class]).replace('"', '""').replace("\\n", "\n") + '"' 19 | bad_class = ('""" '+ bad_class+'\n"""').replace('"', '""') 20 | good_class = ('""" '+good_class+'\n"""').replace('"', '""') 21 | classes_str = '"['+bad_class+", "+good_class+']"' 22 | idx_str = "1" 23 | 24 | csv_line = ','.join([prompt_str, classes_str, idx_str]) 25 | print(csv_line) 26 | 27 | def generate_example(found_func_names, func_node): 28 | #print("%s: %s" % (func_node.name, list(found_func_names))) 29 | 30 | # Randomly choose two builtin functions 31 | found_func_names = sorted(list(found_func_names)) 32 | random.shuffle(found_func_names) 33 | f0, f1 = found_func_names[:2] 34 | 35 | # swap statement 36 | swap_statement_str = "%s, %s = %s, %s" % (f0, f1, f1, f0) 37 | 38 | # bad (unmodified) function 39 | bad_func = astunparse.unparse(func_node).strip() 40 | bad_func_lines = bad_func.split("\n") 41 | 42 | # good function with swapped builtins 43 | good_func_node = copy.deepcopy(func_node) 44 | def traverse(node): 45 | if isinstance(node, ast.Name): 46 | if node.id == f0: 47 | node.id = f1 48 | elif node.id == f1: 49 | node.id = f0 50 | for child in ast.iter_child_nodes(node): 51 | traverse(child) 52 | traverse(good_func_node) 53 | good_func = astunparse.unparse(good_func_node).strip() 54 | good_func_lines = good_func.split("\n") 55 | 56 | #print(f0, f1) 57 | #print(swap_statement_str) 58 | #print(bad_func) 59 | #print(good_func) 60 | 61 | # find docstring line 62 | docstring_repr = repr(ast.get_docstring(func_node, clean=False)) 63 | for i, line in enumerate(bad_func_lines): 64 | if line.strip() == docstring_repr: 65 | docstring_line_num = i 66 | break 67 | 68 | # prepare prompt 69 | pretty_docstring = ' """' + ast.get_docstring(func_node, clean=False) + '"""' 70 | prompt_lines = [swap_statement_str] + bad_func_lines[:docstring_line_num] + [pretty_docstring] 71 | prompt = "\n".join(prompt_lines) 72 | #print(prompt) 73 | 74 | # bad class 75 | bad_class = "\n" + "\n".join(bad_func_lines[docstring_line_num+1:]) 76 | #print(bad_class) 77 | 78 | # good class 79 | good_class = "\n" + "\n".join(good_func_lines[docstring_line_num+1:]) 80 | #print(good_class) 81 | 82 | # output csv 83 | output_csv(prompt, bad_class, good_class) 84 | 85 | target_func_names = set([x for x in dir(builtins) if callable(eval(x))]) 86 | 87 | num_generated_funcs = 0 88 | 89 | def process_function(func_node): 90 | # find calls to two builtin functions 91 | global num_generated_funcs 92 | 93 | found_func_names = set() 94 | def traverse(node): 95 | if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id in target_func_names: 96 | found_func_names.add(node.func.id) 97 | for child in ast.iter_child_nodes(node): 98 | traverse(child) 99 | traverse(func_node) 100 | 101 | # num of builtins check 102 | if len(found_func_names) < 2: 103 | return 104 | func_str = astunparse.unparse(func_node).strip() 105 | # length check 106 | if len(func_str.split()) > 200: 107 | return 108 | # special char check 109 | if ('"""' in func_str) or ('\\\n' in func_str): 110 | return 111 | 112 | # suitable function 113 | generate_example(found_func_names, func_node) 114 | num_generated_funcs += 1 115 | 116 | def process(max_num_examples): 117 | file_str = sys.stdin.read() 118 | file_str = file_str.replace('\x00','').strip() 119 | functions = [f.strip() for f in file_str.split("###\n")] 120 | for f in functions: 121 | if num_generated_funcs >= max_num_examples: 122 | break 123 | try: 124 | func_node = ast.parse(f) 125 | except SyntaxError: 126 | continue 127 | for node in func_node.body: 128 | if not isinstance(node, ast.FunctionDef): 129 | continue 130 | process_function(node) 131 | print("Number of examples: %s" % num_generated_funcs, file=sys.stderr) 132 | 133 | def main(): 134 | if (len(sys.argv) != 2): 135 | usage() 136 | 137 | # print csv preamble 138 | print("prompt,classes,answer_index") 139 | 140 | process(int(sys.argv[1])) 141 | 142 | if __name__ == "__main__": 143 | main() 144 | 145 | -------------------------------------------------------------------------------- /generate_examples_no_builtins.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import sys 4 | import ast 5 | import astunparse 6 | import builtins 7 | import random 8 | random.seed(42) 9 | import copy 10 | 11 | def usage(): 12 | print("Usage:", file=sys.stderr) 13 | print("%s max_num_examples < functions_with_docstrings > examples" % sys.argv[0], file=sys.stderr) 14 | sys.exit(-1) 15 | 16 | def output_csv(prompt, bad_class, good_class): 17 | prompt_str = '"' + prompt.replace('"', '""') + '"' 18 | #classes_str = '"' + repr([bad_class, good_class]).replace('"', '""').replace("\\n", "\n") + '"' 19 | bad_class = ('""" '+ bad_class+'\n"""').replace('"', '""') 20 | good_class = ('""" '+good_class+'\n"""').replace('"', '""') 21 | classes_str = '"['+bad_class+", "+good_class+']"' 22 | idx_str = "1" 23 | 24 | csv_line = ','.join([prompt_str, classes_str, idx_str]) 25 | print(csv_line) 26 | 27 | def generate_example(found_func_names, func_node): 28 | #print("%s: %s" % (func_node.name, list(found_func_names))) 29 | 30 | # Randomly choose two non-builtin functions 31 | found_func_names = sorted(list(found_func_names)) 32 | random.shuffle(found_func_names) 33 | f0, f1 = found_func_names[:2] 34 | 35 | # swap statement 36 | swap_statement_str = "%s, %s = %s, %s" % (f0, f1, f1, f0) 37 | 38 | # bad (unmodified) function 39 | bad_func = astunparse.unparse(func_node).strip() 40 | bad_func_lines = bad_func.split("\n") 41 | 42 | # good function with swapped non-builtins 43 | good_func_node = copy.deepcopy(func_node) 44 | def traverse(node): 45 | if isinstance(node, ast.Name): 46 | if node.id == f0: 47 | node.id = f1 48 | elif node.id == f1: 49 | node.id = f0 50 | for child in ast.iter_child_nodes(node): 51 | traverse(child) 52 | traverse(good_func_node) 53 | good_func = astunparse.unparse(good_func_node).strip() 54 | good_func_lines = good_func.split("\n") 55 | 56 | #print(f0, f1) 57 | #print(swap_statement_str) 58 | #print(bad_func) 59 | #print(good_func) 60 | 61 | # find docstring line 62 | docstring_repr = repr(ast.get_docstring(func_node, clean=False)) 63 | for i, line in enumerate(bad_func_lines): 64 | if line.strip() == docstring_repr: 65 | docstring_line_num = i 66 | break 67 | 68 | # prepare prompt 69 | pretty_docstring = ' """' + ast.get_docstring(func_node, clean=False) + '"""' 70 | prompt_lines = [swap_statement_str] + bad_func_lines[:docstring_line_num] + [pretty_docstring] 71 | prompt = "\n".join(prompt_lines) 72 | #print(prompt) 73 | 74 | # bad class 75 | bad_class = "\n" + "\n".join(bad_func_lines[docstring_line_num+1:]) 76 | #print(bad_class) 77 | 78 | # good class 79 | good_class = "\n" + "\n".join(good_func_lines[docstring_line_num+1:]) 80 | #print(good_class) 81 | 82 | # output csv 83 | output_csv(prompt, bad_class, good_class) 84 | 85 | target_func_names = set([x for x in dir(builtins) if callable(eval(x))]) 86 | 87 | num_generated_funcs = 0 88 | 89 | def process_function(func_node): 90 | # find calls to two non-builtin functions 91 | global num_generated_funcs 92 | 93 | found_func_names = set() 94 | possible_internal_func_names = set() 95 | def traverse(node): 96 | if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id not in target_func_names: 97 | found_func_names.add(node.func.id) 98 | if isinstance(node, ast.FunctionDef): 99 | possible_internal_func_names.add(node.name) 100 | if hasattr(node, "target"): 101 | if isinstance(node.target, ast.Name): 102 | possible_internal_func_names.add(node.target.id) 103 | if hasattr(node, "targets"): 104 | for target in node.targets: 105 | if isinstance(target, ast.Name): 106 | possible_internal_func_names.add(target.id) 107 | for child in ast.iter_child_nodes(node): 108 | traverse(child) 109 | traverse(func_node) 110 | 111 | # remove names of functions that are possibly defined inside the current top-level function 112 | found_func_names = found_func_names - possible_internal_func_names 113 | # num of non-builtins check 114 | if len(found_func_names) < 2: 115 | return 116 | func_str = astunparse.unparse(func_node).strip() 117 | # length check 118 | if len(func_str.split()) > 200: 119 | return 120 | # special char check 121 | if ('"""' in func_str) or ('\\\n' in func_str): 122 | return 123 | 124 | # suitable function 125 | generate_example(found_func_names, func_node) 126 | num_generated_funcs += 1 127 | 128 | def process(max_num_examples): 129 | file_str = sys.stdin.read() 130 | file_str = file_str.replace('\x00','').strip() 131 | functions = [f.strip() for f in file_str.split("###\n")] 132 | for f in functions: 133 | if num_generated_funcs >= max_num_examples: 134 | break 135 | try: 136 | func_node = ast.parse(f) 137 | except SyntaxError: 138 | continue 139 | for node in func_node.body: 140 | if not isinstance(node, ast.FunctionDef): 141 | continue 142 | process_function(node) 143 | print("Number of examples: %s" % num_generated_funcs, file=sys.stderr) 144 | 145 | def main(): 146 | if (len(sys.argv) != 2): 147 | usage() 148 | 149 | # print csv preamble 150 | print("prompt,classes,answer_index") 151 | 152 | process(int(sys.argv[1])) 153 | 154 | if __name__ == "__main__": 155 | main() 156 | 157 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # The Larger They Are, the Harder They Fail: Language Models do not Recognize Identifier Swaps in Python 2 | 3 | Code for data generation and evaluation. Based on a submission to the Inverse Scaling Prize https://github.com/inverse-scaling/prize , task `python_builtins_swap` 4 | by Antonio Valerio Miceli-Barone amiceli@ed.ac.uk and Fazl Barez f.barez@ed.ac.uk. 5 | Paper: https://arxiv.org/abs/2305.15507 6 | 7 | ## Task description 8 | 9 | We ask the model to complete a python function given a declaration and docstring, but with a caveat: before the function declaration, we add a statement (e.g. `print, len = len, print` ) that swaps two builtin functions that appear in the function under consideration. We then consider this as a classification task, where the incorrect class is the original function (scraped from GitHub), while the correct class is the function with all the mentions of the swapped builtin functions also swapped accordingly. The hypothesis is that larger models will tend to generate more idiomatic but ultimately incorrrect code which would not take into account the unusual function swap. 10 | 11 | Drake meme 12 | 13 | ### Why is the task important? 14 | 15 | For this question, explain your hypothesis for why you expect the task described above to demonstrate inverse scaling. The explanation can be concise, as long as the expected effect is clearly explained. 16 | 17 | Example: We expect this task to demonstrate inverse scaling because larger language models are better at picking up on and matching the bias in the question, which will lead them to change their answer more. 18 | 19 | ### Why do you expect to see inverse scaling? 20 | 21 | Larger models may be more prone to reproduce the typical distribution of code seen during training, where unusual swaps (e.g. print with len) are normally not present. 22 | 23 | ### Why is the task novel or surprising? 24 | 25 | Is inverse scaling on the task novel (not shown in prior work) and/or surprising? Why or why not? 26 | 27 | ### Dataset generation procedure 28 | 29 | 1. We scrape python code from GitHub using https://github.com/uclnlp/pycodesuggest 30 | 2. We extract top-level functions with a docstring 31 | 3. We take each function that calls at least two different builtin functions, randomly select two of these, and then we create a prompt (everything up to the docstring) and a correct and incorrect pair "classes" (everything after the docstring, with and without the correct substitution) 32 | 33 | ## Code generation 34 | 35 | In order to generate the dataset, first clone the pycodesuggest repository in the `gen_data` directory and scrape python repositories from GitHub. 36 | For this subission we downloaded 559 repositories from the most recent snapshot of GitHub available on 16 Dec 2022. 37 | 38 | We used the command: 39 | ``` 40 | python3 /path_to_pycodesuggest/github-scraper/scraper.py --mode new --outdir=/full_path_to_scrape_output_dir/scrape/ --dbfile=//full_path_to_scrape_output_dir/cloned_repos.dat --githubuser=amiceli --search="CC-BY-4.0 in:readme size:<=200000" 41 | ``` 42 | which we stopped after getting enough repositories. 43 | We did not use the normalization scripts. 44 | 45 | The generated database and file list are available in the gen_data directory 46 | 47 | After the download is complete, run `filter_functions_with_docstrings_and_shuffle.py` and `generate_examples.py` to generate the dataset. We arbitrary cut off the dataset at 1000 examples. Run `generate_examples_no_builtins.py` to generate the alternate dataset where non-builtin functions are swapped. Both datasets are available in the `cc_4_0_licensed/` directory. 48 | This code depends on `astunparse 1.6.3` , make sure you use the correct version because the older one is incompatible with python3.8 . 49 | 50 | ## Evaluation 51 | 52 | For our main experiments, clone our modified version of the Inverse Scaling Prize repository `inverse-scaling-eval-pipeline` and follow the instructions. The `experiments/` directory contains a jupyter notebook to generate the plots in the paper. 53 | 54 | For our experiments on the Chat LLMs, use the jupyter notebook in the `eval_chat_llms/` directory. 55 | 56 | ## Results 57 | 58 | All the models tested always prefer the incorrect answer to the correct one, hence classification accuracy is zero. For some model families the preference is more prominent in terms of classification loss for bigger models, resulting in inverse scaling. 59 | 60 | ![Main experimental results](experiments/all_models_loss_plot.png?raw=true "Main experimental results") 61 | 62 | Similar results are observed on the Chat LLMs in the OpenAI family and Anthropic family. 63 | 64 | ![Chat LLMs results](eval_chat_llms/chat_llms_classsification_plot.png?raw=true "Chat LLMs results") 65 | 66 | Inverse scaling is also observed when swapping non-builtin top-level functions. 67 | 68 | ![Non-builtin experiment results](experiments/non_builtins_gpt3_loss_plot.png?raw=true "Non-builtin experiment results") 69 | 70 | LLMs prefer incorrect programs that use functions in a common way to out-of-distribution but correct programs. 71 | 72 | ## Copyright takedown 73 | 74 | If you believe that material you own a copyright to has been included into our dataset and you wish it to be removed, please contact the authors by opening an issue on this GitHub repository. 75 | 76 | ## Cite this work 77 | 78 | Please cite this work as: 79 | ``` 80 | @inproceedings{miceli-barone-etal-2023-larger, 81 | title = "The Larger they are, the Harder they Fail: Language Models do not Recognize Identifier Swaps in Python", 82 | author = "Miceli Barone, Antonio Valerio and 83 | Barez, Fazl and 84 | Cohen, Shay B. and 85 | Konstas, Ioannis", 86 | booktitle = "Findings of the Association for Computational Linguistics: ACL 2023", 87 | month = jul, 88 | year = "2023", 89 | address = "Toronto, Canada", 90 | publisher = "Association for Computational Linguistics", 91 | url = "https://aclanthology.org/2023.findings-acl.19", 92 | pages = "272--292", 93 | } 94 | ``` 95 | -------------------------------------------------------------------------------- /eval_chat_llms/eval_chat_llms.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "5d64b071", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import sys, os\n", 11 | "from collections import namedtuple\n", 12 | "import csv\n", 13 | "import ast\n", 14 | "import re\n", 15 | "\n", 16 | "from tqdm import tqdm\n", 17 | "\n", 18 | "import openai, anthropic\n", 19 | "import backoff\n", 20 | "import tiktoken\n", 21 | "\n", 22 | "from langchain.chat_models import ChatOpenAI\n", 23 | "from langchain.chat_models import ChatAnthropic\n", 24 | "from langchain.prompts.chat import (\n", 25 | " ChatPromptTemplate,\n", 26 | " SystemMessagePromptTemplate,\n", 27 | " AIMessagePromptTemplate,\n", 28 | " HumanMessagePromptTemplate,\n", 29 | ")\n", 30 | "from langchain.schema import (\n", 31 | " AIMessage,\n", 32 | " HumanMessage,\n", 33 | " SystemMessage\n", 34 | ")" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 2, 40 | "id": "1d4f2d0b", 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "oai_api_key_file = \"path/to/openai_key\"\n", 45 | "anthropic_api_key_file = \"path/to/anthropic_key\"\n", 46 | "\n", 47 | "with open(oai_api_key_file, \"r\") as api_key_fs:\n", 48 | " os.environ[\"OPENAI_API_KEY\"] = api_key_fs.read().strip()\n", 49 | " \n", 50 | "with open(anthropic_api_key_file, \"r\") as api_key_fs:\n", 51 | " os.environ[\"ANTHROPIC_API_KEY\"] = api_key_fs.read().strip()" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 3, 57 | "id": "bfd3ee1a", 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "test_set_file = \"../cc_4_0_licensed/python_builtins_swap.csv\" # Path to test set" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 4, 67 | "id": "f58467d7", 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "test_set = []\n", 72 | "with open(test_set_file, \"r\") as in_fs:\n", 73 | " reader = csv.reader(in_fs)\n", 74 | " for i, row in enumerate(reader):\n", 75 | " if i == 0:\n", 76 | " continue # skip header\n", 77 | " example = {\"prompt\": row[0], \"classes\": ast.literal_eval(row[1]), \"answer_index\": int(row[2])}\n", 78 | " test_set.append(example)" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 6, 84 | "id": "5700c058", 85 | "metadata": {}, 86 | "outputs": [], 87 | "source": [ 88 | "prompt_template = \\\n", 89 | "\"\"\"\n", 90 | "Consider the following Python programs:\n", 91 | "\n", 92 | "Program 1:\n", 93 | "\n", 94 | "{program_1}\n", 95 | "\n", 96 | "Program 2:\n", 97 | "\n", 98 | "{program_2}\n", 99 | "\n", 100 | "Which program is more likely to be correct? Write only the number of the program and nothing else.\n", 101 | "\"\"\"" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 26, 107 | "id": "43c2dcca", 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "batch_size = 20 # note: actual submitted batches are twice as big since each example is sumbitted in two variants\n", 112 | "max_tokens_per_example = 4000 # Computed using tiktoken, not including the system prompt, assuming gpt-3.5-turbo-0301" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 13, 118 | "id": "08771c15", 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "tokenizer = tiktoken.encoding_for_model(\"gpt-3.5-turbo-0301\")" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": 27, 128 | "id": "c1105335", 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "@backoff.on_exception(backoff.expo, Exception)\n", 133 | "def chat_llm_batch(chat_model, batch_messages, **kwargs):\n", 134 | " return chat_model.generate(batch_messages, **kwargs)\n", 135 | "\n", 136 | "def eval_chat_model_batch(chat_model, batch, switch_order):\n", 137 | " b_accuracy, b_num_valid = 0, 0\n", 138 | " batch_messages = []\n", 139 | " y_batch = []\n", 140 | " for example in batch:\n", 141 | " y = example[\"answer_index\"]\n", 142 | " if switch_order:\n", 143 | " y = 1 - y\n", 144 | " x = prompt_template.format(\n", 145 | " program_1 = example[\"prompt\"] + example[\"classes\"][1-y].rstrip() + \"\\n\",\n", 146 | " program_2 = example[\"prompt\"] + example[\"classes\"][y].rstrip() + \"\\n\")\n", 147 | " if len(tokenizer.encode(x)) > max_tokens_per_example:\n", 148 | " # too long, skip\n", 149 | " continue\n", 150 | " x_messages = [\n", 151 | " SystemMessage(content=\"You are a helpful assistant.\"),\n", 152 | " HumanMessage(content=x)]\n", 153 | " batch_messages.append(x_messages)\n", 154 | " y_batch.append(y)\n", 155 | " llm_batch_response = chat_llm_batch(chat_model, batch_messages)\n", 156 | " for i, gen in enumerate(llm_batch_response.generations):\n", 157 | " match = re.search(r'(\\d+)', gen[0].text)\n", 158 | " if match is None:\n", 159 | " continue\n", 160 | " gen_class = int(match.group(1)) - 1 # generated class ids should be returned by the llm in the 1, 2 range\n", 161 | " if (gen_class < 0) or (gen_class > 1):\n", 162 | " continue\n", 163 | " if gen_class == y:\n", 164 | " b_accuracy += 1\n", 165 | " b_num_valid += 1\n", 166 | " return b_accuracy, b_num_valid \n", 167 | "\n", 168 | "def eval_chat_model(chat_model, test_set):\n", 169 | " raw_total_accuracy, num_valid, num_examples = 0, 0, 0\n", 170 | " batch = []\n", 171 | " for i in tqdm(range(len(test_set) + 1)):\n", 172 | " if (i >= len(test_set)) or (len(batch) >= batch_size):\n", 173 | " b_accuracy, b_num_valid = eval_chat_model_batch(chat_model, batch, False)\n", 174 | " raw_total_accuracy += b_accuracy\n", 175 | " num_valid += b_num_valid\n", 176 | " num_examples += len(batch)\n", 177 | " b_accuracy, b_num_valid = eval_chat_model_batch(chat_model, batch, True)\n", 178 | " raw_total_accuracy += b_accuracy\n", 179 | " num_valid += b_num_valid\n", 180 | " num_examples += len(batch)\n", 181 | " batch = []\n", 182 | " if i < len(test_set):\n", 183 | " batch.append(test_set[i])\n", 184 | " total_accuracy = float(raw_total_accuracy) / num_examples\n", 185 | " total_accuracy_on_valid = float(raw_total_accuracy) / num_valid\n", 186 | " return {\n", 187 | " \"raw_total_accuracy\": raw_total_accuracy,\n", 188 | " \"total_accuracy\": total_accuracy,\n", 189 | " \"total_accuracy_on_valid\": total_accuracy_on_valid,\n", 190 | " \"num_valid\": num_valid,\n", 191 | " \"num_examples\": num_examples}" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": 15, 197 | "id": "73b7d0d6", 198 | "metadata": {}, 199 | "outputs": [], 200 | "source": [ 201 | "eval_results = {}" 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": 16, 207 | "id": "21f91b50", 208 | "metadata": {}, 209 | "outputs": [ 210 | { 211 | "name": "stdout", 212 | "output_type": "stream", 213 | "text": [ 214 | "claude-instant-v1.1\n" 215 | ] 216 | }, 217 | { 218 | "name": "stderr", 219 | "output_type": "stream", 220 | "text": [ 221 | "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1001/1001 [19:59<00:00, 1.20s/it]" 222 | ] 223 | }, 224 | { 225 | "name": "stdout", 226 | "output_type": "stream", 227 | "text": [ 228 | "{'raw_total_accuracy': 206, 'total_accuracy': 0.103, 'total_accuracy_on_valid': 0.103, 'num_valid': 2000, 'num_examples': 2000}\n", 229 | "----\n" 230 | ] 231 | }, 232 | { 233 | "name": "stderr", 234 | "output_type": "stream", 235 | "text": [ 236 | "\n" 237 | ] 238 | } 239 | ], 240 | "source": [ 241 | "model_name = \"claude-instant-v1.1\"\n", 242 | "print(model_name)\n", 243 | "chat = ChatAnthropic(model=model_name, temperature=0.0)\n", 244 | "chat_eval = eval_chat_model(chat, test_set)\n", 245 | "print(chat_eval)\n", 246 | "print(\"----\")\n", 247 | "eval_results[model_name] = chat_eval" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": 17, 253 | "id": "eec967f4", 254 | "metadata": {}, 255 | "outputs": [ 256 | { 257 | "name": "stdout", 258 | "output_type": "stream", 259 | "text": [ 260 | "claude-v1.3\n" 261 | ] 262 | }, 263 | { 264 | "name": "stderr", 265 | "output_type": "stream", 266 | "text": [ 267 | "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1001/1001 [44:04<00:00, 2.64s/it]" 268 | ] 269 | }, 270 | { 271 | "name": "stdout", 272 | "output_type": "stream", 273 | "text": [ 274 | "{'raw_total_accuracy': 350, 'total_accuracy': 0.175, 'total_accuracy_on_valid': 0.175, 'num_valid': 2000, 'num_examples': 2000}\n", 275 | "----\n" 276 | ] 277 | }, 278 | { 279 | "name": "stderr", 280 | "output_type": "stream", 281 | "text": [ 282 | "\n" 283 | ] 284 | } 285 | ], 286 | "source": [ 287 | "model_name = \"claude-v1.3\"\n", 288 | "print(model_name)\n", 289 | "chat = ChatAnthropic(model=model_name, temperature=0.0)\n", 290 | "chat_eval = eval_chat_model(chat, test_set)\n", 291 | "print(chat_eval)\n", 292 | "print(\"----\")\n", 293 | "eval_results[model_name] = chat_eval" 294 | ] 295 | }, 296 | { 297 | "cell_type": "code", 298 | "execution_count": 19, 299 | "id": "70146189", 300 | "metadata": {}, 301 | "outputs": [ 302 | { 303 | "name": "stdout", 304 | "output_type": "stream", 305 | "text": [ 306 | "gpt-3.5-turbo-0301\n" 307 | ] 308 | }, 309 | { 310 | "name": "stderr", 311 | "output_type": "stream", 312 | "text": [ 313 | " 0%| | 0/1001 [00:00._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID c972c4c81559af3b5b87a200dcc30cfb in your message.).\n", 314 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 733f3116a1f0e13399dc761d63e86d5b in your message.).\n", 315 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 648d55296b55331c90f1a708a9bd62c7 in your message.).\n", 316 | " 2%|███▌ | 21/1001 [02:56<2:16:53, 8.38s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 78abc76238809b53f9b07291f1a65258 in your message.).\n", 317 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID cbe735c97c99a54afd478332b013c5ed in your message.).\n", 318 | " 4%|██████▉ | 41/1001 [05:14<2:00:46, 7.55s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 48ddf4dc0e07f85fc3db394a86e6c552 in your message.).\n", 319 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID e9e2af2e93cbc267679bf53fe51a109f in your message.).\n", 320 | " 6%|██████████▎ | 61/1001 [07:44<1:57:51, 7.52s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 7a9b9aee36b37a43e784d0830f35c10a in your message.).\n", 321 | " 8%|█████████████▊ | 81/1001 [09:06<1:34:43, 6.18s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 3ac3ad0f5c4d17c377a7d75a0bdf8157 in your message.).\n", 322 | " 12%|████████████████████▍ | 121/1001 [12:22<1:20:02, 5.46s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID dec996e5bfdf68528d8986637bf3108a in your message.).\n", 323 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID fc39bfc5a6d35e87a62107d35277fca7 in your message.).\n", 324 | " 14%|███████████████████████▊ | 141/1001 [15:24<1:35:14, 6.64s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID d6477786da0b945cea8d95863896c327 in your message.).\n", 325 | " 16%|███████████████████████████▏ | 161/1001 [16:53<1:23:21, 5.95s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID d3833f81fdd9fec2df7701cc1483dc3f in your message.).\n", 326 | " 18%|██████████████████████████████▌ | 181/1001 [18:42<1:19:08, 5.79s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised APIError: Gateway timeout. {\"error\":{\"code\":524,\"message\":\"Gateway timeout.\",\"param\":null,\"type\":\"cf_gateway_timeout\"}} 524 {'error': {'code': 524, 'message': 'Gateway timeout.', 'param': None, 'type': 'cf_gateway_timeout'}} {'Date': 'Fri, 19 May 2023 03:15:11 GMT', 'Content-Type': 'application/json', 'Content-Length': '92', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7c991de88ac554ca-MAN'}.\n", 327 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 496f5b43f10392e79944b269889b42a3 in your message.).\n", 328 | " 20%|█████████████████████████████████▉ | 201/1001 [30:18<3:16:36, 14.75s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 6af8f193ec2819f0363a67c631400d29 in your message.).\n", 329 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 9297c1114defe99a5b9b459d1470d8d0 in your message.).\n", 330 | " 22%|█████████████████████████████████████▎ | 221/1001 [32:28<2:38:51, 12.22s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID bce0e2a4229097fef4d23f837ac4132e in your message.).\n", 331 | " 24%|████████████████████████████████████████▋ | 241/1001 [33:59<2:05:06, 9.88s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 5d1c9c5efcec53a9913a851b0abd6c1c in your message.).\n", 332 | " 26%|████████████████████████████████████████████ | 261/1001 [35:32<1:42:20, 8.30s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID f53ddf65745275ba339510f961c0bcb2 in your message.).\n", 333 | " 28%|███████████████████████████████████████████████▍ | 281/1001 [37:26<1:30:14, 7.52s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 19ea986237046f51342d73572cc6c074 in your message.).\n", 334 | " 30%|██████████████████████████████████████████████████▊ | 301/1001 [39:11<1:19:47, 6.84s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 98ef64371c5e795ad3ca04d4a24ca4fd in your message.).\n", 335 | " 32%|██████████████████████████████████████████████████████▏ | 321/1001 [41:09<1:14:10, 6.54s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 30e3584dbb4e7843696d79b1fa83f665 in your message.).\n", 336 | " 34%|█████████████████████████████████████████████████████████▌ | 341/1001 [43:30<1:13:42, 6.70s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 1bf905f0dac6b4c92702626c230b2f4f in your message.).\n", 337 | " 36%|████████████████████████████████████████████████████████████▉ | 361/1001 [45:17<1:07:09, 6.30s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID f4035d75dd635543f08622cfd86c69ca in your message.).\n", 338 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID e9492f8ec1f661477f4b52449868e0a2 in your message.).\n", 339 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 58ab1ed704edbe3d4a782e3efa1b0804 in your message.).\n", 340 | " 42%|███████████████████████████████████████████████████████████████████████ | 421/1001 [53:32<1:07:23, 6.97s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 72c834add675b33b571e686c2b067d00 in your message.).\n", 341 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 0caddff1ad816fb84e83f49a1e37a7b3 in your message.).\n", 342 | " 44%|██████████████████████████████████████████████████████████████████████████▍ | 441/1001 [55:34<1:02:38, 6.71s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID b324e50d8c4110282a8a9af917f0c724 in your message.).\n" 343 | ] 344 | }, 345 | { 346 | "name": "stderr", 347 | "output_type": "stream", 348 | "text": [ 349 | " 48%|██████████████████████████████████████████████████████████████████████████████████▏ | 481/1001 [58:53<50:31, 5.83s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 19 May 2023 03:51:10 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7c9959abca650763-MAN', 'alt-svc': 'h3=\":443\"; ma=86400, h3-29=\":443\"; ma=86400'}.\n", 350 | " 50%|███████████████████████████████████████████████████████████████████████████████████▌ | 501/1001 [1:05:28<1:23:20, 10.00s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 2020fb8629e9fd48eeb974cc62481fb1 in your message.).\n", 351 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID af7bb2e3b582bddbcd48ea164b36fb2d in your message.).\n", 352 | " 54%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 541/1001 [1:09:15<59:20, 7.74s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 4274e08ee3a21135720cc16c3e0c960e in your message.).\n", 353 | " 56%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 561/1001 [1:10:59<51:11, 6.98s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 6ba0f86850591bd07462a749f88ab058 in your message.).\n", 354 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID c3d5797b533845fd80bbc1393090cb50 in your message.).\n", 355 | " 58%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 581/1001 [1:13:43<51:22, 7.34s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 2a02fb0cd0ce7f57eb623983b3792e0a in your message.).\n", 356 | " 60%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 601/1001 [1:15:25<44:28, 6.67s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 936ac7fe674e2306fab43b20573538f7 in your message.).\n", 357 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 2.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID b294124c9922f82529e844869da4d225 in your message.).\n", 358 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 7373f73a07cfd74e79d5fb1f6d811277 in your message.).\n", 359 | " 62%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 621/1001 [1:18:19<46:05, 7.28s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID a9a7f7388809a45a3fb4dbdfa6a77350 in your message.).\n", 360 | " 64%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 641/1001 [1:20:10<40:32, 6.76s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 2b62b51e0e87101a332677549da0b9a7 in your message.).\n", 361 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 7ebcc84bdaeec7be21746738735456f5 in your message.).\n", 362 | " 66%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 661/1001 [1:22:18<37:41, 6.65s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID fbf712d25b96e0040928232e05bdee5b in your message.).\n", 363 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 35b74a1c994aac9566b6f33741f11e24 in your message.).\n" 364 | ] 365 | }, 366 | { 367 | "name": "stderr", 368 | "output_type": "stream", 369 | "text": [ 370 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID efe51d859e825291eccc62b2e2796018 in your message.).\n", 371 | " 68%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 681/1001 [1:24:56<37:28, 7.03s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 52bf904a49ade142ef428ef051853999 in your message.).\n", 372 | " 72%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 721/1001 [1:27:49<25:58, 5.57s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 42c459ad2eca5e1b81ec8f88c0d7eee9 in your message.).\n", 373 | " 76%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 761/1001 [1:30:44<19:26, 4.86s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 87b82c18a0a1b33f41cbca55c960aeeb in your message.).\n", 374 | " 82%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 821/1001 [1:34:15<11:30, 3.83s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 41a47e96cbe37f3397c5b5166495cef6 in your message.).\n", 375 | " 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 861/1001 [1:36:41<08:28, 3.64s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 3ff19d7b74541c1b5afb2a9f0ac46c1c in your message.).\n", 376 | " 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 901/1001 [1:39:31<06:21, 3.82s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 39b0247737cc2c2050ac03dcdc2c5269 in your message.).\n", 377 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 66943b060398bcc7d95fb44583c4b6c1 in your message.).\n", 378 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID b3f23958dd2cef1b88b8fccb16e63a8c in your message.).\n", 379 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 2cc01eacb10be5cdf644e6d02230edfd in your message.).\n", 380 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 20119bfc788cca651c52d853d1194f4f in your message.).\n", 381 | " 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 941/1001 [1:44:24<05:14, 5.24s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 6dcd037fd7fe193836125a6693c3ae51 in your message.).\n", 382 | " 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 961/1001 [1:45:53<03:19, 5.00s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 624f7b69e1bd6ab0a1b671899826671c in your message.).\n", 383 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 465a7df4df9531babf1321524f807425 in your message.).\n", 384 | "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1001/1001 [1:49:18<00:00, 6.55s/it]" 385 | ] 386 | }, 387 | { 388 | "name": "stdout", 389 | "output_type": "stream", 390 | "text": [ 391 | "{'raw_total_accuracy': 67, 'total_accuracy': 0.0335, 'total_accuracy_on_valid': 0.0393885949441505, 'num_valid': 1701, 'num_examples': 2000}\n", 392 | "----\n" 393 | ] 394 | }, 395 | { 396 | "name": "stderr", 397 | "output_type": "stream", 398 | "text": [ 399 | "\n" 400 | ] 401 | } 402 | ], 403 | "source": [ 404 | "model_name=\"gpt-3.5-turbo-0301\"\n", 405 | "chat = ChatOpenAI(model_name=model_name, temperature=0.0)\n", 406 | "print(model_name)\n", 407 | "chat_eval = eval_chat_model(chat, test_set)\n", 408 | "print(chat_eval)\n", 409 | "print(\"----\")\n", 410 | "eval_results[model_name] = chat_eval" 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": 28, 416 | "id": "cc837ea7", 417 | "metadata": {}, 418 | "outputs": [ 419 | { 420 | "name": "stdout", 421 | "output_type": "stream", 422 | "text": [ 423 | "gpt-4-0314\n" 424 | ] 425 | }, 426 | { 427 | "name": "stderr", 428 | "output_type": "stream", 429 | "text": [ 430 | " 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 721/1001 [37:03<14:16, 3.06s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 19 May 2023 11:33:58 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7c9bff9a1cee0abf-MAN', 'alt-svc': 'h3=\":443\"; ma=86400, h3-29=\":443\"; ma=86400'}.\n", 431 | " 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 741/1001 [43:15<33:26, 7.72s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 19 May 2023 11:39:29 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7c9c07a8dc0b0abf-MAN', 'alt-svc': 'h3=\":443\"; ma=86400, h3-29=\":443\"; ma=86400'}.\n", 432 | "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 19 May 2023 11:45:41 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7c9c10c48a440abf-MAN', 'alt-svc': 'h3=\":443\"; ma=86400, h3-29=\":443\"; ma=86400'}.\n", 433 | " 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 781/1001 [55:46<43:39, 11.91s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 19 May 2023 11:52:04 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7c9c1a1b797e0abf-MAN', 'alt-svc': 'h3=\":443\"; ma=86400, h3-29=\":443\"; ma=86400'}.\n", 434 | " 80%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 801/1001 [1:02:04<46:43, 14.02s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 19 May 2023 11:58:54 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7c9c2414dc6a0abf-MAN', 'alt-svc': 'h3=\":443\"; ma=86400, h3-29=\":443\"; ma=86400'}.\n", 435 | " 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 861/1001 [1:10:26<21:29, 9.21s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 19 May 2023 12:07:25 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7c9c30972daa0abf-MAN', 'alt-svc': 'h3=\":443\"; ma=86400, h3-29=\":443\"; ma=86400'}.\n", 436 | " 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 881/1001 [1:16:48<24:20, 12.17s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 19 May 2023 12:13:40 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7c9c39b60c800abf-MAN', 'alt-svc': 'h3=\":443\"; ma=86400, h3-29=\":443\"; ma=86400'}.\n", 437 | " 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 921/1001 [1:24:10<14:30, 10.88s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 19 May 2023 12:20:28 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7c9c43b4faa10abf-MAN', 'alt-svc': 'h3=\":443\"; ma=86400, h3-29=\":443\"; ma=86400'}.\n", 438 | " 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 961/1001 [1:31:23<06:46, 10.15s/it]Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 1.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 19 May 2023 12:27:50 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7c9c4e6ded4d0abf-MAN', 'alt-svc': 'h3=\":443\"; ma=86400, h3-29=\":443\"; ma=86400'}.\n" 439 | ] 440 | }, 441 | { 442 | "name": "stderr", 443 | "output_type": "stream", 444 | "text": [ 445 | "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1001/1001 [1:38:48<00:00, 5.92s/it]" 446 | ] 447 | }, 448 | { 449 | "name": "stdout", 450 | "output_type": "stream", 451 | "text": [ 452 | "{'raw_total_accuracy': 37, 'total_accuracy': 0.0185, 'total_accuracy_on_valid': 0.018518518518518517, 'num_valid': 1998, 'num_examples': 2000}\n", 453 | "----\n" 454 | ] 455 | }, 456 | { 457 | "name": "stderr", 458 | "output_type": "stream", 459 | "text": [ 460 | "\n" 461 | ] 462 | } 463 | ], 464 | "source": [ 465 | "model_name=\"gpt-4-0314\"\n", 466 | "chat = ChatOpenAI(model_name=model_name, temperature=0.0)\n", 467 | "print(model_name)\n", 468 | "chat_eval = eval_chat_model(chat, test_set)\n", 469 | "print(chat_eval)\n", 470 | "print(\"----\")\n", 471 | "eval_results[model_name] = chat_eval" 472 | ] 473 | }, 474 | { 475 | "cell_type": "code", 476 | "execution_count": null, 477 | "id": "a2c8575a", 478 | "metadata": {}, 479 | "outputs": [], 480 | "source": [] 481 | }, 482 | { 483 | "cell_type": "code", 484 | "execution_count": 69, 485 | "id": "81b9df05", 486 | "metadata": {}, 487 | "outputs": [], 488 | "source": [ 489 | "import json\n", 490 | "import matplotlib.pyplot as plt\n", 491 | "import numpy as np\n", 492 | "from pathlib import Path" 493 | ] 494 | }, 495 | { 496 | "cell_type": "code", 497 | "execution_count": 30, 498 | "id": "2ab4c70d", 499 | "metadata": {}, 500 | "outputs": [], 501 | "source": [ 502 | "out_file_name = \"eval_chat_llms_results.json\"" 503 | ] 504 | }, 505 | { 506 | "cell_type": "code", 507 | "execution_count": 33, 508 | "id": "1438aa8c", 509 | "metadata": {}, 510 | "outputs": [], 511 | "source": [ 512 | "with open(out_file_name, \"w\") as out_fs:\n", 513 | " json.dump(eval_results, out_fs)" 514 | ] 515 | }, 516 | { 517 | "cell_type": "code", 518 | "execution_count": null, 519 | "id": "733358a0", 520 | "metadata": {}, 521 | "outputs": [], 522 | "source": [] 523 | }, 524 | { 525 | "cell_type": "code", 526 | "execution_count": 34, 527 | "id": "9c51c0c9", 528 | "metadata": {}, 529 | "outputs": [], 530 | "source": [ 531 | "plt.rcParams.update({'font.size': 12.0})\n", 532 | "plt.rcParams.update({'figure.titlesize': 'small'})\n", 533 | "plt.rcParams.update({'legend.fontsize': 'small'})" 534 | ] 535 | }, 536 | { 537 | "cell_type": "code", 538 | "execution_count": 73, 539 | "id": "20ab4444", 540 | "metadata": {}, 541 | "outputs": [ 542 | { 543 | "data": { 544 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmwAAAHQCAYAAAAcdTa2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABUaElEQVR4nO3de3zO9eP/8ee12dFsjGGYzTksh4hIzqeSQ3JW+CRKJEX66LSRUFGUdFKIpBCinENOkVMJ5fAxjDmzmdmwvX5/+O36ulzDxq7tvfW4327XrfZ6v97v9+t9uK7r6fV+v1+XzRhjBAAAAMtyy+4GAAAA4NYIbAAAABZHYAMAALA4AhsAAIDFEdgAAAAsjsAGAABgcQQ2AAAAiyOwAQAAWByBDQAAwOIIbNkoKipKNptNvXr1yu6mOImMjJTNZtPq1audpn377beqXr268uXLJ5vNpkGDBkmSwsLCFBYWlqXtvBUr71/kPMuWLVPdunWVP39+2Ww2tWvXLtOWPXXqVNlsNk2dOjXTlgkgdyGwZbK///5bzz//vMLDwxUQECBPT08VK1ZMrVq10pdffqmkpKTsbuJd2bhxo7p3764LFy6oX79+ioiIUMuWLbOtPTabTQ0bNsy29ePfISoqSm3bttXBgwf11FNPKSIiQl26dMnuZmXI6tWrZbPZFBkZmeF5w8LCZLPZFBUVddu6DRs2vOk/9m7Uq1cv2Ww22Ww2vfnmmzetN23aNHs93u/4t8qT3Q3ITUaMGKHhw4crJSVFderUUc+ePeXn56cTJ05o9erVevrpp/XJJ59oy5Yt2d3U2xowYIC6dOmikiVLOpT/9NNPMsbo66+/Vt26dR2mrVy5MiubeFvFixfXnj17FBAQkN1NQQ63YsUKJSYmaty4cerWrVt2NyfXyZMnj6ZMmaKIiAi5u7s7Tf/iiy+UJ08eXb16NRtaB1gDgS2TjBo1ShEREQoJCdHs2bNVu3ZtpzqLFi3SuHHjsqF1GVeoUCEVKlTIqfzYsWOSpGLFijlNK1OmjMvblREeHh665557srsZyAVudd7j7j366KOaP3++lixZolatWjlM27Nnj9avX6/HHntM8+bNy6YWAtmPS6KZICoqSpGRkfLw8NDPP/+cZliTrn0oLVmy5LbL27t3r/773/+qZs2aCgoKkpeXl0JDQ9W3b19FR0c71TfGaNq0aapbt66CgoLk7e2tkJAQtWjRQt99951D3T///FNdu3ZVWFiYvLy8FBQUpPvuu0+DBg3SlStX7PVuvIct9R6bKVOmSJJKlSplv0SRepnkVvewfffdd2rSpIkCAwPl7e2tsLAwde3a1aG3MTY2Vu+9954aN26sEiVKyNPTU0FBQWrTpo02btzosLzU9kjSmjVr7G25/pLPre5hi4mJUf/+/RUWFmZfT/v27bV161anutffX7Rq1So1bNhQ+fLlk7+/v1q1aqU9e/akuc1puXz5siZOnKhHHnlEoaGh8vLyUmBgoJo2barFixffdL7o6GgNHDhQ5cqVk4+PjwIDA1WrVi299dZbd1z3VpeXUi9VXX8J7Pr9uXfvXnXu3FmFCxeWm5ub/TzZunWrXnjhBVWtWtV+rMuVK6fBgwfr3LlzN92+250fn332mWw2m4YPH57m/MePH5eHh4fuvffem67jRt9//73q16+vgIAA+fj46N5779Xo0aMdbltIvYwYEREhSWrUqJH9PEvPJb/0bNuN0nuOZeRzolevXmrUqJEkafjw4Q7vl/Ruhyt1795dPj4++uKLL5ympZY9/fTTac57+fJlffjhh7rvvvtUoEAB+fr6KiwsTG3bttWKFStc2m4gK9HDlgmmTJmiK1euqEuXLgoPD79lXS8vr9su74cfftCnn36qRo0aqW7duvL09NSuXbs0efJkLVy4UFu2bFHx4sXt9V977TWNHj1apUqVUqdOnRQQEKCYmBj9/vvvmj17tjp37izpWlirXbu2bDab2rRpo1KlSikuLk779+/XpEmTNHLkSHl4eKTZpmrVqikiIkLz58/XH3/8oRdeeEH58+eXJPt/02KM0X/+8x9NmzZNhQoVUvv27RUUFKTo6GitWrVKFSpUUM2aNSVd+5f0a6+9pvr166tVq1YqUKCADh8+rB9//FGLFy/WwoUL7ffLpbZn+PDhCg0NdQhlt7vH5eDBg6pXr56OHTumxo0bq2vXrjpy5Ihmz56tn376SXPnztWjjz7qNN+iRYu0YMECPfzww3r22We1e/du/fzzz/r999+1e/fuNHskb3T27Fm98MILqlu3rpo1a6agoCDFxMRo4cKFeuSRR/TFF184fTFt2bJFLVq00NmzZ1W/fn21b99eCQkJ2r17tyIjI/XGG2/cUd07deDAAdWuXVvly5dX9+7ddenSJfn7+0u69uU6b948NWjQQE2bNlVKSoq2bt2q999/X4sXL9amTZuUL18++7LSe350795dQ4cO1ZdffqnXX3/d6bLZV199patXr+qZZ55J1za8+uqrGj16tAoVKqRu3brJz89Pixcv1quvvqqlS5dq2bJl8vT0VFhYmCIiIrR69WqtWbNGPXv2tP+j5HYP2GTk3E+VkXMsI58TqQ9ITJs2TQ0aNHB4j1jhQaH8+fOrY8eOmjlzpo4fP66iRYtKkpKSkvT111+rQYMGKl++fJrz9urVS99++63Cw8PVo0cP+fj46NixY1q3bp2WLFmipk2bZuWmAK5jcNcaN25sJJkvvvgiQ/MdPHjQSDI9e/Z0KI+OjjaJiYlO9ZcuXWrc3NzMs88+61AeGBhoihcvbi5evOg0z6lTp+z//9JLLxlJZv78+U71zp49a5KTk+1/R0REGElm1apVDvV69uxpJJmDBw86LSM0NNSEhoY6lH322WdGkrn//vvN+fPnHaZdvXrVHDt2zP73+fPnHdqb6siRIyY4ONjcc889TtMkmQYNGjiVG3Pz/du8eXMjyYwcOdKhfP369cbd3d0EBgaaCxcu2MunTJliJBl3d3ezYsUKh3n++9//GknmnXfeSbMNN0pMTDRHjhxxKj9//rypXLmyKVCggElISLCXJyUlmbCwMCPJfPPNN07zXb+sjNQ15tb7Lq3jnLo/JZlhw4alOV9UVJS5evWqU/nkyZONJDNmzBiH8oycH/379zeSzMKFCx3qpaSkmFKlShlfX1+nZaRlw4YNRpIJCQkxMTEx9vIrV66YRx991Egyb7/9tsM8N3s/3EpGtu1OzrGMfk6sWrXKSDIRERHp3oZUoaGhN33f36hBgwbp3lep59ny5cvN2rVrjSQzatQo+/Rvv/3WSDIzZsww+/btczpnz58/b2w2m6lRo0aa593p06fTs3lAjsAl0UwQExMjSSpRokSmLK948eJp9sQ1b95clStX1tKlS52meXh4pHmzblq9Pj4+Pk5lBQoUkJtb5p8OH330kaRrl7RuvPnf3d1dwcHB9r8DAgLSbG+JEiXUoUMH/f333zp8+PBdtSc6OlrLli1TyZIlNXToUIdpdevWVdeuXXX27Fn98MMPTvN26dJFTZo0cSjr27evJGnz5s3pWr+Xl1ea50lAQICeeuopnTt3Tr///ru9fOHChYqKilKbNm3SvNn9+mVlpO7dKFKkiP0S4Y1CQ0PTPA+feuop+fv7O527GTk/+vXrZ697vWXLlungwYPq3Llzuh4w+eqrryRJr7/+ur0nR7p24/u4cePk5uamyZMn33Y5t5ORbUuVkXPsTj4nrKxevXq65557NHnyZBljJF3rsS1QoIAef/zxNOex2WwyxsjLyyvNz6+CBQu6tM1AViKwWZAxRjNmzFDTpk0VFBSkPHny2O832blzp44ePepQv3v37oqKilKlSpU0bNgwLVmyRLGxsU7L7dy5s9zd3dWuXTv16NFDX3/9tQ4cOOCy7bh48aL++usvFSlSRNWrV0/XPOvXr1enTp0UEhIiLy8v+3anfvnduO0ZtX37dknSQw89lObl38aNGzvUu96Nl68kKSQkRJJueX/WjXbt2qVevXqpdOnS8vHxsW/j4MGDJTlu42+//SZJevjhh2+73IzUvRtVq1a96aX9K1euaOLEiapXr54CAwPl7u4um80mNzc3xcXFOWxbRs+PypUrq379+lq8eLGOHDliL//8888lSc8++2y62r9t2zZJ/3esr1e+fHmVKFFCBw8eTPM9lF53cu5LGTvHMvo5kRP06dNH//vf//TLL79o//79WrVqlZ588kl5e3unWd/f31+tW7fWhg0bVK1aNY0YMUKrVq1SQkJCFrcccD3uYcsEwcHB2rNnT6Z9QL700ksaP368goOD1aJFCxUvXtzeKzZ16lQdOnTIof4HH3yg0qVLa8qUKRozZozGjBmjPHny6JFHHtG4ceNUtmxZSVKtWrW0du1avf3225ozZ46mT58uSapQoYIiIiLUtWvXTGl/qvPnz0uSw/12tzJv3jx16NBB3t7eatasmcqUKaO8efPab2pfs2bNXY9jl/olnFbvxvXlqW2/Xlr36uXJc+0tlJycnK71//bbb2rcuLGuXr2qJk2aqE2bNvL395ebm5t27NihBQsWOGxjRvZhRvf3nbq+V+pGnTt31rx581S6dGm1bdtWRYsWtYe78ePH3/G2pXruuef066+/avLkyRo+fLiOHz+uH3/8UdWqVVOtWrXStYz0nAOHDx/W+fPn73hImDs9Fhk5xzL6OZET9OjRQ6+++qomT56s0NBQGWPUp0+fW87z3Xff6Z133tHMmTPtPb/e3t7q0KGDxo4dqyJFimRF0wGXI7Blgnr16umXX37RypUr1bt377ta1smTJ/Xhhx8qPDxcGzZscLhBW7r2KwM3cnd316BBgzRo0CCdPHlS69at06xZszR79mzt2rVLu3btsn9p1qlTR4sWLVJSUpK2bt2qJUuW6KOPPlK3bt0UFBSUqTfopn75pDfIvvHGG/L09NSWLVtUsWJFh2nPPPOM1qxZc9dtSv0CPn78eJrTUy9vu2rstpEjR+rSpUv2JwGvN3r0aC1YsMChLCP7MKP722az3XRcq7QC6/XzpWXLli2aN2+e/YnX1KAhSSkpKXr33Xfvqr2S1L59exUpUkRffvml3nzzzQw/bCA5ngNpDUWTGefAnWxbRtzJ50ROUKhQIfvwHf7+/qpTp85tH+Ty8fFRZGSkIiMjdeTIEf3666+aOnWqZsyYoaioKK1duzaLWg+4FpdEM8F//vMfeXh4aO7cudq9e/ct696uh+h///ufUlJS1Lx5c6cP4ejoaP3vf/+75fyFCxdW+/bt9f3336tx48Y6cOCA/vrrL6d6Xl5eqlu3rkaMGKEPP/xQkpzCwt3KmzevwsPDdeLEiTQvMd5o//79qlSpklNYS0lJ0bp169Kcx83NLd29W5Lsl6fWrVuXZlhZtWqVJOm+++5L9zIzYv/+/QoMDEzzSda0AukDDzwgSbcc8uNO6krX7lu8/tJiquTkZO3YsSNdy7je/v37JUlt2rRxCGvStfuvLl265FCW0fNDunav5tNPP62jR49q4cKFmjx5svz8/NS9e/d0tzP1HEhrOIv9+/crOjpapUqVuuXTz7dzJ9uWEXfyOZF6b2FG3i/ZoU+fPkpKStKpU6du27t2o5CQEHXv3l1Lly5V2bJltW7dOp05c8ZFLQWyFoEtE4SFhSkyMlKXL19Wq1atbjq+0pIlS257f1HqI/br1q1z+GCNj49Xnz59nEJGUlKS1q9f77ScK1eu6OzZs5IkX19fSdKGDRucvjQl6cSJEw71MtPAgQMlXeshu/GeoJSUFHtvhnRt2/ft22cfpFS6dp9OZGTkTYNwwYIF0wwdN1OiRAk1a9ZMUVFRGj9+vMO0TZs2aebMmSpQoIAee+yxdC8zI8LCwnT27Fn9+eefDuVffvllmjeJt27dWmFhYfrxxx/T7DW5frytjNSVrl0iP3z4sJYtW+ZQPnLkyDu6nJZ67t4YhE6ePKn+/funOU9Gzo9Uffv2lbu7uwYMGKCDBw+qW7duTqHlVp566ilJ17bz1KlT9vLk5GQNGTJEKSkpd91TLt3ZtqVXRj8npP+7Af9uH9xxtUaNGmnBggWaN2/ebX/+69SpU9q5c6dT+cWLFxUfH688efLI09PTVU0FshSXRDPJq6++qqtXr2r48OG6//77VbduXdWsWdP+01S//vqr9u3bl+ZNxdcrWrSounTpolmzZqlatWpq3ry5YmNjtXz5cnl7e6tatWoOvR+XLl1SvXr1VLZsWdWoUUOhoaFKTEzU8uXLtWfPHrVp08beY/Xuu+/ql19+0UMPPaRSpUrJz89Pu3bt0uLFi1WgQAH702iZ6emnn9batWs1ffp0lStXTm3btlVQUJCOHTumX375RU899ZR9oNsXX3xRzz77rKpXr67HH39cHh4eWr9+vXbv3q3WrVtr4cKFTstv0qSJZs2apdatW+u+++6Th4eH6tevr/r169+0TZ9++qkefPBBvfzyy1q2bJlq1qxpH4fNzc1NU6ZMyVAAyIhBgwZp6dKlqlevnn3MvC1btmjdunXq0KGD5syZ41Df09NTs2fPVvPmzdWtWzd99tlneuCBB5SYmKg9e/Zo5cqV9i/njNSVpCFDhmjp0qVq27atOnfurMDAQG3YsEEHDx5Uw4YNMzyg6v33368HH3xQP/zwg+rWrat69erpxIkTWrx4sSpUqJDmrwRk5PxIVbJkSbVq1Uo//vijJGXocqh07WngoUOH6t1331V4eLg6dOigvHnzavHixfrrr79Ur149vfzyyxlaZlruZNvSK6OfE9K1e1WLFy+uWbNmycPDQ6GhobLZbHryyScVGhqarvUOGTJEfn5+aU4bMWKEw0/ZjRkz5qY/Zj9w4MCb9mKnjhOZHkePHlX16tV17733qkqVKgoJCVFcXJwWLVqk48ePa+DAgS57LwNZLjvHFMmNdu/ebQYMGGAqV65s8uXLZzw8PEzRokVNy5YtzeTJkx3GTbrZOGEXL140r776qilTpozx8vIyJUqUMM8995w5ffq0fYyjVJcvXzbvvPOOadmypQkJCTFeXl6mUKFCpnbt2uaTTz4xSUlJ9rpLly41vXr1MhUrVjT+/v7G19fXlC9f3jz//PMmKirKoQ2ZNQ5bqhkzZpj69esbf39/4+XlZcLCwky3bt3M1q1bHepNmTLFVK1a1fj6+pqCBQuadu3amT///POm7Tlx4oTp2rWrKVy4sHFzc3MYZ+pm+9eYa2NYPfvss6ZkyZLGw8PDFCxY0LRt29Zs3rzZqW7qGFlTpkxJc9t0i/HM0rJw4UJTu3Zt4+fnZwICAkyzZs3MmjVrbrmeQ4cOmX79+pmwsDDj4eFhAgMDTa1atZzGC8to3QULFpgaNWoYLy8vExgYaDp37myioqJuOQ5bWvsz1ZkzZ0y/fv1MaGio8fLyMqVLlzbDhg0zFy9ezJTzI9X8+fONJFOzZs2btuV2vv32W/Pggw8aPz8/4+XlZSpVqmRGjhxpLl265FT3TsZhS5WebbuTcywjnxOpNm/ebBo3bmz8/f2NzWZL9zaljsN2q9f27duNMf83DtutXvPmzTPGOI7DdjtpjcN27tw5M3z4cNOoUSNTrFgx4+npaYoWLWoaNGhgZs6caVJSUm67XCCnsBnz/we8AYAcIjIyUsOHD9fkyZMz5fIlAFgdgQ1AjnLhwgWVK1dOV65c0ZEjR1xy7yUAWA33sAHIEX766Sdt27ZNCxcu1IkTJzR27FjCGoB/DQIbgBxh9uzZmjZtmooUKaJhw4bpxRdfzO4mAUCW4ZIoAACAxTEOGwAAgMUR2AAAACwu19zDlpKSomPHjilfvnw3/a1DAABczRijCxcuqFixYnJzo18EmSPXBLZjx44pJCQku5sBAIAk6ciRIypRokR2NwO5RK4JbKk/P3LkyBH5+/tnc2sAAP9WcXFxCgkJ4WexkKlyTWBLvQzq7+9PYAMAZDtuz0Fm4uI6AACAxRHYAAAALC7DgS0+Pl4RERFq2bKlAgMDZbPZNHXq1DTr7tmzRy1btpSfn58CAwP15JNP6tSpU071UlJS9O6776pUqVLy9vZWlSpV9O2332Z4YwAAAHKjDN/Ddvr0aY0YMUIlS5ZU1apVtXr16jTrRUdHq379+goICNCoUaMUHx+vsWPHaufOndq8ebM8PT3tdV977TWNGTNGffr00f33368FCxaoW7dustls6tKlyx1vHAAA/xYpKSm6fPlydjcD6eTh4SF3d/d0189wYAsODlZMTIyKFi2qLVu26P7770+z3qhRo3Tx4kVt3bpVJUuWlCTVqlVLzZo109SpU9W3b19J0tGjRzVu3Dj1799fEydOlCQ9/fTTatCggV5++WV17NgxQxsEAMC/zeXLl3Xw4EGlpKRkd1OQAfnz51fRokXT9YBKhgObl5eXihYtett6c+fO1aOPPmoPa5LUtGlTlS9fXt9//709sC1YsEBXrlzRc889Z69ns9nUr18/devWTRs3blS9evUy2kwAAP4VjDGKiYmRu7u7QkJCGKw3BzDGKCEhQSdPnpR0rTPsdlwyrMfRo0d18uRJ1axZ02larVq19PPPP9v/3r59u/LmzauKFSs61UudTmADACBtV69eVUJCgooVKyZfX9/sbg7SycfHR5J08uRJFS5c+LZXE10S2GJiYiSlnRiDg4N19uxZJSUlycvLSzExMSpSpIhTd2DqvMeOHUtzHUlJSUpKSrL/HRcXl1nNBwAgx0hOTpYkh3vDkTOkBuwrV67cNrC5pN/00qVLkq5dPr2Rt7e3Q51Lly6lq96NRo8erYCAAPuLn6UCAPybMVBvzpORY+aSwJbazXd9D1iqxMREhzo+Pj7pqnejYcOGKTY21v46cuRIprQdAADAalxySTT1cmbqpdHrxcTEKDAw0N6rFhwcrFWrVskY45A0U+ctVqxYmuvw8vJKs2cOAAAgt3FJD1vx4sUVFBSkLVu2OE3bvHmzqlWrZv+7WrVqSkhI0J49exzqbdq0yT4dAABkjM2Wta87dfz4cT3//PMqXbq0vLy8FBISotatW2vlypWZtzMyydSpU5U/f/5sWbfLnv19/PHHtWjRIodLlStXrtTevXvVsWNHe1nbtm3l4eGhSZMm2cuMMfr0009VvHhx1a1b11VNBAAA2SgqKko1atTQL7/8ovfee087d+7UkiVL1KhRI/Xv3/+OlnmzwYOvXLlyN03NfuYOfPTRR+att94y/fr1M5JM+/btzVtvvWXeeustc/78eWOMMYcPHzYFCxY0ZcqUMR9++KEZNWqUKVCggLn33ntNYmKiw/JefvllI8n07dvXfPHFF6ZVq1ZGkvnmm2/S3abY2FgjycTGxt7JJgEAkCmy+vvo0qVLZvfu3ebSpUsO5VLWvu7Eww8/bIoXL27i4+Odpp07d84YY8yhQ4dMmzZtTN68eU2+fPlMx44dzfHjx+31IiIiTNWqVc0XX3xhwsLCjM1m+//bLzNp0iTTunVr4+vrayIiIowxxsyfP99Ur17deHl5mVKlSpnIyEhz5coVh/X27dvXFC5c2Hh5eZnKlSubhQsXmlWrVhlJDq/UZd6pmx27tNzRLg4NDXVqdOrr4MGD9np//fWXad68ufH19TX58+c33bt3d9jJqZKTk82oUaNMaGio8fT0NJUrVzYzZszIUJsIbAAAKyCwpc+ZM2eMzWYzo0aNummd5ORkU61aNVOvXj2zZcsW89tvv5kaNWqYBg0a2OtERESYvHnzmpYtW5pt27aZP/744/9vv0zhwoXNV199ZQ4cOGAOHTpkfv31V+Pv72+mTp1qDhw4YJYtW2bCwsJMZGSkfX0PPPCAqVy5slm2bJk5cOCAWbhwofn5559NUlKSGT9+vPH39zcxMTEmJibGXLhwIeMbfh2XBzYrIrABQDawYhLIZgS29Nm0aZORZH744Yeb1lm2bJlxd3c3hw8ftpft2rXLSDKbN282xlwLbB4eHubkyZM3bL/MoEGDHMqaNGniFBCnT59ugoODjTHGLF261Li5uZl//vknzfZMmTLFBAQEpHsbbycjgc0lT4kCAADcijHmtnX27NmjkJAQh7FWK1WqpPz582vPnj323zMPDQ1VUFCQ0/w3/uLSH3/8ofXr1+vtt9+2lyUnJysxMVEJCQnasWOHSpQoofLly9/pZrkMgQ0AAGS5cuXKyWaz6e+//77rZeXNmzdd5fHx8Ro+fLjat2/vVNfb2/umY79aAb8QCwAAslxgYKBatGihjz/+WBcvXnSafv78eVWsWFFHjhxxGHFi9+7dOn/+vCpVqpThdd533336559/VLZsWaeXm5ubqlSpoujoaO3duzfN+T09Pe0/BZbVCGwAACBbfPzxx0pOTlatWrU0d+5c7du3T3v27NGHH36oOnXqqGnTprr33nvVvXt3bdu2TZs3b1aPHj3UoEEDp8ud6fHmm2/q66+/1vDhw7Vr1y7t2bNHs2bN0uuvvy5JatCggerXr6/HH39cy5cv18GDB7V48WItWbJEkhQWFqb4+HitXLlSp0+fVkJCQqbuj1shsAHIGaw++ieADCtdurS2bdumRo0aafDgwQoPD1ezZs20cuVKffLJJ7LZbFqwYIEKFCig+vXrq2nTpipdurS+++67O1pfixYttGjRIi1btkz333+/HnjgAX3wwQcKDQ2115k7d67uv/9+de3aVZUqVdLQoUPtvWp169bVs88+q86dOysoKEjvvvtupuyH9LCZ9Nz1lwPExcUpICBAsbGx8vf3z+7mAMhsWRWmcsdHYtbJiuOSw45JVn8fJSYm6uDBgypVqpS8vb1dvj5knowcO3rYAAAALI7ABgAAYHEENgAAAIsjsAEAAFgcgQ0AAMDiCGwAAAAWR2ADAACwOAIbAACAxRHYAAAALI7ABgAAYHEENgAAcqOs+v3dO/wd3l69eqldu3aZv90W0LBhQw0aNChTl0lgAwAA/zqXL192KktOTlZKSko2tOb2CGwAACBbNWzYUAMHDtTQoUMVGBiookWLKjIy0qHO+fPn9cwzz6hIkSLy9vZWeHi4Fi1aZJ8+d+5cVa5cWV5eXgoLC9O4ceMc5g8LC9Nbb72lHj16yN/fX3379tXUqVOVP39+/fjjj6pUqZK8vLx0+PBhJSUlaciQISpevLjy5s2r2rVra/Xq1Q7LW79+vRo2bChfX18VKFBALVq00Llz59SrVy+tWbNGEyZMkM1mk81mU1RU1F3vIwIbAADIdtOmTVPevHm1adMmvfvuuxoxYoSWL18uSUpJSdHDDz+s9evXa8aMGdq9e7fGjBkjd3d3SdLWrVvVqVMndenSRTt37lRkZKTeeOMNTZ061WEdY8eOVdWqVbV9+3a98cYbkqSEhAS98847mjx5snbt2qXChQtrwIAB2rhxo2bNmqU///xTHTt2VMuWLbVv3z5J0o4dO9SkSRNVqlRJGzdu1Lp169S6dWslJydrwoQJqlOnjvr06aOYmBjFxMQoJCTkrvdPnrteAgAAwF2qUqWKIiIiJEnlypXTxIkTtXLlSjVr1kwrVqzQ5s2btWfPHpUvX16SVLp0afu877//vpo0aWIPYeXLl9fu3bv13nvvqVevXvZ6jRs31uDBg+1/r127VleuXNGkSZNUtWpVSdLhw4c1ZcoUHT58WMWKFZMkDRkyREuWLNGUKVM0atQovfvuu6pZs6YmTZpkX1blypXt/+/p6SlfX18VLVo00/YPPWwAACDbValSxeHv4OBgnTx5UtK1Hq0SJUrYw9qN9uzZowcffNCh7MEHH9S+ffuUnJxsL6tZs6bTvJ6eng7r3rlzp5KTk1W+fHn5+fnZX2vWrNGBAwfs7WnSpMmdbegdoocNAABkOw8PD4e/bTab/QEAHx+fTFlH3rx5ncp8fHxku+4p1/j4eLm7u2vr1q32S66p/Pz8MrU9GUEPGwAAsLQqVaooOjpae/fuTXN6xYoVtX79eoey9evXq3z58k6h63aqV6+u5ORknTx5UmXLlnV4pV7irFKlilauXHnTZXh6ejr07GUGAhsAALC0Bg0aqH79+nr88ce1fPlyHTx4UIsXL9aSJUskSYMHD9bKlSv11ltvae/evZo2bZomTpyoIUOGZHhd5cuXV/fu3dWjRw/98MMPOnjwoDZv3qzRo0frp59+kiQNGzZMv//+u5577jn9+eef+vvvv/XJJ5/o9OnTkq49kbpp0yZFRUXp9OnTmTJUCIENAABY3ty5c3X//fera9euqlSpkoYOHWrvxbrvvvv0/fffa9asWQoPD9ebb76pESNGODxwkBFTpkxRjx49NHjwYFWoUEHt2rXT77//rpIlS0q6FuqWLVumP/74Q7Vq1VKdOnW0YMEC5clz7U6zIUOGyN3dXZUqVVJQUJAOHz5819tvM8aYu16KBcTFxSkgIECxsbHy9/fP7uYAyGx3MJL6HckdH4lZJyuOSw47Jln9fZSYmKiDBw+qVKlS8vb2dvn6kHkycuzoYQMAALA4AhsAAIDFMaxHduNyAgAAuA162AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAuV5UVJRsNpt27NghSVq9erVsNpvOnz9/03mmTp2q/PnzZ0n7bodx2AAAyIVsWfVzbv9fRn/pslevXjp//rzmz5/vmgbdRt26dRUTE6OAgIBsWX9GEdgAAMC/jqenp4oWLZrdzUg3LokCAIBs1bBhQw0cOFBDhw5VYGCgihYtqsjISPv0bt26qXPnzg7zXLlyRYUKFdLXX38tSVqyZInq1aun/Pnzq2DBgnr00Ud14MCBm64zrUuiU6dOVcmSJeXr66vHHntMZ86cydTtvBsENgAAkO2mTZumvHnzatOmTXr33Xc1YsQILV++XJLUvXt3LVy4UPHx8fb6S5cuVUJCgh577DFJ0sWLF/XSSy9py5YtWrlypdzc3PTYY48pJSUlXevftGmTevfurQEDBmjHjh1q1KiRRo4cmfkbeoe4JAoAALJdlSpVFBERIUkqV66cJk6cqJUrV6pZs2Zq0aKF8ubNq3nz5unJJ5+UJM2cOVNt2rRRvnz5JEmPP/64w/K++uorBQUFaffu3QoPD7/t+idMmKCWLVtq6NChkqTy5ctrw4YNWrJkSWZu5h2jhw0AAGS7KlWqOPwdHByskydPSpLy5MmjTp066ZtvvpF0rTdtwYIF6t69u73+vn371LVrV5UuXVr+/v4KCwuTJB0+fDhd69+zZ49q167tUFanTp073ZxMRw8bAADIdh4eHg5/22w2h8uZ3bt3V4MGDXTy5EktX75cPj4+atmypX1669atFRoaqi+++ELFihVTSkqKwsPDdfny5SzbBlcisAEAAMurW7euQkJC9N1332nx4sXq2LGjPeSdOXNG//zzj7744gs99NBDkqR169ZlaPkVK1bUpk2bHMp+++23zGl8JiCwAQCAHKFbt2769NNPtXfvXq1atcpeXqBAARUsWFCff/65goODdfjwYf33v//N0LIHDhyoBx98UGPHjlXbtm21dOlSy9y/JnEPGwAAyCG6d++u3bt3q3jx4nrwwQft5W5ubpo1a5a2bt2q8PBwvfjii3rvvfcytOwHHnhAX3zxhSZMmKCqVatq2bJlev311zN7E+6YzWR0aGKLiouLU0BAgGJjY+Xv75/dzUm/rBiJOnccYvzbZdWo7bxfMobPMCdZ/X2UmJiogwcPqlSpUvL29nb5+pB5MnLs6GEDAACwOAIbAACAxRHYAAAALI7ABgAAYHEENgAAcoFc8gzhv0pGjhmBDQCAHMzd3V2Scs2I/v8mCQkJkpx/5SEtDJwLAEAOlidPHvn6+urUqVPy8PCQmxt9MVZnjFFCQoJOnjyp/Pnz20P3rRDYgLQwthSAHMJmsyk4OFgHDx7UoUOHsrs5yID8+fOraNGi6apLYAMAIIfz9PRUuXLluCyag3h4eKSrZy0VgQ0AgFzAzc2NXzrIxVx6oXvfvn3q0qWLSpQoIV9fX91zzz0aMWKE/Sa7VBs2bFC9evXk6+urokWLauDAgYqPj3dl0wAAAHIMl/WwHTlyRLVq1VJAQIAGDBigwMBAbdy4UREREdq6dasWLFggSdqxY4eaNGmiihUr6v3331d0dLTGjh2rffv2afHixa5qHgAAQI7hssA2ffp0nT9/XuvWrVPlypUlSX379lVKSoq+/vprnTt3TgUKFNCrr76qAgUKaPXq1fYfyQ0LC1OfPn20bNkyNW/e3FVNBAAAyBFcdkk0Li5OklSkSBGH8uDgYLm5ucnT01NxcXFavny5nnjiCXtYk6QePXrIz89P33//vauaBwAAkGO4LLA1bNhQktS7d2/t2LFDR44c0XfffadPPvlEAwcOVN68ebVz505dvXpVNWvWdJjX09NT1apV0/bt213VPAAAgBzDZZdEW7ZsqbfeekujRo3Sjz/+aC9/7bXXNHLkSElSTEyMpGu9bjcKDg7W2rVrb7r8pKQkJSUl2f9O7dEDAADIbVw6rEdYWJjq16+vxx9/XAULFtRPP/2kUaNGqWjRohowYIAuXbokSfLy8nKa19vb2z49LaNHj9bw4cNd1nYAAACrcFlgmzVrlvr27au9e/eqRIkSkqT27dsrJSVFr7zyirp27SofHx9JcugpS5WYmGifnpZhw4bppZdesv8dFxenkJCQTN4KAACA7Oeye9gmTZqk6tWr28NaqjZt2ighIUHbt2+3XwpNvTR6vZiYGBUrVuymy/fy8pK/v7/DCwAAIDdyWWA7ceKEkpOTncqvXLkiSbp69arCw8OVJ08ebdmyxaHO5cuXtWPHDlWrVs1VzQMAAMgxXBbYypcvr+3bt2vv3r0O5d9++63c3NxUpUoVBQQEqGnTppoxY4YuXLhgrzN9+nTFx8erY8eOrmoeAABAjmEzxhhXLPjXX39V48aNVbBgQQ0YMEAFCxbUokWLtHjxYj399NP64osvJEnbtm1T3bp1ValSJfXt21fR0dEaN26c6tevr6VLl6Z7fXFxcQoICFBsbGzOujxqs7l+Ha45xLkbx8V6suKYSByXjOK94iTHfh/B0lwW2CRp8+bNioyM1Pbt23XmzBmVKlVKPXv21NChQ5Unz/8977Bu3Tq98sor2rZtm/Lly6dOnTpp9OjRypcvX7rXlWPfIHzYWRPHxXoIbNbEe8VJjv0+gqW5NLBlpRz7BuHDzpo4LtZDYLMm3itOcuz3ESzNZfewAQAAIHMQ2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACzO5YFt27ZtatOmjQIDA+Xr66vw8HB9+OGHDnU2bNigevXqydfXV0WLFtXAgQMVHx/v6qYBAADkCHlcufBly5apdevWql69ut544w35+fnpwIEDio6OttfZsWOHmjRpoooVK+r9999XdHS0xo4dq3379mnx4sWubB4AAECO4LLAFhcXpx49eqhVq1aaM2eO3NzS7sx79dVXVaBAAa1evVr+/v6SpLCwMPXp00fLli1T8+bNXdVEAACAHMFll0RnzpypEydO6O2335abm5suXryolJQUhzpxcXFavny5nnjiCXtYk6QePXrIz89P33//vauaBwAAkGO4LLCtWLFC/v7+Onr0qCpUqCA/Pz/5+/urX79+SkxMlCTt3LlTV69eVc2aNR3m9fT0VLVq1bR9+3ZXNQ8AACDHcFlg27dvn65evaq2bduqRYsWmjt3rp566il9+umn+s9//iNJiomJkSQFBwc7zR8cHKxjx47ddPlJSUmKi4tzeAEAAORGLruHLT4+XgkJCXr22WftT4W2b99ely9f1meffaYRI0bo0qVLkiQvLy+n+b29ve3T0zJ69GgNHz7cNY0HAACwEJf1sPn4+EiSunbt6lDerVs3SdLGjRvtdZKSkpzmT0xMtE9Py7BhwxQbG2t/HTlyJLOaDgAAYCkuC2zFihWTJBUpUsShvHDhwpKkc+fO2S+Fpl4avV5MTIx9GWnx8vKSv7+/wwsAACA3cllgq1GjhiTp6NGjDuWp96UFBQUpPDxcefLk0ZYtWxzqXL58WTt27FC1atVc1TwAAIAcw2WBrVOnTpKkL7/80qF88uTJypMnjxo2bKiAgAA1bdpUM2bM0IULF+x1pk+frvj4eHXs2NFVzQMAAMgxXPbQQfXq1fXUU0/pq6++0tWrV9WgQQOtXr1as2fP1rBhw+yXO99++23VrVtXDRo0UN++fRUdHa1x48apefPmatmypauaBwAAkGPYjDHGVQu/cuWKRo0apSlTpujYsWMKDQ1V//79NWjQIId669at0yuvvKJt27YpX7586tSpk0aPHq18+fKle11xcXEKCAhQbGxszrqfzWZz/Tpcd4hzL46L9WTFMZE4LhnFe8VJjv0+gqW5NLBlpRz7BuHDzpo4LtZDYLMm3itOcuz3ESzNZfewAQAAIHMQ2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMVlWWB7++23ZbPZFB4e7jRtw4YNqlevnnx9fVW0aFENHDhQ8fHxWdU0AAAAS8uTFSuJjo7WqFGjlDdvXqdpO3bsUJMmTVSxYkW9//77io6O1tixY7Vv3z4tXrw4K5oHAABgaVkS2IYMGaIHHnhAycnJOn36tMO0V199VQUKFNDq1avl7+8vSQoLC1OfPn20bNkyNW/ePCuaCAAAYFkuvyT666+/as6cORo/frzTtLi4OC1fvlxPPPGEPaxJUo8ePeTn56fvv//e1c0DAACwPJcGtuTkZD3//PN6+umnde+99zpN37lzp65evaqaNWs6lHt6eqpatWravn27K5sHAACQI7j0kuinn36qQ4cOacWKFWlOj4mJkSQFBwc7TQsODtbatWtvuuykpCQlJSXZ/46Li7vL1gIAAFiTy3rYzpw5ozfffFNvvPGGgoKC0qxz6dIlSZKXl5fTNG9vb/v0tIwePVoBAQH2V0hISOY0HAAAwGJcFthef/11BQYG6vnnn79pHR8fH0ly6ClLlZiYaJ+elmHDhik2Ntb+OnLkyN03GgAAwIJcckl03759+vzzzzV+/HgdO3bMXp6YmKgrV64oKipK/v7+9kuhqZdGrxcTE6NixYrddB1eXl5p9swBAADkNi7pYTt69KhSUlI0cOBAlSpVyv7atGmT9u7dq1KlSmnEiBEKDw9Xnjx5tGXLFof5L1++rB07dqhatWquaB4AAECO4pIetvDwcM2bN8+p/PXXX9eFCxc0YcIElSlTRgEBAWratKlmzJihN954Q/ny5ZMkTZ8+XfHx8erYsaMrmgcAAJCj2IwxJqtW1rBhQ50+fVp//fWXvWzbtm2qW7euKlWqpL59+yo6Olrjxo1T/fr1tXTp0nQvOy4uTgEBAYqNjXUY083ybDbXryPrDnHuwXGxnqw4JhLHJaN4rzjJsd9HsLRs//H3++67TytWrJCPj49efPFFff755+rdu7fmzJmT3U0DAACwhCztYXOlHPsvGv51ak0cF+uhh82aeK84ybHfR7C0bO9hAwAAwK0R2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACzOZYHt999/14ABA1S5cmXlzZtXJUuWVKdOnbR3716nunv27FHLli3l5+enwMBAPfnkkzp16pSrmgYAAJCj5HHVgt955x2tX79eHTt2VJUqVXT8+HFNnDhR9913n3777TeFh4dLkqKjo1W/fn0FBARo1KhRio+P19ixY7Vz505t3rxZnp6ermoiAABAjuCywPbSSy9p5syZDoGrc+fOuvfeezVmzBjNmDFDkjRq1ChdvHhRW7duVcmSJSVJtWrVUrNmzTR16lT17dvXVU0EAADIEWzGGJOVK6xRo4YkaevWrZKkIkWKqEGDBvr+++8d6lWoUEEhISFasWJFupYbFxengIAAxcbGyt/fP3Mb7Uo2m+vXkbWHOHfguFhPVhwTieOSUbxXnOTY7yNYWpY+dGCM0YkTJ1SoUCFJ0tGjR3Xy5EnVrFnTqW6tWrW0ffv2rGweAACAJWVpYPvmm2909OhRde7cWZIUExMjSQoODnaqGxwcrLNnzyopKSnNZSUlJSkuLs7hBQAAkBtlWWD7+++/1b9/f9WpU0c9e/aUJF26dEmS5OXl5VTf29vboc6NRo8erYCAAPsrJCTERS0HAADIXlkS2I4fP65WrVopICBAc+bMkbu7uyTJx8dHktLsRUtMTHSoc6Nhw4YpNjbW/jpy5IiLWg8AAJC9XPaUaKrY2Fg9/PDDOn/+vNauXatixYrZp6VeCk29NHq9mJgYBQYGptn7Jl3rlbvZNAAAgNzEpYEtMTFRrVu31t69e7VixQpVqlTJYXrx4sUVFBSkLVu2OM27efNmVatWzZXNAwAAyBFcdkk0OTlZnTt31saNGzV79mzVqVMnzXqPP/64Fi1a5HBJc+XKldq7d686duzoquYBAADkGC4bh23QoEGaMGGCWrdurU6dOjlNf+KJJyRJR44cUfXq1ZU/f3698MILio+P13vvvacSJUro999/T/dlzxw77g1jGFkTx8V6GIfNmnivOMmx30ewNJcFtoYNG2rNmjU3nX79anft2qWXXnpJ69atk6enp1q1aqVx48apSJEi6V5fjn2D8GFnTRwX6yGwWRPvFSc59vsIlpblv3TgKjn2DcKHnTVxXKyHwGZNvFec5NjvI1halg6cCwAAgIxz+bAeQGbKsk6WrFkNAADpQg8bAACAxRHYAAAALI7ABgAAYHEENgAAAIsjsAEAAFgcgQ0AAMDiCGwAAAAWR2ADAACwOAbOBYDr2LJgdOZc8ouAALIQPWwAAAAWR2ADAACwOAIbAACAxRHYAAAALI7ABgAAYHEENgAAAIsjsAEAAFgcgQ0AAMDiCGwAAAAWR2ADAACwOAIbAACAxRHYAAAALI7ABgAAYHEENgAAAIsjsAEAAFgcgQ0AAMDi8mR3A6zKZsua9ZisWQ0AAMjB6GEDAACwOAIbAACAxRHYAAAALI7ABgAAYHEENgAAAIsjsAEAAFgcgQ0AAMDiCGwAAAAWR2ADAACwOAIbAACAxRHYAAAALI7ABgAAYHEENgAAAIsjsAEAAFgcgQ0AAMDiCGwAAAAWR2ADAACwOAIbAACAxRHYAAAALI7ABgAAYHEENgAAAIsjsAEAAFgcgQ0AAMDiCGwAAAAWlye7GwAg57PZXL8O4/pVAIBl0cMGAABgcQQ2AAAAiyOwAQAAWByBDQAAwOJ46AAAcqGseBBE4mEQIKtYooctKSlJr7zyiooVKyYfHx/Vrl1by5cvz+5mAQAAWIIlAluvXr30/vvvq3v37powYYLc3d31yCOPaN26ddndNAAAgGxnM8Zka4/25s2bVbt2bb333nsaMmSIJCkxMVHh4eEqXLiwNmzYkK7lxMXFKSAgQLGxsfL397/rdmXd5YSsGMAq91y04LhYU9aMw5Y1Bz8r1pIVH7u8V7JPZn8fAZIFetjmzJkjd3d39e3b117m7e2t3r17a+PGjTpy5Eg2tg4AACD7ZXtg2759u8qXL+/0r5BatWpJknbs2JENrQIAALCObH9KNCYmRsHBwU7lqWXHjh1Lc76kpCQlJSXZ/46NjZV0rSs6J8mS1uawfWIFHBfryU17K6d9Tt1KVmxJQBZd3039Hrlbqcc3m+84Qi6T7YHt0qVL8vLycir39va2T0/L6NGjNXz4cKfykJCQzG2giwVkyUqyZC25CsfFenLT3grIRcc+92xJ5h+XCxcu5KpjjeyV7YHNx8fHoacsVWJion16WoYNG6aXXnrJ/ndKSorOnj2rggULypZVd9tmg7i4OIWEhOjIkSPczGoRHBNr4rhY07/huBhjdOHCBRUrViy7m4JcJNsDW3BwsI4ePepUHhMTI0k3PeG9vLyceuby58+f6e2zKn9//1z7YZdTcUysieNiTbn9uNCzhsyW7Q8dVKtWTXv37nW6p2PTpk326QAAAP9m2R7YOnTooOTkZH3++ef2sqSkJE2ZMkW1a9fOcfekAQAAZLZsvyRau3ZtdezYUcOGDdPJkydVtmxZTZs2TVFRUfryyy+zu3mW4+XlpYiIiDQf1ED24JhYE8fFmjguwJ3J9l86kK49YPDGG29oxowZOnfunKpUqaK33npLLVq0yO6mAQAAZDtLBDYAAADcXLbfwwYAAIBb+1cFttWrV8tms2n16tVZvu6oqCjZbDZNnTo105Y5depU2Ww2RUVFZdoy/41y23kB67LZbBowYEB2NwNADvSvCmzImJ9//lmRkZFZsq4NGzYoMjJS58+fz5L1wfXu5Ji++OKLuu+++xQYGChfX19VrFhRkZGRio+PT9f8NpstzdeYMWPSNf+kSZMIz1ngbt/vBw4ckLe3t2w2m7Zs2ZK5jQMsisCWgz355JO6dOmSQkNDXbL8n3/+Oc2f/3KFDRs2aPjw4QS2XOROjunvv/+uhx56SMOHD9eECRPUqFEjjRkzRi1btlRKSkq6ltGsWTNNnz7d4dW6det0zUtgyxp3+35/8cUXlSdPtg9yAGQpzvgczN3dXe7u7tndDCDTrFu3zqmsTJkyGjJkiDZv3qwHHnjgtssoX768nnjiCVc0744YY5SYmHjTn9lDxixdulRLly7V0KFDNXLkyOxuDpBlcl0P29GjR9W7d28VK1ZMXl5eKlWqlPr166fLly+nWX/t2rXq2LGjSpYsKS8vL4WEhOjFF190+tH5hg0bqmHDhk7z9+rVS2FhYQ5l58+fV69evRQQEKD8+fOrZ8+eN/2X5N9//60OHTooMDBQ3t7eqlmzpn788cd0bWta97CFhYXp0Ucf1bp161SrVi15e3urdOnS+vrrrx3mvXLlioYPH65y5crJ29tbBQsWVL169bR8+XL7dn388ceSHC8zpRo7dqzq1q2rggULysfHRzVq1NCcOXOc2ph6z878+fMVHh4uLy8vVa5cWUuWLLHXiYyM1MsvvyxJKlWqlH1dmXlv3r/pvLgbq1evVs2aNeXt7a0yZcros88+U2RkpMOxTz2m33zzjSpUqCBvb2/VqFFDv/76q71OZh7T1P2Ykd6YS5cu2X+POCPr2bVrl9asWWNvb+qxvXEfpLrVe3Dp0qWqWbOmfHx89NlnnznMd6t9l2r79u16+OGH5e/vLz8/PzVp0kS//fZbhrYpM1nh3Lhy5YpeeOEFvfDCCypTpkymbyNgZbmqh+3YsWOqVauWzp8/r759++qee+7R0aNHNWfOHCUkJKQ5z+zZs5WQkKB+/fqpYMGC2rx5sz766CNFR0dr9uzZGW6DMUZt27bVunXr9Oyzz6pixYqaN2+eevbs6VR3165devDBB1W8eHH997//Vd68efX999+rXbt2mjt3rh577LEMr1+S9u/frw4dOqh3797q2bOnvvrqK/Xq1Us1atRQ5cqVJV370Bw9erSefvpp1apVS3FxcdqyZYu2bdumZs2a6ZlnntGxY8e0fPlyTZ8+3WkdEyZMUJs2bdS9e3ddvnxZs2bNUseOHbVo0SK1atXKoe66dev0ww8/6LnnnlO+fPn04Ycf6vHHH9fhw4dVsGBBtW/fXnv37tW3336rDz74QIUKFZIkBQUF3dH234jzIn22b9+uli1bKjg4WMOHD1dycrJGjBiR5nFYs2aNvvvuOw0cOFBeXl6aNGmSWrZsqc2bNys8PPyujunVq1d1/vx5Xb58WX/99Zdef/115cuXT7Vq1UrXdkydOlWTJk2SMUYVK1bU66+/rm7dut12vvHjx+v555+Xn5+fXnvtNUlSkSJF0rXOG/3zzz/q2rWrnnnmGfXp00cVKlSwT7vdvpOunQMPPfSQ/P39NXToUHl4eOizzz5Tw4YNtWbNGtWuXfuO2nWnrHJujB8/XufOndPrr7+uH374IdO3E7A0k4v06NHDuLm5md9//91pWkpKilm1apWRZFatWmUvT0hIcKo7evRoY7PZzKFDh+xlDRo0MA0aNHCq27NnTxMaGmr/e/78+UaSeffdd+1lV69eNQ899JCRZKZMmWIvb9Kkibn33ntNYmKiQzvr1q1rypUrd9vtnTJlipFkDh48aC8LDQ01ksyvv/5qLzt58qTx8vIygwcPtpdVrVrVtGrV6pbL79+/v7nZKXLjfrt8+bIJDw83jRs3diiXZDw9Pc3+/fvtZX/88YeRZD766CN72Xvvvee0LZnl33Ze3KnWrVsbX19fc/ToUXvZvn37TJ48eRzOA0lGktmyZYu97NChQ8bb29s89thj9rI7PaYbN260r0OSqVChgsOxuZW6deua8ePHmwULFphPPvnEhIeHG0lm0qRJ6Zq/cuXKaR7PiIiINN8Lt3oPLlmyxKl+evddu3btjKenpzlw4IC97NixYyZfvnymfv366dqWzGSFcyMmJsbky5fPfPbZZ8aY/9v3ab2vgdwo11wSTUlJ0fz589W6dWvVrFnTaXpalzMkOdxXcvHiRZ0+fVp169aVMUbbt2/PcDt+/vln5cmTR/369bOXubu76/nnn3eod/bsWf3yyy/q1KmTLly4oNOnT+v06dM6c+aMWrRooX379uno0aMZXr8kVapUSQ899JD976CgIFWoUEH/+9//7GX58+fXrl27tG/fvjtax/X77dy5c4qNjdVDDz2kbdu2OdVt2rSpw+WLKlWqyN/f36E9rsJ5kT7JyclasWKF2rVrp2LFitnLy5Ytq4cfftipfp06dVSjRg373yVLllTbtm21dOlSJScn31VbKlWqpOXLl2v+/PkaOnSo8ubNm+6nRNevX68XXnhBbdq00bPPPqutW7cqPDxcr776qtPlbFcqVarUTX+p5Xb7Ljk5WcuWLVO7du1UunRpe73g4GB169ZN69atU1xcnMu3IZVVzo1XXnlFpUuX1tNPP33HywByslxzSfTUqVOKi4uzX1JIr8OHD+vNN9/Ujz/+qHPnzjlMi42NzXA7Dh06pODgYPn5+TmUX39JRLp22dIYozfeeENvvPFGmss6efKkihYtqlOnTjmUBwYGytPT86ZtKFmypFNZgQIFHLZvxIgRatu2rcqXL6/w8HC1bNlSTz75pKpUqXLbbZSkRYsWaeTIkdqxY4eSkpLs5WkFoPS0x1Vy63lRvHjxDLfhVk6ePKlLly6pbNmyTtPSKitXrpxTWfny5ZWQkKBTp06paNGiN13X2bNnHe4d9PHxUUBAgP1vf39/NW3aVJLUtm1bzZw5U23bttW2bdtUtWrVDG2Xp6enBgwYYA9v9erVU3x8vEMAdHd3z7TL76lKlSp102m323eSlJCQ4HRuSFLFihWVkpKiI0eO2G9vcDUrnBu//fabpk+frpUrV8rNLdf0MwAZkmsC251ITk5Ws2bNdPbsWb3yyiu65557lDdvXh09elS9evVyGEbAZrPJpPErXnf6L8bUZQ8ZMuSm/xIvW7asjhw54vThv2rVqjRvdE91sydHr29//fr1deDAAS1YsEDLli3T5MmT9cEHH+jTTz+97b9g165dqzZt2qh+/fqaNGmSgoOD5eHhoSlTpmjmzJl31B4ryQnnRU7Wvn17rVmzxv53z549bzmURvv27fXkk09q1qxZGQ5skhQSEiLpWhiQrj0wc/1wNaGhobe94f1mPbE3O848EXpnbnZuDB06VA899JBKlSplP1anT5+WJMXExOjw4cNp/sMQyE1yTWALCgqSv7+//vrrr3TPs3PnTu3du1fTpk1Tjx497OWpT0per0CBAmlewjt06JDD36GhoVq5cqXi4+MdelP++ecfh3qplzo8PDzsvQlp8fDwcGrPnXxppSUwMFD/+c9/9J///Efx8fGqX7++IiMj7YHtZl9Sc+fOlbe3t5YuXSovLy97+ZQpU+64LTdb193KredFZitcuLC8vb21f/9+p2lplaV1KX3v3r3y9fW191bd7JiOGzfOodfy+stsaUlKSlJKSsod9WxKsh+f1Hb16NFD9erVs0+/PlzdrM0FChSQdO1J1fz589vLbzzO6ZGefefr6+t0bkjXnh52c3Ozh9CsYIVz4/Dhwzp06FCaPZdt2rRRQEAAYzgi18s1fctubm5q166dFi5cmObI12n1gqT2/Fw/zRijCRMmONUtU6aM/v77b4fLk3/88YfWr1/vUO+RRx7R1atX9cknn9jLkpOT9dFHHznUK1y4sBo2bKjPPvtMMTExTutLXY+3t7eaNm3q8Er98rgbZ86ccfjbz89PZcuWdbi8mTdvXknOwym4u7vLZrM59C5ERUVp/vz5d9yem63rbuXW8yKzubu7q2nTppo/f76OHTtmL9+/f78WL17sVH/jxo0O9yseOXJECxYsUPPmze3772bHtEaNGg7nc6VKlez1rly54rSuyZMnS5LDPYgJCQn6+++/7b0sUtr75sKFCxo/frwKFSpkv6+qdOnSDut/8MEH7fXz5s2b5jmYeg/m9cNTXLx4UdOmTXOqezu323fu7u5q3ry5FixY4NDzd+LECc2cOVP16tWTv79/htd7p6xwbnz++eeaN2+ewyv1/s+xY8fqm2++ydRtBqwo1/SwSdKoUaO0bNkyNWjQQH379lXFihUVExOj2bNnpzkg5z333GMflPPo0aPy9/fX3Llz07y36qmnntL777+vFi1aqHfv3jp58qQ+/fRTVa5c2eEG4NatW+vBBx/Uf//7X0VFRalSpUr64Ycf0uwd+Pjjj1WvXj3de++96tOnj0qXLq0TJ05o48aNio6O1h9//JG5O+g6lSpVUsOGDVWjRg0FBgZqy5YtmjNnjsPvHKZ+wQ0cOFAtWrSQu7u7unTpolatWun9999Xy5Yt1a1bN508eVIff/yxypYtqz///POO2pO6rtdee01dunSRh4eHWrdubf9gvxucF+kTGRmpZcuW6cEHH1S/fv2UnJysiRMnKjw8XDt27HCoGx4erhYtWjgM3SDJ4VJjRo/p6tWrNXDgQHXo0EHlypXT5cuXtXbtWv3www+qWbOmw2C4mzdvVqNGjRQREWH/+bSPP/7Y/oBJyZIlFRMTo6+++kqHDx/W9OnTb3nf5/Vt/uSTTzRy5EiVLVtWhQsXVuPGjdW8eXOVLFlSvXv31ssvvyx3d3d99dVXCgoK0uHDhzOym9O170aOHKnly5erXr16eu6555QnTx599tlnSkpK0rvvvpuh9WWG7D43mjdv7lSWGvYaNGiQ5gNFQK6THY+mutKhQ4dMjx49TFBQkPHy8jKlS5c2/fv3N0lJSWkO37B7927TtGlT4+fnZwoVKmT69OljH3bi+qEWjDFmxowZpnTp0sbT09NUq1bNLF261Gn4BmOMOXPmjHnyySeNv7+/CQgIME8++aTZvn17mss8cOCA6dGjhylatKjx8PAwxYsXN48++qiZM2fObbf1ZkMKpDVcx43DT4wcOdLUqlXL5M+f3/j4+Jh77rnHvP322+by5cv2OlevXjXPP/+8CQoKMjabzeHx/S+//NKUK1fOeHl5mXvuucdMmTIlzaEPJJn+/fs7tSc0NNT07NnToeytt94yxYsXN25ubpk+xMe/6by4GytXrjTVq1c3np6epkyZMmby5Mlm8ODBxtvb214n9ZjOmDHDfg5Ur149zaE3MnJM9+/fb3r06GFKly5tfHx8jLe3t6lcubKJiIgw8fHxDnVTj1lERIS9bNmyZaZZs2b2fZY/f37TvHlzs3LlynRv//Hjx02rVq1Mvnz5jCSH98zWrVtN7dq1jaenpylZsqR5//33M/QeNCZj+27btm2mRYsWxs/Pz/j6+ppGjRqZDRs2pHtbMlt2nhtpYVgP/NvYjLHond8ALKFdu3YOQ8DYbDb1799fEydOzOaWIbtxbgBZJ9fcwwbg7t04Vtm+ffv0888/3/KpZPw7cG4A2StX3cMG4O6ULl1avXr1UunSpXXo0CF98skn8vT01NChQ7O7achmnBtA9iKwAbBr2bKlvv32Wx0/flxeXl6qU6eORo0aleZgqPh34dwAshf3sAEAAFgc97ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2ACkm81ms/92Z0ZERUXJZrNp6tSpmd4mAPg3ILABOdDUqVNls9lks9nS/AF7Y4xCQkJks9n06KOPZkMLAQCZicAG5GDe3t6aOXOmU/maNWsUHR0tLy+vbGgVACCzEdiAHOyRRx7R7NmzdfXqVYfymTNnqkaNGipatGg2tQwAkJkIbEAO1rVrV505c0bLly+3l12+fFlz5sxRt27dnOpfvHhRgwcPVkhIiLy8vFShQgWNHTtWN/7gSVJSkl588UUFBQUpX758atOmjaKjo9Nsw9GjR/XUU0+pSJEi8vLyUuXKlfXVV19l7oYCwL8cgQ3IwcLCwlSnTh19++239rLFixcrNjZWXbp0cahrjFGbNm30wQcfqGXLlnr//fdVoUIFvfzyy3rppZcc6j799NMaP368mjdvrjFjxsjDw0OtWrVyWv+JEyf0wAMPaMWKFRowYIAmTJigsmXLqnfv3ho/frxLthkA/pUMgBxnypQpRpL5/fffzcSJE02+fPlMQkKCMcaYjh07mkaNGhljjAkNDTWtWrUyxhgzf/58I8mMHDnSYVkdOnQwNpvN7N+/3xhjzI4dO4wk89xzzznU69atm5FkIiIi7GW9e/c2wcHB5vTp0w51u3TpYgICAuxtOnjwoJFkpkyZkmn7AAD+TehhA3K4Tp066dKlS1q0aJEuXLigRYsWpXk59Oeff5a7u7sGDhzoUD548GAZY7R48WJ7PUlO9QYNGuTwtzFGc+fOVevWrWWM0enTp+2vFi1aKDY2Vtu2bcvELQWAf6882d0AAHcnKChITZs21cyZM5WQkKDk5GR16NDBqd6hQ4dUrFgx5cuXz6G8YsWK9ump/3Vzc1OZMmUc6lWoUMHh71OnTun8+fP6/PPP9fnnn6fZtpMnT97xdgEA/g+BDcgFunXrpj59+uj48eN6+OGHlT9/fpevMyUlRZL0xBNPqGfPnmnWqVKlisvbAQD/BgQ2IBd47LHH9Mwzz+i3337Td999l2ad0NBQrVixQhcuXHDoZfv777/t01P/m5KSogMHDjj0qv3zzz8Oy0t9gjQ5OVlNmzbN7E0CAFyHe9iAXMDPz0+ffPKJIiMj1bp16zTrPPLII0pOTtbEiRMdyj/44APZbDY9/PDDkmT/74cffuhQ78anPt3d3fX4449r7ty5+uuvv5zWd+rUqTvdHADADehhA3KJm12WTNW6dWs1atRIr732mqKiolS1alUtW7ZMCxYs0KBBg+z3rFWrVk1du3bVpEmTFBsbq7p162rlypXav3+/0zLHjBmjVatWqXbt2urTp48qVaqks2fPatu2bVqxYoXOnj3rkm0FgH8bAhvwL+Hm5qYff/xRb775pr777jtNmTJFYWFheu+99zR48GCHul999ZWCgoL0zTffaP78+WrcuLF++uknhYSEONQrUqSINm/erBEjRuiHH37QpEmTVLBgQVWuXFnvvPNOVm4eAORqNmNuGOIcAAAAlsI9bAAAABZHYAMAALA4AhsAAIDFEdgAAAAsjsAGAABgcQQ2AAAAiyOwAQAAWByBDQAAwOIIbAAAABZHYAMAALA4AhsAAIDFEdgAAAAsjsAGAABgcf8PJjORVQ19oPMAAAAASUVORK5CYII=\n", 545 | "text/plain": [ 546 | "
" 547 | ] 548 | }, 549 | "metadata": {}, 550 | "output_type": "display_data" 551 | } 552 | ], 553 | "source": [ 554 | "title = \"Classification accuracy of chat LLMs\"\n", 555 | "\n", 556 | "model_names_succinct = [\"-\".join(model_name.split(\"-\")[:-1]) for model_name in eval_results.keys()]\n", 557 | "ind = np.arange(len(model_names_succinct))\n", 558 | "width = 0.25\n", 559 | "\n", 560 | "pos_vals = [100.0 * eval_results[model_name]['total_accuracy'] for model_name in eval_results.keys()]\n", 561 | "plt.bar(ind, pos_vals, width, label='Correct', color='blue')\n", 562 | "\n", 563 | "neg_vals = [100.0 * (eval_results[model_name]['num_examples'] - eval_results[model_name]['raw_total_accuracy']) / eval_results[model_name]['num_examples'] for model_name in eval_results.keys()]\n", 564 | "plt.bar(ind+width, neg_vals, width, label='Incorrect', color='red')\n", 565 | "\n", 566 | "invalid_vals = [100.0 * (1.0 - (eval_results[model_name]['num_valid'] / eval_results[model_name]['num_examples'])) for model_name in eval_results.keys()]\n", 567 | "plt.bar(ind+2*width, invalid_vals, width, label='Invalid', color='black')\n", 568 | "\n", 569 | "plt.xlabel(\"Model\")\n", 570 | "plt.title(title)\n", 571 | "plt.xticks(ind + (width), model_names_succinct)\n", 572 | "plt.legend(bbox_to_anchor=(1.05, 1.0), loc='upper left')\n", 573 | "plt.tight_layout()\n", 574 | "\n", 575 | "plt.savefig(Path(\"./\", \"chat_llms_classsification_plot.svg\"), format=\"svg\")\n", 576 | "plt.savefig(Path(\"./\", \"chat_llms_classsification_plot.png\"), format=\"png\")\n", 577 | "plt.savefig(Path(\"./\", \"chat_llms_classsification_plot.pdf\"), format=\"pdf\")" 578 | ] 579 | }, 580 | { 581 | "cell_type": "code", 582 | "execution_count": 71, 583 | "id": "e6ce861b", 584 | "metadata": {}, 585 | "outputs": [ 586 | { 587 | "data": { 588 | "text/plain": [ 589 | "{'claude-instant-v1.1': {'raw_total_accuracy': 206,\n", 590 | " 'total_accuracy': 0.103,\n", 591 | " 'total_accuracy_on_valid': 0.103,\n", 592 | " 'num_valid': 2000,\n", 593 | " 'num_examples': 2000},\n", 594 | " 'claude-v1.3': {'raw_total_accuracy': 350,\n", 595 | " 'total_accuracy': 0.175,\n", 596 | " 'total_accuracy_on_valid': 0.175,\n", 597 | " 'num_valid': 2000,\n", 598 | " 'num_examples': 2000},\n", 599 | " 'gpt-3.5-turbo-0301': {'raw_total_accuracy': 67,\n", 600 | " 'total_accuracy': 0.0335,\n", 601 | " 'total_accuracy_on_valid': 0.0393885949441505,\n", 602 | " 'num_valid': 1701,\n", 603 | " 'num_examples': 2000},\n", 604 | " 'gpt-4-0314': {'raw_total_accuracy': 37,\n", 605 | " 'total_accuracy': 0.0185,\n", 606 | " 'total_accuracy_on_valid': 0.018518518518518517,\n", 607 | " 'num_valid': 1998,\n", 608 | " 'num_examples': 2000}}" 609 | ] 610 | }, 611 | "execution_count": 71, 612 | "metadata": {}, 613 | "output_type": "execute_result" 614 | } 615 | ], 616 | "source": [ 617 | "eval_results" 618 | ] 619 | }, 620 | { 621 | "cell_type": "code", 622 | "execution_count": 74, 623 | "id": "5f9b7e1f", 624 | "metadata": {}, 625 | "outputs": [], 626 | "source": [ 627 | "from IPython.display import Image" 628 | ] 629 | }, 630 | { 631 | "cell_type": "code", 632 | "execution_count": 75, 633 | "id": "5865c575", 634 | "metadata": {}, 635 | "outputs": [ 636 | { 637 | "data": { 638 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAHgCAYAAAA10dzkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABU/ElEQVR4nO3deXxM1+P/8fckshEJIQgisReppZRaat9atVTtWnza0ipVLdWPbglVtKWliy60KFUtam0RFLWV2lpFa/kIQuwkggTJ+f3hm/kZEyREJsl9PR+PebQ599x7z11m5u3ce8/YjDFGAAAAsAw3VzcAAAAAmYsACAAAYDEEQAAAAIshAAIAAFgMARAAAMBiCIAAAAAWQwAEAACwGAIgAACAxRAAAQAALIYACAAAYDEEQAAAAIshAAIAAFgMARAAAMBiCIAAAAAWQwAEAACwGAIgAACAxRAAAQAALIYAmIVFRUXJZrOpV69erm6Kk4iICNlsNq1atcpp2vfff69q1aopb968stlsGjhwoCQpNDRUoaGhmdrOW8nK+xfZT2RkpOrUqaN8+fLJZrOpXbt2GbbsKVOmyGazacqUKRm2TADWRgDMZP/8849efPFFhYWFyd/fX56enipatKhatWqlr7/+WomJia5u4l3ZsGGDunfvrvPnz6tv374KDw9Xy5YtXdYem82mhg0bumz9sIaoqCi1bdtWBw4c0NNPP63w8HB16dLF1c1Kl1WrVslmsykiIiLd84aGhspmsykqKuq2dRs2bHjTfzzeqFevXrLZbLLZbHr77bdvWm/q1Kn2erzfgbTJ5eoGWMnw4cM1bNgwJScnq3bt2urZs6d8fX11/PhxrVq1Ss8++6w+//xzbd682dVNva3+/furS5cuKlGihEP5zz//LGOMvv32W9WpU8dh2ooVKzKzibdVrFgx7d69W/7+/q5uCrK55cuXKyEhQWPHjlW3bt1c3ZwcJ1euXJo8ebLCw8Pl7u7uNH3ixInKlSuXrl696oLWAdkTATCTjBw5UuHh4QoODtasWbNUq1YtpzqLFi3S2LFjXdC69CtYsKAKFizoVH706FFJUtGiRZ2mlS5d+p63Kz08PDx03333uboZyAFudd7j7j322GOaN2+elixZolatWjlM2717t9atW6fHH39cc+fOdVELgeyHS8CZICoqShEREfLw8NAvv/ySaviTrn3ILVmy5LbL27Nnj/773/+qRo0aCgwMlJeXl0JCQtSnTx9FR0c71TfGaOrUqapTp44CAwPl7e2t4OBgtWjRQj/88IND3b/++ktdu3ZVaGiovLy8FBgYqAceeEADBw7UlStX7PVuvAcw5R6lyZMnS5JKlixpvySTclnoVvcA/vDDD2rSpIkCAgLk7e2t0NBQde3a1aE3NDY2Vh988IEaN26s4sWLy9PTU4GBgWrTpo02bNjgsLyU9kjS6tWr7W25/hLXre4BjImJUb9+/RQaGmpfT/v27bVlyxanutffn7Vy5Uo1bNhQefPmlZ+fn1q1aqXdu3enus2puXz5sj799FM9+uijCgkJkZeXlwICAtS0aVMtXrz4pvNFR0drwIABKlu2rHx8fBQQEKCaNWvqnXfeueO6t7qclnJp7vpLftfvzz179qhz584qVKiQ3Nzc7OfJli1b9NJLL6lKlSr2Y122bFkNGjRIZ8+even23e78+PLLL2Wz2TRs2LBU5z927Jg8PDx0//3333QdN/rxxx9Vv359+fv7y8fHR/fff79GjRrlcJtGymXT8PBwSVKjRo3s51laLnGmZdtulNZzLD2fE7169VKjRo0kScOGDXN4v6R1O+6l7t27y8fHRxMnTnSallL27LPPpjrv5cuX9fHHH+uBBx5Q/vz5lTt3boWGhqpt27Zavnz5PW03kJXRA5gJJk+erCtXrqhLly4KCwu7ZV0vL6/bLu+nn37SF198oUaNGqlOnTry9PTUzp07NWnSJC1cuFCbN29WsWLF7PXfeOMNjRo1SiVLllSnTp3k7++vmJgY/fHHH5o1a5Y6d+4s6Vr4q1Wrlmw2m9q0aaOSJUsqLi5O+/bt04QJEzRixAh5eHik2qaqVasqPDxc8+bN059//qmXXnpJ+fLlkyT7f1NjjNF//vMfTZ06VQULFlT79u0VGBio6OhorVy5UuXLl1eNGjUkXfuX/htvvKH69eurVatWyp8/vw4dOqQFCxZo8eLFWrhwof1+w5T2DBs2TCEhIQ4h73b3CB04cED16tXT0aNH1bhxY3Xt2lWHDx/WrFmz9PPPP2vOnDl67LHHnOZbtGiR5s+fr0ceeUTPP/+8du3apV9++UV//PGHdu3alWqP6Y3OnDmjl156SXXq1FGzZs0UGBiomJgYLVy4UI8++qgmTpzo9EW3efNmtWjRQmfOnFH9+vXVvn17Xbx4Ubt27VJERITeeuutO6p7p/bv369atWqpXLly6t69uy5duiQ/Pz9J176s586dqwYNGqhp06ZKTk7Wli1b9OGHH2rx4sXauHGj8ubNa19WWs+P7t27a8iQIfr666/15ptvOl0m/Oabb3T16lU999xzadqG119/XaNGjVLBggXVrVs3+fr6avHixXr99de1dOlSRUZGytPTU6GhoQoPD9eqVau0evVq9ezZ0/6PnNs98JSecz9Fes6x9HxOpDywMnXqVDVo0MDhPZIVHtzKly+fOnbsqBkzZujYsWMqUqSIJCkxMVHffvutGjRooHLlyqU6b69evfT9998rLCxMPXr0kI+Pj44ePaq1a9dqyZIlatq0aWZuCpB1GNxzjRs3NpLMxIkT0zXfgQMHjCTTs2dPh/Lo6GiTkJDgVH/p0qXGzc3NPP/88w7lAQEBplixYubChQtO85w8edL+/6+88oqRZObNm+dU78yZMyYpKcn+d3h4uJFkVq5c6VCvZ8+eRpI5cOCA0zJCQkJMSEiIQ9mXX35pJJkHH3zQnDt3zmHa1atXzdGjR+1/nzt3zqG9KQ4fPmyCgoLMfffd5zRNkmnQoIFTuTE337/Nmzc3ksyIESMcytetW2fc3d1NQECAOX/+vL188uTJRpJxd3c3y5cvd5jnv//9r5Fk3nvvvVTbcKOEhARz+PBhp/Jz586ZSpUqmfz585uLFy/ayxMTE01oaKiRZL777jun+a5fVnrqGnPrfZfacU7Zn5LM0KFDU50vKirKXL161al80qRJRpIZPXq0Q3l6zo9+/foZSWbhwoUO9ZKTk03JkiVN7ty5nZaRmvXr1xtJJjg42MTExNjLr1y5Yh577DEjybz77rsO89zs/XAr6dm2OznH0vs5sXLlSiPJhIeHp3kbUoSEhNz0fX+jBg0apHlfpZxny5YtM2vWrDGSzMiRI+3Tv//+eyPJTJ8+3ezdu9fpnD137pyx2WymevXqqZ53p06dSsvmATkSl4AzQUxMjCSpePHiGbK8YsWKpdpT2Lx5c1WqVElLly51mubh4ZHqzdOp9Ur5+Pg4leXPn19ubhl/unzyySeSrl3Cu/FhDHd3dwUFBdn/9vf3T7W9xYsXV4cOHfTPP//o0KFDd9We6OhoRUZGqkSJEhoyZIjDtDp16qhr1646c+aMfvrpJ6d5u3TpoiZNmjiU9enTR5K0adOmNK3fy8sr1fPE399fTz/9tM6ePas//vjDXr5w4UJFRUWpTZs2qT58cP2y0lP3bhQuXNh+SfRGISEhqZ6HTz/9tPz8/JzO3fScH3379rXXvV5kZKQOHDigzp07p+mBn2+++UaS9Oabb9p7mqRrDyKMHTtWbm5umjRp0m2Xczvp2bYU6TnH7uRzIiurV6+e7rvvPk2aNEnGGEnXepTz58+vJ554ItV5bDabjDHy8vJK9fOrQIEC97TNQFZGAMyGjDGaPn26mjZtqsDAQOXKlct+v86OHTt05MgRh/rdu3dXVFSUKlasqKFDh2rJkiWKjY11Wm7nzp3l7u6udu3aqUePHvr222+1f//+e7YdFy5c0N9//63ChQurWrVqaZpn3bp16tSpk4KDg+Xl5WXf7pQv0xu3Pb22bdsmSXr44YdTvdzduHFjh3rXu/FynSQFBwdL0i3vb7vRzp071atXL5UqVUo+Pj72bRw0aJAkx238/fffJUmPPPLIbZebnrp3o0qVKje9leHKlSv69NNPVa9ePQUEBMjd3V02m01ubm6Ki4tz2Lb0nh+VKlVS/fr1tXjxYh0+fNhe/tVXX0mSnn/++TS1f+vWrZL+/7G+Xrly5VS8eHEdOHAg1fdQWt3JuS+l7xxL7+dEdtC7d2/973//06+//qp9+/Zp5cqVeuqpp+Tt7Z1qfT8/P7Vu3Vrr169X1apVNXz4cK1cuVIXL17M5JYDWQ/3AGaCoKAg7d69O8M+cF955RWNGzdOQUFBatGihYoVK2bvtZsyZYoOHjzoUP+jjz5SqVKlNHnyZI0ePVqjR49Wrly59Oijj2rs2LEqU6aMJKlmzZpas2aN3n33Xc2ePVvTpk2TJJUvX17h4eHq2rVrhrQ/xblz5yTJ4X7FW5k7d646dOggb29vNWvWTKVLl1aePHnsDxmsXr36rsdRTPlST6335frylLZfL7V7HXPluvYWS0pKStP6f//9dzVu3FhXr15VkyZN1KZNG/n5+cnNzU3bt2/X/PnzHbYxPfswvfv7Tl3fa3ajzp07a+7cuSpVqpTatm2rIkWK2MPiuHHj7njbUrzwwgv67bffNGnSJA0bNkzHjh3TggULVLVqVdWsWTNNy0jLOXDo0CGdO3fujocQutNjkZ5zLL2fE9lBjx499Prrr2vSpEkKCQmRMUa9e/e+5Tw//PCD3nvvPc2YMcPeM+3t7a0OHTpozJgxKly4cGY0HchyCICZoF69evr111+1YsUKPfPMM3e1rBMnTujjjz9WWFiY1q9f73DDvHTtVzhu5O7uroEDB2rgwIE6ceKE1q5dq5kzZ2rWrFnauXOndu7caf8Srl27thYtWqTExERt2bJFS5Ys0SeffKJu3bopMDAwQ2+YTvkyS2swfuutt+Tp6anNmzerQoUKDtOee+45rV69+q7blPKFfuzYsVSnp1zOv1djB44YMUKXLl2yP+l5vVGjRmn+/PkOZenZh+nd3zab7abjqqUWgK+fLzWbN2/W3Llz7U80pwQXSUpOTtb7779/V+2VpPbt26tw4cL6+uuv9fbbb6f74Q/J8RxIbeiijDgH7mTb0uNOPieyg4IFC9qHe/Hz81Pt2rVv+2Cdj4+PIiIiFBERocOHD+u3337TlClTNH36dEVFRWnNmjWZ1Hoga+EScCb4z3/+Iw8PD82ZM0e7du26Zd3b9WD973//U3Jyspo3b+70oR4dHa3//e9/t5y/UKFCat++vX788Uc1btxY+/fv199//+1Uz8vLS3Xq1NHw4cP18ccfS5JT+LhbefLkUVhYmI4fP57qJdUb7du3TxUrVnQKf8nJyVq7dm2q87i5uaW5902S/XLc2rVrUw0/K1eulCQ98MADaV5meuzbt08BAQGpPqmcWsB96KGHJOmWQ8TcSV3p2n2f119KTZGUlKTt27enaRnX27dvnySpTZs2DuFPunb/2qVLlxzK0nt+SNfudX322Wd15MgRLVy4UJMmTZKvr6+6d++e5namnAOpDX+yb98+RUdHq2TJkrd8uv127mTb0uNOPidS7s1Mz/vFFXr37q3ExESdPHnytr1/NwoODlb37t21dOlSlSlTRmvXrtXp06fvUUuBrI0AmAlCQ0MVERGhy5cvq1WrVjcd32vJkiW3vT8rZUiGtWvXOnxQx8fHq3fv3k6hJTExUevWrXNazpUrV3TmzBlJUu7cuSVJ69evd/oSlqTjx4871MtIAwYMkHStB+/Ge6qSk5PtvS3StW3fu3evfdBd6dp9ThERETcN1gUKFEg1xNxM8eLF1axZM0VFRWncuHEO0zZu3KgZM2Yof/78evzxx9O8zPQIDQ3VmTNn9NdffzmUf/3116netN+6dWuFhoZqwYIFqfbqXD/eW3rqStduCTh06JAiIyMdykeMGHFHlw9Tzt0bg9WJEyfUr1+/VOdJz/mRok+fPnJ3d1f//v114MABdevWzSkE3crTTz8t6dp2njx50l6elJSkwYMHKzk5+a578qU727a0Su/nhPT/H4i42wep7rVGjRpp/vz5mjt37m1/bu/kyZPasWOHU/mFCxcUHx+vXLlyydPT8141FcjSuAScSV5//XVdvXpVw4YN04MPPqg6deqoRo0a9p+C++2337R3795Ub/K+XpEiRdSlSxfNnDlTVatWVfPmzRUbG6tly5bJ29tbVatWdeiduXTpkurVq6cyZcqoevXqCgkJUUJCgpYtW6bdu3erTZs29h61999/X7/++qsefvhhlSxZUr6+vtq5c6cWL16s/Pnz2582zEjPPvus1qxZo2nTpqls2bJq27atAgMDdfToUf366696+umn7QM3v/zyy3r++edVrVo1PfHEE/Lw8NC6deu0a9cutW7dWgsXLnRafpMmTTRz5ky1bt1aDzzwgDw8PFS/fn3Vr1//pm364osvVLduXb366quKjIxUjRo17OMAurm5afLkyekKFOkxcOBALV26VPXq1bOP2bh582atXbtWHTp00OzZsx3qe3p6atasWWrevLm6deumL7/8Ug899JASEhK0e/durVixwv5ln566kjR48GAtXbpUbdu2VefOnRUQEKD169frwIEDatiwYboHCH7wwQdVt25d/fTTT6pTp47q1aun48ePa/HixSpfvnyqv6KRnvMjRYkSJdSqVSstWLBAktJ1+Ve69rT3kCFD9P777yssLEwdOnRQnjx5tHjxYv3999+qV6+eXn311XQtMzV3sm1pld7PCenavb7FihXTzJkz5eHhoZCQENlsNj311FMKCQlJ03oHDx4sX1/fVKcNHz7c4acjR48erSlTpqRad8CAATftZU8ZpzQtjhw5omrVqun+++9X5cqVFRwcrLi4OC1atEjHjh3TgAED7tl7GcjyXDkGjRXt2rXL9O/f31SqVMnkzZvXeHh4mCJFipiWLVuaSZMmOYzbdbNx6i5cuGBef/11U7p0aePl5WWKFy9uXnjhBXPq1Cn7GFspLl++bN577z3TsmVLExwcbLy8vEzBggVNrVq1zOeff24SExPtdZcuXWp69eplKlSoYPz8/Ezu3LlNuXLlzIsvvmiioqIc2pBR4wCmmD59uqlfv77x8/MzXl5eJjQ01HTr1s1s2bLFod7kyZNNlSpVTO7cuU2BAgVMu3btzF9//XXT9hw/ftx07drVFCpUyLi5uTmMc3az/WvMtTHUnn/+eVOiRAnj4eFhChQoYNq2bWs2bdrkVDdljLbJkyenum26xXh6qVm4cKGpVauW8fX1Nf7+/qZZs2Zm9erVt1zPwYMHTd++fU1oaKjx8PAwAQEBpmbNmk7j1aW37vz580316tWNl5eXCQgIMJ07dzZRUVG3HAcwtf2Z4vTp06Zv374mJCTEeHl5mVKlSpmhQ4eaCxcuZMj5kWLevHlGkqlRo8ZN23I733//valbt67x9fU1Xl5epmLFimbEiBHm0qVLTnXvZBzAFGnZtjs5x9LzOZFi06ZNpnHjxsbPz8/YbLY0b1PKOIC3em3bts0Y8//HAbzVa+7cucYYx3EAbye1cQDPnj1rhg0bZho1amSKFi1qPD09TZEiRUyDBg3MjBkzTHJy8m2XC+RUNmP+b0AlAMghIiIiNGzYME2aNClDLtcCQE5DAASQo5w/f15ly5bVlStXdPjw4Xty7yoAZHfcAwggR/j555+1detWLVy4UMePH9eYMWMIfwBwEwRAADnCrFmzNHXqVBUuXFhDhw7Vyy+/7OomAUCWxSVgAAAAi2EcQAAAAIshAAIAAFiMZe4BTE5O1tGjR5U3b96b/lYpAAD3mjFG58+fV9GiReXmRj8MXMMyAfDo0aMKDg52dTMAAJAkHT58WMWLF3d1M2BRlgmAKT/3c/jwYfn5+bm4NQAAq4qLi1NwcDA/QweXskwATLns6+fnRwAEALgctyPBlbj5AAAAwGIIgAAAABaT4QEwPj5e4eHhatmypQICAmSz2TRlypRU6+7evVstW7aUr6+vAgIC9NRTT+nkyZNO9ZKTk/X++++rZMmS8vb2VuXKlfX9999ndNMBAAAsIcPvATx16pSGDx+uEiVKqEqVKlq1alWq9aKjo1W/fn35+/tr5MiRio+P15gxY7Rjxw5t2rRJnp6e9rpvvPGGRo8erd69e+vBBx/U/Pnz1a1bN9lsNnXp0iWjNwEAgBwnOTlZly9fdnUzkEYeHh5yd3e/Z8vP8AAYFBSkmJgYFSlSRJs3b9aDDz6Yar2RI0fqwoUL2rJli0qUKCFJqlmzppo1a6YpU6aoT58+kqQjR45o7Nix6tevnz799FNJ0rPPPqsGDRro1VdfVceOHe/pDgIAILu7fPmyDhw4oOTkZFc3BemQL18+FSlS5J48MJThAdDLy0tFihS5bb05c+boscces4c/SWratKnKlSunH3/80R4A58+frytXruiFF16w17PZbOrbt6+6deumDRs2qF69ehm9GQAA5AjGGMXExMjd3V3BwcEMPp0NGGN08eJFnThxQtK1zrWM5pJhYI4cOaITJ06oRo0aTtNq1qypX375xf73tm3blCdPHlWoUMGpXsp0AiAAAKm7evWqLl68qKJFiyp37tyubg7SyMfHR5J04sQJFSpUKMOvdrokAMbExEhKPdEGBQXpzJkzSkxMlJeXl2JiYlS4cGGn7s+UeY8ePZrqOhITE5WYmGj/Oy4uLqOaDwBAtpGUlCRJDvfWI3tICexXrlzJ8ADokn7gS5cuSbp2ufhG3t7eDnUuXbqUpno3GjVqlPz9/e0vfgYOAGBlDDyd/dzLY+aSAJjSrXl9D12KhIQEhzo+Pj5pqnejoUOHKjY21v46fPhwhrQdAAAgu3PJJeCUy7cpl4KvFxMTo4CAAHuvX1BQkFauXCljjEMSTpm3aNGiqa7Dy8sr1Z5DAAAAq3NJD2CxYsUUGBiozZs3O03btGmTqlatav+7atWqunjxonbv3u1Qb+PGjfbpAAAgfWy2zH3dqWPHjunFF19UqVKl5OXlpeDgYLVu3VorVqzIuJ2RQaZMmaJ8+fK5uhlp4rJnwZ944gktWrTI4dLsihUrtGfPHnXs2NFe1rZtW3l4eGjChAn2MmOMvvjiCxUrVkx16tTJ1HYDAIDMERUVperVq+vXX3/VBx98oB07dmjJkiVq1KiR+vXrd0fLvNlg2FeuXLmbpmY/5h745JNPzDvvvGP69u1rJJn27dubd955x7zzzjvm3LlzxhhjDh06ZAoUKGBKly5tPv74YzNy5EiTP39+c//995uEhASH5b366qtGkunTp4+ZOHGiadWqlZFkvvvuuzS3KTY21kgysbGxGbqtAACkR2Z/H126dMns2rXLXLp0yaFcytzXnXjkkUdMsWLFTHx8vNO0s2fPGmOMOXjwoGnTpo3JkyePyZs3r+nYsaM5duyYvV54eLipUqWKmThxogkNDTU2m+3/tl9mwoQJpnXr1iZ37twmPDzcGGPMvHnzTLVq1YyXl5cpWbKkiYiIMFeuXHFYb58+fUyhQoWMl5eXqVSpklm4cKFZuXKlkeTwSlnmnbrZscsI9yQAhoSEOO2ElNeBAwfs9f7++2/TvHlzkzt3bpMvXz7TvXt3h4OWIikpyYwcOdKEhIQYT09PU6lSJTN9+vR0tYkACADICgiAaXP69Gljs9nMyJEjb1onKSnJVK1a1dSrV89s3rzZ/P7776Z69eqmQYMG9jrh4eEmT548pmXLlmbr1q3mzz///L/tlylUqJD55ptvzP79+83BgwfNb7/9Zvz8/MyUKVPM/v37TWRkpAkNDTURERH29T300EOmUqVKJjIy0uzfv98sXLjQ/PLLLyYxMdGMGzfO+Pn5mZiYGBMTE2POnz+f/g2/TrYLgFkRARAAXCArJgsXIwCmzcaNG40k89NPP920TmRkpHF3dzeHDh2yl+3cudNIMps2bTLGXAuAHh4e5sSJEzdsv8zAgQMdypo0aeIUOKdNm2aCgoKMMcYsXbrUuLm5mX///TfV9kyePNn4+/uneRtv514GQJc8BQwAAHArxpjb1tm9e7eCg4MdxvqtWLGi8uXLp927d+vBBx+UJIWEhCgwMNBp/ht/kezPP//UunXr9O6779rLkpKSlJCQoIsXL2r79u0qXry4ypUrd6eblWUQAAEAQJZTtmxZ2Ww2/fPPP3e9rDx58qSpPD4+XsOGDVP79u2d6np7e9907OHsiF+EBgAAWU5AQIBatGihzz77TBcuXHCafu7cOVWoUEGHDx92GFFk165dOnfunCpWrJjudT7wwAP6999/VaZMGaeXm5ubKleurOjoaO3ZsyfV+T09Pe0/vZfVEQABAECW9NlnnykpKUk1a9bUnDlztHfvXu3evVsff/yxateuraZNm+r+++9X9+7dtXXrVm3atEk9evRQgwYNnC7vpsXbb7+tb7/9VsOGDdPOnTu1e/duzZw5U2+++aYkqUGDBqpfv76eeOIJLVu2TAcOHNDixYu1ZMkSSVJoaKji4+O1YsUKnTp1ShcvXszQ/ZGRCIAAcoasPpotgHQrVaqUtm7dqkaNGmnQoEEKCwtTs2bNtGLFCn3++eey2WyaP3++8ufPr/r166tp06YqVaqUfvjhhztaX4sWLbRo0SJFRkbqwQcf1EMPPaSPPvpIISEh9jpz5szRgw8+qK5du6pixYoaMmSIvdevTp06ev7559W5c2cFBgbq/fffz5D9cC/YTFrusswB4uLi5O/vr9jYWPn5+bm6OQAyWmaFM2t8ZGaczDgu2eyYZPb3UUJCgg4cOKCSJUvK29v7nq8PGedeHjt6AAEAACyGAAgAAGAxBEAAAACLIQACAABYDAEQAADAYgiAAAAAFkMABAAAsBgCIAAAgMUQAAEAACyGAAgAAGAxBEAAAKwos34/+w5/R7tXr15q165dxm93FtCwYUMNHDjQpW0gAAIAANzg8uXLTmVJSUlKTk52QWsyHgEQAABkaQ0bNtSAAQM0ZMgQBQQEqEiRIoqIiHCoc+7cOT333HMqXLiwvL29FRYWpkWLFtmnz5kzR5UqVZKXl5dCQ0M1duxYh/lDQ0P1zjvvqEePHvLz81OfPn00ZcoU5cuXTwsWLFDFihXl5eWlQ4cOKTExUYMHD1axYsWUJ08e1apVS6tWrXJY3rp169SwYUPlzp1b+fPnV4sWLXT27Fn16tVLq1ev1vjx42Wz2WSz2RQVFXWP9tzNEQABAECWN3XqVOXJk0cbN27U+++/r+HDh2vZsmWSpOTkZD3yyCNat26dpk+frl27dmn06NFyd3eXJG3ZskWdOnVSly5dtGPHDkVEROitt97SlClTHNYxZswYValSRdu2bdNbb70lSbp48aLee+89TZo0STt37lShQoXUv39/bdiwQTNnztRff/2ljh07qmXLltq7d68kafv27WrSpIkqVqyoDRs2aO3atWrdurWSkpI0fvx41a5dW71791ZMTIxiYmIUHByceTvy/+TK9DUCAACkU+XKlRUeHi5JKlu2rD799FOtWLFCzZo10/Lly7Vp0ybt3r1b5cqVkySVKlXKPu+HH36oJk2a2ENduXLltGvXLn3wwQfq1auXvV7jxo01aNAg+99r1qzRlStXNGHCBFWpUkWSdOjQIU2ePFmHDh1S0aJFJUmDBw/WkiVLNHnyZI0cOVLvv/++atSooQkTJtiXValSJfv/e3p6Knfu3CpSpEgG76W0owcQAABkeZUrV3b4OygoSCdOnJB0rcetePHi9vB3o927d6tu3boOZXXr1tXevXuVlJRkL6tRo4bTvJ6eng7r3rFjh5KSklSuXDn5+vraX6tXr9b+/fvt7WnSpMmdbWgmoQcQAABkeR4eHg5/22w2+wMZPj4+GbKOPHnyOJX5+PjIdt1TzPHx8XJ3d9eWLVvsl5hT+Pr6Zmh77iV6AAEAQLZWuXJlRUdHa8+ePalOr1ChgtatW+dQtm7dOpUrV84pxN1OtWrVlJSUpBMnTqhMmTIOr5RLupUrV9aKFStuugxPT0+HnkdXIAACAIBsrUGDBqpfv76eeOIJLVu2TAcOHNDixYu1ZMkSSdKgQYO0YsUKvfPOO9qzZ4+mTp2qTz/9VIMHD073usqVK6fu3burR48e+umnn3TgwAFt2rRJo0aN0s8//yxJGjp0qP744w+98MIL+uuvv/TPP//o888/16lTpyRde+J448aNioqK0qlTp1wytAwBEAAAZHtz5szRgw8+qK5du6pixYoaMmSIvZftgQce0I8//qiZM2cqLCxMb7/9toYPH+7wAEh6TJ48WT169NCgQYNUvnx5tWvXTn/88YdKlCgh6VpIjIyM1J9//qmaNWuqdu3amj9/vnLlunbn3eDBg+Xu7q6KFSsqMDBQhw4dypB9kB42Y4zJ9LW6QFxcnPz9/RUbGys/Pz9XNwdARruDXxq4I9b4yMw4mXFcstkxyezvo4SEBB04cEAlS5aUt7f3PV8fMs69PHb0AAIAAFgMARAAAMBiGAYmq+PyCQAAyGD0AAIAAFgMARAAAMBiCIAAAAAWQwAEAACwGAIgAACAxRAAAQAALIYACAAALC8qKko2m03bt2+XJK1atUo2m03nzp276TxTpkxRvnz5MqV9GY1xAAEAsCBbZv184v9J7y/P9urVS+fOndO8efPuTYNuo06dOoqJiZG/v79L1n+vEQABAABu4OnpqSJFiri6GfcMl4ABAECW1rBhQw0YMEBDhgxRQECAihQpooiICPv0bt26qXPnzg7zXLlyRQULFtS3334rSVqyZInq1aunfPnyqUCBAnrssce0f//+m64ztUvAU6ZMUYkSJZQ7d249/vjjOn36dIZuZ2YiAAIAgCxv6tSpypMnjzZu3Kj3339fw4cP17JlyyRJ3bt318KFCxUfH2+vv3TpUl28eFGPP/64JOnChQt65ZVXtHnzZq1YsUJubm56/PHHlZycnKb1b9y4Uc8884z69++v7du3q1GjRhoxYkTGb2gm4RIwAADI8ipXrqzw8HBJUtmyZfXpp59qxYoVatasmVq0aKE8efJo7ty5euqppyRJM2bMUJs2bZQ3b15J0hNPPOGwvG+++UaBgYHatWuXwsLCbrv+8ePHq2XLlhoyZIgkqVy5clq/fr2WLFmSkZuZaegBBAAAWV7lypUd/g4KCtKJEyckSbly5VKnTp303XffSbrW2zd//nx1797dXn/v3r3q2rWrSpUqJT8/P4WGhkqSDh06lKb17969W7Vq1XIoq1279p1ujsvRAwgAALI8Dw8Ph79tNpvD5dvu3burQYMGOnHihJYtWyYfHx+1bNnSPr1169YKCQnRxIkTVbRoUSUnJyssLEyXL1/OtG3ISgiAAAAg26tTp46Cg4P1ww8/aPHixerYsaM9NJ4+fVr//vuvJk6cqIcffliStHbt2nQtv0KFCtq4caND2e+//54xjXcBAiAAAMgRunXrpi+++EJ79uzRypUr7eX58+dXgQIF9NVXXykoKEiHDh3Sf//733Qte8CAAapbt67GjBmjtm3baunSpdn2/j+JewABAEAO0b17d+3atUvFihVT3bp17eVubm6aOXOmtmzZorCwML388sv64IMP0rXshx56SBMnTtT48eNVpUoVRUZG6s0338zoTcg0NpPeobmzqbi4OPn7+ys2NlZ+fn6ubk7aZcZI7dY4BZDTZdavGvB+SR8+w5xk9vdRQkKCDhw4oJIlS8rb2/uerw8Z514eO3oAAQAALIYACAAAYDEEQAAAAIshAAIAAFgMARAAAAuwyDOfOcq9PGYEQAAAcjB3d3dJsuwvXmRnFy9elOT8KygZgYGgAQDIwXLlyqXcuXPr5MmT8vDwkJsbfT9ZnTFGFy9e1IkTJ5QvXz57iM9IBEDgTjC2GYBswmazKSgoSAcOHNDBgwdd3RykQ758+VSkSJF7smwCIAAAOZynp6fKli3LZeBsxMPD4570/KUgAAIAYAFubm78EgjsXHojwN69e9WlSxcVL15cuXPn1n333afhw4fbb3pMsX79etWrV0+5c+dWkSJFNGDAAMXHx7uo1QAAANmby3oADx8+rJo1a8rf31/9+/dXQECANmzYoPDwcG3ZskXz58+XJG3fvl1NmjRRhQoV9OGHHyo6OlpjxozR3r17tXjxYlc1HwAAINtyWQCcNm2azp07p7Vr16pSpUqSpD59+ig5OVnffvutzp49q/z58+v1119X/vz5tWrVKvuPZoeGhqp3796KjIxU8+bNXbUJAAAA2ZLLLgHHxcVJkgoXLuxQHhQUJDc3N3l6eiouLk7Lli3Tk08+aQ9/ktSjRw/5+vrqxx9/zNQ2AwAA5AQuC4ANGzaUJD3zzDPavn27Dh8+rB9++EGff/65BgwYoDx58mjHjh26evWqatSo4TCvp6enqlatqm3btrmg5QAAANmbyy4Bt2zZUu+8845GjhypBQsW2MvfeOMNjRgxQpIUExMj6Vqv4I2CgoK0Zs2amy4/MTFRiYmJ9r9TehwBAACszqXDwISGhqp+/fp64oknVKBAAf38888aOXKkihQpov79++vSpUuSJC8vL6d5vb297dNTM2rUKA0bNuyetR0AACC7clkAnDlzpvr06aM9e/aoePHikqT27dsrOTlZr732mrp27SofHx9JcujJS5GQkGCfnpqhQ4fqlVdesf8dFxen4ODgDN4KAACA7Mdl9wBOmDBB1apVs4e/FG3atNHFixe1bds2+6XflEvB14uJiVHRokVvunwvLy/5+fk5vAAAAODCAHj8+HElJSU5lV+5ckWSdPXqVYWFhSlXrlzavHmzQ53Lly9r+/btqlq1amY0FQAAIEdxWQAsV66ctm3bpj179jiUf//993Jzc1PlypXl7++vpk2bavr06Tp//ry9zrRp0xQfH6+OHTtmdrMBAACyPZsxxrhixb/99psaN26sAgUKqH///ipQoIAWLVqkxYsX69lnn9XEiRMlSVu3blWdOnVUsWJF9enTR9HR0Ro7dqzq16+vpUuXpnl9cXFx8vf3V2xsbPa6HGyz3ft1uOYUyN44LllPZhwTieOSXrxXnGTb7yPkKC4LgJK0adMmRUREaNu2bTp9+rRKliypnj17asiQIcqV6/8/n7J27Vq99tpr2rp1q/LmzatOnTpp1KhRyps3b5rXlW3fcHx4Zk0cl6yHAJg18V5xkm2/j5CjuDQAZqZs+4bjwzNr4rhkPQTArIn3ipNs+32EHMVl9wACAADANQiAAAAAFkMABAAAsBgCIAAAgMUQAAEAACyGAAgAAGAxBEAAAACLIQACAABYDAEQAADAYgiAAAAAFkMABAAAsBgCIAAAgMUQAAEAACyGAAgAAGAxBEAAAACLIQACAABYDAEQAADAYgiAAAAAFkMABAAAsBgCIAAAgMUQAAEAACyGAAgAAGAxBEAAAACLIQACAABYDAEQAADAYgiAAAAAFkMABAAAsBgCIAAAgMUQAAEAACyGAAgAAGAxBEAAAACLIQACAABYDAEQAADAYgiAAAAAFkMABAAAsBgCIAAAgMUQAAEAACyGAAgAAGAxBEAAAACLIQACAABYDAEQAADAYgiAAAAAFkMABAAAsBgCIAAAgMUQAAEAACyGAAgAAGAxBEAAAACLIQACAABYDAEQAADAYgiAAAAAFkMABAAAsBgCIAAAgMUQAAEAACyGAAgAAGAxBEAAAACLIQACAABYDAEQAADAYgiAAAAAFuPyALh161a1adNGAQEByp07t8LCwvTxxx871Fm/fr3q1aun3Llzq0iRIhowYIDi4+Nd1GIAAIDsLZcrVx4ZGanWrVurWrVqeuutt+Tr66v9+/crOjraXmf79u1q0qSJKlSooA8//FDR0dEaM2aM9u7dq8WLF7uw9QAAANmTywJgXFycevTooVatWmn27Nlyc0u9M/L1119X/vz5tWrVKvn5+UmSQkND1bt3b0VGRqp58+aZ2WwAAIBsz2WXgGfMmKHjx4/r3XfflZubmy5cuKDk5GSHOnFxcVq2bJmefPJJe/iTpB49esjX11c//vhjZjcbAAAg23NZAFy+fLn8/Px05MgRlS9fXr6+vvLz81Pfvn2VkJAgSdqxY4euXr2qGjVqOMzr6empqlWratu2ba5oOgAAQLbmsgC4d+9eXb16VW3btlWLFi00Z84cPf300/riiy/0n//8R5IUExMjSQoKCnKaPygoSEePHr3p8hMTExUXF+fwAgAAgAvvAYyPj9fFixf1/PPP25/6bd++vS5fvqwvv/xSw4cP16VLlyRJXl5eTvN7e3vbp6dm1KhRGjZs2L1pPAAAQDbmsh5AHx8fSVLXrl0dyrt16yZJ2rBhg71OYmKi0/wJCQn26akZOnSoYmNj7a/Dhw9nVNMBAACyNZcFwKJFi0qSChcu7FBeqFAhSdLZs2ftl35TLgVfLyYmxr6M1Hh5ecnPz8/hBQAAABcGwOrVq0uSjhw54lCecl9fYGCgwsLClCtXLm3evNmhzuXLl7V9+3ZVrVo1U9oKAACQk7gsAHbq1EmS9PXXXzuUT5o0Sbly5VLDhg3l7++vpk2bavr06Tp//ry9zrRp0xQfH6+OHTtmapsBAAByApc9BFKtWjU9/fTT+uabb3T16lU1aNBAq1at0qxZszR06FD75d13331XderUUYMGDdSnTx9FR0dr7Nixat68uVq2bOmq5gMAAGRbNmOMcdXKr1y5opEjR2ry5Mk6evSoQkJC1K9fPw0cONCh3tq1a/Xaa69p69atyps3rzp16qRRo0Ypb968aV5XXFyc/P39FRsbm73uB7TZ7v06XHcKZF8cl6wnM46JxHFJL94rTrLt9xFyFJcGwMyUbd9wfHhmTRyXrIcAmDXxXnGSbb+PkKO47B5AAAAAuAYBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWEyWCYDvvvuubDabwsLCnKatX79e9erVU+7cuVWkSBENGDBA8fHxLmglAABA9pfL1Q2QpOjoaI0cOVJ58uRxmrZ9+3Y1adJEFSpU0Icffqjo6GiNGTNGe/fu1eLFi13QWgAAgOwtSwTAwYMH66GHHlJSUpJOnTrlMO31119X/vz5tWrVKvn5+UmSQkND1bt3b0VGRqp58+auaDIAAEC25fJLwL/99ptmz56tcePGOU2Li4vTsmXL9OSTT9rDnyT16NFDvr6++vHHHzOxpQAAADmDSwNgUlKSXnzxRT377LO6//77nabv2LFDV69eVY0aNRzKPT09VbVqVW3bti2zmgoAAJBjuPQS8BdffKGDBw9q+fLlqU6PiYmRJAUFBTlNCwoK0po1a2667MTERCUmJtr/jouLu8vWAgAA5Awu6wE8ffq03n77bb311lsKDAxMtc6lS5ckSV5eXk7TvL297dNTM2rUKPn7+9tfwcHBGdNwAACAbM5lAfDNN99UQECAXnzxxZvW8fHxkSSHnrwUCQkJ9umpGTp0qGJjY+2vw4cP332jAQAAcgCXXALeu3evvvrqK40bN05Hjx61lyckJOjKlSuKioqSn5+f/dJvyqXg68XExKho0aI3XYeXl1eqPYcAAABW55IewCNHjig5OVkDBgxQyZIl7a+NGzdqz549KlmypIYPH66wsDDlypVLmzdvdpj/8uXL2r59u6pWreqK5gMAAGRrLukBDAsL09y5c53K33zzTZ0/f17jx49X6dKl5e/vr6ZNm2r69Ol66623lDdvXknStGnTFB8fr44dO2Z20wEAALI9mzHGuLoRKRo2bKhTp07p77//tpdt3bpVderUUcWKFdWnTx9FR0dr7Nixql+/vpYuXZrmZcfFxcnf31+xsbEOYwpmeTbbvV9H1jkFsg+OS9aTGcdE4rikF+8VJ9n2+wg5issHgr6dBx54QMuXL5ePj49efvllffXVV3rmmWc0e/ZsVzcNAAAgW8pSPYD3Urb9Fxf/es6aOC5ZDz2AWRPvFSfZ9vsIOUqW7wEEAABAxiIAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWIzLAuAff/yh/v37q1KlSsqTJ49KlCihTp06ac+ePU51d+/erZYtW8rX11cBAQF66qmndPLkSRe0GgAAIPvL5aoVv/fee1q3bp06duyoypUr69ixY/r000/1wAMP6Pfff1dYWJgkKTo6WvXr15e/v79Gjhyp+Ph4jRkzRjt27NCmTZvk6enpqk0AAADIllwWAF955RXNmDHDIcB17txZ999/v0aPHq3p06dLkkaOHKkLFy5oy5YtKlGihCSpZs2aatasmaZMmaI+ffq4pP0AAADZlc0YY1zdiOtVr15dkrRlyxZJUuHChdWgQQP9+OOPDvXKly+v4OBgLV++PE3LjYuLk7+/v2JjY+Xn55exjb6XbLZ7v46sdQpkDxyXrCczjonEcUkv3itOsu33EXKULPUQiDFGx48fV8GCBSVJR44c0YkTJ1SjRg2nujVr1tS2bdsyu4kAAADZXpYKgN99952OHDmizp07S5JiYmIkSUFBQU51g4KCdObMGSUmJqa6rMTERMXFxTm8AAAAkIUC4D///KN+/fqpdu3a6tmzpyTp0qVLkiQvLy+n+t7e3g51bjRq1Cj5+/vbX8HBwfeo5QAAANlLlgiAx44dU6tWreTv76/Zs2fL3d1dkuTj4yNJqfbyJSQkONS50dChQxUbG2t/HT58+B61HgAAIHtx2VPAKWJjY/XII4/o3LlzWrNmjYoWLWqflnLpN+VS8PViYmIUEBCQau+gdK3X8GbTAAAArMylATAhIUGtW7fWnj17tHz5clWsWNFherFixRQYGKjNmzc7zbtp0yZVrVo1k1oKAACQc7jsEnBSUpI6d+6sDRs2aNasWapdu3aq9Z544gktWrTI4RLuihUrtGfPHnXs2DGzmgsAAJBjuGwcwIEDB2r8+PFq3bq1OnXq5DT9ySeflCQdPnxY1apVU758+fTSSy8pPj5eH3zwgYoXL64//vgjzZd5s+24S4yhlTVxXLIexgHMmnivOMm230fIUVwWABs2bKjVq1ffdPr1zdq5c6deeeUVrV27Vp6enmrVqpXGjh2rwoULp3l92fYNx4dn1sRxyXoIgFkT7xUn2fb7CDlKlvslkHsl277h+PDMmjguWQ8BMGviveIk234fIUfJEsPAAAAAIPO4fBgYICNlWidQ5qwGAIB7gh5AAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQwEDQDpYMuE0cYt8gudAFyIHkAAAACLIQACAABYDAEQAADAYgiAAAAAFkMABAAAsBgCIAAAgMUQAAEAACyGAAgAAGAxBEAAAACLIQACAABYDAEQAADAYgiAAAAAFkMABAAAsBgCIAAAgMUQAAEAACyGAAgAAGAxuVzdgOzKZsuc9ZjMWQ0AALAQegABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDFEAABAAAshgAIAABgMQRAAAAAiyEAAgAAWAwBEAAAwGIIgAAAABZDAAQAALAYAiAAAIDF5HJ1AwDkfDbbvV+HuferAIAcgx5AAAAAiyEAAgAAWAwBEAAAwGIIgAAAABbDQyAAYEGZ8WCOxMM5QFaVLXoAExMT9dprr6lo0aLy8fFRrVq1tGzZMlc3CwAAIFvKFgGwV69e+vDDD9W9e3eNHz9e7u7uevTRR7V27VpXNw0AACDbsRljsnQP/aZNm1SrVi198MEHGjx4sCQpISFBYWFhKlSokNavX5+m5cTFxcnf31+xsbHy8/O763Zl3uWTzBhALUufAunCccmaMmccwMw5+Jmxlsz4WOa94joZ/X0E3Iks3wM4e/Zsubu7q0+fPvYyb29vPfPMM9qwYYMOHz7swtYBAABkP1k+AG7btk3lypVz+ldSzZo1JUnbt293QasAAACyryz/FHBMTIyCgoKcylPKjh49mup8iYmJSkxMtP8dGxsr6VrXe3aSKa3NZvskK+C4ZD05aW9lt8+pW8mMLfHPpOvZKd8jdyvl+GbxO7CQw2X5AHjp0iV5eXk5lXt7e9unp2bUqFEaNmyYU3lwcHDGNvAe88+UlWTKWnIUjkvWk5P2ln8OOvY5Z0sy/ricP38+Rx1rZC9ZPgD6+Pg49OSlSEhIsE9PzdChQ/XKK6/Y/05OTtaZM2dUoEAB2TLr7mcXiIuLU3BwsA4fPszNxVkExyRr4rhkTVY4LsYYnT9/XkWLFnV1U2BhWT4ABgUF6ciRI07lMTExknTTN5CXl5dTz2G+fPkyvH1ZlZ+fX4798MyuOCZZE8cla8rpx4WeP7haln8IpGrVqtqzZ4/TPTEbN260TwcAAEDaZfkA2KFDByUlJemrr76ylyUmJmry5MmqVatWtrunDwAAwNWy/CXgWrVqqWPHjho6dKhOnDihMmXKaOrUqYqKitLXX3/t6uZlOV5eXgoPD0/1wRm4Bscka+K4ZE0cFyBzZPlfApGuPfDx1ltvafr06Tp79qwqV66sd955Ry1atHB10wAAALKdbBEAAQAAkHGy/D2AAAAAyFgEwOusWrVKNptNq1atyvR1R0VFyWazacqUKRm2zClTpshmsykqKirDlmlFOe28QNZls9nUv39/VzcDgAUQAHHHfvnlF0VERGTKutavX6+IiAidO3cuU9aHe+9OjunLL7+sBx54QAEBAcqdO7cqVKigiIgIxcfHp2l+m82W6mv06NFpmn/ChAmE8Uxwt+/3/fv3y9vbWzabTZs3b87YxgE5BAEwB3vqqad06dIlhYSE3JPl//LLL6n+3N69sH79eg0bNowAmIPcyTH9448/9PDDD2vYsGEaP368GjVqpNGjR6tly5ZKTk5O0zKaNWumadOmObxat26dpnkJgJnjbt/vL7/8snLlyvKDXAAuxTskB3N3d5e7u7urmwFkmLVr1zqVlS5dWoMHD9amTZv00EMP3XYZ5cqV05NPPnkvmndHjDFKSEi46c9aIn2WLl2qpUuXasiQIRoxYoSrmwNkWZbrATxy5IieeeYZFS1aVF5eXipZsqT69u2ry5cvp1p/zZo16tixo0qUKCEvLy8FBwfr5Zdf1qVLlxzqNWzYUA0bNnSav1evXgoNDXUoO3funHr16iV/f3/ly5dPPXv2vOm/dP/55x916NBBAQEB8vb2Vo0aNbRgwYI0bWtq9wCGhobqscce09q1a1WzZk15e3urVKlS+vbbbx3mvXLlioYNG6ayZcvK29tbBQoUUL169bRs2TL7dn322WeSHC+rpRgzZozq1KmjAgUKyMfHR9WrV9fs2bOd2phyz9O8efMUFhYmLy8vVapUSUuWLLHXiYiI0KuvvipJKlmypH1dGXlvo5XOi7uxatUq1ahRQ97e3ipdurS+/PJLRUREOBz7lGP63XffqXz58vL29lb16tX122+/2etk5DFN2Y/p6S26dOmS/ffE07OenTt3avXq1fb2phzbG/dBilu9B5cuXaoaNWrIx8dHX375pcN8t9p3KbZt26ZHHnlEfn5+8vX1VZMmTfT777+na5syUlY4N65cuaKXXnpJL730kkqXLp3h2wjkJJbqATx69Khq1qypc+fOqU+fPrrvvvt05MgRzZ49WxcvXkx1nlmzZunixYvq27evChQooE2bNumTTz5RdHS0Zs2ale42GGPUtm1brV27Vs8//7wqVKiguXPnqmfPnk51d+7cqbp166pYsWL673//qzx58ujHH39Uu3btNGfOHD3++OPpXr8k7du3Tx06dNAzzzyjnj176ptvvlGvXr1UvXp1VapUSdK1D+FRo0bp2WefVc2aNRUXF6fNmzdr69atatasmZ577jkdPXpUy5Yt07Rp05zWMX78eLVp00bdu3fX5cuXNXPmTHXs2FGLFi1Sq1atHOquXbtWP/30k1544QXlzZtXH3/8sZ544gkdOnRIBQoUUPv27bVnzx59//33+uijj1SwYEFJUmBg4B1t/404L9Jm27ZtatmypYKCgjRs2DAlJSVp+PDhqR6H1atX64cfftCAAQPk5eWlCRMmqGXLltq0aZPCwsLu6phevXpV586d0+XLl/X333/rzTffVN68eVWzZs00bceUKVM0YcIEGWNUoUIFvfnmm+rWrdtt5xs3bpxefPFF+fr66o033pAkFS5cOE3rvNG///6rrl276rnnnlPv3r1Vvnx5+7Tb7Tvp2jnw8MMPy8/PT0OGDJGHh4e+/PJLNWzYUKtXr1atWrXuqF13KqucG+PGjdPZs2f15ptv6qeffsrw7QRyFGMhPXr0MG5ubuaPP/5wmpacnGxWrlxpJJmVK1fayy9evOhUd9SoUcZms5mDBw/ayxo0aGAaNGjgVLdnz54mJCTE/ve8efOMJPP+++/by65evWoefvhhI8lMnjzZXt6kSRNz//33m4SEBId21qlTx5QtW/a22zt58mQjyRw4cMBeFhISYiSZ3377zV524sQJ4+XlZQYNGmQvq1KlimnVqtUtl9+vXz9zs1Poxv12+fJlExYWZho3buxQLsl4enqaffv22cv+/PNPI8l88skn9rIPPvjAaVsyitXOizvVunVrkzt3bnPkyBF72d69e02uXLkczgNJRpLZvHmzvezgwYPG29vbPP744/ayOz2mGzZssK9DkilfvrzDsbmVOnXqmHHjxpn58+ebzz//3ISFhRlJZsKECWmav1KlSqkez/Dw8FTfC7d6Dy5ZssSpflr3Xbt27Yynp6fZv3+/vezo0aMmb968pn79+mnaloyUFc6NmJgYkzdvXvPll18aY/7/vk/tfQ3AGMtcAk5OTta8efPUunVr1ahRw2l6apdvJDncl3PhwgWdOnVKderUkTFG27ZtS3c7fvnlF+XKlUt9+/a1l7m7u+vFF190qHfmzBn9+uuv6tSpk86fP69Tp07p1KlTOn36tFq0aKG9e/fqyJEj6V6/JFWsWFEPP/yw/e/AwECVL19e//vf/+xl+fLl086dO7V37947Wsf1++3s2bOKjY3Vww8/rK1btzrVbdq0qcPlmsqVK8vPz8+hPfcK50XaJCUlafny5WrXrp2KFi1qLy9TpoweeeQRp/q1a9dW9erV7X+XKFFCbdu21dKlS5WUlHRXbalYsaKWLVumefPmaciQIcqTJ0+anwJet26dXnrpJbVp00bPP/+8tmzZorCwML3++utOl+/vpZIlS970l4xut++SkpIUGRmpdu3aqVSpUvZ6QUFB6tatm9auXau4uLh7vg0pssq58dprr6lUqVJ69tln73gZgJVY5hLwyZMnFRcXZ7+EklaHDh3S22+/rQULFujs2bMO02JjY9PdjoMHDyooKEi+vr4O5ddfApKuXaY1xuitt97SW2+9leqyTpw4oSJFiujkyZMO5QEBAfL09LxpG0qUKOFUlj9/foftGz58uNq2baty5copLCxMLVu21FNPPaXKlSvfdhsladGiRRoxYoS2b9+uxMREe3lqgSot7blXcup5UaxYsXS34VZOnDihS5cuqUyZMk7TUisrW7asU1m5cuV08eJFnTx5UkWKFLnpus6cOeNw76WPj4/8/f3tf/v5+alp06aSpLZt22rGjBlq27attm7dqipVqqRruzw9PdW/f397GKxXr57i4+MdAqW7u3uG3W6QomTJkjeddrt9J0kXL150OjckqUKFCkpOTtbhw4ftt3Pca1nh3Pj99981bdo0rVixQm5ulunXAO6KZQLgnUhKSlKzZs105swZvfbaa7rvvvuUJ08eHTlyRL169XIYdsJms8mk8qt6d/ov2pRlDx48+KY9BWXKlNHhw4edvkxWrlyZ6oMHKW72ZPD17a9fv77279+v+fPnKzIyUpMmTdJHH32kL7744rb/wl6zZo3atGmj+vXra8KECQoKCpKHh4cmT56sGTNm3FF7spLscF5kZ+3bt9fq1avtf/fs2fOWQ6+0b99eTz31lGbOnJnuAChJwcHBkq6FC+naA0zXD28UEhJy2wcQbtZTfLPjzBO/d+Zm58aQIUP08MMPq2TJkvZjderUKUlSTEyMDh06lOo/NAErs0wADAwMlJ+fn/7+++80z7Njxw7t2bNHU6dOVY8ePezlKU/CXi9//vypXrI8ePCgw98hISFasWKF4uPjHXp7/v33X4d6KZd2PDw87L0dqfHw8HBqz518CaYmICBA//nPf/Sf//xH8fHxql+/viIiIuwB8GZfenPmzJG3t7eWLl0qLy8ve/nkyZPvuC03W9fdyqnnRUYrVKiQvL29tW/fPqdpqZWlduvAnj17lDt3bntv2s2O6dixYx16Va+/rJiaxMREJScn31HPqyT78UlpV48ePVSvXj379OvD2s3anD9/fknXnkTOly+fvfzG45wWadl3uXPndjo3pGtPh7u5udlDbWbICufGoUOHdPDgwVR7Vtu0aSN/f3/GEAVuYJm+cjc3N7Vr104LFy5MdWT41HppUnqmrp9mjNH48eOd6pYuXVr//POPw+XYP//8U+vWrXOo9+ijj+rq1av6/PPP7WVJSUn65JNPHOoVKlRIDRs21JdffqmYmBin9aWsx9vbW02bNnV4pXwZ3Y3Tp087/O3r66syZco4XM7NkyePJOfhN9zd3WWz2Rx6P6KiojRv3rw7bs/N1nW3cup5kdHc3d3VtGlTzZs3T0ePHrWX79u3T4sXL3aqv2HDBof7PQ8fPqz58+erefPm9v13s2NavXp1h/O5YsWK9npXrlxxWtekSZMkyeEezosXL+qff/6x9wJJqe+b8+fPa9y4cSpYsKD9vrRSpUo5rL9u3br2+nny5En1HEy5h/X64UwuXLigqVOnOtW9ndvtO3d3dzVv3lzz58936Jk8fvy4ZsyYoXr16snPzy/d671TWeHc+OqrrzR37lyHV8r9s2PGjNF3332XodsM5ASW6QGUpJEjRyoyMlINGjRQnz59VKFCBcXExGjWrFmpDjB733332QeZPXLkiPz8/DRnzpxU7017+umn9eGHH6pFixZ65plndOLECX3xxReqVKmSww3ZrVu3Vt26dfXf//5XUVFRqlixon766adUey8+++wz1atXT/fff7969+6tUqVK6fjx49qwYYOio6P1559/ZuwOuk7FihXVsGFDVa9eXQEBAdq8ebNmz57t8DulKV+YAwYMUIsWLeTu7q4uXbqoVatW+vDDD9WyZUt169ZNJ06c0GeffaYyZcror7/+uqP2pKzrjTfeUJcuXeTh4aHWrVvbvyjuBudF2kRERCgyMlJ169ZV3759lZSUpE8//VRhYWHavn27Q92wsDC1aNHCYagPSQ6XVtN7TFetWqUBAwaoQ4cOKlu2rC5fvqw1a9bop59+Uo0aNRwGd960aZMaNWqk8PBw+88VfvbZZ/YHfkqUKKGYmBh98803OnTokKZNm3bL+2avb/Pnn3+uESNGqEyZMipUqJAaN26s5s2bq0SJEnrmmWf06quvyt3dXd98840CAwN16NCh9OzmNO27ESNGaNmyZapXr55eeOEF5cqVS19++aUSExP1/vvvp2t9GcHV50bz5s2dylLCY4MGDVJ9wAuwPFc8euxKBw8eND169DCBgYHGy8vLlCpVyvTr188kJiamOtzHrl27TNOmTY2vr68pWLCg6d27t32YkuuH5jDGmOnTp5tSpUoZT09PU7VqVbN06VKn4T6MMeb06dPmqaeeMn5+fsbf39889dRTZtu2bakuc//+/aZHjx6mSJEixsPDwxQrVsw89thjZvbs2bfd1psNQZHa8C43DlcyYsQIU7NmTZMvXz7j4+Nj7rvvPvPuu++ay5cv2+tcvXrVvPjiiyYwMNDYbDaH4R6+/vprU7ZsWePl5WXuu+8+M3ny5FSHypBk+vXr59SekJAQ07NnT4eyd955xxQrVsy4ubll+JAwVjov7saKFStMtWrVjKenpyldurSZNGmSGTRokPH29rbXSTmm06dPt58D1apVS3WolvQc03379pkePXqYUqVKGR8fH+Pt7W0qVapkwsPDTXx8vEPdlGMWHh5uL4uMjDTNmjWz77N8+fKZ5s2bmxUrVqR5+48dO2ZatWpl8ubNayQ5vGe2bNliatWqZTw9PU2JEiXMhx9+mK73oDHp23dbt241LVq0ML6+viZ37tymUaNGZv369WnelozmynMjNQwDA9yazZgseqc9gGyhXbt2DkMG2Ww29evXT59++qmLWwZX49wAsi7L3AMI4O7dOFbe3r179csvv9zyqXNYA+cGkL1Y6h5AAHenVKlS6tWrl0qVKqWDBw/q888/l6enp4YMGeLqpsHFODeA7IUACCDNWrZsqe+//17Hjh2Tl5eXateurZEjR6Y6uC+shXMDyF64BxAAAMBiuAcQAADAYgiAAAAAFkMABAAAsBgCIAAAgMUQAAEAACyGAAggw9hsNvtv76ZHVFSUbDabpkyZkuFtAgA4IwACOdCUKVNks9lks9m0du1ap+nGGAUHB8tms+mxxx5zQQsBAK5EAARyMG9vb82YMcOpfPXq1YqOjpaXl5cLWgUAcDUCIJCDPfroo5o1a5auXr3qUD5jxgxVr15dRYoUcVHLAACuRAAEcrCuXbvq9OnTWrZsmb3s8uXLmj17trp16+ZU/8KFCxo0aJCCg4Pl5eWl8uXLa8yYMbrxB4MSExP18ssvKzAwUHnz5lWbNm0UHR2dahuOHDmip59+WoULF5aXl5cqVaqkb775JmM3FACQLgRAIAcLDQ1V7dq19f3339vLFi9erNjYWHXp0sWhrjFGbdq00UcffaSWLVvqww8/VPny5fXqq6/qlVdecaj77LPPaty4cWrevLlGjx4tDw8PtWrVymn9x48f10MPPaTly5erf//+Gj9+vMqUKaNnnnlG48aNuyfbDAC4PQIgkMN169ZN8+bN06VLlyRJ3333nRo0aKCiRYs61FuwYIF+/fVXvfPOO5o4caL69eunBQsWqEOHDho/frz2798vSfrzzz81ffp0vfDCC/ruu+/Ur18/zZkzR2FhYU7rfuONN5SUlKRt27bprbfe0vPPP6/58+erS5cuioiIsLcJAJC5CIBADtepUyddunRJixYt0vnz57Vo0aJUL//+8ssvcnd314ABAxzKBw0aJGOMFi9ebK8nyanewIEDHf42xmjOnDlq3bq1jDE6deqU/dWiRQvFxsZq69atGbilAIC0yuXqBgC4twIDA9W0aVPNmDFDFy9eVFJSkjp06OBU7+DBgypatKjy5s3rUF6hQgX79JT/urm5qXTp0g71ypcv7/D3yZMnde7cOX311Vf66quvUm3biRMn7ni7AAB3jgAIWEC3bt3Uu3dvHTt2TI888ojy5ct3z9eZnJwsSXryySfVs2fPVOtUrlz5nrcDAOCMAAhYwOOPP67nnntOv//+u3744YdU64SEhGj58uU6f/68Qy/gP//8Y5+e8t/k5GTt37/fodfv33//dVheyhPCSUlJatq0aUZvEgDgLnAPIGABvr6++vzzzxUREaHWrVunWufRRx9VUlKSPv30U4fyjz76SDabTY888ogk2f/78ccfO9S78aled3d3PfHEE5ozZ47+/vtvp/WdPHnyTjcHAHCX6AEELOJml2FTtG7dWo0aNdIbb7yhqKgoValSRZGRkZo/f74GDhxov+evatWq6tq1qyZMmKDY2FjVqVNHK1as0L59+5yWOXr0aK1cuVK1atVS7969VbFiRZ05c0Zbt27V8uXLdebMmXuyrQCAWyMAApAkubm5acGCBXr77bf1ww8/aPLkyQoNDdUHH3ygQYMGOdT95ptvFBgYqO+++07z5s1T48aN9fPPPys4ONihXuHChbVp0yYNHz5cP/30kyZMmKACBQqoUqVKeu+99zJz8wAA17GZG4f4BwAAQI7GPYAAAAAWQwAEAACwGAIgAACAxRAAAQAALIYACAAAYDEEQAAAAIshAAIAAFgMARAAAMBiCIAAAAAWQwAEAACwGAIgAACAxRAAAQAALIYACAAAYDEEQAAAAIshAAIAAFgMARAAAMBiCIAAAAAWQwAEAACwGAIgAACAxRAAAQAALIYACAAAYDH/D+uEhJ0eW8zoAAAAAElFTkSuQmCC\n", 639 | "text/plain": [ 640 | "" 641 | ] 642 | }, 643 | "execution_count": 75, 644 | "metadata": {}, 645 | "output_type": "execute_result" 646 | } 647 | ], 648 | "source": [ 649 | "Image(\"./chat_llms_classsification_plot.png\")" 650 | ] 651 | }, 652 | { 653 | "cell_type": "code", 654 | "execution_count": null, 655 | "id": "c715b0ca", 656 | "metadata": {}, 657 | "outputs": [], 658 | "source": [] 659 | } 660 | ], 661 | "metadata": { 662 | "kernelspec": { 663 | "display_name": "Python 3 (ipykernel)", 664 | "language": "python", 665 | "name": "python3" 666 | }, 667 | "language_info": { 668 | "codemirror_mode": { 669 | "name": "ipython", 670 | "version": 3 671 | }, 672 | "file_extension": ".py", 673 | "mimetype": "text/x-python", 674 | "name": "python", 675 | "nbconvert_exporter": "python", 676 | "pygments_lexer": "ipython3", 677 | "version": "3.8.13" 678 | } 679 | }, 680 | "nbformat": 4, 681 | "nbformat_minor": 5 682 | } 683 | --------------------------------------------------------------------------------