├── .gitignore ├── LICENSE ├── README.md ├── configs ├── miniwob │ └── eval_openai_agent.yml └── webarena │ └── eval_openai_agent.yml ├── notebooks ├── others │ ├── eval_airlinecrm_metrics.ipynb │ ├── eval_compression.ipynb │ ├── eval_miniwob_llama_metrics.ipynb │ └── eval_miniwob_metrics.ipynb └── webarena │ ├── context_len_hist_webarena_step_vs_flat.ipynb │ ├── final_webarena_step_vs_all.ipynb │ └── plots_webarena_step_vs_all.ipynb ├── requirements.txt ├── scripts ├── evaluate │ ├── eval_miniwob.py │ └── eval_webarena.py └── setup │ └── auto_login_webarena.py ├── setup.py └── src └── webagents_step ├── agents ├── agent.py ├── finetuned_agent.py ├── keyboard_agent.py ├── playback_agent.py ├── prompt_agent.py └── step_agent.py ├── environment ├── env.py ├── liveweb.py ├── miniwob.py └── webarena.py ├── parser ├── miniwob_parser.py └── playwright_parser_webarena.py ├── prompts ├── miniwob │ ├── flat_fewshot_template.py │ └── step_fewshot_template.py └── webarena │ ├── flat_fewshot_template.py │ └── step_fewshot_template.py └── utils ├── data_prep.py ├── llm.py └── stack.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # OS generated files 132 | .DS_Store 133 | .DS_Store? 134 | 135 | # Custom dirs 136 | local/ 137 | data/ 138 | outputs/ 139 | api_key.py 140 | 141 | # Data files 142 | *.csv 143 | # *.txt 144 | 145 | *chrome* 146 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 ASAPP Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SteP: Stacked LLM Policies for Web Actions 2 | 3 | Paper link: [https://arxiv.org/abs/2310.03720](https://arxiv.org/abs/2310.03720) 4 | 5 | ## Installation 6 | 7 | To set up the project, clone the repository and create a virtual environment: 8 | 9 | ```bash 10 | cd webagents-step 11 | pyenv virtualenv webagents-step 12 | pyenv activate webagents-step 13 | ``` 14 | 15 | Install the required packages: 16 | 17 | ```bash 18 | pip install -r requirements.txt 19 | ``` 20 | 21 | ## WebArena Evaluation 22 | 23 | ### WebArena Results 24 | 25 | We break down the success rates across different websites and provide links to the trajectory logs below, containing the observations, model predictions, and evaluator outputs for each task. 26 | 27 | The latest runs with `gpt-4-turbo-2024-04-09` model and WebArena code (last commit May 29, 2024) are linked below 28 | 29 | | Website | Number of tasks | Success Rate | Trajectory Logs | 30 | |---------|--------------------|--------------|------------------| 31 | | Gitlab | 180 | 31.7% | [logs](https://drive.google.com/drive/folders/1znkg8aQoEVLTvSyQ8iebb_bsOJL2DrKl?usp=share_link) | 32 | | Reddit | 106 | 59.4% | [logs](https://drive.google.com/drive/folders/1Ek9cMz344tKXbEchakPyPXoTU14FYSlm?usp=share_link) | 33 | | Shopping | 187 | 36.9% | [logs](https://drive.google.com/drive/folders/1ztCP7JH18XS_mGlPCIrP7cKc2eF6Yf8S?usp=share_link) | 34 | | Shopping admin (CMS) | 182 | 24.2% | [logs](https://drive.google.com/drive/folders/1quti9851rBO49alYYL9C1NZNcpRI_Cg-?usp=share_link) | 35 | | Map | 109 | 30.3% | [logs](https://drive.google.com/drive/folders/1V7c122QKNAIVdbskLFNwTJcwILGIf_kS?usp=share_link) | 36 | | Multisite | 48 | 12.5% | [logs](https://drive.google.com/drive/folders/1JmvrY1Ys_bHHY8eQmJocnyZGiPeG7BpV?usp=share_link) | 37 | | All | 812 | 33.5% | [logs](https://drive.google.com/drive/folders/1AKXlClGbFU4RQtfWN9f6jva7MbbGCbur?usp=share_link) | 38 | 39 | ### Installing WebArena 40 | Install WebArena from [WebArena github repository](https://github.com/web-arena-x/webarena). This code uses the last commit 4c741b4b20a3e183836e58f383f9be1785248160 on May 29, 2024. 41 | 42 | Generate test data configs: 43 | ```bash 44 | python scripts/generate_test_data.py 45 | ``` 46 | You will see `*.json` files generated in config_files/ folder. Copy these over to a `tasks/webarena` directory in the `webagents-step/` root directory. 47 | 48 | You will also need to setup authentication for all websites as per instructions in the WebArena README (See instructions for *Obtain the auto-login cookies for all websites*). This will generate a `.auth` folder. Copy this over to `webagents-step/` root directory. 49 | 50 | ### Running Evaluation 51 | 52 | To run WebArena evaluation: 53 | ```bash 54 | python scripts/evaluate/eval_webarena.py --config configs/webarena/eval_openai_agent.yml 55 | ``` 56 | 57 | Important: 58 | * Set up each website as a docker as listed in [WebArena instructions](https://github.com/web-arena-x/webarena/blob/main/environment_docker/README.md) 59 | * Reset the website state before running an evaluation. This matters since the initial state of the website affects the success of the task. 60 | * For Reddit tasks, there is a rate limit on making more than 3 posts in an hour. You need to add a sleep of 21 minutes before every new task. This can be done by adding `time.sleep(1260)` inside the for loop in `eval_webarena.py` 61 | 62 | ## MiniWoB++ Evaluation 63 | 64 | ### Installing MiniWob++ 65 | Install MiniWoB++ from [this repository](https://github.com/Farama-Foundation/miniwob-plusplus). Use commit 43bd1fe. 66 | 67 | ### Running Evaluation 68 | 69 | To run MiniWoB++ evaluation: 70 | ```bash 71 | python scripts/evaluate/eval_miniwob.py --config configs/miniwob/eval_openai_agent.yml 72 | ``` 73 | 74 | ## Contact 75 | This project is still in active development. For any questions or issues, please contact us at [psodhi@asapp.com](mailto:psodhi@asapp.com). 76 | -------------------------------------------------------------------------------- /configs/miniwob/eval_openai_agent.yml: -------------------------------------------------------------------------------- 1 | dataset: "miniwob" 2 | logging: True 3 | verbose: 1 4 | debug: False 5 | logdir: "data/miniwob/eval" 6 | agent: 7 | type: "step" #"flat_fewshot" 8 | root_action: 'miniwob_agent' 9 | low_level_action_list: ['click', 'type', 'stop'] 10 | model_name: "gpt-4-turbo-preview" 11 | model_host: "openai" 12 | prompt_mode: "chat" 13 | max_target_len: 100 14 | env: 15 | max_env_steps: 30 16 | max_browser_rows: 150 17 | headless: False 18 | start_seed: 50 19 | end_seed: 51 20 | num_samples_per_task: 0 21 | tasks: ['book-flight'] 22 | # tasks: ['click-link', 'click-option', 'focus-text', 'click-button', 'click-button-sequence', 'click-dialog', 'click-dialog-2', 'click-tab', 'click-test', 'click-test-2', 'enter-text', 'focus-text-2', 'enter-text-dynamic', 'enter-password', 'login-user', 'click-pie', 'enter-date', 'grid-coordinate', 'click-widget', 'multi-orderings', 'choose-date', 'click-collapsible-2', 'simple-arithmetic', 'click-tab-2', 'click-tab-2-hard', 'multi-layouts', 'copy-paste', 'click-collapsible', 'choose-date-easy', 'copy-paste-2', 'simple-algebra', 'click-checkboxes', 'click-checkboxes-transfer', 'login-user-popup', 'click-checkboxes-soft', 'enter-text-2', 'email-inbox-forward-nl', 'search-engine', 'find-word', 'choose-date-medium', 'click-checkboxes-large', 'book-flight', 'email-inbox-nl-turk', 'email-inbox-forward-nl-turk', 'email-inbox'] -------------------------------------------------------------------------------- /configs/webarena/eval_openai_agent.yml: -------------------------------------------------------------------------------- 1 | dataset: "webarena" 2 | logging: True 3 | verbose: 1 4 | debug: False 5 | logdir: "data/webarena/eval" 6 | agent: 7 | type: "step" # "flat_fewshot8k", "flat_fewshot4k" 8 | root_action: 'shopping_agent' 9 | low_level_action_list: ['click', 'type', 'scroll', 'stop', 'goto', 'hover', 'note', 'go_back'] 10 | model_name: "gpt-4-turbo-2024-04-09" 11 | # model_name: "gpt-3.5-turbo-0125" 12 | model_host: "openai" 13 | prompt_mode: "chat" 14 | max_target_len: 100 15 | env: 16 | max_env_steps: 20 17 | max_browser_rows: 500 18 | headless: False 19 | task_ids: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811] 20 | -------------------------------------------------------------------------------- /notebooks/others/eval_compression.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 7, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "import pandas as pd\n", 11 | "import matplotlib.pyplot as plt\n", 12 | "import seaborn as sns\n", 13 | "\n", 14 | "from tabulate import tabulate\n", 15 | "\n", 16 | "plt.rcParams['font.size'] = 18" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 8, 22 | "metadata": {}, 23 | "outputs": [ 24 | { 25 | "name": "stdout", 26 | "output_type": "stream", 27 | "text": [ 28 | " method not_compressed compressed\n", 29 | "0 Miniwob 182 182\n", 30 | "1 CRM 1367 539\n", 31 | "2 Liveweb 11485 1933\n" 32 | ] 33 | } 34 | ], 35 | "source": [ 36 | "import pandas as pd\n", 37 | "\n", 38 | "# Define the data\n", 39 | "data = {\n", 40 | " \"method\": [\"Miniwob\", \"CRM\", \"Liveweb\"],\n", 41 | " \"not_compressed\": [182, 1367, 11485],\n", 42 | " \"compressed\": [182, 539, 1933]\n", 43 | "}\n", 44 | "\n", 45 | "# Create the DataFrame\n", 46 | "df = pd.DataFrame(data)\n", 47 | "\n", 48 | "# Display the DataFrame\n", 49 | "print(df)\n" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 10, 55 | "metadata": {}, 56 | "outputs": [ 57 | { 58 | "data": { 59 | "text/plain": [ 60 | "
" 61 | ] 62 | }, 63 | "metadata": {}, 64 | "output_type": "display_data" 65 | }, 66 | { 67 | "data": { 68 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl4AAAG+CAYAAABCjQqZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB7iElEQVR4nO3dd3xUVf7/8dek90YL0lE6uCLorhIEpCMoUgQEQVRAVL6CYGGVKgooiO7KotJRlKqISEdUiriIKAoIigkdhJBMer+/P/KbuzPJTEhCmBB4Px+PPJjccz73nrkJdz4599xzLIZhGIiIiIjIVedR2g0QERERuVEo8RIRERFxEyVeIiIiIm6ixEtERETETZR4iYiIiLiJEi8RERERN1HiJSIiIuImSrxERERE3ESJl4iIiIibKPESERERcZMyk3ilpKSwYcMGpkyZQo8ePahRowYWiwWLxcLEiRMLjD19+jT/+c9/6N27N7fccgv+/v74+/tTq1Yt+vXrx1dffVWoNpw/f57Ro0dTr149/P39iYiIoGXLlsybN4/CrLx07Ngxhg0bRq1atfDz86NChQp07NiR1atXF+r4P/74IwMGDKBq1ar4+vpSuXJlHnzwwUK3X0REREqZUUZs377dAJx+TZgwwWXciRMnDIvF4lA/ICDA8Pf3d9j22GOPGVlZWS7388MPPxjlypUz6wcFBRleXl7m9x07djTS09Ndxn/55ZdGQECAWT8kJMTw8PAwvx88eLCRk5PjMn7u3LkOxwsNDXV4XwWdAxEREbk2lJkeL4Dw8HDatm3L888/zyeffEJkZORlY7KzszEMg7Zt27J48WJOnz5NcnIySUlJHDx4kAceeACABQsWuOw5s1qtdO3aldjYWOrXr8/evXtJTEwkOTmZd999F29vbzZt2sTIkSOdxkdHR/PQQw+RkpJCixYtOHLkCFarFavVyvjx4wFYuHAhb775ptP47777jieffJKsrCy6d+/OyZMniY+P58KFCwwbNgyASZMmsWLFisueDxERESlFpZ35FZaz3qgaNWpctrcnPj7e2Ldvn8vynJwco1OnTmYvVmpqar46r7zyigEY/v7+xp9//pmv/PXXXzcAw9PT0zhy5Ei+8gEDBhiAERkZacTFxeUrHzp0qNkLdunSpXzlUVFRBmA0adLEyMjIyFfesWNHAzBq1qxZYK+diIiIlK4y0+Pl6elZrLjQ0FBuv/12l+UWi4XHHnsMgKSkJA4fPpyvzpIlSwDo27cvtWrVylc+YsQIgoKCyM7OZunSpQ5lycnJ5hiu4cOHExYWli9+7NixACQkJLBmzRqHsj///JOdO3cCMGbMGLy9vV3Gx8TE8O2337p8ryIiIlK6ykzidTX5+fmZr7Ozsx3Kjhw5wokTJwDo3Lmz0/igoCBatmwJwObNmx3Kdu7cSWpqaoHxNWvWpEGDBk7jt2zZYr7u1KmT0/ioqCiCg4OdxouIiMi1w6u0G3At+PrrrwHw8fGhbt26DmW//vqr+bpx48Yu99G4cWM2bNjAoUOHih1/+PBhDh486DS+YsWKVKxY0Wmsp6enOfYsb/zl5OTkcObMGYKDg7FYLEWKFRERETAMg8TERG666SY8PAru07rhE6/o6Gjee+89APr06UNISIhD+ZkzZ8zXVapUcbkfW1lCQgJJSUkEBQU5xIeHh+Pv73/ZePvj2X9f0LFt5Xv37s0Xn1d6ejrp6enm96dPn6Zhw4YFxoiIiMjlnTx5kqpVqxZY54ZOvFJTU+nduzcpKSmUL1+eadOm5auTmJhovg4ICHC5L/uyxMREM/GyxRcUa19uf7ySiM9r6tSpTJo0Kd/2kydP5ks6RURE5PISEhKoVq2aOeynIDds4pWVlcXDDz/Mvn378Pb2ZunSpdx0002l3ayrbuzYsTz33HPm97ZflpCQECVeIiIiV6AwQ3ZuyMQrOzub/v37s2bNGry8vPj444/p0KGD07r22WtKSorL5CQlJcVpjO21fXlB8Xmz5SuNz8vX1xdfX98C64iIiMjVccM91Zidnc2AAQNYsWIFnp6efPTRR/Tq1ctlfftesNOnT7usZysLCQkxbzPax8fFxZlPNxYUn7fXzfZ9QccuKF5ERESuHTdU4mXr6Vq2bJmZdPXp06fAGPsnEe2fUMzLVpZ3oHpR4xs1auQ0/q+//uLChQtOY7Ozs/ntt9+cxouIiMi144ZJvLKzs3n44YdZvny5mXT17dv3snF169alevXqAGzcuNFpneTkZHbs2AGQ75ZlVFSU+TSjq/jjx4+bE7fmjW/fvr352lX8rl27zEH1rm6ZioiISOm7IRIvW0/XihUr8PLyYunSpYVKuiB3oNzAgQMBWLZsGTExMfnqzJ49m6SkJDw9Penfv79DWWBgID179gRgzpw5WK3WfPHTp08Hcsdnde/e3aGsdu3aREVFATBz5kwyMzPzxduexqxRowb33HNPod6XiIiIuF+ZSrzi4uK4ePGi+ZWTkwPkDiy3356UlGTG2MZ0LV++3BxIf7nbi3mNGTOGyMhIUlJSuO+++9i3bx8AGRkZzJkzh3HjxgEwdOjQfBOwAkyePJnAwEDOnj1Lt27d+P3334HcnrLJkyeb84i98sorhIeH54ufPn06np6e/Pzzz/Tt29ccz3Xp0iWeeuopNmzYAMAbb7xR7KWVRERExA1Ke7HIorAtin25r0GDBpkx33zzjbnd29vbqFSpUoFfy5Ytc3rsH374wShXrpy5r+DgYMPb29v8vkOHDkZaWprLtn/55ZdGQECAWT80NNTw9PQ0vx88eLCRk5PjMn7u3LmGl5eXWT8sLMywWCzm9wUtFF4Qq9VqAIbVai1WvIiIyI2uKJ+l1/10ErZeMYDMzEzOnz9fYH1XTx42a9aMgwcPMn36dNatW8fJkycJDAykcePGDBo0iMcee6zAZQK6dOnCgQMHmD59Olu2bOHs2bOEh4fTtGlThg0bZt6OdOWJJ57g9ttvZ+bMmXzzzTdcuHCBihUrctdddzFixAjuvffeAuNFRESk9FkMwzBKuxFSehISEggNDcVqtRZrAlXDMMjMzHRIcEVE3MXDwwNvb2+tNSulqiifpdd9j5dcHdnZ2Vy8eJHExESnA/5FRNzF29ub4OBgypcvr3Gucs1T4iVFlp2dzcmTJ0lPTyc0NJSgoCA8PT31F6eIuJVhGGRnZ5OUlER8fDypqalUq1ZNyZdc05R4SZFdvHiR9PR0qlevbs5RJiJSWoKCgggNDeXEiRNcvHiRSpUqlXaTRFwqU9NJSOkzDIPExERCQ0OVdInINcPf35+QkBASExPR0GW5linxkiLJzMwkMzPTYT1KEZFrQXBwsHmNErlW6VajFInt6UWNoRCRa43tulSST1lP23+xxPZV2l5qWr60myCox0uKSQPpReRao+uSlAVKvERERETcRImXiIiIiJso8RIRERFxEyVeIiJyzYiJicFisWCxWIiJiSnt5oiUOD3VKHIDWrNmDT/99BO33XYb3bt3L+3miIjcMJR4yVVzvTyGfT0+gr1mzRoWL17MoEGDlHiJiLiRbjWKiIiIuIkSLxERERE3UeIlUopat26NxWJh4sSJGIbB3Llz+fvf/05ISAjBwcHcddddfPTRRwXu49NPP6Vr165UqlQJHx8fKlWqRNeuXfnss8/y1f3666+xWCwsXrwYgMWLF5sDmW1fX3/99RW/r4yMDObNm0enTp2oVKkSvr6+VK5cmbvuuovJkycTHR3tNG7//v0MHDiQGjVq4OfnR3h4OHfffTdvv/026enpTmMWLVqExWKhZs2aAOzYsYNu3bpRsWJFAgMDadq0KfPnz3eI+fLLL2nfvj0VKlQgICCAO+64g+XLl7t8P/bn5ty5czzzzDPUqlULPz8/IiMj6d+/P7/99pvT2LyDxY8dO8bQoUOpVasWvr6+ZrttcnJyWLp0KV26dDF/phUqVKBDhw588sknLtchzMrK4oMPPqB169aUL18eb29vypUrR7169ejTp0++c2CzfPlyOnfuTKVKlfD29iYsLIw6depw//33M3v2bNLS0pzGXbhwgVdeeYWmTZsSGhqKn58ftWvX5vHHH+fgwYMuzyXA6dOnGTZsGNWqVcPX15eqVasyePBg/vjjjwLjRK4HGuMlcg3Izs7mwQcf5PPPP8fLy4uAgAASExPZs2cPe/bs4ffff2fSpEkOMRkZGQwcONBMGDw8PAgNDeXixYt8+eWXfPnll/Tr14/Fixfj7e0NYCZmVquVtLQ0/Pz8CA0Nddivj4/PFb2X6Oho7r//fn799VcgN2kJCwsjISHBfD+XLl3i7bffdoibNWsWo0ePNhOL0NBQkpOT+e677/juu+9YuHAhGzdupHLlyi6PPW/ePIYNG4ZhGISEhJCSksJPP/3EE088wR9//MHUqVOZMGECkydPxsPDg+DgYFJTU/nhhx/o27cvcXFxPPnkkwW+t379+nHu3Dn8/f3x9vbm/PnzfPzxx3z66ad89tlndOrUyWX87t27GTZsGElJSQQEBJg/F5tLly7x4IMP8u2335rbbD/TLVu2sGXLFpYtW8bKlSsdfk7Z2dl06dKFLVu2OMQlJydz6dIljh49yooVK3j88ccdjvfYY4+xcOFC8/ugoCAyMzP5448/+OOPP/jiiy+477778iWHW7dupXfv3sTHxwPg7e2Nj48P0dHRREdH89FHHzF37lwGDhyY7xz8+OOPtGvXjri4OCB3cWur1cqiRYv49NNPmTt3rsvzJ3I9UI+XyDVg9uzZfP311yxatIiEhASsVisnT56kW7duAEyZMoXff//dIeaf//wny5cvx2KxMG7cOGJjY7l06RIXL17kn//8JwCffPIJ48aNM2Puvvtuzp07R58+fQDo06cP586dc/i6++67i/0+EhIS6NixI7/++ivh4eF88MEHxMXFcenSJZKTkzl27BgzZ86kRo0aDnHr1q3jueeewzAMHnjgAf7880/i4+NJSkpiyZIlBAcHc+DAAXr16kV2drbTY1+4cIGnn36aZ555hvPnzxMfH09sbCyDBg0C4I033uCNN97gtddeY8qUKVy6dIn4+HjOnDljJktjxozBarW6fH+jRo3Cx8eHzZs3k5ycTGJiIt9//z1NmjQhLS2NPn36cOrUKZfxw4YNo1GjRuzdu5fk5GSSkpLYvHkzkJs89ejRg2+//ZbbbruNL774guTkZPM8LF68mIoVK7J27VpefPFFh/1+8sknbNmyBT8/P+bNm0diYiLx8fGkpqZy/vx5Pv30U3r16uUQs3PnThYuXIiHhwfTp08nNjaWxMREkpOTuXjxIps2bWLQoEH5EvFffvmF+++/n/j4eIYMGcKhQ4dITU0lKSmJ48eP89RTT5GRkcHjjz/ODz/84BCbmJjIgw8+SFxcHNWrV3c4j7t376ZatWoMGzbM5fkTuR4o8RK5BsTFxfHZZ58xaNAg/P39AahatSorV67kpptuIicnhxUrVpj1T58+zTvvvAPASy+9xOTJkwkLCwMgPDyc1157jeeeew6At956i7Nnz7rlfbz55pv8/vvv+Pr6sm3bNoYMGeLQo1a7dm2ee+45Ro0a5RD3wgsvANCyZUtWr15NrVq1gNzet0ceeYSlS5cCuT1Gzm6hAqSkpDBw4EDeeecdKlSoAEBERATz58+nVq1a5OTk8OKLLzJp0iRefvlls12VK1dm+fLlBAYGkpyczBdffOHy/aWmprJx40bat29vrgt45513snXrViIiIkhISGDq1Kku48uVK8fWrVtp3ry5ua1u3boAfPzxx3zzzTfUr1+fr7/+mq5duxIQEABAYGAgAwcOZP369VgsFv7zn//w119/mfvYvXs3AAMHDuTxxx8nKCgIyO1trFixIg8++CArV650aIstpl27drzwwgtEREQ4tLNDhw4sWrSIm266ySFu5MiRpKamMnbsWD744AMaNGhgLk5dvXp1Zs+ezf/93/+RlZXFlClTHGLnzJnDiRMn8PHxyXce77rrLrZu3ar1FuW6p8RL5BrQokUL2rRpk2+7r68vHTt2BODAgQPm9tWrV5OVlYWfnx8vvfSS032+8sor+Pr6kpmZyapVq65Ow/NYsGABAE888QRNmzYtVMyBAwc4fPgwkNtm24e4vW7dunHnnXcCub07rjg7F56enrRt2xYAPz8/Ro4cma9OSEgId911l9keV3r37k2DBg3yba9YsaJ5i7KgsWLPPPOMmRTlZRuDNXz48Hy3f22aNWtGo0aNyMjIYPv27eZ2W9J97tw5l8fOyxZz4cIFl72IecXExPDVV1/h5eXFmDFjXNaz3WLcunWrw76XLVsGuD6PkZGRBd7qFbkeKPESuQb8/e9/d1lm63G4dOmSuc12C+eOO+4gJCTEaVx4eLjZs5L3ls/VcPz4cc6cOQNg3iItDFvbvLy8aNWqlct67du3d6ifV0REBDfffLPTskqVKgHQsGFDAgMDC6xjG3vkzL333nvZstjYWJcPD7Ro0cLp9uzsbPbs2QPAxIkTiYyMdPl15MgRIPd823Tp0gWLxcLatWvp3Lkzn3zyifmzcKVt27b4+fmxf/9+WrZsyfz5812222bXrl1A7gMADRs2dNlG263b5ORkYmNjgdwxib/88ovDuXKmoDKR64EG14tcA4KDg12WeXnl/jfNzMw0t9luM1WpUqXA/VatWtWh/tVk39uSdwxXQWxtK1++PL6+vi7rXe69FOYcFvU851XQ+bYv++uvv8zbpfYqVqzoNPbSpUvmU5sFJX72UlJSzNdRUVFMnz6dV155hY0bN7Jx40Yg95y1a9eOgQMH5utRvfnmm5k3bx5PPvmk+QADQIUKFWjTpg0PP/ww999/v8OtP1syl5OTw/nz54vUzkuXLpGVlQUUfB5tP2eR65V6vESkRGhszuU5u40KONyO27BhA4ZhXPZr4sSJDvt4/vnniY6OZtasWXTv3p2KFSty6tQpFi1axL333kvv3r3zJZX9+/fn+PHjvPfee/Tp04dq1apx4cIFVqxYQffu3WnVqhUJCQn52lmpUqVCtdEwjHxPRIrc6JR4iZRBtp6Tgp6gsy931dNSkiIjI83X9rfBLsfWtosXL7qcqwvc+15cOX36dKHKitrGcuXKmT1uRTl3ed10002MHDmSzz77jPPnz3PgwAGeeOIJAFatWsWcOXPyxURERDBs2DCWLVvGiRMn+OOPP3jppZewWCzs2LHDIcGz/YwvXrxIcnJykdoWERFhJp6FPY8i1yMlXiJlkP3YLVfTH8THxzuMBbPn4ZH7X9/VZJzFUb16dfMWUkFPBuZley9ZWVl88803Lutt3boVyP9e3Ml+QLursoiICKe3GQvi7e1tPjxQlHN3OU2aNGHu3Lnm2DL7eb5cufnmm5k6dSoPP/xwvhjbfrKzs9mwYUOR2uLj48Ott94KFHwev/rqqyLtV6SsUeIlUgb17NkTLy8v0tLSmD59utM6r7/+Ounp6Xh7e9OzZ0+HMtuAfNsEmCXFNkHnvHnz2L9/f6Fibr31Vho2bAjkzlfm7Am79evX8/333wPQr1+/Empt0a1cudIc3G7v4sWLvP/++wDmHGlFNXToUCD3va5fv77AuvYPWgAF9hQC5hQltoS7uDF16tShdevWALz88ssFznnmrJ22c+PqPP7111+89957Be5TpKxT4iVSBlWpUoVnn30WgGnTpjFhwgQziYqPj2fcuHG8+eabADz33HP5Zntv3LgxkLu8jqulbopjzJgx1KlTh/T0dNq2bcvcuXMdxggdO3aMyZMnM2PGDIc4W/K4Y8cOevXqZT5dl5mZydKlS81k6+6776Z79+4l1t6i8vPzo1OnTmzdutXsLdy7dy/t2rXj4sWLBAcHu5ze43IGDBhAu3btMAyDBx98kClTpjg8mZicnMz27dt5+umnqV27tkNs9+7deeyxx9iwYYNDMn3p0iWmTJnCtm3bALjvvvvMsmeeeYaHHnqI1atXOzywkJSUxHvvvceSJUvyxQD8+9//JigoiKNHj/KPf/yDzz//3GFZodOnT/Phhx/Stm3bfBO9Dh8+nKpVq5Kenk6nTp3Ytm2beR6///572rVrR05OTnFOn0iZoacaRcqo119/nZMnT7JixQomT57MlClTCA0NxWq1mh9e/fr149VXX80X27NnT/75z39y4cIFGjRoQPny5c1pFpYtW8Y//vGPYrUpODiYjRs30q1bNw4dOsTQoUN58sknCQsLIy0tzXzCzZY02nTt2pW33nqL0aNHs2bNGtasWUNYWBgpKSlkZGQAubfNVq5c6XKAujvMmjWLf/7zn7Rv356AgAA8PDxISkoCcudc++STT6hevXqx9u3p6cnq1avp378/69atY9y4cYwbN46QkBA8PDywWq1mkmIbD2aTmprKwoULzeV/bD2a9klvr169zPFekJvUrly50pxYNSgoCC8vL4fELSoqipdfftnhWI0bN2bjxo306tWL3377je7du+Pp6Wn+vFJTU826eRPEkJAQPvvsM9q3b09MTAzt2rVzOI/BwcHMmzev2L2GImWBerxEyigfHx+WL1/OqlWr6Ny5M+XKlSMxMZFy5crRuXNnPv30Uz7++ON86wFC7hxf3377LX379qVKlSpYrVaOHz/O8ePHXS6KXFi1a9dm//79/Oc//6F169aEh4eTmJhIWFgYd911F6+++mq+meshdzmeH374gQEDBlCtWjVSUlLw9/fnH//4B7NmzWLv3r35ZlF3t1q1arF//36efvppKlSoQEZGBhUrVqRfv37s378/X+9QUYWEhPDFF1+wfv16+vTpQ/Xq1UlPTyclJYUqVarQoUMHpk6dmu823b///W+mT59Oly5dqFOnDoZhkJqayk033cT999/P6tWrWblypcNtw3HjxvGvf/2LBx98kPr16+Pl5UVSUhIVK1akffv2LFiwgK+//trpvGctWrTg6NGjzJgxg3vuuYewsDDi4+Px9PSkQYMGDBgwgKVLl+ZbjxNyx/TZBv1XqVKFrKwsQkNDGTRoED/++KM51k3kemUxSnJ0rZQ5CQkJZi+Jq4k47aWlpREdHU2tWrXw8/NzQwtFSp9tqozt27ebY5zk2nM1rk/T9l8skf1cC15qWr60m3DdKspnqXq8RERERNxEiZeIiIiImyjxEhEREXETPdUoIg6WL1+e76nDy+nTpw/vvPPOVWqRiMj1Q4mXiDhITU0t9ALINpebSLOs0zNIIlJSlHiJiINHH32URx99tLSbISJyXdIYLxERERE3UeIlIiIi4iZKvERERETcRImXiIiIiJuUmcQrJSWFDRs2MGXKFHr06EGNGjWwWCxYLBYmTpxYqH2cP3+e0aNHU69ePfz9/YmIiKBly5bMmzevUE8tHTt2jGHDhpnLUVSoUIGOHTuyevXqQh3/xx9/ZMCAAVStWhVfX18qV67Mgw8+yFdffVWo+O3bt/Pggw9SuXJlfH19qVq1KgMGDODHH38sVLyIiIiUrjLzVON///tfunTpUuz4ffv20bFjR2JjYwEICgoiMTGRnTt3snPnTlatWsXatWvx8fFxGr9+/Xp69+5NSkoKkLuY7aVLl9i8eTObN29m8ODBzJ8/31zTLa958+YxfPhwsrKyAAgNDeX8+fOsWbOGNWvWMGHChAITyIkTJzJp0iQgd924kJAQTp8+zdKlS1m+fDlz5szhiSeeKO7pERERETcoMz1eAOHh4bRt25bnn3+eTz75hMjIyELFWa1WunbtSmxsLPXr12fv3r0kJiaSnJzMu+++i7e3N5s2bWLkyJFO46Ojo3nooYdISUmhRYsWHDlyBKvVitVqZfz48QAsXLiQN99802n8d999x5NPPklWVhbdu3fn5MmTxMfHc+HCBYYNGwbApEmTWLFihdP4FStWmEnXsGHDuHDhAvHx8Zw8eZLu3buTlZXFk08+yXfffVeo8yEiIiKlw2KUkZkBs7Oz8fT0dNhWs2ZNjh8/ftneonHjxjFlyhT8/f05ePAgtWrVciifOnUq//znP/H09OTQoUPUrVvXofyRRx7ho48+IjIyksOHDxMWFuZQPmzYMD744ANCQkKIiYkhPDzcobxly5bs3LmTJk2asG/fPry9vR3KO3XqxKZNm6hZsyZ//PGHw/vMzs7m5ptv5vjx43Tq1IkNGzY4xGZkZNCsWTN+/fVXoqKi2LFjh8vz4ExRVlQHSEtLIzo62rzdKiJyrbga16dp+y+WyH6uBS81LV/aTbhuFeWztMz0eOVNuopiyZIlAPTt2zdf0gUwYsQIgoKCyM7OZunSpQ5lycnJ5hiu4cOH50u6AMaOHQvknvg1a9Y4lP3555/s3LkTgDFjxuRLuuzjY2Ji+Pbbbx3KvvnmG44fP+5Qz56Pjw9jxowBYOfOnURHR+erIyIiIteGMpN4FdeRI0c4ceIEAJ07d3ZaJygoiJYtWwKwefNmh7KdO3eSmppaYHzNmjVp0KCB0/gtW7aYrzt16uQ0PioqiuDg4ALjg4ODadGihdN4+3bljRcREZFrx3WfeP3666/m68aNG7usZys7dOjQFcUfPHjQaXzFihWpWLGi01hPT0/q169fYHyDBg1c9vpVrFiRChUqOI0XERGRa8d1n3idOXPGfF2lShWX9WxlCQkJJCUl5YsPDw/H39//svH2x7P/vqBjX834vNLT00lISHD4EhEREfe47hOvxMRE83VAQIDLevZl9jG21wXF2pfbx14L8XlNnTqV0NBQ86tatWoF1heRa9fEiROxWCy0bt26tJsiIoVUZubxkpIxduxYnnvuOfP7hISEq5Z8Wf//FBhlXeiECaXdBBERuU5c94mXbdA65M5+7+oxT9vEqHljbK/tywuKt4+9FuLz8vX1xdfXt8A6IiIicnVc97cab7rpJvP16dOnXdazlYWEhBAUFJQvPi4uzny6saB4++PZf1/Qsa9mvIiIiFw7rvvEy/5JRPsnFPOylTVs2PCK4hs1auQ0/q+//uLChQtOY7Ozs/ntt98KjD98+DDZ2dlO4+33nTdeRERErh3XfeJVt25dqlevDsDGjRud1klOTjZnfO/QoYNDWVRUlPk0o6v448ePc/jwYafx7du3N1+7it+1a5c5KN5VfGJiIrt373Yab7/fvPFSNpw8eZIXXniB2267jdDQUPz9/bn55pt54IEHWLJkCWlpaQ71s7OzWbBgAffeey/ly5fH19eXKlWq0Lt3b77++muXx2ndurW5sHxWVhazZs2iadOmBAUFUbFiRbp3787PP/9s1k9JSWHKlCk0btyYwMBAypUrR58+fTh27JjT/S9atAiLxULNmjWB3HnoOnfuTIUKFfD396dRo0ZMmTIl3/uxyTtYfPXq1XTo0IGKFSvi4eGRb4WKCxcu8Morr9C0aVNCQ0Px8/Ojdu3aPP744wVOrXLq1ClGjRpFo0aNCAwMxNfXl5tuuolmzZoxatQo9u7dmy8mLi6O8ePHc/vttxMSEoKPjw+RkZHceuutPPnkk2zbts3l8Xbt2sWAAQOoUaMGfn5+hIaGcueddzJ9+nSHp6id2bBhA+3btycsLIygoCD+9re/8cYbb5CZmVlgnIhcm677xMtisTBw4EAAli1bRkxMTL46s2fPJikpCU9PT/r37+9QFhgYSM+ePQGYM2cOVqs1X/z06dOB3PFV3bt3dyirXbs2UVFRAMycOdPpxXLatGkA1KhRg3vuucehrFWrVtSoUcOhnr3MzExmzpwJ5CaJzmbml2vbhx9+SN26dXnzzTf5+eefSUtLIzAwkBMnTrB27VoGDRpk9ohC7tqj7dq14/HHH2f79u3Ex8cTEBDA2bNnWbVqFW3atOH5558v8JiZmZl06tSJ5557zpy77sKFC3z++edERUXxww8/EBsbS1RUFOPGjePYsWMYhsGlS5dYsWIFd999tzkxsSv/+c9/6NixIxs3biQrK4usrCwOHTrEuHHjuPvuu4mLiyswfvTo0fTq1YutW7eSlZWFh4fj5Wrr1q3UrVuX1157jZ9++onU1FS8vLyIjo5mwYIF3H777eaqFfZ+/vlnbr31Vt5++20OHTpEeno6gYGBnDt3jh9//JG3336b2bNnO8ScOnWK2267jVdffZX9+/eTnJxMUFAQFy9e5JdffuH999/n1VdfzXesnJwcnn32WaKioli6dCknTpzA29ub5ORk9u7dy0svvUTz5s3N1SnymjhxIl26dGHr1q1YrVa8vb05dOgQL774Iu3atSMjI6PAcygi154ylXjFxcVx8eJF8ysnJwfI/avcfnvevyDHjBlDZGQkKSkp3Hfffezbtw/IXedwzpw5jBs3DoChQ4fmW6cRYPLkyQQGBnL27Fm6devG77//DuT2lE2ePJn33nsPgFdeeSXfOo2Qm5h5enry888/07dvX3M81qVLl3jqqafM9RffeOONfJOkenp68sYbbwCwfv16nnrqKS5dugTkjuvq27cvBw4ccKgnZceXX37JoEGDSEtLo0WLFuzYsYPU1FQuXrxo9sQOGTIEHx8fM+bxxx/n66+/xsfHh3/9618kJCQQFxfHmTNneOyxxwCYMWOG+XvpzH/+8x9++uknVq5cSVJSEomJifz3v/+ldu3aJCUl8eyzzzJkyBDi4uLYtGkTycnJJCUlsXXrVipUqMBff/3FP//5T5f7v3DhAiNHjqRXr16cOHGCuLg4EhISmDNnDr6+vuzfv5/HH3/cZfy+fft46623ePHFFzl//jyXLl0iOTmZwYMHA/DLL79w//33Ex8fz5AhQzh06BCpqakkJSVx/PhxnnrqKTIyMnj88cf54YcfHPY9evRo4uLiuP322/nuu+/IzMzk0qVLpKWlcfToUWbMmJHvlv3EiRM5ceIENWvWZOvWrWRkZHDp0iXS09OJiYlhzpw5/OMf/8j3PiZMmMC//vUvKlasyOzZs4mNjSUxMZHU1FS2b99O06ZNOXLkCD169DCvZzZr165l0v9/Mrh3794O53H27Nns2bOHOXPmuDyHInJtKjOLZMP/FsW+nEGDBrFo0SKHbfv27aNjx47ExsYCub1TaWlpZg9Uhw4dWLt2rcsn/tavX0/v3r3NpwdDQ0NJSkoyx10NHjyY+fPnY7FYnMbPmzeP4cOHk5WVBUBYWBhWqxXb6b/cQt8TJ040L8IWi4XQ0FDi4+MB8PLyYs6cOTzxxBOXOTP5Xc1FsjWdRMGysrKoW7cu0dHRREVFsW3bNocEy5nvv//e/IB///33GTp0aL46vXr1YvXq1ZQvX56TJ086/Jxat27NN998A8COHTvM3libr776irZt2wLg7+/PgQMHuOWWWxzqLFiwgMcffxx/f3+zF8Zm0aJFZnLUqlUrvvrqq3w9VfPnzzd/V//73/9yxx13mGX2v+fPPfec2ZubV9u2bfnqq68YO3Ysr7/+utM6zz77LP/617944IEHHNZQDQgIIDU1ld27d3PXXXc5jc2rYcOGHD58mI8//ph+/foVKiYmJoZbbrkFHx8fvvvuO/72t7/lq5OYmEjDhg05deoUn332mUOPeaNGjTh06JDL8/j+++/z5JNPArnnuqBbzDcKLZJdMC2SffVcl4tkX6lmzZpx8OBBRo0aRZ06dcjMzCQwMJCoqCjmzp3Lhg0bCpxmoUuXLhw4cIAhQ4ZQs2ZN0tLSCA8Pp3379qxatYoFCxa4TLoAnnjiCb7//nsefvhhqlSpQkpKijmmZtu2bQUmXZD7gbRt2za6d+9OxYoVSUlJoUqVKjz88MPs2bOnWEmXlK7t27ebi5rPmjXrskkXwPLlywGoWrWqy5+57ZbXxYsXHdYKtRcVFZUv6YLcD3Db/4NevXrlS7oAOnbsCEBqaqrZ++vMK6+8ki9ZgNw/UqpWrQrk3v53xsPDgxdffNFpWUxMDF999RVeXl7mAvHO2IYYbN261eHBFNtC92fPnnUZm1dxYhYtWkR2djadOnVymnSB4/CETZs2mdsPHDhg3gJ2dR6HDBly2RUtROTaU6bm8XI2PqsoKlWqxFtvvcVbb71VrPibb76ZDz74oNjHv/3221m6dGmx4++9917uvffeYsfLtcX2sERkZCTNmzcvVIzttlmbNm2cfhhD7rqeVapU4fTp0/zwww9069YtX50777zTaaynpyfly5fn9OnTDj1R9ipVqmS+djVOy8vLy1x4Pi8PDw9at27NRx99lO82oM0tt9zicm3TXbt2Abnjp/I+hWzPlmwlJycTGxtr7q9r167MnTuXQYMGsWvXLu6//37uuOOOAleH6Nq1K9999x0vvfQSv/32Gz169ODuu+8u8C9bWzs3b95MZGSky3q2oRH2vfm281KY83gl1xQRcb8ylXiJXE/OnTsHYD48URh//fUXcPm1O6tWrcrp06fN+nkVNNGul5dXgXVs5YDLJ+tsT1q6Ymu/q/a5Srrgf+uR5uTkcP78eZf17NlPQPzGG2/wxx9/sH37dvMPMU9PT2677Tbuu+8+hg4dmu/8Pv/88/z888+sWLGCuXPnMnfuXCwWC40aNaJTp0488cQT1KtXz2k7k5OTSU5OLlIbbeflcufR1nMoImXHDXOrUeRaU9Ct6Rtd3odM7Nl6sipVqoRhGIX6sk1vAbm3Db/66it27NjBCy+8QIsWLfDy8mLfvn1MnjyZOnXq8Mknnzgc09vbm+XLl/PTTz8xfvx47r33XgICAvj111/Nwfh5x6PZ2vniiy8Wqo0aoyVyY1DiJVJKbLefCvPAiI2tJ+jUqVMF1rOVF9RzdDVdvHixwKkObE/2Fqd9tvNme/KzuKKiopg+fTo7d+4kPj6ezz//nCZNmpCamspjjz3mtDftb3/7G5MmTWLbtm3Ex8ezdetW7rnnHrKzs81esbztLMrP18Z2Xgp7HkWk7FDiJVJK7r77biD3lqOrsU552caCbd++Pd/0Aza//fab+YHsapzW1ZaVlWVOSpyXYRjmk5WFHdtmr0WLFkBuj5JtKpYr5efnx/3338+nn34K5D4dt3PnzgJjvLy8aNu2LV9++SW+vr4YhsHWrVvztXPr1q0uJ4x1xXZeCjqPOTk56iUTKYOUeImUkjZt2lC7dm0ARo0aVajJMPv27Qvk9nTMmzfPaZ3x48cDueOD2rVrV0KtLbrXXnvNaXK4ePFiTp48CUCfPn2KvN86deqYM9u//PLLTic1tmeb9w5yExlXCStgrlIBODy8kJ6e7jLG19fXvDVqH/PYY4/h5eXFxYsXmXCZKUkyMjIc5h+89dZbadCgAeD6PC5YsOCyPZ8icu1R4iVSSjw9PXn33XexWCzs3LmTtm3bsnPnTvNDNiMjg6+//poBAwaYUwvceeed5koKI0aM4N133zUHZZ87d44hQ4awcuVKIHdaiZKay6ioAgIC2LlzJw8//LCZHKSlpfHBBx8wfPhwAB544AGXT1dezr///W+CgoI4evQo//jHP/j8888depVOnz7Nhx9+SNu2bR2mpTh16hR16tRhypQp7N+/35xXD3KncBgwYACQu2JFq1atzLIaNWowduxY9uzZ45CE/fHHH/Tv35+UlBQ8PDzMqTYg9ylo2+TMb7zxBgMHDnRY7zUrK4uffvqJyZMnc8stt/DTTz85vMfXXnsNyO3dzHse33vvPZ555hlzmgsRKTv0VKNIKercuTOLFi1i6NCh7Ny5k5YtW+Lr60tQUBBWq9VMDOznq5o/fz4XL17km2++YcSIEYwaNYrg4GDi4+PNCXnHjBljTq5ZGipUqMDzzz/PiBEjWL58OeHh4SQlJZlPQf7tb39j/vz5xd5/48aN2bhxI7169eK3336je/fueHp6EhYWRkpKCqmpqWZdW6+izZ9//sm4ceMYN24cnp6e5mTIth5HHx8fFi1aREREhBlz/vx5pk2bxrRp0/Dw8CA0NJTU1FQz2bNYLMycOTPf9Bbjxo0jKyuLKVOm8OGHH/Lhhx/i7+9PQEAA8fHxDvOL5X3Y4sEHH+Tll1/mtddeY/ny5eZ5TExMJCsri5YtWxIVFcXUqVOLfR5FxP3U4yVSygYOHMhvv/3GyJEjadiwIV5eXqSmplKjRg26d+/Ohx9+aN52gtxVE7Zt28b8+fNp3bo1wcHBJCUlERkZSc+ePdm+fTtvvvlmKb6jXE8//TSbNm2iU6dOeHh44OHhQf369Zk8eTLfffcd5cqVu6L9t2jRwlzi55577iEsLIz4+Hg8PT1p0KABAwYMYOnSpbz99ttmTJUqVVi7di2jRo3iH//4B5UrVyYpKQkvLy8aNmzI008/za+//kqvXr0cjrV582bGjh1Ly5YtqVatmpnY3XLLLQwePJi9e/cycuTIfG20WCxMnjyZAwcO8NRTT9GgQQM8PT2xWq2Eh4dz99138/zzz7N7925zTJi9KVOmsG7dOu69915CQkJIT0+nQYMGTJs2rVArHYjItadMLRkkJe9qLhkkNx7bkkE1atS44gmPRYpKSwYVTEsGXT1aMkhERETkGqTES0RERMRNlHiJiIiIuIkSLxERERE3UeIlIiXm0UcfxTAMDawXEXFBiZeIiIiImyjxEhEREXETJV4iIiIibqLES0RERMRNlHhJsWjBAxG51ui6JGWBEi8pEg+P3F8Z+8V9RUSuBbbrku06JXIt0m+nFIm3tzfe3t4kJSWVdlNERBwkJiaa1yiRa5USLykSi8VCcHAwVquV1NTU0m6OiAgAqampJCQkEBwcjMViKe3miLjkVdoNkLKnfPnypKamcuLECUJCQggODsbT01MXOxFxK8MwyM7OJjExkYSEBHx9fSlfvnxpN0ukQEq8pMg8PT2pVq0aFy9eJDExkfj4+NJukojcwLy9vQkLC6N8+fJ4enqWdnNECqTES4rF09OTSpUqUbFiRTIzM8nJySntJonIDcjDwwNvb2/1uEuZocRLrojFYsHHx6e0myEiIlImaHC9iIiIiJso8RIRERFxEyVeIiIiIm6ixEtERETETZR4iYiIiLiJEi8RERERN1HiJSIiIuImSrxERERE3ESJl4iIiIibKPESERERcRMlXiIiIiJuosRLRERExE2UeImIiIi4iRIvERERETe54RKvLVu28NBDD1GjRg38/Pzw9/endu3a9O/fn2+++abA2MTERCZOnEiTJk0ICgoiNDSUO+64g5kzZ5KRkXHZY58/f57Ro0dTr149/P39iYiIoGXLlsybNw/DMC4bf+zYMYYNG0atWrXw8/OjQoUKdOzYkdWrVxf6/YuIiEjpsRiF+cS/DhiGwfDhw3n//ffNbf7+/gCkpqaa20aNGsVbb72VL/748eO0bt2amJgYAAICAsjOziY9PR2Apk2bsm3bNsLDw50ef9++fXTs2JHY2FgAgoKCSEtLIysrC4COHTuydu1afHx8nMavX7+e3r17k5KSAkBISAhJSUnk5OQAMHjwYObPn4/FYin0OQFISEggNDQUq9VKSEhIkWJFRK530/ZfLO0mlJiXmpYv7SZct4ryWXrD9HgtWrTITLp69erF0aNHSUlJISUlhd9++40HHngAgFmzZvHZZ585xGZlZdGtWzdiYmKoXLkyW7ZsITk5mZSUFJYtW0ZwcDD79+9nwIABTo9ttVrp2rUrsbGx1K9fn71795KYmEhycjLvvvsu3t7ebNq0iZEjRzqNj46O5qGHHiIlJYUWLVpw5MgRrFYrVquV8ePHA7Bw4ULefPPNEjpbIiIicjXcMInXkiVLALjlllv45JNPqFOnjllWr149Vq5cSe3atQFYsWKFQ+zixYv55ZdfAFi9ejXt2rUDwMPDgz59+pgJ3fr169m2bVu+Y8+YMYNz587h7+/P+vXrad68OQA+Pj48/fTTTJo0CYAPPviAo0eP5osfP348ycnJREZGsm7dOurWrQvk9ppNmjSJoUOHAvDaa68RFxdXzDMkIiIiV9sNk3idPXsWgL/97W94eXnlK/f29ua2224DICkpyaFs8eLFALRp04a77rorX2zfvn2pVasW8L8Ez55tm309eyNGjCAoKIjs7GyWLl3qUJacnGyO4Ro+fDhhYWH54seOHQvkdnWuWbMmX7mIiIhcG26YxMvWm/Xzzz+b46rsZWZm8tNPPwGYPVIAKSkp7Nq1C4DOnTs73bfFYqFTp04AbN682aHsyJEjnDhxosD4oKAgWrZs6TR+586d5hg0V/E1a9akQYMGTuNFRETk2nHDJF7Dhw8H4I8//qBfv3788ccfZtmRI0d46KGH+PPPP7n55psZNWqUWXb48GFzAHvjxo1d7t9Wdu7cOS5dumRu//XXX/PVKSj+0KFDDtuLGn/w4EGXdURERKR03TCJV7du3Zg1axY+Pj6sWrWKOnXqEBAQQEBAAPXr1+frr79m+PDh/Pe//3V4IuHMmTPm6ypVqrjcv32ZfUxR4xMSEhxuddriw8PDzacwC4q3P54z6enpJCQkOHyJiIiIe9wwiRfAyJEj+fTTT6lYsSKQO42E7TZeRkYGSUlJWK1Wh5jExETzdUBAgMt925fZx5RUfEGx9uX2sc5MnTqV0NBQ86tatWoF1hcREZGSc8MkXikpKfTp04euXbtSvXp1Nm/ezIULF7hw4QKbN2+mYcOGfPjhh9x5550cOHCgtJt71YwdO9acisJqtXLy5MnSbpKIiMgNI//jfdep559/nhUrVlCvXj127NiBn5+fWda+fXuioqK47bbbOHr0KE8//TQ7duwAIDg42Kxnm7zUGfsy+5i88a4mVrtcfEHHti+3j3XG19cXX1/fAuuIiIjI1XFD9HglJibywQcfAPD00087JF02/v7+PPPMM0Duk4R//fUXADfddJNZ5/Tp0y6PYV9mH1PU+JCQEIKCgvLFx8XFOcyw7yre/ngiIiJybbkhEq+jR4+aU0jcfPPNLuvZT6oaHR0NQIMGDfDwyD1N9k8Y5mUri4yMJCIiwtxu/yRiYeIbNmzosL2o8Y0aNXJZR0RERErXDZF42RInyF1z0ZXz58+br2237AICAmjRogUAGzdudBpnGAabNm0CoEOHDg5ldevWpXr16gXGJycnm7c288ZHRUWZTzO6ij9+/DiHDx92Gi8iIiLXjhsi8apfv76ZvMybN8/pBKrZ2dnm7cjw8HDq1atnlg0aNAiA7du38/333+eLXblyJX/++ScAAwcOdCizWCzmtmXLlpmLbNubPXs2SUlJeHp60r9/f4eywMBAevbsCcCcOXPyPXUJMH36dCA3WezevXv+EyAiIiLXhBsi8fL39+eJJ54A4Mcff6Rbt2788ssv5OTkkJOTw4EDB+jSpQu7d+8Gcqed8PT0NOMHDRpEkyZNMAyDnj17musx5uTksHLlSoYMGQLkzizftm3bfMcfM2YMkZGRpKSkcN9997Fv3z4gdwqLOXPmMG7cOACGDh1qrsNob/LkyQQGBnL27Fm6devG77//DuT2lE2ePJn33nsPgFdeeYXw8PASOWciIiJS8iyGYRil3Qh3SE1NpUePHg6362xP96Wnp5vb+vXrx4cffuiQeAHExMTQpk0bs8cqICCAnJwc0tLSAGjatCnbtm1zmfjs27ePjh07EhsbC+T2TqWlpZGZmQnk3iJcu3atyycO169fT+/evc2nF0NDQ0lKSiI7OxuAwYMHM3/+fCwWS5HOS0JCAqGhoVitVpdPXIqI3Kim7b9Y2k0oMS81LV/aTbhuFeWz9Ibo8YLcXq/169ezcuVKHnjgAapWrYot56xWrRo9e/Zk3bp1fPzxx/mSLshdD/HAgQOMHz+exo0bY7FY8Pb2plmzZsyYMYM9e/YU2NvUrFkzDh48yKhRo6hTpw6ZmZkEBgYSFRXF3Llz2bBhQ4HTPHTp0oUDBw4wZMgQatasSVpaGuHh4bRv355Vq1axYMGCIiddIiIi4l43TI+XOKceLxER19TjJYWhHi8RERGRa5ASLxERERE3UeIlIiIi4iZKvERERETcRImXiIiIiJso8RIRERFxk6uaeMXFxTld4kZERETkRlTsxOvMmTMsWbLE6cLNBw8epHnz5pQvX56IiAhatmzJ0aNHr6ihIiIiImVdsROvBQsWMHjwYL7++muH7ampqXTp0oX9+/djGAaGYbBr1y7atWtHQkLClbZXREREpMwqduK1detWAPr06eOwffHixZw8eZKIiAjmzp3LRx99RNWqVTl9+jSzZ8++staKiIiIlGHFTrxsi0XXr1/fYfunn36KxWLh9ddf5/HHH+fhhx9m7ty5GIbB2rVrr6ixIiIiImVZsROvixcvEhISgr+/v7ktJyeH3bt3Y7FY6NWrl7m9ffv2eHh4cOTIkStrrYiIiEgZVuzEKzs7m/T0dIdtv/zyCykpKTRq1Ijw8PD/HcTDg/DwcJKTk4vfUhEREZEyrtiJV+XKlUlPTyc6OtrctmnTJgDuvvvufPWTkpKIiIgo7uFEREREyrxiJ1533XUXAJMmTSInJ4cLFy4wZ84cLBYLHTt2dKgbHR1Neno6lStXvrLWioiIiJRhxU68nn32WQA+/PBDwsLCqFatGsePH6dWrVp07drVoe6WLVsAuP3226+gqSIiIiJlW7ETrzvvvJMFCxYQFBREUlISGRkZ1K9fn08//RQvLy+HukuWLAGgTZs2V9ZaERERkTLMYhiGcSU7SE1N5ddffyUsLIybb74ZDw/HXC4jI4Nly5ZhGAYPPPAAYWFhV3I4KWEJCQmEhoZitVoJCQkp7eaIiFxTpu2/WNpNKDEvNS1f2k24bhXls9SrwNJC8Pf354477nBZ7uPjw8CBA6/0MCIiIiJl3lVdJFtERERE/ueKe7xs0tLSiIuLIzMzs8B61atXL6lDioiIiJQpV5R4paSk8MYbb/DJJ5/wxx9/XLa+xWIhKyvrSg4pIiIiUmYVO/GKj4/nnnvu4eDBgxR2fP4VjuMXERERKdOKnXi9+uqr/Prrr3h7ezNixAgeeOABbrrppnxTSYiIiIhIrmJnSWvWrMFisfD2228zfPjwkmyTiIiIyHWp2E81nj59Gg8PDwYPHlyS7RERERG5bhW7xysiIoK0tDT8/PxKsj0iIiIi161i93hFRUVhtVo5ffp0SbZHRERE5LpV7MTrxRdfxMvLi1dffbUk2yMiIiJy3Sp24tWsWTMWLVrE4sWLefzxx/nzzz9Lsl0iIiIi151ij/GqXbs2AJ6enixatIhFixYRERFBcHCwyxiLxcKxY8eKe0gRERGRMq3YiVdMTEy+bbGxscTGxrqMsVgsxT2ciIiISJlX7MRr4cKFJdkOERERketesROvQYMGlWQ7RERERK57xR5cLyIiIiJFo8RLRERExE2uOPE6deoUzz33HI0aNSIoKCjfItlxcXG8/vrrTJ06laysrCs9nIiIiEiZVewxXgBbtmzhoYceIiEhAcMwgPxPLoaHh7NmzRr27dtHo0aNuP/++6/kkCIiIiJlVrF7vE6ePEmvXr2wWq1069aNVatWER4e7rTuY489hmEYfPnll8VuqIiIiEhZV+zEa+bMmSQmJvLQQw+xZs0aevTogY+Pj9O6HTt2BGDv3r3FPZyIiIhImVfsxGvTpk1YLJZCrdVYq1YtfH19iY6OLu7hSlRCQgLTp0/n7rvvpkKFCvj6+lK1alXatGnDxIkTiY+PdxqXmJjIxIkTadKkCUFBQYSGhnLHHXcwc+ZMMjIyLnvc8+fPM3r0aOrVq4e/vz8RERG0bNmSefPmmbdqC3Ls2DGGDRtGrVq18PPzo0KFCnTs2JHVq1cX9RSIiIhIKbAYhfnEdyIwMBCLxUJSUpK5rXLlyvz1119kZ2fnq1+hQgWsVmuhEpSrafv27fTr14/z588D4OPjQ0BAgEOytX//fm677TaHuOPHj9O6dWtzxv6AgACys7NJT08HoGnTpmzbts3l7dZ9+/bRsWNHc2b/oKAg0tLSzAcOOnbsyNq1a132Gq5fv57evXuTkpICQEhICElJSeTk5AAwePBg5s+fX+TVARISEggNDcVqtRISElKkWBGR6920/RdLuwkl5qWm5Uu7CdetonyWFrvHy8PDw/zQv5ysrCwSEhJK/YN9165d3HfffZw/f54ePXqwd+9e0tLSiIuLIzk5mf/+97+8/PLLhIaGOsRlZWXRrVs3YmJiqFy5Mlu2bCE5OZmUlBSWLVtGcHAw+/fvZ8CAAU6Pa7Va6dq1K7GxsdSvX5+9e/eSmJhIcnIy7777Lt7e3mzatImRI0c6jY+Ojuahhx4iJSWFFi1acOTIEaxWK1arlfHjxwO5Kwm8+eabJXq+REREpGQVO/GqUaMG6enpnDhx4rJ1v/32WzIzM6lTp05xD3fFUlJSGDhwIKmpqYwYMYLVq1fTvHlzs4coICCAO+64gylTplCrVi2H2MWLF/PLL78AsHr1atq1awfkJp99+vTh/fffB3J7pbZt25bv2DNmzODcuXP4+/uzfv16mjdvDuT2tj399NNMmjQJgA8++ICjR4/mix8/fjzJyclERkaybt066tatC+T2mk2aNImhQ4cC8NprrxEXF3fF50pERESujmInXrbk47333iuwXmZmJi+//DIWi4XOnTsX93BX7MMPP+TPP/8kMjKSN954o0ixixcvBqBNmzbcdddd+cr79u1rJmtLlizJV27bZl/P3ogRIwgKCiI7O5ulS5c6lCUnJ5tjuIYPH05YWFi++LFjxwK5XZ1r1qwp/BsTERERtyp24jVq1Ch8fHyYOXMm8+fPd1rnxx9/pF27dnz//fcEBwfz1FNPFbuhV8qW/PTu3Rs/P79Cx6WkpLBr1y4Al4mjxWKhU6dOAGzevNmh7MiRI2avoKv4oKAgWrZs6TR+586dpKamFhhfs2ZNGjRo4DReRERErh1XdKtx3rx5ZGdnM3ToUCpVqmTe5rr77rupUqUKd9xxBzt27MDLy4slS5ZQvnzpDOxLT0/nhx9+AKBZs2acOHGCoUOHUq1aNXx8fKhUqRLdunVzOs/Y4cOHzbFsjRs3dnkMW9m5c+e4dOmSuf3XX3/NV6eg+EOHDjlsL2r8wYMHXdYRERGR0nVFSwb179+fDRs2cPPNN3PhwgUyMjIwDIM9e/Zw9uxZDMPglltuYePGjaU6Y31MTIz5NOWff/5J48aNmTt3Ln/99ReBgYH89ddfrFu3jq5duzJkyBCHqR3OnDljvq5SpYrLY9iX2ccUNT4hIcHhSVFbfHh4OP7+/peNtz+eM+np6SQkJDh8iYiIiHtc0ZJBAO3bt+fIkSN8++237Nq1izNnzpCdnU1kZCQtWrSgTZs2eHp6lkRbi81+wPmUKVMICwtj5cqVPPDAA3h7e3PixAnGjBnDypUrmTdvHg0aNOC5554DcufusgkICHB5DPsy+5jixgcFBTnEFxRrX25/PGemTp1qDuYXERER9yp24nXhwgUqVKgA5I5xatWqFa1atSowZuvWreagfHeyn/YiJyeH+fPn0717d3Nb9erVWbZsGUePHuXnn3/m9ddf5//+7//yLfh9PRg7dqyZVEJuD1u1atVKsUUiIiI3jmLfauzcubM5mWdhfPXVVzzwwAPFPdwVCQ4ONl/XqVPHIemy8fDwYMyYMQDExsayb9++fLEFvV/7MvuYkoq/3Lm2ldvHOuPr60tISIjDl4iIiLhHsROvH3/8kR49epgzrxfkm2++oVu3bqSlpRX3cFfEfmxV/fr1XdZr2LCh+fr48eMA3HTTTea206dPu4y1L7OPKWp8SEiIeZvRPj4uLs58urGgePvjiYiIyLWl2IlXnTp12LJlC48++miB9Xbs2EG3bt1ITU2ld+/exT3cFYmIiChwYLuN/aB628SqDRo0wMMj9zTZP2GYl60sMjKSiIgIc7v9k4iFibdP/ooT36hRI5d1REREpHRd0SLZFStW5JNPPjFv0eW1e/duunbtSlJSEj179uTjjz8udkOvVIcOHYDc6SFcsZ/KwTbRaUBAAC1atABg48aNTuMMw2DTpk0Ox7GpW7cu1atXLzA+OTmZHTt2OI2Piooyn2Z0FX/8+HHzfeWNFxERkWtHsROvmjVrsmHDBoKCgpg1axYzZ850KN+zZw9dunQhMTGRBx54gGXLlpk9R6Vh8ODBAPzxxx9OZ3fPyclhxowZQO6tydtvv90sGzRoEJC7wPb333+fL3blypX8+eefAAwcONChzGKxmNuWLVtmLrJtb/bs2SQlJeHp6Un//v0dygIDA+nZsycAc+bMwWq15oufPn06kDu+y9n4NREREbk2XFEmdNttt7FmzRq8vb158cUXzeVu9u7dS+fOnUlISKBr166sXLmy1KeUaNmyJb169QLgiSeeYPXq1eb4tBMnTtCvXz8OHDgA5K55aJ8kDho0iCZNmmAYBj179jTXY8zJyWHlypUMGTIEyH3goG3btvmOPWbMGCIjI0lJSeG+++4zB+5nZGQwZ84cxo0bB8DQoUPNdRjtTZ48mcDAQM6ePUu3bt34/fffgdyessmTJ5vLNr3yyiuEh4df+ckSERGRq8Ji2A9sKqbly5fz8MMP4+XlxdSpU5kyZQrx8fF06dKFzz77DG9v75Jo6xVLTk6mS5cufPvtt0DuE34BAQEO83xNmDCBiRMn5ouNiYmhTZs2Zo9VQEAAOTk55gMDTZs2Zdu2bS4Tn3379tGxY0diY2OB3N6ptLQ0MjMzgdxbhGvXrsXX19dp/Pr16+ndu7f59GJoaChJSUlkZ2cDuT168+fPN8emFVZCQgKhoaFYrVY94Sgikse0/RdLuwkl5qWmpbN6zI2gKJ+lJXLvr0+fPsyaNYvMzEyef/554uPj6dChA6tXr75mki7IvW23fft25s6dyz333ENgYCBJSUlUqVKFvn37smvXLqdJF+TeWj1w4ADjx4+ncePGWCwWvL29adasGTNmzGDPnj0F9jY1a9aMgwcPMmrUKOrUqUNmZiaBgYFERUUxd+5cNmzY4DLpAujSpQsHDhxgyJAh1KxZk7S0NMLDw2nfvj2rVq1iwYIFRU66RERExL1KpMfLZuzYsUyfPp127drxxRdfFJhIyLVBPV4iIq6px0sKoyifpYWamr127dqFPrjFYuHgwYM0aNDAadmxY8cKvS8RERGR60mhEi9nT+IV5OzZs06361aYiIiI3MgKlXgtXLjwardDRERE5LpXqMTLNo+ViIiIiBRf6c1oKiIiInKDUeIlIiIi4iZXnHgZhsGnn35K7969qVWrFoGBgQQGBlKrVi0eeugh1qxZQwnOWCEiIiJSZhVqjJcr58+fp1evXuzevRvAIcE6fvw4J06cYPXq1bRo0YIVK1YQGRl5Za0VERERKcOKnXhlZGTQsWNHfvnlFwzD4M4776R9+/ZUrVoVgFOnTrF161a+//57du3aRefOnfnvf/97Tc1kLyIiIuJOxU685syZw4EDBwgJCeGjjz6ia9eu+eq8+uqrrF+/nocffpgDBw7w3nvvMWLEiCtqsIiIiEhZVewxXitWrMBisTB79mynSZdNly5dmD17NoZhsGzZsuIeTkRERKTMK3bidfjwYby9venTp89l6/bp0wcfHx8OHz5c3MOJiIiIlHnFTrxSU1MJCAjAy+vydyu9vLwICAggNTW1uIcTERERKfOKnXhVqlQJq9XKiRMnLls3JiaG+Ph4KlWqVNzDiYiIiJR5xU687rnnHgzDYNSoUQXO02UYBs899xwWi4VWrVoV93AiIiIiZV6hE68lS5awcuVK83tbMrVmzRruvfdetm3bRmZmplmemZnJ1q1badOmDWvWrMFisTBq1KiSbb2IiIhIGWIxCjmtvIeHB5UrV+b06dPmtlmzZjF69GgsFguQO5arfPnyAFy8eJGsrCyzN+ytt95i5MiRJdx8uVIJCQmEhoZitVoJCQkp7eaIiFxTpu2/WNpNKDEvNS1f2k24bhXls7RItxrz5mijRo1i7dq11KtXD8MwyMzM5OzZs5w9e5bMzEwMw6Bhw4Z88cUXSrpERETkhndFSwYBdO3ala5du/LLL7/www8/8NdffwFQsWJFmjdvTpMmTa64kSIiIiLXgytOvGyaNGmiJEtERESkAMV+qlFEREREikaJl4iIiIibFOlW4/nz5/H09Cz2wSwWC1lZWcWOFxERESnLijzGq5CzT4iIiIhIHkVKvAIDAxk9evTVaouIiIjIda1IiVdQUBATJky4Wm0RERERua5pcL2IiIiImyjxEhEREXETJV4iIiIibqLES0RERMRNlHiJiIiIuEmhn2rMycm5mu0QERERue6px0tERETETZR4iYiIiLiJEi8RERERN1HiJSIiIuImSrxERERE3ESJl4iIiIibKPESERERcRMlXiIiIiJucsMnXtOmTcNisZhfBUlMTGTixIk0adKEoKAgQkNDueOOO5g5cyYZGRmXPdb58+cZPXo09erVw9/fn4iICFq2bMm8efMwDOOy8ceOHWPYsGHUqlULPz8/KlSoQMeOHVm9enWh36+IiIiUHotRmE/869SRI0e47bbbSEtLM7e5Oh3Hjx+ndevWxMTEABAQEEB2djbp6ekANG3alG3bthEeHu40ft++fXTs2JHY2FgAgoKCSEtLIysrC4COHTuydu1afHx8nMavX7+e3r17k5KSAkBISAhJSUnmigKDBw9m/vz5l00e80pISCA0NBSr1UpISEiRYkVErnfT9l8s7SaUmJeali/tJly3ivJZesP2eOXk5PDYY4+RlpbGXXfdVWDdrKwsunXrRkxMDJUrV2bLli0kJyeTkpLCsmXLCA4OZv/+/QwYMMBpvNVqpWvXrsTGxlK/fn327t1LYmIiycnJvPvuu3h7e7Np0yZGjhzpND46OpqHHnqIlJQUWrRowZEjR7BarVitVsaPHw/AwoULefPNN6/onIiIiMjVdcMmXv/+97/ZvXs3/fv3p0OHDgXWXbx4Mb/88gsAq1evpl27dgB4eHjQp08f3n//fSC3V2rbtm354mfMmMG5c+fw9/dn/fr1NG/eHAAfHx+efvppJk2aBMAHH3zA0aNH88WPHz+e5ORkIiMjWbduHXXr1gVye80mTZrE0KFDAXjttdeIi4srzukQERERN7ghE6/o6GhefvllypUrx6xZsy5bf/HixQC0adPGae9Y3759qVWrFgBLlizJV27bZl/P3ogRIwgKCiI7O5ulS5c6lCUnJ5tjuIYPH05YWFi++LFjxwK5XZ1r1qy57PsRERGR0nFDJl5DhgwhOTmZt956iwoVKhRYNyUlhV27dgHQuXNnp3UsFgudOnUCYPPmzQ5lR44c4cSJEwXGBwUF0bJlS6fxO3fuJDU1tcD4mjVr0qBBA6fxIiIicu244RKvuXPnsm3bNtq1a8fAgQMvW//w4cPmAPbGjRu7rGcrO3fuHJcuXTK3//rrr/nqFBR/6NAhh+1FjT948KDLOiIiIlK6vEq7Ae50+vRpnn/+efz9/c1xWZdz5swZ83WVKlVc1rMvO3PmDBEREcWKT0hIICkpiaCgIIf48PBw/P39Lxtvfzxn0tPTzScxbccTERER97iheryGDRuG1Wpl4sSJ1K5du1AxiYmJ5uuAgACX9ezL7GNKKr6gWPty+1hnpk6dSmhoqPlVrVq1AuuLiIhIyblhEq+PPvqIL7/8kttuu43nnnuutJtTasaOHWtORWG1Wjl58mRpN0lEROSGcUPcajx//jwjR47E09OTuXPn4uVV+LcdHBxsvrZNXuqMfZl9TN54VxOrXS6+oGPbl9vHOuPr64uvr2+BdUREROTquCF6vF566SViY2MZOnQo9evXJykpyeHLfrmfvNtuuukms+z06dMuj2FfZh9T1PiQkBBzfJd9fFxcnPl0Y0Hx9scTERGRa8sNkXhFR0cDMGfOHIKDg/N9TZ061axr2/bCCy8A0KBBAzw8ck+T/ROGednKIiMjzYH14PgkYmHiGzZs6LC9qPGNGjVyWUdERERK1w2ReF2JgIAAWrRoAcDGjRud1jEMg02bNgHkmwW/bt26VK9evcD45ORkduzY4TQ+KirKfJrRVfzx48c5fPiw03gRERG5dtwQidfXX3+NYRguvyZMmGDWtW17++23zW2DBg0CYPv27Xz//ff59r9y5Ur+/PNPgHxzg1ksFnPbsmXLzEW27c2ePZukpCQ8PT3p37+/Q1lgYCA9e/YEcnvsrFZrvvjp06cDub113bt3v8zZEBERkdJyQyReV2rQoEE0adIEwzDo2bOnuR5jTk4OK1euZMiQIUDuzPJt27bNFz9mzBgiIyNJSUnhvvvuY9++fQBkZGQwZ84cxo0bB8DQoUPNdRjtTZ48mcDAQM6ePUu3bt34/fffgdyessmTJ/Pee+8B8MorrxAeHl7yJ0BERERKxA3xVOOV8vLyYu3atbRp04aYmBjatWtHQEAAOTk5pKWlAdC0adN86yzahIaGsm7dOjp27MihQ4do3rw5wcHBpKWlkZmZCeTeInS1bmStWrVYsWIFvXv3ZseOHdStW5fQ0FCSkpLIzs4GYPDgwTz//PNX4d2LiIhISVGPVyHVrFmTAwcOMH78eBo3bozFYsHb25tmzZoxY8YM9uzZU2BvU7NmzTh48CCjRo2iTp06ZGZmEhgYSFRUFHPnzmXDhg0FTvPQpUsXDhw4wJAhQ6hZsyZpaWmEh4fTvn17Vq1axYIFC7BYLFfjrYuIiEgJsRiGYZR2I6T0JCQkEBoaitVqdTnHmIjIjWra/oul3YQS81LT8qXdhOtWUT5L1eMlIiIi4iZKvERERETcRImXiIiIiJso8RIRERFxEyVeIiIiIm6ixEtERETETZR4iYiIiLiJEi8RERERN1HiJSIiIuImSrxERERE3ESJl4iIiIibKPESERERcRMlXiIiIiJuosRLRERExE2UeImIiIi4iRIvERERETdR4iUiIiLiJkq8RERERNxEiZeIiIiImyjxEhEREXETJV4iIiIibqLES0RERMRNlHiJiIiIuIkSLxERERE3UeIlIiIi4iZKvERERETcRImXiIiIiJso8RIRERFxEyVeIiIiIm6ixEtERETETZR4iYiIiLiJEi8RERERN1HiJSIiIuImSrxERERE3ESJl4iIiIibKPESERERcRMlXiIiIiJuosRLRERExE2UeImIiIi4yQ2TeMXGxrJw4UIGDBhAw4YNCQwMxNfXl6pVq9K9e3c+++yzy+4jMTGRiRMn0qRJE4KCgggNDeWOO+5g5syZZGRkXDb+/PnzjB49mnr16uHv709ERAQtW7Zk3rx5GIZx2fhjx44xbNgwatWqhZ+fHxUqVKBjx46sXr26UOdARERESpfFKMwn/nXA29ubrKws83s/Pz88PT1JTk42t3Xu3JlVq1YREBCQL/748eO0bt2amJgYAAICAsjOziY9PR2Apk2bsm3bNsLDw50ef9++fXTs2JHY2FgAgoKCSEtLM9vUsWNH1q5di4+Pj9P49evX07t3b1JSUgAICQkhKSmJnJwcAAYPHsz8+fOxWCxFOS0kJCQQGhqK1WolJCSkSLEiIte7afsvlnYTSsxLTcuXdhOuW0X5LL1heryysrK48847+c9//sOxY8dITU0lKSmJ6OhoHn/8cQA2bNjAsGHDnMZ269aNmJgYKleuzJYtW0hOTiYlJYVly5YRHBzM/v37GTBggNNjW61WunbtSmxsLPXr12fv3r0kJiaSnJzMu+++i7e3N5s2bWLkyJFO46Ojo3nooYdISUmhRYsWHDlyBKvVitVqZfz48QAsXLiQN998s2ROloiIiFwVN0yP1/bt22nTpo3L8ieffJL3338fgBMnTlCtWjWzbP78+TzxxBMA7N69m7vuussh9pNPPuHhhx8GYOvWrbRt29ahfNy4cUyZMgV/f38OHjxIrVq1HMqnTp3KP//5Tzw9PTl06BB169Z1KH/kkUf46KOPiIyM5PDhw4SFhTmUDxs2jA8++ICQkBBiYmJc9ro5ox4vERHX1OMlhaEeLycKSroAs9cL4IcffnAoW7x4sbmPvEkXQN++fc1kasmSJfnKbdvs69kbMWIEQUFBZGdns3TpUoey5ORkcwzX8OHD8yVdAGPHjgVyf/Br1qxx9RZFRESklN0widfl+Pn5ma+zs7PN1ykpKezatQvIHQPmjMVioVOnTgBs3rzZoezIkSOcOHGiwPigoCBatmzpNH7nzp2kpqYWGF+zZk0aNGjgNF5ERESuHUq8/r+vv/7afN2kSRPz9eHDh80B7I0bN3YZbys7d+4cly5dMrf/+uuv+eoUFH/o0CGH7UWNP3jwoMs6IiIiUrqUeAHx8fFMnToVgJYtW1KvXj2z7MyZM+brKlWquNyHfZl9TFHjExISSEpKyhcfHh6Ov7//ZePtjyciIiLXFq/SbkBpy8nJ4ZFHHuHs2bP4+fnx7rvvOpQnJiaar51NM+GszD6muPFBQUEO8QXF2pfbH8+Z9PR0cwoMyE30RERExD1u+B6vZ599lnXr1gEwe/Zsbr311lJu0dU1depUQkNDzS/7pzdFRETk6rqhE68xY8aYPVyzZs3isccey1cnODjYfG2bvNQZ+zL7mJKKLyjWvtw+1pmxY8eac4BZrVZOnjxZYH0REREpOTfsrcYXXniBmTNnAjBjxgyXk5fedNNN5uvTp0+77BE7ffq005i88a7m97DFh4SEmLcZ7ePj4uJITU11Oc7LFm9/PGd8fX3x9fUtsI6IiFx/rJMmlXYTSkzohAml3YRiuyF7vJ5//nlzlvc33niD0aNHu6zboEEDPDxyT5P9E4Z52coiIyOJiIgwt9s/iViY+IYNGzpsL2p8o0aNXNYRERGR0nXDJV5jxoxhxowZQG7S9fzzzxdYPyAggBYtWgCwceNGp3UMw2DTpk0AdOjQwaGsbt26VK9evcD45ORkduzY4TQ+KirK7OVyFX/8+HEOHz7sNF5ERESuHTdU4jVmzBiH24uXS7psBg0aBOQuO/T999/nK1+5ciV//vknAAMHDnQos1gs5rZly5aZi2zbmz17NklJSXh6etK/f3+HssDAQHr27AnAnDlzsFqt+eKnT58O5I7v6t69e6Hek4iIiLjfDZN42Y/peuuttwq8vZjXoEGDaNKkCYZh0LNnT7Zt2wbkTkWxcuVKhgwZAuTOLJ93nUbITfgiIyNJSUnhvvvuY9++fQBkZGQwZ84cxo0bB8DQoUPzrdMIMHnyZAIDAzl79izdunXj999/B3J7yiZPnsx7770HwCuvvFKkdRpFRETEvW6IRbJPnDhBjRo1APDw8KBChQoF1h8zZgxjxoxx2BYTE0ObNm3MHquAgABycnJIS0sDoGnTpmzbts1l4rNv3z46duxIbGwskNs7lZaWRmZmJpB7i3Dt2rUuB76vX7+e3r17m08vhoaGkpSUZC5vNHjwYObPn4/FYrnc6XCgRbJFRFy7nhbJHr52dmk3ocRca4PrtUh2HrYlf2yvz58/X+CX/czxNjVr1uTAgQOMHz+exo0bY7FY8Pb2plmzZsyYMYM9e/YU2NvUrFkzDh48yKhRo6hTpw6ZmZkEBgYSFRXF3Llz2bBhQ4FPG3bp0oUDBw4wZMgQatasSVpaGuHh4bRv355Vq1axYMGCIiddIiIi4l43RI+XuKYeLxER19TjdW1Sj5eIiIiIXJYSLxERERE3UeIlIiIi4iZKvERERETcRImXiIiIiJso8RIRERFxEyVeIiIiIm6ixEtERETETbxKuwEiIlfD9TTx5UtNy5d2E0SkhKjHS0RERMRNlHiJiIiIuIkSLxERERE3UeIlIiIi4iZKvERERETcRImXiIiIiJso8RIRERFxEyVeIiIiIm6ixEtERETETZR4iYiIiLiJEi8RERERN1HiJSIiIuImSrxERERE3ESJl4iIiIibKPESERERcRMlXiIiIiJuosRLRERExE2UeImIiIi4iRIvERERETdR4iUiIiLiJkq8RERERNxEiZeIiIiImyjxEhEREXETJV4iIiIibuJV2g0QEZGCWSdNKu0mlJjQCRNKuwkipUo9XiIiIiJuosRLRERExE2UeImIiIi4iRIvERERETdR4iUiIiLiJkq8RERERNxEiVcZk5iYyMSJE2nSpAlBQUGEhoZyxx13MHPmTDIyMkq7eSIiIlIAzeNVhhw/fpzWrVsTExMDQEBAAOnp6fzwww/88MMPLF26lG3bthEeHl66DRURERGn1ONVRmRlZdGtWzdiYmKoXLkyW7ZsITk5mZSUFJYtW0ZwcDD79+9nwIABpd1UERERcUE9XmXE4sWL+eWXXwBYvXo1d911FwAeHh706dOHnJwcHn74YdavX8+2bdto27ZtaTb3ikzbf7G0m1Bihq+dXdpNKDGacVxE5Mqpx6uMWLx4MQBt2rQxky57ffv2pVatWgAsWbLErW0TERGRwlHiVQakpKSwa9cuADp37uy0jsVioVOnTgBs3rzZbW0TERGRwlPiVQYcPnyYnJwcABo3buyynq3s3LlzXLp0yS1tExERkcJT4lUGnDlzxnxdpUoVl/Xsy+xjRERE5NqgwfVlQGJiovk6ICDAZT37MvsYe+np6aSnp5vfW61WABISEq60mSUmLcl528uihLS00m5CibFcQ78jhaHfo2uTfo9Kj36Prh7bZ6hhGJetq8TrBjN16lQmTZqUb3u1atVKoTXXv/xnugybNq20W3DD0u+RlAT9Hl19iYmJhIaGFlhHiVcZEBwcbL5OSUlxWc++zD7G3tixY3nuuefM73Nycrh06RLlypXDYrGUQGvFJiEhgWrVqnHy5ElCQkJKuzlSRun3SEqCfo+uLsMwSExM5KabbrpsXSVeZYD9D/L06dPceuutTuudPn3aaYw9X19ffH19HbaFhYVdeSPFpZCQEF3o5Irp90hKgn6Prp7L9XTZaHB9GdCgQQM8PHJ/VL/++qvLerayyMhIIiIi3NI2ERERKTwlXmVAQEAALVq0AGDjxo1O6xiGwaZNmwDo0KGD29omIiIihafEq4wYNGgQANu3b+f777/PV75y5Ur+/PNPAAYOHOjWtolzvr6+TJgwId+tXZGi0O+RlAT9Hl07LEZhnn2UUpeVlcXtt9/OL7/8QpUqVVi8eDFt27YlJyeH1atX88QTT5CQkEDnzp1Zv359aTdXREREnFDiVYbExMTQpk0bYmJigNxbkDk5OaT9/7lZmjZtyrZt2wgPDy/FVoqIiIgrSrzKmMTERGbMmMGnn35KdHQ0Hh4e1K1bl379+jFixAh8fHxKu4kiIiLighIvERERETfR4Hq5oVgsFiwWC19//XWJ7jcmJsbct+1W8LVq4sSJWCwWWrduXdpNEbmuPfroo1gsFh599NHSbso1q3Xr1lgsFiZOnFjaTXEbJV5yzbIlCLavZcuWXTbmvvvuc4i51pMgub5kZ2ezYsUKBg4cSN26dQkLC8PHx4eKFSsSFRXF2LFj883FZ5+02395enoSFhZG8+bNefHFFzlx4kSBx7Z9gNli7SdUdiY9Pd1cscJisVCzZs0rffvXNfvrkciVUOIlZcbChQsLLD9z5ow5l5kr9erVo169egUuNl4c3t7e5r69vb1LdN9SNuzZs4eGDRvSp08fPvzwQ37//XdSUlIIDg4mNjaWXbt2MW3aNJo0aULPnj3JyMjIt4+QkBAqVapEpUqVCAsLw2q1sm/fPt544w0aNWrEhg0bCtWWnJwclixZUmCdNWvWcOnSpWK9VymcypUrU69ePSpXrlzaTZFriBIvueaVL1+ewMBAtm7dyqlTp1zWW7JkCdnZ2QX+5f7bb7/x22+/ceedd5ZoG6tUqWLuu0qVKiW6b7n2ffHFF7Ru3ZqjR49Srlw5pk6dytGjR8nIyCA2NpaMjAz27t3LSy+9REhICJ9++qnTdVffeecdzp07x7lz54iNjSU5OZmFCxcSFhZGUlIS/fr1u2yyZPv9X7RoUYH1bH/IqKfr6pk6dSq//fYbU6dOLe2myDVEiZdc8wIDA+nVqxc5OTkFfpjYPkg0nkLc6ffff2fAgAGkp6fTsGFDfvrpJ1566SXq1Klj1vH09KR58+ZMnTqV6OhoHnjggULtOyAggEcffZR//etfAFitVlatWlVgzD333EPNmjU5evQoO3fudFrn1KlTbNmyhaCgIHr16lXIdyoiJUGJl5QJgwcPBlz/Fb9z506OHj1K7dq1ueeee1zux9Xg+ryD48+fP8+zzz5LrVq18PPzo1KlSvTt25fffvvN6X5dDa5fvXo1FouFChUq4OwB4o4dO5pxztbhnDp1KhaLhZYtWzo97tdff03v3r2pUqUKvr6+lC9fnrZt27Jw4UKys7Ndngd7K1asoFWrVkRERBAYGEizZs149913Cx1/o3vllVdISEjAz8+Pzz77jKpVqxZYPyIigjVr1hR6QV2ATp06ma8PHjxYYF37wdyubs8vWrSInJwcevfuTWBgYKHbIUXjbHD9X3/9hbe3NxaLhbVr1xYYP378eCwWC7fccovT8l27djFgwABq1KiBn58foaGh3HnnnUyfPp2kpKR89bt164bFYmHMmDH5ys6ePWtei5o3b+70ePXq1cNisTB//nyn5V9++SU9e/Y0r0fh4eHcc889zJkzx+mt9bwyMjKYNm0at956K4GBgYSHh9O+fftC32IvMwyRa9SECRMMwKhRo4aRk5Nj3HzzzQZgfPPNN/nqPvbYYwZgTJ482di+fbsBGIARHR3tUM+2ffv27Q7bo6OjzbJ169YZFStWNAAjICDA8PX1NctCQkKMn376Kd/x7ePtj3nhwgXDYrEYgPHzzz87xGRkZBgBAQFm3DvvvJNvv+3atTMAY/z48fnKRo0aZcZaLBYjLCzM8PT0NLfde++9RkJCgsvz2qpVK+OFF14w48PDww0PDw8zvmPHjkZaWlq+ePmfc+fOmefs8ccfL9Y+7H93Fi5c6LTO+fPnzTpPP/200zqtWrUyAGPQoEFGTEyMYbFYjKCgICMpKSlfXdv/pW+//dbh/5m4ZjtPRfnYHDRokPkzsXffffcZgNGrVy+XsTk5OUatWrUMwJg4caJDWXZ2tvF///d/ZnsAIygoyOH/f7169YyYmBiHuJkzZxqA0bRp03zH++ijj8xYDw8PIy4uzqH81KlTZvmff/7pUJaSkmL06tXLoT0hISHmtQ8w/vGPfxiXLl3Kd1zb7+3YsWONli1bGoDh5eVlhIWFOexvwoQJLs9VWaMeLykT7P9qXLBggUNZcnIyK1aswMPDo0RuMz7yyCPUqVOHvXv3kpycTFJSElu2bKFy5cokJCQwYsSIQu+rfPnyNGnSBICvvvrKoez7778nJSWFkJAQp+UZGRns2rULgDZt2jiUvfvuu8yaNQuAoUOHcubMGeLi4rBarcyaNQsvLy+++uorhgwZ4rJtP/30E2+88QbPPPMM58+f59KlS8TFxfHqq69isVjYtGkTY8eOLfR7vRFt376dnJwcAB588MGrdpyNGzear2vXrn3Z+jVq1ODee+8lKSmJlStXOpR98803HDt2jDp16rjsSZWry7ae7hdffEF8fLzTOrt27SI6OhqLxcIjjzziUDZhwgT+9a9/UbFiRWbPnk1sbCyJiYmkpqayfft2mjZtypEjR+jRo4f5+wn/u478/PPP+cYKbt++Hch9wCMnJyffXQFbeY0aNahVq5ZD2dChQ1m1ahW1a9dm6dKlWK1WrFYrKSkpfP7559SuXZs9e/bw2GOPuTwn//nPf/jvf//Le++9R2JiInFxcZw4ccK8FT5p0qTL9hCWGaWd+Ym4kvcv8RMnThgeHh5GYGCgkZiYaNZbsGCBARjt27c3DMO44h6v+vXrGykpKfnas3btWrPOyZMnXcbnPebIkSMNwOjWrZvD9kmTJpl/6Xl7exthYWFGdna2Wf7NN98YgOHn5+fQ85SSkmJEREQYgNGvXz+n5+5f//qX2Z4ffvjBocz+L/dHHnnEafwrr7xi/uV5+vRpp3Xkf+cJKPZ5KqjHKzk52Vi0aJH517+vr69x5swZp/ux7/EyjP/1YNxzzz0O9QYOHGgAxmuvvWYYRv7/Z+JcSfZ4paamGqGhoQZgvP/++05jhw4dagBGVFSUw/bo6GjD09PT8Pf3d9r7bhiGkZCQYFStWtUAjM8++8zcnp2dbV47Vq9e7RBj610bP368ARgjRoxwKB88eLABGI8++qjD9m+//dYAjIoVKxonTpxw2p6TJ08agYGBBmDs37/focz2ewsY8+fPzxebnZ1t3HPPPQZgNGrUyOn+yxr1eEmZUa1aNdq1a2f2cNnYxrEU9NdUUYwePRp/f/982zt37mwuyfTLL78Uen+2vzK//fZbh3FTtr8gu3btyt///nfi4+P58ccf85Xfdddd+Pr6mtu3bNli/rXqatLBp556ynyE/eOPP3bZtvHjxzvd/vzzz+Pv709WVharV6++3Fu8YcXGxpqvIyIirnh/zz77LJGRkURGRlKuXDkCAwN59NFHiY+Px9vbm8WLFxd6aoIePXoQGhrKjh07OHbsGJC75NiqVavw8PBg0KBBV9xeKR4/Pz969+4NwIcffpivPD093bzG5e3tWrRoEdnZ2XTq1Im//e1vTvcfHBxM9+7dARym2PHw8KBVq1aAYw/78ePHiY6Opk6dOmZvXN4eeNv1KG/vu228V//+/alWrZrT9lStWtWMczXlT7Vq1cyxvPY8PDx45ZVXgNzxjUW59l6rlHhJmWL7j2m73fjHH3+wY8cOwsPDzQvNlfr73//udLuXlxcVKlQAKNL8R61atcLT09OckwkgLS2N7777jqCgIO68807zomR/sbO9znuh++GHH4DcC1XdunWdHtPT05N7773XoX5e1apVczloNyQkhGbNmhUYLyUvISGB8+fPm7d+bapXr86BAwfo06dPoffl7+9P3759MQzD/ONk+fLlpKSk0KFDB017UspsCY7tlqK9devWER8fj5+fHw899JBDmW34webNm80k3dmX7Wd+/Phxh3jbdcHZtebee+/l5ptvpnr16hw8eJC//voLgOjoaPOhobzXI1t75s+fX2B7tm7d6rQ9NrYJgJ1p2bIlXl5ewPVxPVLiJWXKgw8+SHh4OLt27eL33383Ly79+vXDz8+vRI4RHBzsssz2nz8zM7PQ+wsNDaVp06bA/y5wu3fvJj093byg5L0YpqamsmfPHiD/hc52MbzcB6ft6Tpb/bwuF28rdxUvUK5cOfN1SUxGunDhQgzDwDAMrFYr27dvp0WLFpw4cYLBgwc7fVKtILZe4CVLlpCTk1PivcNSfFFRUdSqVQvDMPjoo48cymy9YN26dSMsLMyh7MyZM0Du2FZbku7sKzk5GSDffHG268nhw4c5d+4c8L/eLNt1KO8fgrbym2++OV+vlq099n80OPtKS0tz2h6bgq5Hfn5+5v+16+F6pMRLyhRfX1/69esHwLx588zZuZ11UV9L8iZW9n9hQu7tRD8/P3bu3ElmZia7du0iIyODgIAAlz1wUvoaNWpkvt6/f3+J7jskJITWrVuzefNmGjVqxJ49e3jmmWeKtI8777yThg0bcvLkSWbPns3u3buJiIjg/vvvL9G2StHZD5q3v90YGxvL+vXrgfy3GQFzuMKLL75oJukFfeUdJN+oUSMqVaoEOCZWFovFTLhcXa/y/hFo3545c+YUqj2Xm9j3RqDES8ocW5L19ttvc+rUKRo3buxy3plrhe2CZUuo8iZevr6+3H333SQnJ/P999+b5VFRUfmWIKpYsSJAgbP425fb6ud1ubX8bOWu4iX35+rhkXsZ/eyzz67KMQICAvj3v/8NwOLFi9m9e3eR4m3/X2xzNz388MMOYwal9NgSq99//93s4V6+fDmZmZlUqFCBzp0754uJjIwEXN+yK4zWrVsDuQnV0aNHzeuobSiFqx4v2/WqpNsDBV+P0tPTzfGU18P1SImXlDnNmzenSZMm5oR8ZeG2iS2BSklJYevWrezdu5eIiAhuu+02s479X5muBrICZpJ56tQpjh496vR42dnZ5j7uuOMOp3VOnjxpDrrOKzEx0RyPdq0ntaWpUqVK9OzZE8h9iMHVz8MZw8mEuq60adPGHBT90ksvFamNjzzyCF5eXmXq/8uN4pZbbuGuu+4C/tfrZfu3X79+5tAGey1atABg69at5u27orJPrJwlVbbxn8eOHWPLli3m7URbwuasPevWrStWW2y++eYbl/8nduzYQVZWFnB9XI+UeEmZNH36dEaPHs3o0aMZMGBAaTfnsoKCgswEaPLkyWRlZdGqVSuztwT+dzFcu3atOYDUWeLVvn17c7yDq6ca33//ffNiabs168yrr77qdPvMmTNJTU3Fy8vLTCzEuSlTphAUFERqaio9evS4bE9iXFwcPXv2xGq1Fuk4L7/8MpD7IbRly5ZCx1WqVIlZs2YxevRoJk2aZI43lGuDbZD98uXLOXjwoNnzZdue12OPPYaXlxcXL15kwoQJBe47IyPD6bhAW5IVHR1tjvvL25tlu/aMGzcOgPr16zt9onbo0KEA/Prrr8yZM6fA9iQnJ7ucwf7EiRMsXrw43/acnBxef/11ABo2bGjOi1imuW/mCpGiKe78Qlc6j1feGHs1atRwOt9SYeJffvllh5mY//3vfzuUZ2ZmGkFBQWZ5cHCwkZmZ6XRf//73v816w4YNM86dO2cYRu68T++8847h7e1tAEafPn3yxdrOq20eof/7v/8zLly4YBhG7vw/r732mjnj9LPPPuvyXMj/fPbZZ4aPj48BGOXLlzemTZtm/P7772Z5VlaW8eOPPxrjxo0z5+SyzQxemJnrbe644w5zFvC88s7jVViax6tw7OfxunDhQoFftp+tq3m87F26dMn83WnevLkBGA0bNiywLbY5APn/c/H98ssvZllmZqaxf/9+Y9KkSUa1atWMHTt2ON2HbZ4vwPD09DTi4+Mdyj/55BOH69Xw4cNdtsc2x5fFYjFGjhxpHDt2zCxLS0szvvvuO+P55583ypUrl28ORNvvbWhoqOHn52d88MEHRmpqqmEYuXM3PvTQQ2YbPv300wLPS1mhxEuuWddb4rVt2zaHC9nBgwfz1enUqZNZ3qVLlwLfZ94lg8LDww0vLy9zW5s2bYq8ZJD9kiPt2rUzL4ByeTt37jRuueUWh5+xj4+PERER4bAUk8ViMfr162dkZGQYhlG0xOuzzz4z665bt86hTInX1WWfeF3u629/+5thGIVLvAzDMHr06OEQP3Xq1ALr5+TkGOPGjXNYksff398oV66cw/9hwNi5c6fTfTzyyCNmnTvuuCNf+blz5xz2s2LFCpftSU9PN5544gmH+kFBQfmWIQOMU6dOOcTaLxkUFRVlAIa3t7cRHh7uEPfKK68UeE7KEt1qFHGTu+++2xzUHBkZScOGDfPVse/ud3ab0d5bb73FV199Rc+ePalUqRJJSUkEBwfTpk0bFixYwJYtWwqcGgNyb9kuW7aMqKgoDMPAx8eH2267jXfeeYeNGzeW2BQdN4IWLVrw22+/8cknn9C/f39uueUW/Pz8SExMJCIigqioKF5++WUOHz7Mxx9/nO+hicJ44IEHzCcpXU1+K2WP/W1FDw+Pyw6fsFgsTJ48mQMHDvDUU0/RoEEDc67A8PBw7r77bp5//nl2795tjsHKy/764mzQfKVKlcxrlMVicTq+y8bHx4e5c+eye/duHn30UW6++Ways7NJSkqiYsWKtG7dmvHjx3PgwAGX00b4+Piwbds2Xn/9derVq0d6ejqhoaG0bduWL7/80uWwiLLIYhhFGOEpIiIiIsWmHi8RERERN1HiJSIiIuImSrxERERE3ESJl4iIiIibKPESERERcRMlXiIiIiJuosRLRERExE2UeImIiIi4iRIvERERETdR4iUiUsbFxMRgsViwWCzExMSUdnNEpABKvERECmBLaIrztWjRotJuvohcY7xKuwEiIteySpUqOd2elJREcnJygXX8/f2vWrtEpGxS4iUiUoBz58453T5x4kQmTZpUYB0Rkbx0q1FERETETZR4iYiUsMzMTNauXcvQoUNp3rw5lStXxsfHh4oVK9KxY0c++eQTDMNwGX/q1ClGjRpFo0aNCAwMxNfXl5tuuolmzZoxatQo9u7dW6T2pKam0r17dywWC+XLl2fPnj0O5Zs2baJHjx5UrVoVHx8fQkJCqF27Nh06dGDGjBlcunSpWOdBRPLTrUYRkRK2a9cuHnjgAfP7kJAQ/Pz8uHDhAps3b2bz5s189tlnLFu2DA8Px79/f/75Z9q0aUNcXBwAnp6ehISEcO7cOc6ePcuPP/5IXFxcoQfuX7p0iW7durF7926qV6/Opk2bqF+/vlk+efJkJkyYYH4fEBCAYRhER0cTHR3Nli1baN68Oa1bty7+CRERk3q8RERKWEBAAMOGDWPLli1YrVasVisJCQnExsbyzjvvEBISwsqVK3n33XfzxY4ePZq4uDhuv/12vvvuOzIzM7l06RJpaWkcPXqUGTNm0KhRo0K14+TJk0RFRbF7926aNGnCd99955B0HT9+3Byn9txzz3H69GmSk5NJTEwkPj6eHTt28NRTTxEcHFwyJ0ZEsBgF9XeLiIhT9oPri3oZXbVqFb179+bmm2/mjz/+cCgLCAggNTWV3bt3c9dddxVqfzExMdSqVQuA6Ohoatasya+//krnzp05deoU99xzD2vXriU0NNQhbsWKFfTp04e6dety5MiRIr0HESke9XiJiLjZfffdB8CxY8fyPREZFhYGwNmzZ4u9/x07dtCyZUtOnTpFjx492Lx5c76ky/5YiYmJ5tQYInJ1KfESEbkKEhMTefPNN2nVqhUVK1bEx8fHnFg1ICDArHfq1CmHuK5duwIwaNAgRo8ezTfffENKSkqhj/vZZ5/RoUMH4uPjGT58OCtXrsTX19dp3TvvvJPy5ctz9uxZ/v73v/Puu+/y22+/FbkHT0QKT7caRUSKoaBbjUePHqVt27YOSVVAQACBgYHmYPrz588D8PXXX9OqVSuzXnx8PD169GD79u3mNk9PT2677Tbuu+8+hg4dSpUqVRyOZ3+r0ea+++5j3bp1l30fW7du5eGHH+bChQvmttDQUO655x4eeugh+vTpg7e392X3IyKFox4vEZESNnjwYE6dOkXNmjVZuXIlsbGxJCcn89dff3Hu3DlOnz5t1s2btIWFhfHVV1+xY8cOXnjhBVq0aIGXlxf79u1j8uTJ1KlTh08++cTlsQcMGADA+vXree+99y7b1nbt2hEdHc2SJUsYNGgQderUwWq18sUXX/DII4/QtGlTh/aKyBUyRESkyCZMmGAARt7L6IkTJ8zt3333ndPYU6dOmXW2b99+2WOlpqYan3/+udGkSRMDMPz8/Ixz586Z5dHR0eb+oqOjjXHjxhmAYbFYjHfffbfI7+3UqVPG9OnTDT8/PwMwHnzwwSLvQ0ScU4+XiEgJOnnypPm6adOmTuts3bq1SPv08/Pj/vvv59NPPwUgLS2NnTt3uqw/efJkJk6ciGEYPPPMM7zzzjtFOl6VKlV44YUXGD16NABbtmwpUryIuKbES0SkBNk/Pfjzzz/nK09MTGTKlClOY7OyssjJyXG5b/tFt/NOvJrXhAkTzOOMHDmSt956K1+d9PT0AvdhO97ljiUihaf/TSIiJahBgwZUr14dgMcee4x9+/aZZd999x2tW7c2Z6XP69SpU9SpU4cpU6awf/9+srKyzLIDBw6Y47cCAwMdBuS78vLLLzNt2jQgd2LWN954w6F8+vTpdO7cmQ8//NDhQYD09HRWrFjBm2++Cfxv+gsRuXJaMkhEpAR5eHgwe/ZsHnzwQQ4ePEjz5s3N6SNSUlIIDAzk888/p127dk7j//zzT8aNG8e4cePw9PQkNDSUpKQkMjIyAPDx8WHRokVEREQUqj0vvvgiXl5ejBkzhhdffJGsrCz++c9/ApCTk8PGjRvZuHEjkNvD5e/vT1xcnDnov0GDBk57y0SkeJR4iYiUsK5du/Ltt9/y2muvsWvXLlJSUoiMjKRt27a8+OKL1KtXz2lclSpVWLt2Ldu3b+e7777j1KlT/PXXX3h5eXHLLbfQpk0bnn32WerUqVOk9owePRovLy9GjhzJyy+/TFZWFuPHjzenpti+fTu//PILZ8+exWq1Eh4eTqNGjejZsyfDhg3Dz8+vJE6LiKB5vERERETcRmO8RERERNxEiZeIiIiImyjxEhEREXETJV4iIiIibqLES0RERMRNlHiJiIiIuIkSLxERERE3UeIlIiIi4iZKvERERETcRImXiIiIiJso8RIRERFxEyVeIiIiIm6ixEtERETETZR4iYiIiLjJ/wMEs9ylnBQOXQAAAABJRU5ErkJggg==", 69 | "text/plain": [ 70 | "
" 71 | ] 72 | }, 73 | "metadata": {}, 74 | "output_type": "display_data" 75 | } 76 | ], 77 | "source": [ 78 | "# Create the DataFrame\n", 79 | "df = pd.DataFrame(data)\n", 80 | "\n", 81 | "# Set the colors\n", 82 | "colors = ['skyblue', 'lightcoral']\n", 83 | "\n", 84 | "plt.figure(figsize=(10, 6))\n", 85 | "\n", 86 | "# Plot the data\n", 87 | "ax = df.plot(x=\"method\", kind=\"bar\", color=colors, width=0.7)\n", 88 | "# plt.title(\"Comparison of Compressed v/s Not Compressed\")\n", 89 | "plt.ylabel(\"Tokens\")\n", 90 | "plt.xlabel(\"Tasks\")\n", 91 | "plt.xticks(rotation=0)\n", 92 | "plt.tight_layout()\n", 93 | "plt.legend(loc='upper left')\n", 94 | "\n", 95 | "# Display the plot\n", 96 | "plt.show()" 97 | ] 98 | } 99 | ], 100 | "metadata": { 101 | "kernelspec": { 102 | "display_name": "webactions-lm", 103 | "language": "python", 104 | "name": "python3" 105 | }, 106 | "language_info": { 107 | "codemirror_mode": { 108 | "name": "ipython", 109 | "version": 3 110 | }, 111 | "file_extension": ".py", 112 | "mimetype": "text/x-python", 113 | "name": "python", 114 | "nbconvert_exporter": "python", 115 | "pygments_lexer": "ipython3", 116 | "version": "3.9.13" 117 | }, 118 | "orig_nbformat": 4 119 | }, 120 | "nbformat": 4, 121 | "nbformat_minor": 2 122 | } 123 | -------------------------------------------------------------------------------- /notebooks/webarena/final_webarena_step_vs_all.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 10, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import pandas as pd\n", 10 | "import ast" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 11, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "# Download model outputs from https://drive.google.com/drive/folders/1P8HxK9tF623bJ9c6xIxn10UnryVdVzsc\n", 20 | "# and save them to data/results_step/webarena/2405_all_tasks/" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 12, 26 | "metadata": {}, 27 | "outputs": [ 28 | { 29 | "data": { 30 | "text/html": [ 31 | "
\n", 32 | "\n", 45 | "\n", 46 | " \n", 47 | " \n", 48 | " \n", 49 | " \n", 50 | " \n", 51 | " \n", 52 | " \n", 53 | " \n", 54 | " \n", 55 | " \n", 56 | " \n", 57 | " \n", 58 | " \n", 59 | " \n", 60 | " \n", 61 | " \n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | "
tasktask_idsitesmodellogfiledonerewardgpt4_step_successgpt4_step_num_actionstypecategory
0tasks/webarena/0.json0[shopping_admin]gpt-4-turbo-preview20240505-204634/0.jsonTrue0.00.01stepshopping_admin
1tasks/webarena/1.json1[shopping_admin]gpt-4-turbo-preview20240505-204634/1.jsonTrue0.00.06stepshopping_admin
2tasks/webarena/2.json2[shopping_admin]gpt-4-turbo-preview20240505-204634/2.jsonTrue0.00.01stepshopping_admin
3tasks/webarena/3.json3[shopping_admin]gpt-4-turbo-preview20240505-204634/3.jsonTrue0.00.01stepshopping_admin
4tasks/webarena/4.json4[shopping_admin]gpt-4-turbo-preview20240505-204634/4.jsonTrue0.00.01stepshopping_admin
....................................
799tasks/webarena/807.json807[gitlab]gpt-4-turbo-preview20240508-201838/807.jsonTrue0.00.011stepgitlab
800tasks/webarena/808.json808[gitlab]gpt-4-turbo-preview20240508-201838/808.jsonTrue0.00.08stepgitlab
801tasks/webarena/809.json809[gitlab]gpt-4-turbo-preview20240324-091310/809.jsonTrue1.01.011stepgitlab
802tasks/webarena/810.json810[gitlab]gpt-4-turbo-preview20240508-201838/810.jsonTrue0.00.07stepgitlab
803tasks/webarena/811.json811[gitlab]gpt-4-turbo-preview20240508-201838/811.jsonTrue0.00.05stepgitlab
\n", 219 | "

804 rows × 11 columns

\n", 220 | "
" 221 | ], 222 | "text/plain": [ 223 | " task task_id sites model \\\n", 224 | "0 tasks/webarena/0.json 0 [shopping_admin] gpt-4-turbo-preview \n", 225 | "1 tasks/webarena/1.json 1 [shopping_admin] gpt-4-turbo-preview \n", 226 | "2 tasks/webarena/2.json 2 [shopping_admin] gpt-4-turbo-preview \n", 227 | "3 tasks/webarena/3.json 3 [shopping_admin] gpt-4-turbo-preview \n", 228 | "4 tasks/webarena/4.json 4 [shopping_admin] gpt-4-turbo-preview \n", 229 | ".. ... ... ... ... \n", 230 | "799 tasks/webarena/807.json 807 [gitlab] gpt-4-turbo-preview \n", 231 | "800 tasks/webarena/808.json 808 [gitlab] gpt-4-turbo-preview \n", 232 | "801 tasks/webarena/809.json 809 [gitlab] gpt-4-turbo-preview \n", 233 | "802 tasks/webarena/810.json 810 [gitlab] gpt-4-turbo-preview \n", 234 | "803 tasks/webarena/811.json 811 [gitlab] gpt-4-turbo-preview \n", 235 | "\n", 236 | " logfile done reward gpt4_step_success \\\n", 237 | "0 20240505-204634/0.json True 0.0 0.0 \n", 238 | "1 20240505-204634/1.json True 0.0 0.0 \n", 239 | "2 20240505-204634/2.json True 0.0 0.0 \n", 240 | "3 20240505-204634/3.json True 0.0 0.0 \n", 241 | "4 20240505-204634/4.json True 0.0 0.0 \n", 242 | ".. ... ... ... ... \n", 243 | "799 20240508-201838/807.json True 0.0 0.0 \n", 244 | "800 20240508-201838/808.json True 0.0 0.0 \n", 245 | "801 20240324-091310/809.json True 1.0 1.0 \n", 246 | "802 20240508-201838/810.json True 0.0 0.0 \n", 247 | "803 20240508-201838/811.json True 0.0 0.0 \n", 248 | "\n", 249 | " gpt4_step_num_actions type category \n", 250 | "0 1 step shopping_admin \n", 251 | "1 6 step shopping_admin \n", 252 | "2 1 step shopping_admin \n", 253 | "3 1 step shopping_admin \n", 254 | "4 1 step shopping_admin \n", 255 | ".. ... ... ... \n", 256 | "799 11 step gitlab \n", 257 | "800 8 step gitlab \n", 258 | "801 11 step gitlab \n", 259 | "802 7 step gitlab \n", 260 | "803 5 step gitlab \n", 261 | "\n", 262 | "[804 rows x 11 columns]" 263 | ] 264 | }, 265 | "execution_count": 12, 266 | "metadata": {}, 267 | "output_type": "execute_result" 268 | } 269 | ], 270 | "source": [ 271 | "# Load results summarygit\n", 272 | "srcdir = \"../../data/results_step/webarena/2405_all_tasks\"\n", 273 | "df = pd.read_csv(f\"{srcdir}/summary.csv\")\n", 274 | "df['sites'] = df['sites'].apply(ast.literal_eval)\n", 275 | "df['category'] = df['sites'].apply(lambda x: 'multisite' if len(x) > 1 else x[0])\n", 276 | "df" 277 | ] 278 | }, 279 | { 280 | "cell_type": "code", 281 | "execution_count": 16, 282 | "metadata": {}, 283 | "outputs": [ 284 | { 285 | "name": "stdout", 286 | "output_type": "stream", 287 | "text": [ 288 | " Site Mean SteP success (%)\n", 289 | "0 gitlab 32.77\n", 290 | "1 map 31.19\n", 291 | "2 multisite 10.42\n", 292 | "3 reddit 53.77\n", 293 | "4 shopping 52.20\n", 294 | "5 shopping_admin 23.63\n", 295 | "Overall mean: 36.32%\n" 296 | ] 297 | } 298 | ], 299 | "source": [ 300 | "df_category = df.groupby('category')['gpt4_step_success'].mean().reset_index()\n", 301 | "df_category['gpt4_step_success'] = (df_category['gpt4_step_success'] * 100).round(2) # convert to %\n", 302 | "\n", 303 | "df_category.columns = ['Site', 'Mean SteP success (%)']\n", 304 | "print(df_category)\n", 305 | "\n", 306 | "overall_mean = (df['gpt4_step_success'].mean() * 100).round(2)\n", 307 | "print(f\"Overall mean: {overall_mean}%\")" 308 | ] 309 | } 310 | ], 311 | "metadata": { 312 | "kernelspec": { 313 | "display_name": "autoui", 314 | "language": "python", 315 | "name": "python3" 316 | }, 317 | "language_info": { 318 | "codemirror_mode": { 319 | "name": "ipython", 320 | "version": 3 321 | }, 322 | "file_extension": ".py", 323 | "mimetype": "text/x-python", 324 | "name": "python", 325 | "nbconvert_exporter": "python", 326 | "pygments_lexer": "ipython3", 327 | "version": "3.10.12" 328 | } 329 | }, 330 | "nbformat": 4, 331 | "nbformat_minor": 2 332 | } 333 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.26.2 2 | pandas==2.1.3 3 | torch==2.1.1 4 | openai 5 | transformers==4.30 6 | datasets==2.10.1 7 | bert-score==0.3.13 8 | bitsandbytes==0.37.2 9 | playwright==1.32.1 10 | beautifulsoup4 11 | colorama 12 | matplotlib==3.8.2 13 | tensorboard==2.15.1 14 | requests 15 | tqdm>=4.31.1 16 | gradio==4.7.1 17 | huggingface-hub==0.17.3 18 | setuptools==69.0.2 19 | peft @ git+https://github.com/huggingface/peft.git@b4faffea8ae031e5bd69a76b55418b3650c04c80 20 | accelerate==0.24.0 21 | ctranslate2 22 | PyYAML==6.0.1 23 | sentencepiece==0.1.97 24 | tiktoken -------------------------------------------------------------------------------- /scripts/evaluate/eval_miniwob.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pandas as pd 3 | import time 4 | import re 5 | import argparse 6 | import itertools 7 | from tqdm import tqdm 8 | 9 | import os 10 | import sys 11 | import argparse 12 | from typing import List 13 | import shutil 14 | 15 | import openai 16 | 17 | from webagents_step.utils.data_prep import * 18 | from webagents_step.agents.prompt_agent import PromptAgent 19 | from webagents_step.agents.step_agent import StepAgent 20 | from webagents_step.prompts.miniwob import flat_fewshot_template, step_fewshot_template 21 | from webagents_step.environment.miniwob import MiniWoBEnvironmentWrapper 22 | 23 | openai.api_key = os.environ.get("OPENAI_API_KEY") 24 | 25 | def run(): 26 | parser = argparse.ArgumentParser( 27 | description="Only the config file argument should be passed" 28 | ) 29 | parser.add_argument( 30 | "--config", type=str, required=True, help="yaml config file location" 31 | ) 32 | args = parser.parse_args() 33 | with open(args.config, "r") as file: 34 | config = DotDict(yaml.safe_load(file)) 35 | 36 | dstdir = f"{config.logdir}/{time.strftime('%Y%m%d-%H%M%S')}" 37 | os.makedirs(dstdir, exist_ok=True) 38 | shutil.copyfile(args.config, os.path.join(dstdir, args.config.split("/")[-1])) 39 | random.seed(42) 40 | 41 | tasks = config.env.tasks 42 | seeds = range(config.env.start_seed, config.env.end_seed) 43 | sampled_seeds = ( 44 | random.sample(seeds, config.env.num_samples_per_task) 45 | if (config.env.num_samples_per_task > 0) 46 | else seeds 47 | ) 48 | 49 | ##### 50 | # Initialize agent 51 | ##### 52 | if config.agent.type == "step": 53 | action_to_prompt_dict = {k: v for k, v in step_fewshot_template.__dict__.items() if isinstance(v, dict)} 54 | low_level_action_list = config.agent.low_level_action_list 55 | agent_init = lambda: StepAgent( 56 | root_action = config.agent.root_action, 57 | action_to_prompt_dict = action_to_prompt_dict, 58 | low_level_action_list = low_level_action_list, 59 | max_actions=config.env.max_env_steps, 60 | verbose=config.verbose, 61 | logging=config.logging, 62 | debug=config.debug, 63 | model=config.agent.model_name, 64 | prompt_mode=config.agent.prompt_mode, 65 | ) 66 | elif config.agent.type == "flat_fewshot": 67 | agent_init = lambda: PromptAgent( 68 | prompt_template=flat_fewshot_template.flat_fewshot_agent, 69 | model=config.agent.model_name, 70 | prompt_mode=config.agent.prompt_mode, 71 | max_actions=config.env.max_env_steps, 72 | verbose=config.verbose, 73 | logging=config.logging, 74 | debug=config.debug, 75 | ) 76 | else: 77 | raise NotImplementedError(f"{config.agent.type} not implemented") 78 | 79 | ##### 80 | # Evaluate 81 | ##### 82 | 83 | for task_seed_pair in tqdm(itertools.product(tasks, sampled_seeds)): 84 | task, seed = task_seed_pair 85 | env = MiniWoBEnvironmentWrapper( 86 | task=task, 87 | seed=seed, 88 | max_browser_rows=config.env.max_browser_rows, 89 | max_steps=config.env.max_env_steps, 90 | headless=config.env.headless, 91 | ) 92 | agent = agent_init() 93 | objective = env.get_objective() 94 | status = agent.act(objective=objective, env=env) 95 | env.close() 96 | 97 | if config.logging: 98 | log_file = os.path.join(dstdir, f"{task}_{seed:03d}.json") 99 | log_data = { 100 | "task": task, 101 | "seed": seed, 102 | "model": config.agent.model_name, 103 | "trajectory": agent.get_trajectory(), 104 | } 105 | summary_file = os.path.join(dstdir, "summary.csv") 106 | summary_data = { 107 | "task": task, 108 | "seed": seed, 109 | "model": config.agent.model_name, 110 | "logfile": re.search(r"/([^/]+/[^/]+\.json)$", log_file).group(1), 111 | } 112 | summary_data.update(status) 113 | log_run( 114 | log_file=log_file, 115 | log_data=log_data, 116 | summary_file=summary_file, 117 | summary_data=summary_data, 118 | ) 119 | 120 | if __name__ == "__main__": 121 | run() 122 | -------------------------------------------------------------------------------- /scripts/evaluate/eval_webarena.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pandas as pd 3 | import time 4 | import re 5 | import argparse 6 | import itertools 7 | from tqdm import tqdm 8 | 9 | import os 10 | import sys 11 | import argparse 12 | from typing import List 13 | import shutil 14 | 15 | import openai 16 | import time 17 | 18 | from webagents_step.utils.data_prep import * 19 | from webagents_step.agents.prompt_agent import PromptAgent 20 | from webagents_step.agents.step_agent import StepAgent 21 | from webagents_step.prompts.webarena import flat_fewshot_template, step_fewshot_template 22 | from webagents_step.environment.webarena import WebArenaEnvironmentWrapper 23 | 24 | openai.api_key = os.environ.get("OPENAI_API_KEY") 25 | 26 | def run(): 27 | parser = argparse.ArgumentParser( 28 | description="Only the config file argument should be passed" 29 | ) 30 | parser.add_argument( 31 | "--config", type=str, required=True, help="yaml config file location" 32 | ) 33 | args = parser.parse_args() 34 | with open(args.config, "r") as file: 35 | config = DotDict(yaml.safe_load(file)) 36 | 37 | dstdir = f"{config.logdir}/{time.strftime('%Y%m%d-%H%M%S')}" 38 | os.makedirs(dstdir, exist_ok=True) 39 | shutil.copyfile(args.config, os.path.join(dstdir, args.config.split("/")[-1])) 40 | random.seed(42) 41 | 42 | config_file_list = [] 43 | 44 | # ids covered in gitlab 45 | task_ids = config.env.task_ids 46 | 47 | for task_id in task_ids: 48 | config_file_list.append(f"tasks/webarena/{task_id}.json") 49 | 50 | action_to_prompt_dict = {k: v for k, v in step_fewshot_template.__dict__.items() if isinstance(v, dict)} 51 | low_level_action_list = config.agent.low_level_action_list 52 | 53 | if config.agent.type == "step": 54 | agent_init = lambda: StepAgent( 55 | root_action = config.agent.root_action, 56 | action_to_prompt_dict = action_to_prompt_dict, 57 | low_level_action_list = low_level_action_list, 58 | max_actions=config.env.max_env_steps, 59 | verbose=config.verbose, 60 | logging=config.logging, 61 | debug=config.debug, 62 | model=config.agent.model_name, 63 | prompt_mode=config.agent.prompt_mode, 64 | ) 65 | elif config.agent.type == "flat_fewshot8k": 66 | agent_init = lambda: PromptAgent( 67 | prompt_template=flat_fewshot_template.flat_fewshot_agent8k, 68 | model=config.agent.model_name, 69 | prompt_mode=config.agent.prompt_mode, 70 | max_actions=config.env.max_env_steps, 71 | verbose=config.verbose, 72 | logging=config.logging, 73 | debug=config.debug, 74 | ) 75 | elif config.agent.type == "flat_fewshot4k": 76 | agent_init = lambda: PromptAgent( 77 | prompt_template=flat_fewshot_template.flat_fewshot_agent4k, 78 | model=config.agent.model_name, 79 | prompt_mode=config.agent.prompt_mode, 80 | max_actions=config.env.max_env_steps, 81 | verbose=config.verbose, 82 | logging=config.logging, 83 | debug=config.debug, 84 | ) 85 | else: 86 | raise NotImplementedError(f"{config.agent.type} not implemented") 87 | 88 | ##### 89 | # Evaluate 90 | ##### 91 | 92 | for config_file in config_file_list: 93 | env = WebArenaEnvironmentWrapper(config_file=config_file, 94 | max_browser_rows=config.env.max_browser_rows, 95 | max_steps=config.env.max_env_steps, 96 | slow_mo=1, 97 | observation_type="accessibility_tree", 98 | current_viewport_only=True, 99 | viewport_size={"width": 1920, "height": 1080}, 100 | headless=config.env.headless) 101 | 102 | agent = agent_init() 103 | objective = env.get_objective() 104 | status = agent.act(objective=objective, env=env) 105 | env.close() 106 | 107 | if config.logging: 108 | with open(config_file, "r") as f: 109 | task_config = json.load(f) 110 | log_file = os.path.join(dstdir, f"{task_config['task_id']}.json") 111 | log_data = { 112 | "task": config_file, 113 | "id": task_config['task_id'], 114 | "model": config.agent.model_name, 115 | "type": config.agent.type, 116 | "trajectory": agent.get_trajectory(), 117 | } 118 | summary_file = os.path.join(dstdir, "summary.csv") 119 | summary_data = { 120 | "task": config_file, 121 | "task_id": task_config['task_id'], 122 | "model": config.agent.model_name, 123 | "type": config.agent.type, 124 | "logfile": re.search(r"/([^/]+/[^/]+\.json)$", log_file).group(1), 125 | } 126 | summary_data.update(status) 127 | log_run( 128 | log_file=log_file, 129 | log_data=log_data, 130 | summary_file=summary_file, 131 | summary_data=summary_data, 132 | ) 133 | 134 | # For reddit: Sleep for 21 minutes (720 seconds) 135 | # time.sleep(1260) 136 | 137 | if __name__ == "__main__": 138 | run() 139 | -------------------------------------------------------------------------------- /scripts/setup/auto_login_webarena.py: -------------------------------------------------------------------------------- 1 | """Script to automatically login each website""" 2 | import argparse 3 | import glob 4 | import os 5 | import time 6 | from concurrent.futures import ThreadPoolExecutor 7 | from itertools import combinations 8 | from pathlib import Path 9 | 10 | from playwright.sync_api import sync_playwright 11 | 12 | os.makedirs('./.auth', exist_ok=True) 13 | 14 | 15 | SLEEP = 1.5 16 | 17 | # set the URLs of each website, we use the demo sites as an example 18 | os.environ[ 19 | "SHOPPING" 20 | ] = "https://webarena-env-shopping.awsdev.asapp.com" 21 | os.environ[ 22 | "SHOPPING_ADMIN" 23 | ] = "https://webarena-env-shopping.awsdev.asapp.com/admin" 24 | os.environ[ 25 | "REDDIT" 26 | ] = "https://webarena-env-reddit.awsdev.asapp.com" 27 | os.environ[ 28 | "GITLAB" 29 | ] = "https://webarena-env-github.awsdev.asapp.com" 30 | os.environ[ 31 | "MAP" 32 | ] = "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:3000" 33 | os.environ[ 34 | "WIKIPEDIA" 35 | ] = "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:8888/wikipedia_en_all_maxi_2022-05/A/User:The_other_Kiwix_guy/Landing" 36 | os.environ[ 37 | "HOMEPAGE" 38 | ] = "PASS" # The home page is not currently hosted in the demo site 39 | print("Done setting up URLs") 40 | 41 | from browser_env.env_config import ( 42 | ACCOUNTS, 43 | GITLAB, 44 | REDDIT, 45 | SHOPPING, 46 | SHOPPING_ADMIN, 47 | ) 48 | 49 | HEADLESS = False 50 | SLOW_MO = 0 51 | 52 | 53 | SITES = ["gitlab", "shopping", "shopping_admin", "reddit"] 54 | URLS = [ 55 | f"{GITLAB}/-/profile", 56 | f"{SHOPPING}/wishlist/", 57 | f"{SHOPPING_ADMIN}/dashboard", 58 | f"{REDDIT}/user/{ACCOUNTS['reddit']['username']}/account", 59 | ] 60 | EXACT_MATCH = [True, True, True, True] 61 | KEYWORDS = ["", "", "Dashboard", "Delete"] 62 | 63 | 64 | def is_expired( 65 | storage_state: Path, url: str, keyword: str, url_exact: bool = True 66 | ) -> bool: 67 | """Test whether the cookie is expired""" 68 | if not storage_state.exists(): 69 | return True 70 | 71 | context_manager = sync_playwright() 72 | playwright = context_manager.__enter__() 73 | browser = playwright.chromium.launch(headless=True, slow_mo=SLOW_MO) 74 | context = browser.new_context(storage_state=storage_state) 75 | page = context.new_page() 76 | page.goto(url) 77 | time.sleep(1) 78 | d_url = page.url 79 | content = page.content() 80 | context_manager.__exit__() 81 | if keyword: 82 | return keyword not in content 83 | else: 84 | if url_exact: 85 | return d_url != url 86 | else: 87 | return url not in d_url 88 | 89 | 90 | def renew_comb(comb: list[str], auth_folder: str = "./.auth") -> None: 91 | context_manager = sync_playwright() 92 | playwright = context_manager.__enter__() 93 | browser = playwright.chromium.launch(headless=HEADLESS) 94 | context = browser.new_context() 95 | page = context.new_page() 96 | 97 | if "shopping" in comb: 98 | username = ACCOUNTS["shopping"]["username"] 99 | password = ACCOUNTS["shopping"]["password"] 100 | page.goto(f"{SHOPPING}/customer/account/login/") 101 | page.get_by_label("Email", exact=True).fill(username) 102 | page.get_by_label("Password", exact=True).fill(password) 103 | page.get_by_role("button", name="Sign In").click() 104 | 105 | if "reddit" in comb: 106 | username = ACCOUNTS["reddit"]["username"] 107 | password = ACCOUNTS["reddit"]["password"] 108 | page.goto(f"{REDDIT}/registration") 109 | page.goto(f"{REDDIT}/login") 110 | page.get_by_label("Username").fill(username) 111 | page.get_by_label("Password").fill(password) 112 | page.get_by_role("button", name="Log in").click() 113 | 114 | if "shopping_admin" in comb: 115 | username = ACCOUNTS["shopping_admin"]["username"] 116 | password = ACCOUNTS["shopping_admin"]["password"] 117 | page.goto(f"{SHOPPING_ADMIN}") 118 | page.get_by_placeholder("user name").fill(username) 119 | page.get_by_placeholder("password").fill(password) 120 | page.get_by_role("button", name="Sign in").click() 121 | 122 | if "gitlab" in comb: 123 | username = ACCOUNTS["gitlab"]["username"] 124 | password = ACCOUNTS["gitlab"]["password"] 125 | page.goto(f"{GITLAB}/users/sign_in") 126 | page.get_by_test_id("username-field").click() 127 | page.get_by_test_id("username-field").fill(username) 128 | page.get_by_test_id("username-field").press("Tab") 129 | page.get_by_test_id("password-field").fill(password) 130 | page.get_by_test_id("sign-in-button").click() 131 | 132 | context.storage_state(path=f"{auth_folder}/{'.'.join(comb)}_state.json") 133 | 134 | context_manager.__exit__() 135 | 136 | 137 | def get_site_comb_from_filepath(file_path: str) -> list[str]: 138 | comb = os.path.basename(file_path).rsplit("_", 1)[0].split(".") 139 | return comb 140 | 141 | 142 | def main(auth_folder: str = "./.auth") -> None: 143 | pairs = list(combinations(SITES, 2)) 144 | 145 | max_workers = 8 146 | with ThreadPoolExecutor(max_workers=max_workers) as executor: 147 | for pair in pairs: 148 | # TODO[shuyanzh] auth don't work on these two sites 149 | if "reddit" in pair and ( 150 | "shopping" in pair or "shopping_admin" in pair 151 | ): 152 | continue 153 | executor.submit( 154 | renew_comb, list(sorted(pair)), auth_folder=auth_folder 155 | ) 156 | 157 | for site in SITES: 158 | executor.submit(renew_comb, [site], auth_folder=auth_folder) 159 | 160 | futures = [] 161 | cookie_files = list(glob.glob(f"{auth_folder}/*.json")) 162 | with ThreadPoolExecutor(max_workers=max_workers) as executor: 163 | for c_file in cookie_files: 164 | comb = get_site_comb_from_filepath(c_file) 165 | for cur_site in comb: 166 | url = URLS[SITES.index(cur_site)] 167 | keyword = KEYWORDS[SITES.index(cur_site)] 168 | match = EXACT_MATCH[SITES.index(cur_site)] 169 | future = executor.submit( 170 | is_expired, Path(c_file), url, keyword, match 171 | ) 172 | futures.append(future) 173 | 174 | for i, future in enumerate(futures): 175 | assert not future.result(), f"Cookie {cookie_files[i]} expired." 176 | 177 | 178 | if __name__ == "__main__": 179 | parser = argparse.ArgumentParser() 180 | parser.add_argument("--site_list", nargs="+", default=[]) 181 | parser.add_argument("--auth_folder", type=str, default="./.auth") 182 | args = parser.parse_args() 183 | if not args.site_list: 184 | main() 185 | else: 186 | if "all" in args.site_list: 187 | main(auth_folder=args.auth_folder) 188 | else: 189 | renew_comb(args.site_list, auth_folder=args.auth_folder) 190 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | from io import open 3 | 4 | 5 | def read_requirements_file(filename): 6 | with open(filename) as f: 7 | return [line.strip() for line in f] 8 | 9 | 10 | setup( 11 | name="webagents_step", 12 | version="0.0.1", 13 | author="Paloma Sodhi", 14 | author_email="psodhi@asapp.com", 15 | long_description=open("README.md", "r", encoding="utf-8").read(), 16 | long_description_content_type="text/markdown", 17 | url="", 18 | package_dir={'': 'src'}, 19 | packages=find_packages( 20 | exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), 21 | install_requires=read_requirements_file("requirements.txt"), 22 | entry_points={}, 23 | include_package_data=True, 24 | python_requires=">=3.6", 25 | tests_require=["pytest"], 26 | ) 27 | -------------------------------------------------------------------------------- /src/webagents_step/agents/agent.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | 4 | class Agent: 5 | def __init__( 6 | self, 7 | max_actions, 8 | verbose=0, 9 | logging=False, 10 | previous_actions: List = None, 11 | previous_reasons: List = None, 12 | previous_responses: List = None, 13 | ): 14 | self.previous_actions = [] if previous_actions is None else previous_actions 15 | self.previous_reasons = [] if previous_reasons is None else previous_reasons 16 | self.previous_responses = [] if previous_responses is None else previous_responses 17 | self.max_actions = max_actions 18 | self.verbose = verbose 19 | self.logging = logging 20 | self.trajectory = [] 21 | self.data_to_log = {} 22 | 23 | def reset(self): 24 | self.previous_actions = [] 25 | self.previous_reasons = [] 26 | self.previous_responses = [] 27 | self.trajectory = [] 28 | self.data_to_log = {} 29 | 30 | def get_trajectory(self): 31 | return self.trajectory 32 | 33 | def update_history(self, action, reason): 34 | if action: 35 | self.previous_actions += [action] 36 | if reason: 37 | self.previous_reasons += [reason] 38 | 39 | def predict_action(self, objective, observation, url=None): 40 | pass 41 | 42 | def receive_response(self, response): 43 | self.previous_responses += [response] 44 | 45 | def act(self, objective, env): 46 | while not env.done(): 47 | observation = env.observation() 48 | action, reason = self.predict_action( 49 | objective=objective, observation=observation, url=env.get_url() 50 | ) 51 | status = env.step(action) 52 | 53 | if self.logging: 54 | self.log_step( 55 | objective=objective, 56 | url=env.get_url(), 57 | observation=observation, 58 | action=action, 59 | reason=reason, 60 | status=status, 61 | ) 62 | 63 | if len(self.previous_actions) >= self.max_actions: 64 | print(f"Agent exceeded max actions: {self.max_actions}") 65 | break 66 | 67 | return status 68 | 69 | async def async_act(self, objective, env): 70 | while not env.done(): 71 | observation = await env.observation() 72 | action, reason = self.predict_action( 73 | objective=objective, observation=observation, url=env.get_url() 74 | ) 75 | status = await env.step(action) 76 | 77 | if self.logging: 78 | self.log_step( 79 | objective=objective, 80 | url=env.get_url(), 81 | observation=observation, 82 | action=action, 83 | reason=reason, 84 | status=status, 85 | ) 86 | 87 | if len(self.previous_actions) >= self.max_actions: 88 | print(f"Agent exceeded max actions: {self.max_actions}") 89 | break 90 | 91 | return status 92 | 93 | def log_step(self, objective, url, observation, action, reason, status): 94 | self.data_to_log['objective'] = objective 95 | self.data_to_log['url'] = url 96 | self.data_to_log['observation'] = observation 97 | self.data_to_log['previous_actions'] = self.previous_actions[:-1] 98 | self.data_to_log['previous_responses'] = self.previous_responses[:-1] 99 | self.data_to_log['previous_reasons'] = self.previous_reasons[:-1] 100 | self.data_to_log['action'] = action 101 | self.data_to_log['reason'] = reason 102 | for (k, v) in status.items(): 103 | self.data_to_log[k] = v 104 | self.trajectory.append(self.data_to_log) 105 | self.data_to_log = {} 106 | -------------------------------------------------------------------------------- /src/webagents_step/agents/finetuned_agent.py: -------------------------------------------------------------------------------- 1 | from webagents_step.agents.agent import Agent 2 | from webagents_step.utils.llm import fill_prompt_template, construct_llm_message_hf, generate_prediction, parse_action_reason 3 | from typing import Dict 4 | 5 | 6 | class FineTunedAgent(Agent): 7 | def __init__(self, max_actions=10, verbose=0, logging=False, debug=False, model=None, tokenizer=None, prompt_template=None, prompt_mode="completion", model_type="llama2", model_kwargs: Dict = None): 8 | super().__init__(max_actions=max_actions, verbose=verbose, logging=logging) 9 | self.debug = debug 10 | self.model = model 11 | self.tokenizer = tokenizer 12 | self.prompt_template = prompt_template 13 | self.prompt_mode = prompt_mode 14 | self.model_type = model_type 15 | self.model_kwargs = model_kwargs 16 | 17 | self.model.eval() 18 | 19 | def previous_history(self): 20 | previous_history = [] 21 | 22 | if len(self.previous_actions) == len(self.previous_responses): 23 | for action, response in zip(self.previous_actions, self.previous_responses): 24 | if response: 25 | previous_history.append(f"{response} = {action}") 26 | else: 27 | previous_history.append(action) 28 | previous_history="\n".join(previous_history) 29 | else: 30 | previous_history="\n".join(self.previous_actions) 31 | 32 | return previous_history 33 | 34 | def predict_action(self, objective, observation, url=None): 35 | prompt = fill_prompt_template(prompt_template=self.prompt_template, objective=objective, 36 | observation=observation, url=url, 37 | previous_history=self.previous_history()) 38 | messages = construct_llm_message_hf(prompt=prompt, prompt_mode=self.prompt_mode, model_type=self.model_type) 39 | 40 | inputs = messages[0]["content"] # todo: generalize to 'chat' mode 41 | model_response = generate_prediction(inputs=inputs, model=self.model, tokenizer=self.tokenizer, **self.model_kwargs) 42 | action, reason = parse_action_reason(model_response) 43 | 44 | if self.logging: 45 | self.data_to_log['input'] = inputs 46 | self.data_to_log['model_response'] = model_response 47 | 48 | if self.verbose > 0: 49 | if self.verbose > 1: 50 | print(f"\n OBSERVATION: {observation}") 51 | print(f"\n RESPONSE: {model_response}") 52 | print(f"\n OBJECTIVE: {objective}") 53 | print(f"\n PREVIOUS HISTORY: {self.previous_history()}") 54 | print(f"\n REASON: {reason}") 55 | print(f"\n ACTION: {action}") 56 | 57 | if self.debug: 58 | human_input = input() 59 | if human_input != "c": 60 | action = human_input 61 | reason = "None" 62 | 63 | self.update_history(action=action, reason=reason) 64 | return action, reason -------------------------------------------------------------------------------- /src/webagents_step/agents/keyboard_agent.py: -------------------------------------------------------------------------------- 1 | from webagents_step.agents.agent import Agent 2 | 3 | class KeyboardAgent(Agent): 4 | def __init__(self, max_actions: int = 10, verbose: bool = False, logging: bool = False): 5 | super().__init__(max_actions=max_actions, verbose=verbose, logging=logging) 6 | 7 | def predict_action(self, objective, observation, url=None): 8 | print(f"\n OBJECTIVE: {objective}") 9 | print(f"\n OBSERVATION: {observation}") 10 | print(f'\n PREVIOUS ACTIONS: {self.previous_actions}') 11 | action = input() 12 | 13 | self.update_history(action=action, reason="") 14 | return action, None -------------------------------------------------------------------------------- /src/webagents_step/agents/playback_agent.py: -------------------------------------------------------------------------------- 1 | from webagents_step.agents.agent import Agent 2 | from typing import List 3 | 4 | class PlaybackAgent(Agent): 5 | def __init__(self, max_actions: int = 1e6, verbose: bool = False, logging: bool = False, 6 | debug: bool = False, playback_trajectory=None, previous_actions: List = None): 7 | super().__init__(max_actions=max_actions, verbose=verbose, logging=logging, previous_actions=previous_actions) 8 | self.debug = debug 9 | # List of dictionary with atleast the action, reason key 10 | self.playback_trajectory = playback_trajectory 11 | self.max_actions = len(self.playback_trajectory) 12 | 13 | def predict_action(self, objective, observation, url=None): 14 | index = len(self.previous_actions) 15 | 16 | if index < len(self.playback_trajectory): 17 | action = self.playback_trajectory[index]['action'] 18 | reason = self.playback_trajectory[index]['reason'] 19 | 20 | if self.verbose > 0: 21 | if self.verbose > 1: 22 | print(f"\n OBSERVATION: {observation}") 23 | print(f"\n OBJECTIVE: {objective}") 24 | print(f'\n PREVIOUS ACTIONS: {self.previous_actions}') 25 | print(f"\n REASON: {reason}") 26 | print(f"\n ACTION: {action}") 27 | 28 | if self.debug: 29 | human_input = input() 30 | if human_input != "c": 31 | action = human_input 32 | reason = "None" 33 | 34 | self.update_history(action=action, reason=reason) 35 | return action, reason 36 | return None, None -------------------------------------------------------------------------------- /src/webagents_step/agents/prompt_agent.py: -------------------------------------------------------------------------------- 1 | from webagents_step.agents.agent import Agent 2 | from typing import List 3 | from webagents_step.utils.llm import fill_prompt_template, construct_llm_message_openai, call_openai_llm, parse_action_reason, calculate_cost_openai 4 | 5 | class PromptAgent(Agent): 6 | def __init__(self, max_actions: int = 10, verbose: bool = False, logging: bool = False, 7 | debug: bool = False, prompt_template: str = None, model: str = "gpt-3.5-turbo", 8 | prompt_mode: str = "chat", previous_actions: List = None, previous_reasons: List = None, previous_responses: List = None): 9 | super().__init__(max_actions=max_actions, verbose=verbose, logging=logging, previous_actions=previous_actions, previous_reasons=previous_reasons, previous_responses=previous_responses) 10 | self.debug = debug 11 | self.prompt_template = prompt_template 12 | self.model = model 13 | self.prompt_mode = prompt_mode 14 | 15 | def previous_history(self): 16 | previous_history = [] 17 | 18 | if len(self.previous_actions) == len(self.previous_responses): 19 | for action, response in zip(self.previous_actions, self.previous_responses): 20 | if response: 21 | previous_history.append(f"{response} = {action}") 22 | else: 23 | previous_history.append(action) 24 | previous_history="\n".join(previous_history) 25 | else: 26 | previous_history = "\n".join(action for action in self.previous_actions if action is not None) if self.previous_actions is not None else "" 27 | 28 | 29 | return previous_history 30 | 31 | def predict_action(self, objective, observation, url=None): 32 | prompt = fill_prompt_template(prompt_template=self.prompt_template, objective=objective, 33 | observation=observation, url=url, 34 | previous_history=self.previous_history()) 35 | messages = construct_llm_message_openai(prompt=prompt, prompt_mode=self.prompt_mode) 36 | model_response = call_openai_llm(messages=messages, model=self.model) 37 | action, reason = parse_action_reason(model_response) 38 | 39 | if self.logging: 40 | self.data_to_log['prompt'] = messages 41 | 42 | if self.verbose > 0: 43 | if self.verbose > 1: 44 | print(f"\n OBSERVATION: {observation}") 45 | print(f"\n RESPONSE: {model_response}") 46 | print(f"\n OBJECTIVE: {objective}") 47 | print(f"\n URL: {url}") 48 | print(f"\n PREVIOUS HISTORY: {self.previous_history()}") 49 | print(f"\n REASON: {reason}") 50 | print(f"\n ACTION: {action}") 51 | 52 | if self.debug: 53 | human_input = input() 54 | if human_input != "c": 55 | action = human_input 56 | reason = "None" 57 | 58 | self.update_history(action=action, reason=reason) 59 | return action, reason -------------------------------------------------------------------------------- /src/webagents_step/agents/step_agent.py: -------------------------------------------------------------------------------- 1 | from webagents_step.agents.agent import Agent 2 | from webagents_step.utils.stack import Stack 3 | from webagents_step.agents.prompt_agent import PromptAgent 4 | 5 | from typing import List, Dict 6 | import re 7 | 8 | class StepAgent(Agent): 9 | def __init__(self, max_actions: int = 10, verbose: bool = False, logging: bool = False, 10 | debug: bool = False, 11 | root_action: str = None, 12 | action_to_prompt_dict: Dict = None, 13 | low_level_action_list: List = None, 14 | model: str = "gpt-3.5-turbo", 15 | prompt_mode: str = "chat", previous_actions: List = None): 16 | super().__init__(max_actions=max_actions, verbose=verbose, logging=logging, previous_actions=previous_actions) 17 | self.debug = debug 18 | self.root_action = root_action 19 | self.action_to_prompt_dict = {} if action_to_prompt_dict is None else action_to_prompt_dict 20 | self.low_level_action_list = [] if low_level_action_list is None else low_level_action_list 21 | self.model = model 22 | self.prompt_mode = prompt_mode 23 | self.stack = Stack() 24 | 25 | def is_done(self, action): 26 | if "stop" in action: 27 | return True 28 | return False 29 | 30 | def is_low_level_action(self, action): 31 | action_type = action.split()[0] 32 | return (action_type in self.low_level_action_list) 33 | 34 | def is_high_level_action(self, action): 35 | action_type = action.split()[0] 36 | return (action_type in self.action_to_prompt_dict) 37 | 38 | def init_root_agent(self, objective): 39 | root_prompt_template = self.action_to_prompt_dict[self.root_action] 40 | agent = PromptAgent( 41 | prompt_template=root_prompt_template, 42 | model=self.model, 43 | prompt_mode=self.prompt_mode, 44 | max_actions=self.max_actions, 45 | verbose=self.verbose, 46 | logging=self.logging, 47 | debug=self.debug, 48 | previous_actions=[], 49 | previous_reasons=[], 50 | previous_responses=[] 51 | ) 52 | return {'agent': agent, 'objective': objective} 53 | 54 | def init_agent(self, action): 55 | pattern = r'(\w+)\s+\[(.*?)\]' 56 | matches = re.findall(pattern, action) 57 | action_type, _ = matches[0] 58 | objective = action 59 | prompt_template = self.action_to_prompt_dict[action_type] 60 | agent = PromptAgent( 61 | prompt_template=prompt_template, 62 | model=self.model, 63 | prompt_mode=self.prompt_mode, 64 | max_actions=self.max_actions, 65 | verbose=self.verbose, 66 | logging=self.logging, 67 | debug=self.debug, 68 | previous_actions=[], 69 | previous_reasons=[], 70 | previous_responses=[] 71 | ) 72 | return {'agent': agent, 'objective': objective} 73 | 74 | def predict_action(self, objective, observation, url=None): 75 | if self.stack.is_empty(): 76 | new_element = self.init_root_agent(objective=objective) 77 | self.stack.push(new_element) 78 | 79 | action, reason = None, None 80 | while not self.stack.is_empty(): 81 | element = self.stack.peek() 82 | action, reason = element['agent'].predict_action(objective=element['objective'], observation=observation, url=url) 83 | if (not self.is_done(action)) and self.is_low_level_action(action): 84 | element['agent'].receive_response("") 85 | return action, reason 86 | if (not self.is_done(action)) and self.is_high_level_action(action): 87 | new_element = self.init_agent(action) 88 | self.stack.push(new_element) 89 | if self.logging: 90 | self.log_step(objective=element['objective'], url=url, observation=observation, action=action, reason=reason, status={}) 91 | continue 92 | if self.is_done(action): 93 | self.stack.pop() 94 | if not self.stack.is_empty(): 95 | self.stack.peek()['agent'].receive_response(re.search(r"\[(.*?)\]", action).group(1)) 96 | if self.logging: 97 | self.log_step(objective=element['objective'], url=url, observation=observation, action=action, reason=reason, status={}) 98 | continue 99 | return action, reason -------------------------------------------------------------------------------- /src/webagents_step/environment/env.py: -------------------------------------------------------------------------------- 1 | class WebEnvironment(): 2 | def __init__(self): 3 | pass 4 | 5 | def reset(self): 6 | pass 7 | 8 | def observation(self): 9 | pass 10 | 11 | def get_url(self): 12 | pass 13 | 14 | def step(self, action): 15 | pass 16 | 17 | def done(self): 18 | pass 19 | -------------------------------------------------------------------------------- /src/webagents_step/environment/liveweb.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | import pandas as pd 3 | import re 4 | 5 | from webagents_step.parser import ( 6 | heihei_web_parser, 7 | playwright_parser_nat, 8 | playwright_parser_webarena, 9 | ) 10 | from webagents_step.environment.env import WebEnvironment 11 | 12 | 13 | class LiveWebEnvironmentWrapper(WebEnvironment): 14 | def __init__( 15 | self, 16 | url=None, 17 | objective=None, 18 | parser_type="heihei", 19 | observation_type="text", 20 | text_observation_type="accesibility_tree", 21 | max_browser_rows=1000, 22 | max_steps=50, 23 | step_delay=2, 24 | headless=False, 25 | ): 26 | self.url = url 27 | self.objective = objective 28 | self.headless = headless 29 | self.parser_type = parser_type 30 | self.observation_type = observation_type 31 | self.text_observation_type = text_observation_type 32 | self.max_browser_rows = max_browser_rows 33 | self.max_steps = max_steps 34 | 35 | self.steps = 0 36 | self.is_done = False 37 | self.parse_timeout = 5 38 | self.step_delay = step_delay 39 | self.response = "" 40 | 41 | async def init_parser(self): 42 | if self.parser_type == "heihei": 43 | self.parser = heihei_web_parser.HeiHeiWebParser() 44 | await self.parser.init() 45 | elif self.parser_type == "playwright_webarena": 46 | self.parser = playwright_parser_webarena.PlaywrightParserWebArena( 47 | headless=self.headless, 48 | observation_type=self.observation_type, 49 | text_observation_type=self.text_observation_type, 50 | ) 51 | self.parser.init() 52 | elif self.parser_type == "playwright_nat": 53 | self.parser = playwright_parser_nat.PlaywrightParserNat( 54 | headless=self.headless 55 | ) 56 | await self.parser.init() 57 | else: 58 | raise NotImplementedError(f"{self.parser_type} not implemented.") 59 | 60 | if self.url is not None: 61 | await self.parser.go_to_page(self.url) 62 | self.clear_page_presets() 63 | await self.parser.parse_page() 64 | 65 | def clear_page_presets(self): 66 | pass 67 | 68 | async def reset(self): 69 | await self.close() 70 | await self.init_parser() 71 | 72 | async def close(self): 73 | await self.parser.close() 74 | 75 | async def observation(self, tab_id=None, format=None): 76 | format = self.text_observation_type if format is None else format 77 | if self.parser_type == "heihei": 78 | try: 79 | browser_content = await self.parser.parse_page( 80 | format=format, tab_id=tab_id 81 | ) 82 | except: 83 | sleep(self.parse_timeout) 84 | browser_content = await self.parser.parse_page( 85 | format=format, tab_id=tab_id 86 | ) 87 | else: 88 | browser_content = await self.parser.parse_page() 89 | 90 | if format not in ["htree", "html", "json"]: 91 | browser_content = [str(w) for w in browser_content] 92 | browser_content = browser_content[: self.max_browser_rows] 93 | browser_content = "\n".join(browser_content) 94 | 95 | return browser_content 96 | 97 | def get_log(self): 98 | return self.df_log 99 | 100 | def get_response(self): 101 | return self.response 102 | 103 | def get_url(self): 104 | return self.parser.get_url() 105 | 106 | async def execute_action(self, action): 107 | """ 108 | Execute a given action based on the action type, 109 | - click [id]: Clicks an element based on the provided id. 110 | - type [id] [content]: Types the provided content into the element with the specified id. 111 | - goto [url]: Navigates to an existing tab at that URL 112 | - open [url]: Opens a new tab with provided URL 113 | - copy [content]: Copies content, but no-op action 114 | - stop [response]: Stops execution and optionally provides a response. 115 | """ 116 | click_match = re.match(r"click \[(\S+)\]", action, re.IGNORECASE) 117 | type_match = re.match(r"type \[(\S+)\] \[(.+)\]", action, re.IGNORECASE) 118 | goto_match = re.match(r"goto \[(\S+)\]", action, re.IGNORECASE) 119 | open_match = re.match(r"open \[(\S+)\]", action, re.IGNORECASE) 120 | copy_match = re.match(r"copy \[(\S+)\]", action, re.IGNORECASE) 121 | stop_match = re.match(r"stop \[([^\]]*)\]", action, re.IGNORECASE) 122 | 123 | if click_match: 124 | id = click_match.group(1) 125 | if not id.isdigit(): 126 | raise Exception("Id not a valid integer") 127 | await self.parser.click(int(id)) 128 | 129 | elif type_match: 130 | id = type_match.group(1) 131 | content = type_match.group(2) 132 | if not id.isdigit(): 133 | raise Exception("Id not a valid integer") 134 | await self.parser.type(int(id), content) 135 | 136 | elif goto_match: 137 | url = goto_match.group(1) 138 | tab_id, tab_url = await self.parser.get_tab_from_url(url) 139 | await self.parser.go_to_page(url) 140 | 141 | elif open_match: 142 | url = open_match.group(1) 143 | await self.parser.go_to_page(url) 144 | 145 | elif copy_match: 146 | pass 147 | 148 | elif stop_match: 149 | self.response = stop_match.group(1) 150 | self.is_done = True 151 | 152 | else: 153 | print(f"[execute_action] Error {action} not defined") 154 | 155 | async def step(self, action, delay=None): 156 | delay = self.step_delay if delay is None else delay 157 | 158 | if self.steps > self.max_steps: 159 | print(f"Steps {self.steps} exceeded maximum {self.max_steps}") 160 | self.is_done = True 161 | return 162 | 163 | print(f"[Step {self.steps+1}] {action}") 164 | try: 165 | await self.execute_action(action) 166 | except Exception as e: 167 | print(f"Error while executing action '{action}'. Details: {e}") 168 | 169 | sleep(delay) 170 | self.steps = self.steps + 1 171 | 172 | return {"done": self.is_done, "response": self.response} 173 | 174 | def done(self): 175 | return self.is_done 176 | -------------------------------------------------------------------------------- /src/webagents_step/environment/miniwob.py: -------------------------------------------------------------------------------- 1 | from webagents_step.parser import miniwob_parser 2 | from webagents_step.environment.env import WebEnvironment 3 | 4 | import re 5 | from miniwob.action import create_element_click_action, create_focus_and_type_action 6 | from miniwob.environment import MiniWoBEnvironment 7 | 8 | import logging 9 | logging.basicConfig(level=logging.ERROR) 10 | 11 | class MiniWoBEnvironmentWrapper(WebEnvironment): 12 | def __init__(self, task, seed=None, max_browser_rows=125, 13 | max_steps=50, wait_ms=600, headless=False): 14 | 15 | render_mode = None if headless else "human" 16 | self.miniwob_env = MiniWoBEnvironment(subdomain=task, wait_ms=wait_ms, render_mode=render_mode) 17 | obs, _ = self.miniwob_env.reset(seed) 18 | 19 | self.obs = obs 20 | self.objective = obs["utterance"] 21 | self.url = "" 22 | self.max_browser_rows = max_browser_rows 23 | self.max_steps = max_steps 24 | self.steps = 0 25 | self.is_done = False 26 | self.reward = 0.0 27 | 28 | def reset(self, seed=None): 29 | obs, _ = self.miniwob_env.reset(seed) 30 | self.obs = obs 31 | 32 | def close(self): 33 | self.miniwob_env.close() 34 | 35 | def observation(self): # browser content 36 | dom_elements = self.obs['dom_elements'] 37 | browser_content = miniwob_parser.parse_dom_browser_content( 38 | dom_elements, process_dates=True) 39 | browser_content = browser_content[:self.max_browser_rows] 40 | browser_content = "\n".join(browser_content) 41 | return browser_content 42 | 43 | def get_url(self): 44 | return self.url 45 | 46 | def done(self): 47 | if self.is_done: 48 | return True 49 | return False 50 | 51 | def parse_action(self, action): 52 | """ 53 | Parse a given action based on the action type, 54 | - click [id]: Clicks an element based on the provided id. 55 | - type [id] [content]: Types the provided content into the element with the specified id. 56 | - stop [response]: Stops execution and optionally provides a response. 57 | """ 58 | if not action: 59 | print("Action text is None") 60 | return None 61 | 62 | click_match = re.match(r"click \[(\S+)\]", action, re.IGNORECASE) 63 | type_match = re.match(r"type \[(\S+)\] \[(.+)\]", action, re.IGNORECASE) 64 | stop_match = re.match(r"stop \[([^\]]*)\]", action, re.IGNORECASE) 65 | 66 | action_cmd = None 67 | if click_match: 68 | id = click_match.group(1) 69 | action_cmd = create_element_click_action(int(id)) if id.isdigit() else None 70 | elif type_match: 71 | id = type_match.group(1) 72 | content = type_match.group(2) 73 | action_cmd = create_focus_and_type_action(int(id), content) if id.isdigit() else None 74 | elif stop_match: 75 | self.response = stop_match.group(1) 76 | self.is_done = True 77 | action_cmd = None 78 | else: 79 | print(f"[MiniWoBEnvironmentWrapper::parse_action] Error {action} not defined") 80 | 81 | return action_cmd 82 | 83 | def status(self): 84 | return {'done': self.is_done, 'reward': self.reward, 'success': float(self.reward > 0), 'num_actions': self.steps} 85 | 86 | def step(self, action): 87 | self.steps = self.steps + 1 88 | print(f"[Step {self.steps}] {action}") 89 | 90 | if self.steps > self.max_steps: 91 | print(f"Steps {self.steps} exceeded maximum {self.max_steps}") 92 | self.is_done = True 93 | self.reward = -1 94 | return self.status() 95 | 96 | action_cmd = self.parse_action(action) 97 | if action_cmd: 98 | try: 99 | obs, reward, done, truncated, info = self.miniwob_env.step(action_cmd) 100 | self.obs = obs 101 | if "raw_reward" in info: 102 | self.reward = info['raw_reward'] 103 | else: 104 | self.reward = reward 105 | self.is_done = done 106 | except Exception as e: 107 | print(f"Error occurred while taking step: {e}") 108 | 109 | return self.status() 110 | 111 | def get_objective(self): 112 | return self.objective -------------------------------------------------------------------------------- /src/webagents_step/environment/webarena.py: -------------------------------------------------------------------------------- 1 | import os 2 | os.environ[ 3 | "SHOPPING" 4 | ] = "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:7770" 5 | os.environ[ 6 | "SHOPPING_ADMIN" 7 | ] = "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:7780/admin" 8 | os.environ[ 9 | "REDDIT" 10 | ] = "https://webarena-env-reddit.awsdev.asapp.com" 11 | os.environ[ 12 | "GITLAB" 13 | ] = "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:8023/" 14 | os.environ[ 15 | "MAP" 16 | ] = "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:3000" 17 | os.environ[ 18 | "WIKIPEDIA" 19 | ] = "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:8888/wikipedia_en_all_maxi_2022-05/A/User:The_other_Kiwix_guy/Landing" 20 | os.environ[ 21 | "HOMEPAGE" 22 | ] = "PASS" # The home page is not currently hosted in the demo site 23 | 24 | 25 | from webagents_step.environment.env import WebEnvironment 26 | import json 27 | import re 28 | # Init an environment 29 | from browser_env import ( 30 | create_id_based_action, 31 | StateInfo, 32 | Trajectory, 33 | ActionTypes, 34 | ScriptBrowserEnv 35 | ) 36 | from evaluation_harness.evaluators import evaluator_router 37 | 38 | class WebArenaEnvironmentWrapper(WebEnvironment): 39 | def __init__(self, config_file, max_browser_rows=300, max_steps=50, slow_mo=1, observation_type="accessibility_tree", current_viewport_only=False, viewport_size={"width": 1280, "height": 720}, headless=False): 40 | self.webarena_env = ScriptBrowserEnv( 41 | headless=headless, 42 | slow_mo=slow_mo, 43 | observation_type=observation_type, 44 | current_viewport_only=current_viewport_only, 45 | viewport_size=viewport_size 46 | ) 47 | self.config_file = config_file 48 | with open(self.config_file, "r") as f: 49 | self.config = json.load(f) 50 | 51 | self.obs, self.info = self.webarena_env.reset(options={"config_file": self.config_file}) 52 | self.terminated = False 53 | self.objective = self.config["intent"] 54 | self.url = self.config["start_url"] 55 | self.max_browser_rows = max_browser_rows 56 | self.max_steps = max_steps 57 | self.steps = 0 58 | self.is_done = False 59 | self.reward = 0.0 60 | self.action_limit_exceeded = False 61 | 62 | self.trajectory: Trajectory = [] 63 | self.update_webarena_metrics() 64 | 65 | def reset(self): 66 | self.obs, self.info = self.webarena_env.reset(options={"config_file": self.config_file}) 67 | 68 | def close(self): 69 | self.webarena_env.close() 70 | 71 | def get_url(self): 72 | return self.url 73 | 74 | def get_objective(self): 75 | return self.objective 76 | 77 | def observation(self): 78 | self.obs = self.webarena_env._get_obs() 79 | self.url = self.webarena_env.page.url 80 | browser_content = self.obs["text"] 81 | browser_content = browser_content.split("\n")[:self.max_browser_rows] 82 | browser_content = "\n".join(browser_content) 83 | return browser_content 84 | 85 | def done(self): 86 | if self.is_done: 87 | return True 88 | return False 89 | 90 | def status(self): 91 | return {'done': self.is_done, 'reward': self.reward, 'success': float(self.reward > 0), 'num_actions': self.steps, 'action_limit_exceeded': self.action_limit_exceeded} 92 | 93 | def step(self, action): 94 | self.steps = self.steps + 1 95 | print(f"[Step {self.steps}] {action}") 96 | 97 | if self.steps > self.max_steps: 98 | print(f"Steps {self.steps} exceeded maximum {self.max_steps}") 99 | self.action_limit_exceeded = True 100 | self.is_done = True 101 | action_cmd = create_id_based_action("stop [N/A]") 102 | self.update_webarena_metrics(action_cmd) 103 | return self.status() 104 | 105 | if "stop [" in action: 106 | action = action.replace('\\', '') 107 | 108 | if action is None or action is "" or ("note [" in action): 109 | action_cmd = None 110 | else: 111 | action_cmd = create_id_based_action(action) 112 | 113 | if action_cmd: 114 | try: 115 | self.obs, _, self.terminated, _, self.info = self.webarena_env.step(action_cmd) 116 | self.update_webarena_metrics(action_cmd) 117 | except Exception as e: 118 | print(f"Error occurred while taking step: {e}") 119 | 120 | return self.status() 121 | 122 | def update_webarena_metrics(self, action_cmd=None): 123 | # Append action (if any) and resulting sate 124 | if action_cmd: 125 | self.trajectory.append(action_cmd) 126 | if action_cmd["action_type"]== ActionTypes.STOP: 127 | self.is_done = True 128 | 129 | if not self.is_done: # If we are done, no need to append state 130 | state_info: StateInfo = {"observation": self.obs, "info": self.info} 131 | self.trajectory.append(state_info) 132 | 133 | if self.is_done: 134 | try: 135 | evaluator = evaluator_router(self.config_file) 136 | self.reward = evaluator(trajectory=self.trajectory, config_file=self.config_file, page=self.webarena_env.page, client=self.webarena_env.get_page_client(self.webarena_env.page)) 137 | except Exception as e: 138 | print(f"Got excepetion: {e}") 139 | self.reward = 0 -------------------------------------------------------------------------------- /src/webagents_step/parser/miniwob_parser.py: -------------------------------------------------------------------------------- 1 | import re 2 | import datetime 3 | 4 | from miniwob.action import create_element_click_action, create_focus_and_type_action, create_coord_click_action 5 | 6 | def parse_dom_browser_content(dom, ignore_tags=["tr", "td"], format='html', process_dates=False): 7 | """ 8 | Parse dom observations into a browser action set 9 | """ 10 | if format == "html": 11 | output = parse_dom_browser_content_html( 12 | dom, ignore_tags=ignore_tags, process_dates=process_dates) 13 | else: 14 | raise NotImplementedError 15 | 16 | return output 17 | 18 | 19 | def parse_dom_browser_content_html(dom, ignore_tags, process_dates): 20 | action_set = [] 21 | 22 | for node in dom: 23 | if node['tag'] in ignore_tags: 24 | continue 25 | attrs = f" id={node['ref']}" 26 | if len(node['text']) > 0: 27 | value = f" val={node['text']}" 28 | elif 'value' in node and (len(node['value']) > 0): 29 | value = f" val={node['value']}" 30 | else: 31 | value = f" val={node['id']}" 32 | action_set.append(f"<{node['tag']}{attrs}{value} />") 33 | 34 | if process_dates: 35 | action_set = parse_dates_table(action_set) 36 | 37 | return action_set 38 | 39 | 40 | def parse_dates_table(action_set): 41 | action_set_text = "\n".join(action_set) 42 | if "val=ui-datepicker-div" not in action_set_text: 43 | return action_set 44 | 45 | pattern = r"\b(January|February|March|April|May|June|July|August|September|October|November|December)\b" 46 | month = re.findall(pattern, action_set_text)[0] # month name 47 | month = datetime.datetime.strptime(month, '%B').month # month number 48 | pattern = r"20[0-4][0-9]|2050" 49 | year = re.findall(pattern, action_set_text)[0] 50 | 51 | pattern = r'' 52 | action_set_text = re.sub( 53 | pattern, lambda m: f'', action_set_text) 54 | 55 | action_set = action_set_text.split("\n") 56 | 57 | return action_set 58 | -------------------------------------------------------------------------------- /src/webagents_step/parser/playwright_parser_webarena.py: -------------------------------------------------------------------------------- 1 | from sys import platform 2 | from playwright.sync_api import sync_playwright 3 | from browser_env.processors import TextObervationProcessor, ImageObservationProcessor 4 | 5 | 6 | class PlaywrightParserWebArena: 7 | def __init__( 8 | self, 9 | headless=True, 10 | observation_type="text", 11 | text_observation_type="accessibility_tree", 12 | viewport_size={"width": 1280, "height": 1080}, 13 | current_viewport_only=True, 14 | ): 15 | self.headless = headless 16 | self.viewport_size = viewport_size 17 | self.current_viewport_only = current_viewport_only 18 | self.observation_type = observation_type 19 | self.text_observation_type = text_observation_type 20 | 21 | self.playwright = sync_playwright().start() 22 | self.browser = self.playwright.chromium.launch(headless=self.headless) 23 | self.context = self.browser.new_context( 24 | viewport=self.viewport_size, 25 | device_scale_factor=1, 26 | ) 27 | 28 | self.page = self.context.new_page() 29 | client = self.page.context.new_cdp_session(self.page) 30 | if (self.observation_type == "text") and ( 31 | self.text_observation_type == "accessibility_tree" 32 | ): 33 | client.send("Accessibility.enable") 34 | self.page.client = client 35 | 36 | ## scratch ## 37 | # initialize with html string 38 | # self.page.goto(url if "://" in url else "http://" + url) 39 | # potentially later 40 | # self.page.goto("https://www.google.com", wait_until='networkidle') 41 | # print(self.page.accessibility.snapshot()) 42 | # self.page = self.page.accessibility.snapshot() 43 | 44 | self.text_processor = TextObervationProcessor( 45 | observation_type=self.text_observation_type, 46 | current_viewport_only=self.current_viewport_only, 47 | viewport_size=self.viewport_size, 48 | ) 49 | self.image_processor = ImageObservationProcessor(observation_type="image") 50 | 51 | def clear_page_presets(): 52 | pass 53 | 54 | def observation_processor(self): 55 | if self.observation_type == "text": 56 | return self.text_processor 57 | elif self.observation_type == "image": 58 | return self.image_processor 59 | else: 60 | raise ValueError("Invalid observation type") 61 | 62 | def get_url(self): 63 | return self.page.url 64 | 65 | def go_to_page(self, url: str): 66 | self.page.goto(url if "://" in url else "http://" + url) 67 | 68 | def close(self): 69 | self.browser.close() 70 | self.playwright_context.stop() 71 | 72 | def click_xy(self, x: float, y: float) -> None: 73 | viewport_size = self.page.viewport_size 74 | self.page.mouse.click(x * viewport_size["width"], y * viewport_size["height"]) 75 | 76 | def click(self, id: int) -> None: 77 | element_center = self.observation_processor().get_element_center(id) 78 | self.click_xy(element_center[0], element_center[1]) 79 | 80 | def type(self, id: int, text: str, clear: bool = True): 81 | if clear: 82 | self.clear(id) 83 | self.click(id) 84 | self.page.keyboard.type(text) 85 | 86 | def clear(self, id: int) -> None: 87 | self.click(id) 88 | select_key = "Meta" if platform.startswith("darwin") else "Control" 89 | self.page.keyboard.down(select_key) 90 | self.page.keyboard.press("a") 91 | self.page.keyboard.up(select_key) 92 | self.page.keyboard.press("Backspace") 93 | 94 | def parse_page(self): 95 | observation = self.observation_processor().process( 96 | page=self.page, client=self.page.client 97 | ) 98 | 99 | return observation 100 | -------------------------------------------------------------------------------- /src/webagents_step/prompts/miniwob/flat_fewshot_template.py: -------------------------------------------------------------------------------- 1 | flat_fewshot_agent = { 2 | "instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions. 3 | 4 | You can only interact with web elements like links, inputs, buttons in the browser content. You can issue any one of the actions below: 5 | click [id]: Clicks an element corresponding to the provided id. 6 | type [id] [content]: Types the provided content into the element corresponding to the provided id. 7 | stop [answer]: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. Otherwise, leave it empty. 8 | 9 | Examples of actions are click [7], type [11] [New York]. Please issue only one action at a time. 10 | 11 | You will be provided with the following, 12 | OBJECTIVE: 13 | The goal you need to achieve. 14 | OBSERVATION: 15 | A simplified text description of the current browser content, without formatting elements. 16 | URL: 17 | The current webpage URL 18 | PREVIOUS ACTIONS: 19 | A list of your past actions 20 | 21 | You need to generate response containing, 22 | REASON: 23 | A rationale for selecting the action below 24 | ACTION: 25 | A single action 26 | 27 | Please generate in the following format: 28 | 29 | REASON: 30 | Your reason here 31 | ACTION: 32 | Your action here 33 | """, 34 | 35 | "input": """ 36 | OBJECTIVE: 37 | {objective} 38 | OBSERVATION: 39 | {observation} 40 | URL: 41 | {url} 42 | PREVIOUS ACTIONS: 43 | {previous_actions} 44 | """, 45 | 46 | "response": "", 47 | 48 | "examples": [ 49 | { 50 | "input": """ 51 | OBJECTIVE: 52 | Book the shortest one-way flight from: LEB to: RDG on 12/26/2016. 53 | OBSERVATION: 54 | 55 |
56 |
57 |
58 |

59 |
60 | 61 |
62 | 63 |
64 | URL: 65 | 66 | PREVIOUS ACTIONS: 67 | 68 | """, 69 | "response": """ 70 | REASON: 71 | I have no previous actions. 72 | I have to first type "LEB" in the field flight-from corresponding to id 7 73 | ACTION: 74 | type [7] [LEB] 75 | """}, 76 | { 77 | "input": """ 78 | OBJECTIVE: 79 | Book the shortest one-way flight from: LEB to: RDG on 12/26/2016. 80 | CURRENT BROWSER CONTENT: 81 | 82 |
83 |
84 |
85 |

86 |
87 | 88 |
89 |
90 |