├── .gitignore ├── CODE_OF_CONDUCT.md ├── Demo.ipynb ├── LICENSE ├── README.md ├── SECURITY.md ├── SUPPORT.md ├── assets ├── sample_bill_behr.png ├── sample_jordan.png └── sample_mc.png ├── factual_queries ├── __init__.py ├── data │ ├── basketball_players.pkl │ ├── books_multiconstraint.pkl │ ├── counterfact_citizenship.pkl │ ├── counterfact_headquarter_location.pkl │ ├── counterfact_mother_tongue.pkl │ ├── football_teams.pkl │ ├── movies.pkl │ ├── nobel_multiconstraint.pkl │ ├── schools.pkl │ ├── songs.pkl │ └── word_multiconstraint.pkl ├── multi_constraint.py ├── single_constraint.py └── verifiers │ ├── .ipynb_checkpoints │ ├── Verifier Books -checkpoint.ipynb │ └── Verifier Nobel Winner-checkpoint.ipynb │ ├── Verifier Books .ipynb │ └── Verifier Nobel Winner.ipynb ├── main_flow_collection.py ├── main_probe.py ├── model_lib ├── __init__.py ├── attention_tools.py ├── hf_tooling.py ├── hooks.py └── misc_utils.py ├── requirements.txt └── viz_tools ├── __init__.py └── attention_viz.py /.gitignore: -------------------------------------------------------------------------------- 1 | **.pyc 2 | __pycache__/ -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /Demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "cf3545a3-3f3c-4221-a066-e63361003626", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "%load_ext autoreload\n", 11 | "%autoreload 2" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 2, 17 | "id": "1bb1c7d8-292d-4a9c-8ea0-64cb8a2d54a3", 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "import os\n", 22 | "import argparse\n", 23 | "import torch\n", 24 | "import transformers \n", 25 | "\n", 26 | "from tqdm import tqdm\n", 27 | "import numpy as np\n", 28 | "\n", 29 | "from model_lib.hf_tooling import HF_Llama2_Wrapper\n", 30 | "from model_lib.attention_tools import run_attention_monitor\n", 31 | "from factual_queries import load_constraint_dataset\n", 32 | "from viz_tools import plot_attention_flow" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 3, 38 | "id": "d821ff8d-5aa3-4a6e-9844-e85fa4a0f3d1", 39 | "metadata": {}, 40 | "outputs": [], 41 | "source": [ 42 | "import argparse\n", 43 | "def config():\n", 44 | " parser = argparse.ArgumentParser()\n", 45 | " parser.add_argument(\"--model_name\", type=str, default=\"meta-llama/Llama-2-13b-hf\")\n", 46 | " parser.add_argument(\"--max_new_tokens\", type=int, default=15, help=\"Number of tokens to generate for each prompt.\")\n", 47 | " parser.add_argument(\"--dataset-name\", type=str, default=\"basketball_players\")\n", 48 | " parser.add_argument(\"--load-in-8bit\", action=\"store_true\", help=\"Whether to load the model in 8-bit mode. We used this only for Llama-2 70B.\")\n", 49 | " parser.add_argument(\"--subsample-count\", type=int, default=None, help=\"Number of items to run for, mostly for testing mode.\")\n", 50 | " parser.add_argument(\"--output-dir\", type=str, default=\"./outputs\", help=\"Output directory to save the attention flow.\")\n", 51 | " parser.add_argument(\"-f\")\n", 52 | "\n", 53 | " return parser.parse_args()\n", 54 | "\n", 55 | "args = config()" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 4, 61 | "id": "5cad648c-d63e-4bf2-adcb-2bbb448e145d", 62 | "metadata": {}, 63 | "outputs": [ 64 | { 65 | "name": "stderr", 66 | "output_type": "stream", 67 | "text": [ 68 | "Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████| 2/2 [00:03<00:00, 1.94s/it]\n" 69 | ] 70 | } 71 | ], 72 | "source": [ 73 | "# Load the models\n", 74 | "tokenizer = transformers.AutoTokenizer.from_pretrained(\"meta-llama/Llama-2-7b-hf\")\n", 75 | "model = transformers.AutoModelForCausalLM.from_pretrained(\"meta-llama/Llama-2-7b-hf\", torch_dtype=torch.bfloat16, \n", 76 | " device_map=\"cuda\")\n", 77 | "\n", 78 | "model_wrapped = HF_Llama2_Wrapper(model, tokenizer, device=\"cuda\")\n" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 5, 84 | "id": "0213294a-6332-4423-a95b-5021e5653d42", 85 | "metadata": {}, 86 | "outputs": [], 87 | "source": [ 88 | "# Load the dataset to explore existing examples\n", 89 | "items = load_constraint_dataset(args.dataset_name, subsample_count=args.subsample_count)\n", 90 | "items = sorted(items, key=lambda x: x[\"popularity\"], reverse=True)\n", 91 | "item = items[0]\n", 92 | "prompt_info = {\"prompt\": item[\"prompt\"], \n", 93 | " \"constraints\": [f\" {item['constraint']}\"]}" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 6, 99 | "id": "86aefeb7-032e-4b08-b109-534045a58eb7", 100 | "metadata": {}, 101 | "outputs": [ 102 | { 103 | "name": "stderr", 104 | "output_type": "stream", 105 | "text": [ 106 | "LlamaModel is using LlamaSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation=\"eager\"` when loading the model.\n" 107 | ] 108 | } 109 | ], 110 | "source": [ 111 | "# This function populates attention contribution / related data\n", 112 | "data = run_attention_monitor(prompt_info,\n", 113 | " model_wrapped)\n" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 7, 119 | "id": "0b6ecf03-ca75-412c-9600-4cc4a103b701", 120 | "metadata": {}, 121 | "outputs": [ 122 | { 123 | "data": { 124 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAApQAAAHdCAYAAACjcOkFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAB7CAAAewgFu0HU+AACUzklEQVR4nOzdd1gUx/8H8PfRq6IggnSxdxSwIojd2DUxRiPYTdQYE1tMbF9NYi/RxBYVNXaNGnsFGxos2HsHRcWOFGn7+4MfG07ujluO4+7w/eK55zl2Z+dmd+/2PjezMyMTBEEAEREREVE+Gem6AERERERk2BhQEhEREZFGGFASERERkUYYUBIRERGRRhhQEhEREZFGGFASERERkUYYUBIRERGRRhhQEhEREZFGGFASERERkUYYUBIRERGRRhhQEhEREZFGGFASERERkUYYUBIRERGRRhhQEhEREZFGGFASERERkUYYUBIRERGRRhhQEhEREZFGGFASERERkUYYUBIRERGRRhhQkkZkMpn4INJUUFCQ+H6KiIhQmCY0NFRMExYWVqjl07b79++L++bp6anr4hBRHvgd+B+DDih79uwpdzKnTZum6yIRERERfXQMNqBMSEjA1q1b5ZatXLlSUh75qekoyrUj/KVFikycOFF8X0ycOFHXxTEYrG1UX1G+rpLh4HegZgw2oNy0aROSkpLkll27dg2nT5/WUYmIiIiIPk4mui5AfuWsjbS0tERycrK43M/PT1fF+ugIgqDrIlARouy+yY+Fp6cnP1NEBoSf1/8YZA3lvXv3cOzYMQBZVdQzZ84U161btw6pqam6KhoRERHRR8cgA8pVq1aJvwoCAwMxYMAAlCpVCgDw8uVL7Ny5U5fFIyIiIvq4CAYmMzNTKFu2rABAACAsW7ZMEARBGDp0qLisffv2KvPw8PAQ0+b1mDBhQr63USQ1NVVYtWqV8OmnnwpeXl6CjY2NYGVlJXh6egqff/658PfffwuZmZkqyx8eHi6+VmBgoLj80KFDQrdu3QQvLy/B3NxcKFmypBAQECDMnz9fSE1NzTMvdR737t2T2z7nOnVcvnxZGDFihFCrVi3B3t5eMDMzE5ydnYXAwEBh6tSpwvPnz/PMY8WKFeJrhoSEiMv//vtvoW3btoKbm5tgZmYmlCpVSmjevLmwevXqPI+pJnbv3i0MGDBAqFq1qlCyZEnBxMREKF68uODj4yMMGDBA2L59u5CWlqYyj8zMTGHjxo3C559/LpQtW1awtrYWrK2thbJlywrdu3cXNm3apNY+BAYGiscmPDxcEARBePHihTB16lTB19dXsLe3FywsLAQvLy+hT58+wqVLl9TKK69HzvMgCIIQEhIirluxYoUgCILw6tUrYe7cuUJAQIBQpkwZwdjYWAAgvHr1SmX5P6Qo7+fPnwtTp04V/Pz8BAcHB8HCwkIoW7as0L9/f+HcuXN5HrcJEyao9fnNpuwzKAjy7091Hjndu3dPXO7h4ZFnOQRBEE6ePCkMHjxYqFKlimBnZyeYm5sLLi4uQsuWLYX58+cL7969y9f+p6WlCStXrhSaNm0qlClTRjAzMxOcnJyEDh06CDt27FCrbHkpqOvq/fv3hXHjxgl169YVHB0dBVNTU8HR0VGoW7euMH78eOHhw4cFUl5F/v33X6FPnz6Cl5eXYGFhITg4OAh+fn5y1zNl1yxVoqKihG+//VaoWbOm4ODgIJiamgqlS5cWGjduLEydOlV4+fJlnnnkPL7Z1+6YmBjhp59+EmrUqCEUL15csLKyEipWrCgMGTJEuH//vqR91+b32a5du4TPP/9cKFeunGBtbS0AEObMmZPr9ffu3SuMHDlSCAoKEpydnQVzc3PBwsJCcHFxEVq1aiXMmTNHSEhIUOv1+R2oGYMLKI8ePSoeSAsLC+HNmzeCIGR9+LKXm5qaCs+ePVOah64CyvDwcMHb2zvP7evVqyfExsYqLf+HH8D3798L/fv3V5ln7dq1hfj4eJV5afPDlJaWJgwdOlQMIpQ97OzshLCwMJV5ffhhev36tdC+fXuV+bZq1UpISkpSma9Uly9fFnx9fdU6bt26dVOaz82bNwUfH58886hTp45w584dlWX6MCA7fvy44OLiojRPY2NjYcmSJXnmldcjr4Dy+PHjgpubm8JtNQ0oIyMjhTJlyqjcx7yCREMMKN+9eyd069Ytz/ydnZ2F3bt3S9r/2NhYoUGDBirz7d27t5CRkZHnsVKlIK6rU6ZMESwsLFRua2FhIUydOlWjsn4oMzNTGDFihGBkZKT0dV1cXISTJ09KCihfvnwpdOnSJc/jYWdnJ2zatEllXh8GlFu3bhWKFy+uNE9LS0th586dau2/tr7PXr9+LXTq1ElhXjkDyocPHwr29vZqvXfs7e2F/fv35/n66jz4HaicwXXKydkZp0OHDihWrBgAwM/PD5UqVcL169eRlpaGtWvXYtiwYQrzCAkJwYsXL3Do0CFcv34dANC0aVNUqlQpV1p/f/98b5PTpk2b0KNHD6SlpQHI6khUr149eHp6wsjICDdv3sTJkyeRnp6OU6dOoX79+jh9+jRKly6d5zEZMGAAVq5cCSMjI9StWxeVKlVCZmYmTp06hRs3bgAAzp07h169emH37t1y27q4uGDw4MEAgN9//11cnr3sQ9nHW4rMzEx06dIF//zzj7isZMmSCAoKQsmSJRETE4Pw8HCkpqbi9evXCA0NxevXr5Wev5zS09PRpUsXHDp0CGZmZmjQoAG8vb2RkpKCY8eO4eHDhwCAvXv34rvvvsPChQsll1+RiIgItG/fHgkJCeIyd3d3+Pv7o2TJkkhMTMSNGzdw4cIFpKWlISUlRWE+165dQ2BgIOLj48Vl1atXR61atSCTyRAdHY1Lly4BAM6ePYsGDRrg6NGjqFChQp5lvHz5Mn744Qe8e/cOjo6OCAgIgL29PR49eoTDhw8jOTkZGRkZGDRoEKpXr4569erJbd+pUydUq1YNUVFR4ugJfn5+Ct/fH26b0+3bt/Htt9/izZs3sLW1RePGjVGmTBm8evUKR48ezXM/VHnw4AG+++47vHr1CjY2NggODkbp0qXx+PFjhIeHIykpCRkZGZg0aRIyMzPxv//9T6PXU0flypUxePBgJCQkYNWqVQAAW1tb9OrVq8BeIykpCcHBwYiKihKXlSlTBgEBAbCxscHt27dx/PhxZGRkIC4uDu3bt8e6devQtWvXPPN+9+4dWrVqhcuXL8PKygoBAQFwc3NDQkICwsPD8ezZMwDAihUrULFiRYwePTrf+6HpdXXIkCFy1y0bGxs0adIETk5OePLkCcLDw/Hu3TukpKRgzJgxePLkCebMmZPv8ub0/fffy+WV87WfPn2K8PBwPHr0CJ988gm+/fZbtfJ88uQJgoODce3aNXFZ1apVUbNmTdjY2ODZs2c4duwYXrx4gdevX+Ozzz7D6tWr0aNHjzzzPnjwIAYNGoSMjAy4u7ujfv36KFasGO7du4eIiAikp6cjOTkZn332GS5fvgwvLy+leWnr+0wQBPTs2RM7d+6ETCaDr68vqlSpAkEQcPnyZbnhfBITE/HixQsAQIkSJVC1alV4eHjAxsYGqampuHfvHk6dOoWUlBS8ePECbdq0wZEjR9CgQQO51+R3YAHSarhawJKSkoRixYqJEfeHv6R+/vlncZ2Pj0+e+SlqOtPGNpcvXxYsLS0FAIJMJhNGjBghVyuT7c6dO0KjRo3E/Fu3bq0wv5y/qMzNzQUAgp+fn3Dt2jW5dJmZmcLcuXPlfqUcOXJEaTlzplOXOttMmzZNLt2YMWOE9+/fy6WJi4sTWrRoIaYxMTERTp06pTC/nL/Osve/devWuX4Fp6WlCSNGjBDTymSyXL8u8+Phw4eCg4ODmK+Xl5ewZ88ehWlfvnwpLFq0SBgxYkSude/fvxdq1qwp5uPo6CgcOHAgV7p9+/bJvV7t2rWV3sKQs4bP3NxcMDY2FmbNmpWryf3hw4dCtWrVxLRNmjRRur9Sa+8EQf5zYmJiIgAQBg8enKvpKTU1Va6WS2oNpZmZmQBA6NGjh9hake3ly5dC586dxbRGRkbCiRMnCmQfVdVQZstP87W623z11VdiOmNjY2Hu3Lm5agtv3rwp1KlTR0xXrFgxpe//nPuf/ZkKCQkRXrx4IZcuMTFR6N69u5jWxsZGrSb1vOTnurphwwa560poaGiu98CbN2+Enj17yqXbsmWLxuU9ePCgXJ6K3n8JCQlCaGio3DHNPq6KZGRkCE2aNBHT+fv7K7xdIzk5WZg4caIgk8kEAIK1tbVw9+5dhXnmrKE0NzcXrK2tFTZ/Xr58Wa4lo3fv3kr3XZvfZ9nXiurVqwsXL17MlTYlJUV8fv/+fWHo0KHCv//+q7Sm/M2bN8L3338v5l+hQgWVter8DtSMQQWUa9asEQ9MqVKlcn1J3r9/X/yQAVD4hsypsALK4OBgcZvZs2erTPvu3TuhSpUqYnpFb6gPq+jLly+v8h6Rrl27imkHDRqkNJ02Pkxv3rwRbGxsxDSKAqtsKSkpgp+fX55BzodNigEBAUrvUczMzJTLsyCavXr06CH3pf/kyZN85bN8+XIxH1NTU5X3+kVFRYkXWwDCypUrFab7sJl68eLFSvO8dOmS+HmRyWTC48ePFabTNKAEIPTr10+t7aQGlACENm3aKP2SSEtLE4KCguTeK4oYUkB5+/ZtuWbWBQsWKM3v5cuXgqenZ56BQs79ByB0795daZ7Jyclyty+sX79erX1TRep1NSMjQ/Dy8hK3+fTTT5XeI5aZmSl06NBBTOvt7a1xU33dunXlAiVl+X342qoCylWrVolp6tWrl2fzZM5zpuy6njOglMlkSn/4CoIg7Ny5U0xrY2Oj9Jqq7e8zJycnhbdnaWLQoEFi/qpu/+B3oGYMqpd3zubu7t27w8REvsXew8MDjRs3VpheVy5cuIDDhw8DAHx8fPJs+rC2tsa4cePE/9esWZPna0ydOhU2NjZK1/fp00d8nrOJrDCsXbsW7969AwCULl1aZZOjubk5FixYIP4fHh4uNtmrMnfu3FzvhWwymQy9e/cW/9d0/x89eoQNGzaI/y9atEit2xIUWbx4sfj8q6++go+Pj9K0fn5+6N+/v/i/Os0W1atXx4ABA5Sur1atmjhmqyAIOHPmjDrFlszCwgLTp0/XSt4ymQy//fYbjIwUX8pMTEzw22+/if8fO3ZMrfeUPlu6dCkyMzMBALVq1cLXX3+tNG2JEiXkpqRdu3Yt3rx5ozJ/MzMzzJ49W+l6CwsLdO/eXfy/sK8pALB//37cu3cPQFZ5f/vtN6Wzm8hkMvz+++8wNTUFANy5cwcHDhzI92tfuXIF//77r/j/3Llzlb7/ZDKZyvU55TzmixYtgqWlpcr0Y8aMgZ2dHYCs4fKy3xPKtG3bFq1atVK6vk2bNnBycgKQddtDzmb3bIXxfTZ+/Hg4ODjkmU6KnN8BBw8eLNC881LUvgNVMZiA8tGjR3JvhC+//FJhupz3Ka1ZswYZGRlaL5sqOe9Z7N69u1pTOgUHB4vPjx8/rjKthYUF2rVrpzJNzkDl/v37eb5+Qcq++ABZ+5/XRdLf3x/Vq1cX/w8PD1eZvmzZsqhdu7bKNAW5/wcPHkR6ejoAoHz58iov0KokJCTIBXA5g35l+vXrJz4/ffo0EhMTVab/9NNP88yzMN4bLVq0QIkSJbSSd/b9QqpUr15dbj/zek/pu5yfqewpC1Xp1KkTSpYsCQB4//49Tp48qTJ9o0aNxMBCGV1eUwD5Y5AzEFLGxcVF7rOqyXsg5+D7/v7+ed7P7OnpiYYNG6pMExcXh/PnzwMAqlSpgpo1a+ZZDgsLC9SvXx8A8ObNG1y+fFll+ryuBzKZTO51FZ1XbX+fAUC3bt3yTPOhtLQ0HD9+HL///jvGjRuHb7/9FkOGDBEfOe+NzD7OhaWofQeqYjCdcv766y/xF1ilSpXg6+urMF3Xrl0xePBgpKSk4MmTJ9i3bx/atGlTmEWVk/PiHR4ejgcPHuS5jZBj5P2YmBiVaStWrCj+8lbG3t5efP727ds8X78gRUdHi88/vBlamYYNG4odUc6dO6cybc4PnjIFuf+nTp0SnwcFBeU7n4sXL4o/dmxsbFCjRo08t6lVqxasra2RmJiIjIwMXLhwQeUxLexjo0ydOnW0ki8A8QtVnXTZ78Wc70lDIwiC3BeiOp8pU1NT+Pv7Y+/evQCyPlOqfgjpy/tGlfxeV3bs2AEg7+uKKjmPf926ddXapm7duuJkHIrk/J5ITk7GkCFD1Mr3zp074vOYmBiV15GCOK/a/j7z8vISf/yoIzk5Gb/88gsWLVqE58+fq7WNuukKSlH7DlTFYALKnM3XymongaweWB06dBCbJVeuXKnTgPLx48fi8z179kje/tWrVyrXFy9ePM88cgac2bVrhSVn72UPDw+1tvH09BSf5/Xhl7r/2b0S8+vp06fi87Jly+Y7n5zHxc3NTa1f+kZGRnBzcxN7w+rbsVEme9IBbXB3d5ecLuexNzRv3ryRO09F4TOVH9q+rqj72m5ubmpt4+rqqnJ9zu+Je/fuydWoqaugvysUnVdtf59JuVa8evUKwcHBkmscc47MURiK2negKgbR5H369Gnxfg6ZTJbnEAk5A85//vkHr1+/1mbxVMrrfqW85NVkr04gokvZ944AWffTqCNnurw+/IW9/znLo+q+1bzk57h8mFbfjo0yeTXxaMLKykqtdFKOmz7L+b4BisZnKj+0fV1R97XVff/lda3Q9HsCyLuyoCDOq7a/z6RcKwYPHiwGk2ZmZujXrx+2b9+OmzdvIiEhAenp6RCyOh6L99sCyPNe04JW1L4DVTGIgDJn7aQgCPD09IRMJlP6aNu2rZg+JSVFrhNFYcv5xvj777/FN7iUhyHLeSHN654/RelsbW0LvEyayFmeD7/cpcjPcfkwrb4dG11ISkpSK11BH7fC/lLK9mFgUhQ+U/mhy+tKztfOz/tPkZzfE+3bt8/X90RoaGi+9kcKffk+e/ToEdavXw8gq+Vm7969WLp0Kdq3b4/y5cvDxsYGxsbGYnpd/ogsat+Bquh9QJmamop169ZplIcue3vn7AH85MkTnZVDV3I2YWQPsJqXnDcNF3RvP03lPJ85f/VKlfO4xMbGqnWhzczMlLsHSd+OjS6o+57K67hJvS2kIGqU8qN48eJyZS0Kn6n80OV1Jee2sbGxam2TVzpD+Z7Ql3IePnxYvGa2bt0aTZo0UZlenXs9taWofQeqovcB5c6dO/Hy5UsAWUOA1K1bV61H9nAoQNaNxDdv3syVd36qiqVuk/Om7RMnTkh+PUOXs3dZZGSkWtvkTJdX77XClnNGGE16itaoUUP8BZ2QkCDegK3KhQsXxF+uxsbGavUELQj61KTyoZydpFTJ2ZlA0Xsq5+wX2bNvqKLO+dLGcZPJZKhVq5b4vzqfqfT0dHGmI0D/PlOA9GOly+tKzuOfc/ggVfIaqiXn98T58+cltVoUJn35Pst5L6c6nVI0nZFLE0XtO1AVvQ8oc9Yutm7dGqdOnVLrERUVhWrVqonbZk+BlpOFhYX4XN0bVaVuk7P5/e+//5br1KFv8nM88pJzyIj169crnYIw25kzZ3Dx4kXx/7x+eRa25s2bi+N93bp1C/v27ctXPra2tnIjFYSFheW5zbJly8Tn/v7+ku691IQ23hcF5cSJE3nWFF+5ckWup6Si3vk5b4JX5yb/jRs35plGW8ct52dq5cqVedZub9u2TQyScw41o0+kHqucx2D37t3idJDKPH78WK4TSc7tpcr5/omKisLt27dVpn/48KHKHt5AVge/ypUrA8hqlcv5Wdcn+vJ9lnNcz7xuO0hKSlL4/a8IvwM1o9cBZXx8vNxFoGfPnpK2z5l+9erVuS68ObvSP3r0SK08pW7j7+8vXoCSk5Px5ZdfIjU1Va3XSk1NzbNXXEHKz/HIyxdffCHeQxIXF4dJkyYpTZuamoqhQ4eK/zdp0gQVK1YskHIUlDJlysiNkzZw4MB8X1QHDhwoPv/999/lLiIfOnv2rNxA6IMGDcrXa+aHNt4XBUUQBAwbNkxpUJWRkYFvvvlG/L9Ro0YK54n28/MTa8n+/fdfhYM6Z/vjjz9w5cqVPMtmZ2cnfvHFx8cX2BdU//79xXzPnTuHJUuWKE37+vVrjBo1Svy/e/fuavUKLWxS32MtWrQQ55p+//69ygG2BUHA0KFDxePv7e2NZs2a5busH04I8O2336oM6ocPH67WPbc550T/6aef1KoFz1ZYzc/68n2Wc4SN3bt3q+zs8/3336t9jeZ3oGb0OqBcu3ateBGwtbXNcwDvD+UcePXhw4e5mihz1mBu375drQ9GfraZP3+++IY6cOAAGjdurLKp5ObNm5g8eTI8PT0LtVkh575t2rSpQPIsVqyY3EwJU6dOxbhx43Idt6dPn6JDhw5iE6aJiQl+/fXXAilDQfv111/FsdIePHiA+vXrK62pfP36NZYsWSL3pZ6tR48eYrN1amoqWrZsqbAZ/eDBg2jdurV4b1/t2rXlZirRtpzvi/379+vs/kFFzMzMsGPHDoSGhua68f7Vq1fo3r27OLCwTCZT+p5ycnISaxIEQUD37t1z3feWnp6OWbNm4ZtvvoG5uXmeZTM3N0f58uUBZNV2bNu2TeruKeTt7S33YyR74OYPg5bbt2+jRYsWYg1usWLFMH78+AIpQ0GTel01MjLC1KlTxf/XrVuH/v375+ool5CQgN69e+Pvv/8Wl02fPl2tmWtU+fnnn8Xnu3btQkhISK7x/d69e4d+/frh77//Vuv90rNnT/E9mJCQgEaNGmHx4sVKj8Xbt2+xZs0aBAUFyQUh2qYP32fBwcFiD/vbt28jJCQk12gub9++xYABA7Bo0SK1W3P4HagZvR6HMmdzd+fOnSUPP+Lu7o6AgADx/omVK1fKVT+3bt0alpaWSE5Oxvnz51G5cmUEBQXBzs5ODERbtGiBFi1aaLRNtWrVsG7dOnTr1g1JSUn4999/Ua9ePXh7e6N27dooWbIkUlJS8OzZM1y8eFFntUBdunQRA6PRo0djz549qFq1qtzF8Mcff5Q868mIESNw/PhxcVDhKVOmYOHChWjSpAlKlCiBmJgYhIeH4/379+I2M2bMUHvQ4MLm5uaGjRs3omPHjnj37h3u3buHVq1awcPDA/7+/ihZsiTevXuHmzdv4vz580hLS0OHDh1y5WNmZoZ169YhMDAQ8fHxePLkCYKDg1GzZk3xPq3z58/jwoUL4jaOjo5Yt25dnoPZFyR/f3+4ubkhJiYGcXFxqFSpElq0aAEHBwfxPe/n55evGS409cMPP2DevHlYtWoVtm7diuDgYDg6OuLJkyc4fPiw3L1oP/zwAxo1aqQ0r59//hnh4eHIzMzEhQsXUKFCBQQHB8PFxQUvX77E0aNH8ezZM9jY2ODXX39V60u8S5cu+OWXXwBk/YAICwtDuXLl5M7fzJkzJe/3zJkzcebMGZw+fRrp6ekYMmQIpk6dikaNGsHGxgZ37tzB0aNHxZobExMTLFu2TK5pX5/k57r62Wef4ejRo+KYjX/++Sc2bNiAJk2aoHTp0nj27BkOHTokF2R+++236Ny5s8blbd68Ob755htxWs/Vq1dj27Ztcq8dHh6Ot2/fomTJkvj222/FYF5ZMGtsbIyNGzeiefPmiI6Oxtu3bzFo0CCMGjUK9evXh4uLC4yNjfHq1SvcuHED165dE39kdunSReN9Upc+fJ+VKFECI0aMEKcxXLNmDfbs2YO6devCxcUFcXFxiIiIQGJiIkxMTPDHH38gJCQkz3z5HaihgpsWvGBdvHhRbvLzAwcO5CufJUuWiHlYW1sLCQkJcusXLlwoyGQyudfK+ZgwYUKuPPOzjSAIwvnz54U6deoo3e7Dh6enpxAdHZ0rn/DwcDFNYGCgWschZ77KpKamCo0bN1ZZpnv37knOVxAEIS0tTRgyZIhgbGysMv/ixYsLK1asUJnXihUrxPQhISF57vu9e/fE9B4eHnmmV9f58+eFmjVrqnUue/TooTSfGzduCD4+PnnmUbt2beH27dsqyxQYGCimDw8Pz3MfJkyYkOf7VhAEYceOHYKZmZnSsn14HkJCQsR1eZ1PqeX/MO8TJ04Izs7OSstmbGws/Pjjj2q9/rJly1S+R52dnYWjR4+q/Rl8/fq1UKlSJZXnNScp79WEhAThs88+y/N94+zsLOzevVtlXuq+D7Ll5xqUl/xeVydPniyYm5urPAYWFhbCL7/8UiDlzJaZmSkMHz5cZZnLlCkjnDx5Uu57aNiwYSrzTUpKEgYNGiSYmJiodW2xtLRUum8eHh5Kr92KSPnc6vL7TBAEIT09XejVq5fK17SzsxO2bt2q9ueK34Ga0dsm75y1k87Ozvm+ibpr167ir4vExERs3rxZbv2gQYNw7NgxfPnll6hQoQKsra3z7HGYn20AoGbNmjhz5gz27duHr776CjVq1ICDgwNMTExgbW0NT09PtGzZEuPHj8eJEydw9+5duR6F2mZqaoqDBw9i4cKFaNasGZycnGBmZlYgeZuYmGD+/Pm4cOECvvvuO9SsWRMlS5aEqakpSpcujYCAAPz666+4c+dOoYynVhBq1qyJ6Oho/P333+jVqxfKly+PYsWKwdjYGCVKlECdOnXw1VdfYffu3SpvCq9QoQLOnDmDDRs24LPPPoOnpyesrKxgZWUFT09PdOvWDZs2bcKZM2fynLdaW9q2bYszZ85g4MCBqFq1KmxtbfWm93eDBg1w4cIFTJkyBXXq1IG9vT3Mzc3h5eWFvn37IioqClOmTFErrz59+uDixYvo27cvvLy8YGFhATs7O/j4+GDKlCm4ePEiAgIC1C5b8eLFcfr0aUybNg2NGzdGqVKlCqx22cbGBhs2bEBkZCS++uorVK5cGcWLF4eZmRnKlCmDFi1a4LfffsOtW7fQunXrAnlNbcrvdfWnn37CjRs38NNPP8HPz0+8pjo4OMDf3x/jxo3DjRs38MMPPxRoeWUyGWbPno2TJ08iNDQUnp6eMDc3h729PXx9ffHrr7/i4sWLqFevnjhSCZB1b60qlpaWWLhwIW7duoXJkyejSZMmcHFxgYWFBczMzFCqVCnUrVsXAwYMwIYNG/DkyZMC3zd16Pr7zNjYGCtXrsSOHTvQrl07ODo6wtTUFI6OjvD19cXkyZNx5coVdOzYUe08+R2oGZkgGPjI2URERHqsR48eWLt2LYCsnr66uD2ESNsYUBIREWnJu3fv4OrqKnZmu3PnjlwvZaKiQm+bvImIiAzd2LFjxWCybt26DCapyGJASUREJNGCBQswefJkpdMqPnv2DAMGDMD8+fPFZTnHmiQqatjkTUREJNHEiRMxadIkyGQyVKlSBVWrVkWJEiWQkpKC27dv4/Tp03JjDYaEhKg1IxaRodLrcSiJiIj0mSAIuHLlitLZk0xMTDBs2DBMnz69kEtGVLhYQ0lERCRRYmIidu3ahYMHD+LixYt49uwZnj9/jpSUFJQsWRJly5ZFUFAQ+vTpg3Llyum6uERax4CSiIiIiDTCTjlEREREpBEGlERERESkEQaURERERKQRBpREREREpBEGlERERESkEQaURERERKQRDmxOWpWSkoJLly4BAEqVKgUTE77liIgMUXp6OuLj4wEA1atXh4WFRYHm/eTJE43zcXJy4veMjvCok1ZdunQJ/v7+ui4GEREVoKioKPj5+RVYfk+ePIGbm5vG+cTExMDV1bUASkRSscmbiIiIiDTCGkrSqlKlSonPI479Cydn5zy3yZQweZOJsZZ/E0mZSEomNXMpG2h7QisJZZE6uZY2j4vksqifd2amtLyNjCTvqNZI2E0AWfNRS8hdWuZScpZcbu3lr828peevvc9/eob6eT+Ji0Nw43oA5K/tBc2sQlfITKzVTi+kJyL15matlYfUw4CStCrnvSxOzs5wUaMpQkpAaarlgFLSF63ELxSZhA0ELQeUksqi5YBSm2WRSfjWz5AYUBobcEAp5TMn5fxIxYBSSVo9CShz0uZ9ijJTW8jMbCRswMZWfcCAkoiIiPSHDNIidP35LfdRY1ivBffv34dMJoNMJsP9+/dVpvX09IRMJkNYWFihlI2IiEivyYykP0jnWENJRERE+kMmk1hDySpKfcCAkoiIiPSH1FpH1lDqBZ4FIiIiItIIA0o99ObNG/z000+oXr06rK2tYW5ujjJlyqBhw4YYP3480tLScm3z6tUrTJgwAbVq1YKtrS2srKxQvXp1TJkyBUlJSbnST5w4ETKZDBMnTsTDhw/Rt29fuLm5wdTUFKGhoYWwl0RERApkN3lLeZDOsclbzyQlJaFRo0a4fPkySpUqhaZNm8La2hpPnjzB9evXERkZie+++w52dnbiNlevXkWrVq0QExMDZ2dnNGrUCKampoiKisK4ceOwZcsWREREoHjx4rle79atW/Dx8YGZmRkaNmwIQRDg4OCgdnljY2NVro+Li1M7LyIiIkBqRxvWjekDBpR6ZvPmzbh8+TJat26N7du3w9TUVFyXmZmJY8eOwcrKSlyWnJyM9u3bIyYmBj/99BPGjRsHMzMzAFnBab9+/bBu3ToMHz4cy5cvz/V6a9euRc+ePfHnn3/C3NxccnkLYqosIiIiEYcNMkgM6/XM06dPAQDNmzeXCyYBwMjICIGBgWLACAArV67EnTt30LZtW0yePFlunZWVFZYsWQJHR0esXr0ar169yvV6JUuWxIIFC/IVTBIRERU4DhtkkFhDqWf8/PwAANOnT4e9vT3atm2LkiVLKk2/a9cuAEC3bt0UrrexsYGvry92796N06dPo0WLFnLrmzVrprApXF0xMTEq18fFxcHf3z/f+RMR0UeGwwYZJAaUWpBzere8pobLXp+9TVBQEEaPHo0ZM2YgJCQEMpkM5cuXR8OGDdGhQwe0a9cORkb//Rq7e/cuAODLL7/El19+qfK14uPjcy3z9PRUa5+UcVVjKkUiIiIq2hhQaoG19X+T2icmJqpM++7dOwBZNYnZpk6dikGDBmHHjh04fvw4Tpw4gRUrVmDFihXw8/NDeHi4+BqZmZkAgFatWqF06dIqX8vDwyPXMktLS/V2ioiIqDBwHEqDxIBSC0qWLAkbGxu8e/cOt2/fRrVq1RSme/nyJV6+fAkAcHd3l1vn6emJoUOHYujQoQCA06dPo2fPnjh9+jSmT5+OSZMmAcjqFHP9+nX07dsXXbt21eJeERERFQI2eRskhvVakN15BgC2bNmiNN3mzZsBACVKlECtWrVU5unn54evv/4aAHD+/HlxeevWrQEAGzdu1KDEREREeoKdcgwSz4KWjBo1CjKZDGvWrMGyZctyrT958iTGjh0LAPj+++/FHt1bt27F0aNHxabsbGlpadi7dy8A+abrAQMGwMPDA5s2bcLo0aORkJCQ67WePHmCpUuXFti+ERERaY1MJjGgZA2lPmCTt5Y0btwYc+fOxXfffYd+/frhl19+Qe3atWFiYoLbt2/j7NmzEAQBn3/+OcaMGSNud+TIEcybNw8ODg7w8fGBo6MjEhIScOrUKTx79gwuLi4YNWqUmN7a2hq7du1C27ZtMX36dCxZsgQ1atSAq6srkpKScPPmTVy7dg2Ojo7o37+/Lg4FERGR+mQywIhN3oaGAaUWffPNN2jcuDEWLFiAo0ePYvfu3UhPT0epUqXQoUMH9O7dG+3bt5fbJjQ0FJaWljh+/DiuXr2KI0eOoHjx4nB3d8e3336LAQMGwN7eXm6bqlWr4uLFi1i0aBG2bt2Kixcv4uTJk3BwcICrqytGjBiBTp06FeauExER0UdEJuQ1rg2RBmJjY8XZdK7ffggXNYYZypTwljQ11u5dG5I+HhJ/JMskbCBAux9TSWWResnQ5nGRWBaZhJqMjExpeRtLqVHRMqkVNlI+c1LOj1RSyy35rSghf23mLTV/bX7+0zPUz/tRbCyqlM+65SomJqZAh43L+V1hXn8kZBbqj48spLzB+5MztFIuUh9rKImIiEh/sJe3QWJASYUmPTMT6R90NlJESg2Itms09IU2a4UAqcdRfy7eUmocs9Krn1afahy1TsrnQmqNsxYPo17VCgpa3FGJRZFSuS7lfV5onwmOQ2mQGFASERGR/mANpUFiWE9EREREGmENJREREemP7HEopaQnnWNASURERPqDTd4GiQElERER6Q92yjFIDCiJiIhIf7CG0iAxoCQiIiI9IrGGkv2L9QLPAhERERFphDWUhLCwMPTu3RseHh64f/++pG1jY2NVro+Li9OgZERE9NFhk7dBYkBJGsmee5WIiKhAcNggg8SAklC8eHFUrFgRLi4uui4KERF97NjL2yAxoCR06tQJnTp1yte2MTExKtfHxcXB398/X3kTEdFHiE3eBokBJWnE1dVV10UgIqKihDWUBolngYiIiIg0whpKIiIi0h9s8jZIDCiJiIhIf7CXt0Fikzdh69atqFSpEpo2barrohAR0ccuu4ZSyoN0jjWUhDdv3uDGjRtISUnRdVGIiOgjJ4MMMklBIgNKfcCAkoiIiPSHTFpAKbCGUi+wyZsQGhoKQRAkT7tIREREBLCGkoiIiPSJDNJasVlBqRcYUFKhMZLJYKRG04SJkfauDtJbRtTfgK0uimnzuAiC9tILkJa5TOK3mpTjInU/pZJ2v5r+yJR4YNS5/mSTesylHsK0jExpG0jw8l2a2mnNTdRvqHyXkp6f4kgmk9jkbajv36KGASURERHpDQaUhokBJREREekNBpSGiZ1yiIiISG9kB5RSHtr27Nkz7Ny5E+PHj0fr1q3h4OAgvnZoaKhaeYSFham9P2FhYVrdH21gDaWeyM8HIjAwEBEREWqnj4iIQJMmTRRul/36grZv1iIiIjIwpUuX1nUR9B4DSj0REhKSa9mTJ0+wb98+pesrVaqk9XIREREVKj3v5e3u7o5KlSph//79+c5j3759KFOmjNL1rq6u+c5bVxhQ6glF1dsRERFiQGmI1d9ERERS6eM9lOPHj4efnx/8/PxQunRp3L9/H15eXvnOr0KFCvD09Cy4AuoBBpRERESkN7Km59avIdsmTZqk/RcxcOyUY8CSk5Mxa9Ys1KtXD3Z2drCwsEDFihUxatQovHjxQtfFIyIikix7Lm+1HxzZXC+whtJAPX78GK1atcKlS5dQsmRJ+Pn5wdbWFufOncOMGTOwadMmREREwMPDQ6vliI2NVbk+Li5Oq69PRERFiz42eVPeGFAaIEEQ8Nlnn+HSpUvo27cv5syZA1tbWwBAeno6xowZg1mzZqF37944fPiwVsvi5uam1fyJiIiKmt69e+PGjRt4/vw5ihUrhnLlyqFZs2b46quv4OLiouvi5QsDSgO0b98+nDhxArVq1cKiRYtgYvLfaTQxMcH06dOxb98+hIeH4/Lly6hWrZoOS0tERCSBBr281WkV04ce1DmH7nvx4gVevHiBf//9F7NmzcLcuXMxcOBA3RUunxhQGqBdu3YBALp06SIXTGYzMjJC48aNcfnyZURGRmo1oIyJiVG5Pi4uDv7+/lp7fSIiKmKkDlaeI6063ze6HG+5bNmy6Ny5M+rXry+28N29exdbtmzB5s2bkZKSgkGDBkEmk2HAgAE6K2d+MKA0QHfv3gUAjBs3DuPGjVOZNj4+Xqtl0YdfekREVHQU1XsoO3XqhJCQkFzl9fPzQ7du3bBz50507twZaWlpGD58ONq3bw8nJycdlVY6BpQGKDMzEwDQqFEjeHt7q0xbtWrVwigSERFRgdAkoIyKioKzs7M2iqWx4sWLq1zftm1bjB8/HuPGjUNSUhKWLVuGH3/8sZBKpzkGlAYou5q8Q4cOGDFihI5LQ0REVIA0uIfS2dnZoFvOBgwYgPHjx0MQBBw5csSgAkqOQ2mAWrduDQDYtGkT594mIiIqIhwdHWFvbw8AePTokY5LIw0DSgPUoUMH+Pn5ISoqCr1791Z4n+SrV6+waNEipKen66CERERE+SNpUHOpHXgMgKHuD5u8DZCRkRG2bduGTz75BCtXrsTmzZtRs2ZNuLu7IzU1FXfv3sWlS5eQkZGB0NBQhT3BiYiI9FFR7ZSjjvj4eDx//hwAUKZMGR2XRhpGGgaqTJkyOHXqFMLCwrBhwwZcvHgRUVFRKFmyJMqUKYNBgwahffv2sLCw0HVRiYiI1JY99aKU9EXFkiVLxFvZAgMDdVwaaRhQ6rGgoCCV90iam5tj4MCBag+Aqio/3otJRET6oCjWUN6/fx+vXr2Cj4+P0jQ7d+7E//73PwCApaUlevfuXVjFKxAMKImIiEh/aNDLW1uOHz+O27dvi/9nN0sDwO3btxEWFiaXPjQ0VO7/+/fvo0mTJqhfvz7atWuHmjVrwtHREUDW2NKbN2/G5s2bxcqdmTNnGtwUjAwoyaBJrVjV5g9ZfSqLIdNmZbmUY67tSnsp+QuQVhhtNgFKfd9K2U+peUvdT0ll0WLeAGAkqUlXmlK2ZmqnzZRQbmvzjzdk+PPPP7Fy5UqF606cOIETJ07ILfswoMx28uRJnDx5UunrWFlZYc6cOQY3Sw7AgJKIiIj0iEwmrRnbEH6c16lTB3/99RdOnjyJM2fOIC4uDs+fP0d6ejpKlCiBqlWromnTpujXr59Yc2loGFASERGR3tDHeyjDwsJyNWtLYWtrix49eqBHjx4FVyg9w4CSiIiI9IfUsSUNoYryI8CAkoiIiPSHHnbKobwxoCQiIiK9oY9N3pQ3Tr1IRERERBphDSURERHpDdZQGibWUOqhnB+mv/76C/7+/rCxsUGpUqXQvXt3PHz4EEDW7DYLFixArVq1YG1tDQcHB4SGhuLZs2dK87558yYGDhwIb29vWFhYoHjx4mjcuDH++uuvQtk3IiIiVbKnXlT7wZso9QIDSj32ww8/oHfv3rC1tUXr1q1hZWWF9evXo1GjRnj16hU+//xzjBw5Es7OzmjZsiWMjY2xcuVKNG/eHKmpqbny27RpE2rWrIklS5bAzMwMbdq0ga+vL86dO4cvv/wSffr0kVzG2NhYlY+4uLiCOBRERPSRkBRMSu0RTlrDJm89tnTpUpw5cwY1a9YEACQnJ6NFixY4fvw4AgMDkZSUhOvXr8PDwwNA1lRQ9evXx8WLF7Fp0ya58a4uXbqEL7/8EjKZDFu2bEHnzp3FdQ8ePEC7du2wYsUKBAUFoVevXmqX0c3NrYD2loiICOzlbaBYQ6nH/ve//4nBJJA1Wfx3330HICtA/O2338RgEgAcHBzw1VdfAQAOHTokl9fPP/+M9+/fY8qUKXLBJAB4eHhg2bJlAIDffvtNK/tCRESkDtZQGibWUOqxNm3a5FpWvnx5AICJiQlatGihdP3jx4/FZZmZmdizZw8AoFu3bgpfy9fXFzY2NoiOjkZKSgosLCzUKmNMTIzK9XFxcfD391crLyIiIjJMDCj1mLu7e65lNjY2AABnZ2eYmOQ+fba2tgCAlJQUcdmLFy/w9u1bAOo1Ub948QIuLi5qldHV1VWtdEREROpgL2/DxIBSjxkZKb8jQdW6D2VmZorPQ0JC8kxvbm6udt5EREQFSSaTNpsi40n9wIDyI+Dg4ABLS0skJydj5syZcHBw0HWRiIiIFMoKKKXUUGqxMKQ2dsr5CBgbG6N58+YAgI0bN+q4NERERCrI/qulVOfBXt76gQHlR2LChAkwMzPDyJEjsXLlSrlm8GyXL1/G33//rYPSERERZWEvb8PEgPIjUbt2bXE2nNDQUHh4eKBly5bo2bMn2rRpAzc3N1SvXp01mERERCQZ76H8iHz66afw8/PDb7/9hgMHDuDEiRPIyMhA6dKlUa5cOQwZMgRdu3bVdTGJiOgjJoPETjlaKwlJwYBSDwmCoHSdp6enyvVBQUF5bj979myNykdERKQtRkYyGBmpHyZKSUvaw4CSiIiI9AaHDTJMDCip0KRlZCI1PXdnoA/FJ6epnadLCUtNipQnXqgKnwDlNewfMtLiCZJJbEjT5ntFaln0iVaPi+EeFhhp8ZxKOS4ZalyTs6lq/SpIHNjcMDGgJCIiIr3BGkrDxF7eRERERKQR1lASERGR3mCTt2FiDaUBiYiIgEwmQ1BQkK6LQkREpB1SBzVnQKkXGFDqGU9PT8hkMty/f1/XRSEiIip0UqZdZDypP9jkTURERHpDBolN3gY8CkJRwoCSiIiI9AZ7eRsmNnnribCwMMhkMjx48AAA4OXlJXePSEREhFz6tLQ0TJs2DVWrVoWlpSXs7e3RuXNnXLt2TelrvHr1ChMmTECtWrVga2sLKysrVK9eHVOmTEFSUpI2d4+IiIiKMNZQ6oly5cohJCQEmzdvRmJiIrp06QIbGxtxvZOTE548eQIgK5hs06YNIiMj0bhxY1SuXBlRUVHYunUrwsPDER0dDU9PT7n8r169ilatWiEmJgbOzs5o1KgRTE1NERUVhXHjxmHLli2IiIhA8eLFJZU7NjZW5fq4uDhJ+RER0ceNvbwNEwNKPdGoUSM0atQIERERSExMxMyZM3MFhdkBZWRkJHx8fHDnzh04OTkBAFJSUtCxY0fs27cPv/76KxYvXixul5ycjPbt2yMmJgY//fQTxo0bBzMzMwBAUlIS+vXrh3Xr1mH48OFYvny5pHK7ublpsNdERETy2ORtmNjkbYBkMhlWrFghBpMAYGFhgUmTJgEADh48KJd+5cqVuHPnDtq2bYvJkyeLwSQAWFlZYcmSJXB0dMTq1avx6tWrwtkJIiIiBaQMGSS1NpO0hzWUBsjd3R01a9bMtbxy5coAgEePHskt37VrFwCgW7duCvOzsbGBr68vdu/ejdOnT6NFixZqlyUmJkbl+ri4OPj7+6udHxERfdxYQ2mYGFAaIHd3d4XLixUrBgB4//693PK7d+8CAL788kt8+eWXKvOOj4+XVBZXV1dJ6YmIiKjoYUBpgIyMpN2pkJmZCQBo1aoVSpcurTKth4dHvstFRESkqawaSimdcrRYGFIbA8qPgJubG65fv46+ffuia9euui4OERGRclJnv2FAqRfYKUfPZHeYSU9PL7A8W7duDQDYuHFjgeVJRESkDeyUY5gYUOqZ7HsSr1y5UmB5DhgwAB4eHti0aRNGjx6NhISEXGmePHmCpUuXFthrEhER5YcMEufy1nWBCQCbvPVOly5dEB4ejp49e6JFixYoUaIEAGDkyJH5ztPa2hq7du1C27ZtMX36dCxZsgQ1atSAq6srkpKScPPmTVy7dg2Ojo7o379/Qe0KERGRZBzY3DAxoNQzX331FRISEvDXX39h9+7dSElJAQD07NlTo3yrVq2KixcvYtGiRdi6dSsuXryIkydPwsHBAa6urhgxYgQ6depUELtAREREHxkGlHrGyMgIY8aMwZgxYxSuFwRB5faq1tva2mLkyJEa1XYSERFpE8ehNEwMKImIiEhvsMnbMDGgpEJjYmQEU+O8+4E5FbdQO888Kmxzp4e0DdIz1E+fkSmxMBKYGEu7YP7/0KNqS01Xf4N0iZkbG0kre2q6+sexuJW0S9irxDS100r9irrw+LWk9A28HNROe/ahtClRa7gUl5TewtRY7bSmJtKOjEzCkUzPkPbeSpHwvgUAazP191Pqp9lIYlAjZV+llsVEwmcuXcJ1K13qBTefGFAaJgaUREREpDfY5G2YGFASERGR3mANpWHiOJREREREpBHWUBIREZH+4NSLBokBJREREekNNnkbJgaUREREpDeyp16Ukp50jwElERER6Q0jmUzSMExSh2wi7WCnHC24c+cOjI2NUaJECSQlJSlNV7VqVchkMuzevVtclp6ejj///BNBQUEoWbIkzM3N4eXlha+++goxMTEK8/n777/Rr18/VKtWDSVKlICFhQW8vLzQp08f3LhxQ+E2oaGhkMlkCAsLw+XLl9GtWzc4OzvD2NgYEydO1Gj/iYiI8it72CApD9I91lBqgbe3Nz755BPs2LEDa9asQf/+/XOlCQ8Px9WrV+Ht7Y3WrVsDABISEtC+fXtERETAxsYGderUQalSpXDp0iUsWrQImzZtwoEDB+Dj4yOX12effQZzc3NUqVIFwcHBSE9Px+XLl7FixQps3LgR+/fvR4MGDRSWNTIyEoMGDYKzszMaN26M5ORk2Nraqr2vsbGxKtfHxcWpnRcREREZJgaUWvLNN99gx44d+P333xUGlL///jsA4OuvvxZvKB40aBAiIiLQtm1bLFu2DI6OjmL6uXPnYvjw4ejWrRuuXbsGY+P/ZnxYs2YN2rZtC2tra3GZIAhYuHAhBg8ejAEDBuDSpUsKb1xeunQpxowZg59//hlGRtIrrN3c3CRvQ0REpAw75RgmNnlrSbNmzVC1alVcuHABx48fl1sXGxuL7du3w8rKCn369AEAXLt2DevWrUOZMmWwdu1auWASAL799lu0adMGt27dwp49e+TWdevWTS6YBLI+YF9//TXq16+PK1eu4Nq1awrLWaFCBUyZMiVfwSQREVFBM5JJf5DusYZSi7755hsMHDgQCxYsQKNGjcTlixcvRnp6Onr37g07OzsAwO7duyEIAlq3bq20yTkoKAi7d+9GZGQk2rZtK7fu9u3b2Lt3L27fvo2EhARkZGQAAJ4+fQoAuHHjBqpUqZIrz44dO8rVdkql7L7ObHFxcfD39893/kRE9JGRSax1ZECpFxhQalHPnj0xZswY/P3334iLi4OzszNSU1OxdOlSAMCQIUPEtHfv3gUALFu2DMuWLVOZb3x8vPg8IyMDQ4YMweLFiyEIgtJt3r59q3C5p6enurujkKurq0bbExER5cS5vA0TA0otsrKyQv/+/TF9+nQsWbIEEyZMwJYtW/D06VMEBASgRo0aYtrMzEwAQK1atVCzZk2V+datW1d8Pm/ePCxatAhOTk6YPXs2GjRogNKlS8PCwgIA8MUXX2DdunVKg01LS0tNd5OIiKjAyP7/T0p60j0GlFo2ePBgzJo1C0uWLMHYsWOxYMECAPK1k8B/nVsaNmwoplHHxo0bAWQ1o7dv3z7X+lu3buW36ERERERqYU8MLXN3d0fHjh3x+PFjjB8/HpGRkShTpgw6d+4sly576KB//vkHKSkpauf/8uVLAICHh0eudVeuXMH58+fzX3giIqJCxk45hokBZSEYNmwYAGDq1KkAgIEDB8LERL5y2MfHB126dEFMTAw6d+6M+/fv58onMTERa9asETvaAEDlypUBZA1DlN1sDmR1hunVqxfS09MLeneIiIi0JnvYICkP0j02eReCgIAA+Pj4IDo6GqamphgwYIDCdCtWrMDr16+xZ88eVKxYETVr1oSXlxcEQcD9+/dx4cIFpKam4tq1ayhdujQAYOzYsdi7dy+WLl2K8PBw1K5dG2/fvsWRI0dQtmxZdOrUCVu3bi3M3SUiIso3zuVtmFhDWUhatGgBAOjatSucnJwUprG1tcX+/fuxdu1aNGvWDA8fPsTWrVtx+PBhJCcno0ePHti6dSu8vb3FberWrYszZ86gffv2SExMxD///IM7d+5g6NChOHnyJIoVK1Yo+0dERFQQsufylvIg3WMNZSHIyMjA+vXrAQBDhw5VmdbIyAjdu3dH9+7d1c6/evXq2L59u8J1YWFhCAsLU3s5ERGRTkmdn5vxpF5gDWUhWLJkCR48eID69eujfv36ui4OERERSfDs2TPs3LkT48ePR+vWreHg4CDevxkaGio5vz179qBTp05wdXWFubk5XF1d0alTp1wz4RkS1lBqyY0bNzBjxgw8efIEe/fuhZGREWbOnKnrYulUemYm0jIy80yXkJKhdp4OtubSCiFI+ylrZqJ++kwVA8srkpqe97HIJnWctUxB/bwBwMJM/d+WxjJpMyu9l7CfAGBhqv6+Gkvs3mltrn7ZzU2k7WdwBce8E+UgpSNBo3IOkvLWJm22LpqaSKvjkJpeCqmfZ6nHxcRY/bILkFgWCdcLKZ9PKdcsTejjXN7Z/RY0lZmZiQEDBuSawOTRo0d49OgRtm3bhn79+mHx4sUGNyWyYZXWgMTFxWHZsmU4cOAAqlatis2bN6NBgwa6LhYREZFey54pR8qjMLm7u4v9IqT68ccfxWDSx8cH69atQ1RUFNatWwcfHx8AwJ9//omffvqpwMpbWFhDqSVBQUEqp0IkIiKi3KR2tCmMTjnjx4+Hn58f/Pz8ULp0ady/fx9eXl6S8rh586bYUunr64ujR4+Ks9X5+fmhffv2CAwMxJkzZzBjxgz06dMH5cqVK/B90RbWUBIREZHekOXjoW2TJk1C27ZtNWr6njt3rjg29Pz583NNfWxlZYX58+cDANLT0zFnzpz8F1gHGFAakIiICMhkMgQFBem6KERERFpRFAc2FwRBHI2lUqVKqFevnsJ09erVQ8WKFQEA27dvN6iWTgaUesbT0xMymUzhTDlERERkeO7du4fHjx8DAAIDA1WmzV7/6NEjg4oFGFASERGR3iiKc3lfvXpVfF6pUiWVaXOuv3btmtbKVNDYKYeIiIj0hibDBsXFxeWZ3tXVNV/l0kRsbKzar+/m5iY+j4mJ0VqZChprKPVEWFgYZDIZHjx4AADw8vKSuz8kIiJCLn1aWhqmTZuGqlWrwtLSEvb29ujcubPKXzOvXr3ChAkTUKtWLdja2sLKygrVq1fHlClTkJSUpM3dIyIiUlt+hwzy9/eHm5ubyocuJCQkiM9tbGxUprW2thafv3v3TmtlKmisodQT5cqVQ0hICDZv3ozExER06dJF7k3n5OSEJ0+eAMgKJtu0aYPIyEg0btwYlStXRlRUFLZu3Yrw8HBER0fD09NTLv+rV6+iVatWiImJgbOzMxo1agRTU1NERUVh3Lhx2LJlCyIiIlC8eHFJ5c75q0sRdX4tEhERZdPHgc01lZKSIj43MzNTmdbc/L8JO5KTk7VWpoLGgFJPNGrUCI0aNUJERAQSExMxc+bMXEFhdkAZGRkJHx8f3LlzB05OTgCy3qwdO3bEvn378Ouvv2Lx4sXidsnJyWjfvj1iYmLw008/Ydy4ceIbOikpCf369cO6deswfPhwLF++XFK5dfVrj4iIiiap90XmTBsVFQVnZ+eCL5SGLCwsxOepqakq075//158/uHQQvqMTd4GSCaTYcWKFWIwCWS9WSdNmgQAOHjwoFz6lStX4s6dO2jbti0mT54s9+vIysoKS5YsgaOjI1avXo1Xr14Vzk4QEREVMGdnZ7i6uqp86IKtra34PK9m7MTERPF5Xs3j+oQ1lAbI3d0dNWvWzLW8cuXKALKGGshp165dAIBu3bopzM/Gxga+vr7YvXs3Tp8+LWlKqbxuGI6Li4O/v7/a+RER0cct695IKU3eWixMAckZyOZ1q1jO71VDagVkQGmA3N3dFS4vVqwYAPnqcgC4e/cuAODLL7/El19+qTLv+Ph4SWXR1a89IiIqmqTOfmMA8SSqVKkiPr9+/brKtDnXZ1cUGQIGlAbIyEjanQqZmZkAgFatWuU5bZSHh0e+y0VERKQpfZzLW1NeXl4oU6YMHj9+jCNHjqhMe/ToUQCAi4tLrr4U+owB5UfAzc0N169fR9++fdG1a1ddF4eIiEgpRcMB5ZVe38lkMnTo0AELFy7E9evXcerUKYXTL546dUqsoezQoYNB9GDPxk45eia7w0z2BPIFoXXr1gCAjRs3FlieREREWiF1Hm8DCbq+/fZbGBsbAwCGDh2aa0ig5ORkDB06FABgYmKCb7/9trCLqBHWUOoZV1dX3Lp1C1euXEG5cuUKJM8BAwZg3rx52LRpE0aPHo2ffvpJrscZkDUk0Y4dO9C/f/8CeU0iIqKi4vjx47h9+7b4//Pnz8Xnt2/fRlhYmFz60NDQXHlUqFABI0eOxNSpU3HmzBk0bNgQo0ePhre3N+7cuYNp06YhOjoaADBy5EiUL19eK/uiLQwo9UyXLl0QHh6Onj17okWLFihRogSArDdXfllbW2PXrl1o27Ytpk+fjiVLlqBGjRpwdXVFUlISbt68iWvXrsHR0ZEBJRER6ZQ+Nnn/+eefWLlypcJ1J06cwIkTJ+SWKQooAeDnn3/Gs2fPsHz5ckRHR+Pzzz/PlaZv376YMmWKxmUubAwo9cxXX32FhIQE/PXXX9i9e7c4un7Pnj01yrdq1aq4ePEiFi1ahK1bt+LixYs4efIkHBwc4OrqihEjRqBTp04FsQtERET5VhQ75WQzMjLCsmXL0KVLFyxZsgSnT5/G8+fP4eDgAD8/PwwcOFC8Tc3QyARBEHRdCCq6YmNjxXG0Lly/hzIueQ8zlJKWoXb+DrbmeSfKQeq7Xcp1KlNi5qnpmWqnNZHYsz8tQ/28AcDEWP0dNZZ48X4vYT8BwFjCFBlSyg0Ayanqv7fMTYwl5S1lZg/AMKaLU8RAiy2Z1M+z1KBGSvYCpJVFJmEgnTfJaWqnffwoFnWqlAWQNVZiQQ4bl/O74sslh2Bj75THFv959+IJVg9oqpVykfpYQ0lERER6QwaJc3kbxEiURR8DSio0ZsZGMDfJu6bN3FT92jiptQiSayglZS4tb1Nj9fdT6uVSSt6AtNo1qU0aUs4nAGRkqv8KUs+nlOMipQYZkF5bKqVxSHrekpJr9fx/LF/22mzry8iQlrmU94uJlBYBqdXw+WQEaUPQcLga/cDzQEREREQaYQ0lERER6Q1xfEkJ6Un3GFASERGR3pDJpN2GwXhSP2ityVvqL4zCFBoaCplMlmsg0sJ2//59yGSyQpurMyIiAjKZDEFBQbnW6fP5IiKij4eRTPqDdI/3UGpRUFAQZDIZIiIidF0UIiIigyBl2kVWhugPNnkTERGR3pBa68gaSv3AGkoiIiIi0kihBJRLly5FnTp1YG1tDTs7O7Rp0wanTp1SmPbq1auYMGECGjZsCBcXF5iZmcHe3h7NmjXDxo0blb7GwYMH0a5dO5QuXRqmpqYoUaIEypcvj549e+Lo0aNql3Xv3r0oVqwYLCwssH79erl1Z8+eRY8ePeDu7g5zc3OULFkSLVu2xO7du+XSZd+reOTIEQBAkyZN5KrmFd27mZ6ejunTp6Nq1aqwtLSEg4MDPvvsM1y/fl1hOaOiojBq1Cj4+/vDyckJZmZmKF26NNq1a4eDBw+qvb9ERET6JHsubykP0j2tN3l/9913mDt3Lho2bIgOHTrg0qVL2LNnDw4cOICNGzfmmj969uzZWLZsGSpVqoTq1avDzs4ODx8+RHh4OA4dOoRTp05h9uzZctusXLkSvXv3BgD4+/ujSZMmSE5ORmxsLNavXw8HBwc0btw4z7IuXrwYgwcPRvHixbF79240atRIXDdv3jx89913yMzMRK1atVC3bl08efIEERER2L9/PyZNmoTx48cDAJycnBASEoK9e/fi6dOnaNmyJZyc/ptGqly5crleu1u3btixYwcCAwNRo0YNREVFYdOmTdizZw/279+P+vXry6UfO3YswsPDUbVqVTFYv3PnDnbu3ImdO3di7ty5GDZsWJ77rKnY2FiV6+Pi4rReBiIiKjpkEufy5j2U+kHrAeWiRYtw8OBBBAcHi8tmzJiBUaNGoXfv3mjYsCEcHR3FdV9++SXGjh2LsmXLyuVz48YNNGvWDHPmzMHnn38Of39/cd2kSZMgCAKOHTsmFwQCwLNnz/Do0SOVZRQEAWPGjMH06dPh7e2N3bt3o0KFCuL6ffv2Yfjw4bC3t8eWLVvkgtNLly6hTZs2mDBhAgIDAxEYGIhKlSohLCwMQUFBePr0KcaMGaOwZ3W2Bw8eIDExEWfOnEGNGjUAABkZGRg+fDjmz5+P7t2748aNGzA3/2/e6u+//x6rV6+Gs7OzXF4nT55Eq1atMHLkSHTt2hUuLi4q911T2XOvEhERFQTOlGOYtH4eBg4cKBdMAsDIkSPh6+uLN2/e4M8//5RbFxgYmCuYBICKFSti3LhxAIDNmzfLrXv69CmKFy+eK5gEAEdHR/j4+CgtX0pKCj7//HNMnz4d9erVw8mTJ+WCSQCYMGECBEHAokWLctV0Vq9eXawxnT9/vtLXyctPP/0kBpMAYGxsjBkzZsDFxQUPHjzAli1b5NK3bt06VzAJAPXr18fgwYORlpaG7du357s8REREusAmb8Ok9RrKkJAQhct79eqFM2fOICIiAmPHjpVb9+7dO+zZswfR0dF4/vw5UlNTAfzXfHrjxg259P7+/oiIiECvXr0wbNgw+Pj4wMgo71j5+fPnaNq0KSIjI9G5c2f89ddfsLS0zJUmKioKlpaWaNeuncJ8smsfIyMj83xNZRQdJ3Nzc3Tr1g2zZ89GREQEvvjiC7n1L168wK5du3D58mW8evUKaWlpAIBbt24ByH2ctCEmJkbl+ri4OLnaZCIiIlWMIK3J2+gjmS9e32k9oPTy8lK5/MN78Hbs2IHevXvjxYsXSvN8+/at3P9//PEH2rZti9WrV2P16tWwtbWFn58fgoOD8eWXX8Ld3V1hPj/88APS09PRokULbNq0SWEQeu/ePQiCgOTkZLkmZ0Xi4+NVrlfGzs4OdnZ2CtcpO05Lly7F8OHDkZiYqDTfD4+TNri6umr9NYiIiEi/6XwcSkEQxOePHj1Ct27dkJycjFGjRqFHjx7w9PSEjY0NjIyMsH//frRs2VJuGwCoXLkybty4gf379+Pw4cOIjIzEsWPHcPjwYfzvf//DsmXL0LNnz1yv/emnn2Lbtm04ePAgwsLC0KdPn1xpMjMzAQA2Njbo0qVLAe+9+nLu89mzZzFw4EAYGxtj2rRpaNeuHdzd3WFlZQWZTIYlS5Zg4MCBuY4TERGRvpPajM0mb/2g9YDy3r17qFWrVq7l9+/fByBfw7Vjxw4kJyejU6dOmDZtWq5tsptyFTExMUGbNm3Qpk0bAFm1c7Nnz8akSZMwcOBAdOrUCdbW1nLbtGjRAoMGDULbtm3Rr18/vHv3Dt98841cmuxOJzKZDMuXL1erKV2q169f4/Xr1wprKRUdp02bNkEQBAwdOhSjRo3KtY2q40RERKTPOLC5YdJ6p5zVq1erXJ6z9/PLly8BAB4eHrnSC4KAtWvXqv26xYoVw8SJE2FnZ4ekpCTcvHlTYbrGjRvj0KFDKFGiBIYNG4ZffvlFbn2ZMmVQo0YNJCQkYO/evWq/PgCYmZkByBpjMi+KjlNqaio2bNgAQP3jlJKSkqsDDxERkaGQyQCj/x86SJ0Hayj1g9YDyoULF+aay3rOnDmIioqCra0t+vbtKy6vXLkygKxe3DnHL8zIyMD48eMVdnpJSkrC7NmzFd6/eOzYMbx+/RrGxsYq7/Xz8/NDREQEnJyc8OOPP2LMmDFy66dMmQIA6N27N3bs2JFre0EQ8O+//2L//v1yy7Nf88qVK0pfO9vkyZNx+fJl8f/MzEyMHj0asbGxcHNzk2tuzz5OK1euREJCgrg8JSUFX3/9Ne7du5fn6xEREekj9vI2TFpv8s4eNiggIAAuLi64fPkyLl26BGNjYyxfvlxuwO927dqhTp06OHv2LCpUqIDAwEBYW1vj33//xePHjzF69OhcTeGpqan4/vvvMXLkSFSvXh3ly5eHqakp7t+/L87G8+OPP6JUqVIqy1m9enUcO3YMTZs2xbRp05CQkIAFCxZAJpOhXbt2mDdvHr7//nu0b98e5cqVQ8WKFVG8eHHEx8fjwoULePbsGUaPHo0WLVqIeXbp0gUrVqzAqFGjcPDgQTg6OkImk6FPnz5o0KCBmM7d3R116tRB7dq1ERQUBHt7e5w+fRp37tyBtbU11q5dCwsLCzF97969MW/ePERHR8PLywsBAQEwNjbGsWPHkJycjGHDhmHevHkanTciIiJdYJO3YdJ6DeWcOXPwxx9/4O3bt9i2bRsePHiAVq1a4ejRo+jatatcWhMTE3EYIRcXFxw6dAgRERHw8fERB+z+kI2NDRYtWoRu3brh/fv3OHDgALZt24Znz56hc+fOOHToECZNmqRWWcuVK4fjx4+jQoUK+OOPPxAaGoqMjAwAwDfffIPo6GgMGDAAMpkMhw4dwrZt23Dnzh34+Pjgt99+y3X/5SeffIKlS5eiWrVqOHz4MJYvX45ly5blan6XyWTYuHEjJk6ciJiYGGzduhWvXr1Cly5dEBUVlWt8TTs7O5w5cwZff/017OzssGfPHpw8eRItWrTAuXPnFN6zSkRERKQtMoFdgUmLspvsAeDarQdwUWeYIQm/No0l/jSV+m6X1JQiMW8pyaX+AM+UWBYph1HqBUNqc1SGhMJLGatOat7pGdL21MRYe+9FbeYNSKzhkXg+ZQY6RqAg8Z2uzf1Mz8iUlN7EWP26osT3ed/jn+3xo1jUrJQ1lF1MTEyBDhuX87ti1LpjKF4q98QdyryJj8P07gFaKRepT+fDBhERERFlY5O3YWJASYVGgHq1W1JmPZBcQyW16kaL9feSiiLxgim14SExTUINiMRjYm1uLCm9lJpBmUxaYSTNviHxhiCp70WZhPyl1vJKLYuU94vUmjgpRWF7mWIyqedTwofUzET9N6KphJpPTcgkBpTslKMfGFASERGR3pDJZJKCaKkBN2lH4fzcICIiIqIiizWUREREpDeMIPEeSq2VhKTgedBQUFAQZDJZrsHbiYiISDoObG6YWENJREREeiN7SkUp6Un3GFASERGR3uCwQYaJASURERHpD6nN2Awo9QLvofx/OYcpWLp0KerUqQNra2vY2dmhTZs24rzg6oqPj8dvv/2GNm3awMvLC5aWlihWrBh8fX0xbdo0pKSkyKW/c+cOjI2NUaJECSQlJSnNt2rVqpDJZNi9e7fc8vT0dPz5558ICgpCyZIlYW5uDi8vL3z11VeIiYnJlU9ERARkMhmCgoKQlJSE8ePHo3LlyrCysoKnp6ekfSUiIqKPGwPKD3z33XcYOHAgrKys0KFDB7i5uWHPnj0ICAjA1q1b1c5n3759GDZsGC5evAgPDw907NgR/v7+uHHjBsaMGYPg4GC8f/9eTO/t7Y1PPvkEr1+/xpo1axTmGR4ejqtXr8Lb2xutW7cWlyckJKB58+bo378/zp49ixo1aqB9+/YwNzfHokWL4OPjg+joaIV5pqSkICgoCLNnz4aXlxfat2+P8uXLq72fsbGxKh9xcXFq50VERGQEmeQH6R6bvD+waNEiHDx4EMHBweKyGTNmYNSoUejduzcaNmwIR0fHPPOpU6cOTp48iXr16sktf/XqFT7//HPs378fv/32G0aOHCmu++abb7Bjxw78/vvv6N+/f648f//9dwDA119/LTeQ66BBgxAREYG2bdti2bJlcuWbO3cuhg8fjm7duuHatWswNpafteTff/9FjRo1cPv2bTg5OeW5Xx/KnnuViIioIEjtuc0+OfqBNZQfGDhwoFwwCQAjR46Er68v3rx5gz///FOtfCpXrpwrmASAEiVKYP78+QCATZs2ya1r1qwZqlatigsXLuD48eNy62JjY7F9+3ZYWVmhT58+4vJr165h3bp1KFOmDNauXZsr2P3222/Rpk0b3Lp1C3v27FFY1gULFuQrmCQiIipo2Z1ypDxI91hD+YGQkBCFy3v16oUzZ84gIiICY8eOVSuvjIwMREREIDIyEnFxcUhOToYgCOK8uTdu3Mi1zTfffIOBAwdiwYIFaNSokbh88eLFSE9PR+/evWFnZycu3717NwRBQOvWrWFra6uwHEFBQdi9ezciIyPRtm1buXWOjo4ICAhQa38UUXR/Zk5xcXHw9/fPd/5ERPRx4bBBhokB5Qe8vLxULo+NjVUrn1u3bqFTp064cuWK0jRv377Ntaxnz54YM2YM/v77b8TFxcHZ2RmpqalYunQpAGDIkCFy6e/evQsAWLZsGZYtW6ayTPHx8bmWadoBx9XVVaPtiYiIcmKTt2FiQClRdu1iXrp27YorV66gbdu2GDVqFKpUqYJixYrB1NQUqampMDc3V7idlZUV+vfvj+nTp2PJkiWYMGECtmzZgqdPnyIgIAA1atSQS5+ZmQkAqFWrFmrWrKmyTHXr1s21zNLSUq39ISIiIlKGAeUH7t27h1q1auVafv/+fQDq1chdv34dFy9ehKOjI7Zu3QoTE/nDfOvWLZXbDx48GLNmzcKSJUswduxYLFiwAEDu2kngv04xDRs2FNMREREZKhmkNXnL2MtbL7BTzgdWr16tcnlQUFCeebx8+RIAUKZMmVzBJAD89ddfKrd3d3dHx44d8fjxY4wfPx6RkZEoU6YMOnfunCtt9vBB//zzT66xLYmIiAwN5/I2TAwoP7Bw4UJERETILZszZw6ioqJga2uLvn375plHhQoVYGxsjEuXLuXKa8eOHZgzZ06eeQwbNgwAMHXqVABZvc8VBac+Pj7o0qULYmJi0LlzZ7EmNafExESsWbMGT58+zfN1iYiIdMkoHw/SPTZ5fyB72KCAgAC4uLjg8uXLuHTpEoyNjbF8+XK1htdxcHDAkCFDMG/ePDRt2hQBAQEoU6YMbty4gXPnzuGnn37ClClTVOYREBAgDkhuamqKAQMGKE27YsUKvH79Gnv27EHFihVRs2ZNeHl5QRAE3L9/HxcuXEBqaiquXbuG0qVLSz4mREREhSXnzHXqpifdY2D/gTlz5uCPP/7A27dvsW3bNjx48ACtWrXC0aNH0bVrV0n5LFu2DD4+Pjh79ix2794NKysrrF+/HpMnT1YrjxYtWgDI6uCjKpC1tbXF/v37sXbtWjRr1gwPHz7E1q1bcfjwYSQnJ6NHjx7YunUrvL291S4/ERGRLsjy8SDdkwnqdlsu4rJ/4ejL4cjIyIC3tzcePHiAyMhI1K9fX9dFypfY2Fix49DVWw/gokanJik3Y5sYS7uUZGRKO7/avFBJKYrUH+CZEvczJT1T/cQSPyLW5sZ5J8rhvYSySD0uUt5bmRKvBSZG0n6fa3NYFKmdFKRc96TWBklJrieXXwCAIPGNrs2OIVKvW1LeiukZ6uf9KDYWVcp7AMgag7ggh43L+V0xZ+e/KFnaWe1tXz6Nw/C2dbVSLlIfm7z11JIlS/DgwQPUr1/fYINJIiIiqTiwuWFiQKlHbty4gRkzZuDJkyfYu3cvjIyMMHPmTF0Xq9Bps0ZD8mVHT2pXJOctuUZLfWam0mri0iXWrkipjZFaQ50qofZTat5pGRJqeSWmtzKTdqmOfZkkKX3p4orHxVXE1Fja+TeWMC/enafvJOVdtrS1pPRSPkcpaRmS8rY0lVYTL+VDl5CSJinr4pamaqd9l5KudtrE9+qn1RRDRMPDgFKPxMXFYdmyZTAzM0PVqlUxceJENGjQQNfFIiIiKjScKccwMaD8f/pw72RQUJBelIOIiEhXsgJKKb28tVgYUhsDSiIiItIbUseW5HA1+qFInQdPT09x/KrsgcGVmTFjhpj2wwHDg4KCIJPJcg1KLlVYWBhkMhlCQ0M1ykfbso+bokHRiYiIiPJSpALKnNasWYPU1FSl65cvX16IpSEiIiJ1ZFf2SHmQ7hXJgNLX1xcvXrzA9u3bFa6PjIzE9evX4efnp3D9qlWrcO3aNfj7+2uzmERERPQBDmxumIpkQNmnTx8Aymshly1bJpfuQ+7u7qhUqRKsrKy0U0AiIiJSiDWUhqlIBpTVq1eHr68v9u/fj0ePHsmte/fuHTZu3AhXV1dxasMP5XUP5eHDh/Hpp5/C1dUV5ubmKFWqFPz8/DBhwgS8ePFC4TaJiYn44YcfUK5cOZibm8PJyQkhISG5ypft77//Rr9+/VCtWjWUKFECFhYW8PLyQp8+fXDjxg2V+3/o0CF07twZzs7OMDMzg6OjIzp16oSTJ0+q3I6IiEjXjPLxIN0rsuehT58+yMzMRFhYmNzyjRs34t27dwgJCYGRxKnSAOCbb75B06ZNsXnzZpQqVQqdO3eGn58fXr58if/973+4dOlSrm3evHmDBg0aYNGiRahSpQpat24NQRCwatUqNGzYEG/evMm1zWeffYZ169bB0tISwcHBaNmyJYyMjLBixQrUqVMHkZGRCss3YsQINGvWDNu3b4e7uzs6duyIsmXLYvv27QgICMCKFSsk77MqsbGxKh9xcXEF+npERFTESa2dZA2lXiiywwZ98cUX+P777xEWFoYff/xRXL58+XLIZDKlzd2qzJ8/H/Pnz4e9vT02bdqEJk2ayK2PioqCs3Pu+Ue3bduGli1b4tixYyhWrBgA4NWrVwgODsb58+fxxx9/4IcffpDbZs2aNWjbti2srf+bCUIQBCxcuBCDBw/GgAEDcOnSJbmq/qVLl2LWrFkoV64ctmzZgho1aojrjh49irZt22LQoEFo1KgRypcvL3n/Fcmee5WIiIg+XkW2hrJ48eLo3Lkzbt++jSNHjgDImtrwxIkTCAwMRNmyZSXll56ejsmTJwPImmf7w2ASAPz9/RUGWNbW1lixYoUYTAJAiRIlMGbMGADAwYMHc23TrVs3uWASyLqv5Ouvv0b9+vVx5coVXLt2TVyXmZmJiRMnAgDWr18vF0wCQOPGjTFu3DikpqZi8eLFau41ERFR4WKnHMNUZGsogaxm7zVr1mD58uUIDAwUO+nkp3by7NmziI+Ph4ODAzp16iRpW19fX4U1l5UrVwYApfdR3r59G3v37sXt27eRkJCAjIysuWWfPn0KICtArlKlCgAgOjoajx8/hre3N+rUqaMwv6CgIABQ2lyeHzExMSrXx8XFsbc8ERGpTQaJUy9qrSQkRZEOKJs0aQIvLy9s3rwZc+fOxapVq1CsWDF07dpVcl4PHjwAAFSsWFFyjzJ3d3eFy7NrLFNSUuSWZ2RkYMiQIVi8eLHKqRjfvn0rPr979y4A4M6dO3mWLz4+Xq1yq8PV1bXA8iIiIjKCDEYSwkQpaUl7inRAmT1LzYQJExASEoInT55gwIABsLS0LNRySO38M2/ePCxatAhOTk6YPXs2GjRogNKlS8PCwgJA1v2h69atkws2MzMzAQBOTk5o2bKlyvwdHBwk7gEREVHhkNrPhn1y9EORDigBIDQ0FJMmTcKOHTsA5K+5G/ivlvHmzZsQBEGr415t3LgRALB48WK0b98+1/pbt27lWpZ976a9vX2unu1ERESGQvb/f1LSk+4V2U452dzd3dGhQwfY29ujXr16qFu3br7y8fX1hYODA+Lj47Ft27aCLeQHXr58CQDw8PDIte7KlSs4f/58ruV+fn5wcHDA1atXceXKFa2Wj4iIiCinIh9QAlmDhD9//lyjgb1NTEzE4YcGDBiAo0eP5kpz+vRpxMbG5vs1smV31vn999/Fpmwgq4NLr169kJ6enmsbU1NTTJgwAYIgoFOnTjh+/HiuNBkZGTh8+DBOnTqlcRmJiIi0IbvJW8pD+2VSb0zM7M6vH6Mi3+RdkIYNG4YbN25g0aJFCAwMhI+PDypWrIi3b9/i+vXruHv3LsLDwzXuqDJ27Fjs3bsXS5cuRXh4OGrXro23b9/iyJEjKFu2LDp16oStW7fm2m7IkCF4+PAhZsyYgYCAAFStWhXlypWDpaUlnjx5gvPnz+P169dYuHAh6tWrp1EZiYiItEEmsVMOm7z1AwNKCWQyGRYuXIgOHTpg0aJFOHXqFC5fvgw7Ozt4eXkhJCQk1/iP+VG3bl2cOXMGP/30E06fPo1//vkHbm5uGDp0KH766ScMHTpU6bbTp09Hx44d8ccff+D48ePYu3cvzMzM4OzsjKCgILRt2xadO3fWuIxERETaoM+dcr766it8/fXXStd/OH70x0QmqBqXhkhDsbGxYoehq7cewEWN2ltjIwnDRUi8kkh+u0vIXmrW2vzkCZCW+fu0zLwT/T8zE+3eKZOarn5ZTIylnf/0DPWPi9S8pZ7PtAz199PKTNpv/9iXyZLSly5urnZaU2Np51/K5/nO03eS8i5bWtqXt5RzlJKWISlvS1NjSemlXFveJKVJyrq4panaaV9LyPvxo1j4VfMGkDUGcUEOG5fzu+Kvw+dRyqmM2tvGP3mMnsG1tFKubNkdcSdMmCBOIkLyWENJREREeoO9vA0TA0r6qEiuFJSwQabEKioptWXaJqU2TkoNIgBYSKy5eSShds3MVFpt2aT9N9VO28dfWi3HF9MOSUrfOKC82mn3LFghKW+Ulja17JllA9VOK/V97mBrpnZaz1ISmwu11+Ag2ZtkabWIxSTUItqYa++rWsoh1J8rFukjBpRERESkN4xkWQ8p6Un3Pophg4iIiMgwyPLxV1g2bdqEKlWqwMrKCra2tihfvjxCQkIQHh5eaGXQV6yhJCIiIr2hSS/vuLi4PNNr0mnn6tWrcv/fvn0bt2/fxqpVq9CxY0eEhYWhePHi+c7fkDGg/H+enp548OABVqxYgdDQUF0XRyzPvXv34OnpqeviEBERFQoZpHW0yZnS398/z/T5GdzGysoK7du3R9OmTVGpUiXY2NggPj4eR44cwaJFi/DixQts27YNHTp0wIEDB2Bqqv49skUFA0oiIiLSGzKJ91AWxjiUjx49gp2dXa7lzZs3x9ChQ9G6dWtER0fjyJEjWLhwIb755hvtF0rPMKAkIiKiIiEqKgrOzs4Fnq+iYDJb6dKlsXnzZlSqVAlpaWmYP38+A0oiIiIiXdJkHEpnZ2etDGyel7Jly6J58+bYvXs3bt++jcePH6NMGfUHZy8K2MtbDevXr0fTpk1RsmRJmJubw8PDA3369MHNm4rHtPP09IRMJsP9+/exfft2BAcHo2TJkpDJZIiIiBDTXb16FZ9++ikcHBxgaWmJatWqYebMmcjIUD5Dw9WrVzFhwgQ0bNgQLi4uMDMzg729PZo1a4aNGzcq3CYiIkKctD4tLQ3Tpk1D1apVYWlpCXt7e3Tu3BnXrl3T6BgREREVhOxOOVIe+qBKlSri80ePHumwJLrBGkoVBEFAaGgoVq1aBRMTEzRu3BiOjo44d+4cVqxYgQ0bNmDLli1o1aqVwu1nzZqFBQsWwNfXF61atcLjx49hbJw1yPPx48fRqlUrJCYmir9snj9/jrFjx+LUqVNKyzR79mwsW7YMlSpVQvXq1WFnZ4eHDx8iPDwchw4dwqlTpzB79myF26alpaFNmzaIjIxE48aNUblyZURFRWHr1q0IDw9HdHS05A5AsbGxKter0+OOiIgomwzSBqHXk3hSnJ7xY8WAUoXFixdj1apVcHBwwIEDB1CrVi0AWYHmpEmTMGnSJHTv3h03b95EqVKlcm2/cOFCbN++He3bt5dbnpKSgi+++AKJiYn49ttvMXPmTDHQvHjxIpo2bYrnz58rLNOXX36JsWPHomxZ+Zkwbty4gWbNmmHOnDn4/PPPFfZ0i4yMhI+PD+7cuQMnJyexLB07dsS+ffvw66+/YvHixZKOUfbcq0RERAXBCDIYSQjOjPQkpMw5pNDH1twNsMlbpZkzZwIAxo8fLwaTQNavkAkTJqBGjRp4/fo1li5dqnD7kJCQXMEkAGzZsgUxMTFwc3PD9OnTxWASAGrUqIEff/xRaZkCAwNzBZMAULFiRYwbNw4AsHnzZoXbymQyrFixQgwmAcDCwgKTJk0CABw8eFDp6xIRERUGWT4eunbv3j0cOHAAAODt7Q0XFxcdl6jwsYZSidjYWNy5cwdAVmD4IZlMht69e2P48OEIDw/H2LFjc6Xp2rWrwryz76P87LPPFI5VFRISguHDhyst27t377Bnzx5ER0fj+fPnSE1NBfBf8/KNGzcUbufu7o6aNWvmWl65cmUA+bvnIyYmRuX6uLg4tcYFIyIi0kc7duxA69atYWKiOGR6+vQpunTpIn4Xf/3114VZPL3BgFKJ7ODK3t4exYoVU5jG29tbLu2HlN2PmH3foZeXl8L1JUqUQPHixfHmzZtc63bs2IHevXvjxYsXSsv+9u1bhcvd3d0VLs/ev/fv3yvNUxld9KYjIqIiTM9uohw6dCjS0tLQpUsX1K9fH56enrC0tMTz588RERGBxYsXi7epNWrUCIMHD9ZugfQUA0otsrS0LND8Hj16hG7duiE5ORmjRo1Cjx494OnpCRsbGxgZGWH//v1o2bKl0lkAjIx4hwMREek3TYYN0pbHjx9j/vz5mD9/vtI0Xbp0wZ9//glzc3Otl0cfMaBUIvv+hxcvXuDt27cKaynv3r0rl1Zq3vfv31e4/vXr10prJ5OTk9GpUydMmzYt1/pbt25JKgcREZHekToUkJbjyZUrV+LIkSM4efIk7t69i+fPn+Pt27ewsbGBm5sbGjRogJCQENSvX1+7BdFzDCiVcHV1hbe3N+7cuYOwsLBco94LgoCwsDAAQJMmTSTlHRgYiGXLlmHjxo345Zdfct1HuWrVKoXbvXz5EgDg4eGRa50gCFi7dq2kchAREekbPWvxRmBgIAIDA7X8KoaPbaAqjBgxAgAwefJkXLhwQVwuCAKmTJmC8+fPw87ODv3795eUb9euXeHi4oKHDx/ihx9+QGZmprju8uXLmDJlisLtsjvPbN68WW58x4yMDIwfPx6RkZGSykFERKR3DLGbN7GG8kM57zMcOHAgIiMjsXr1avj6+iIwMFAc2PzGjRuwtLTE2rVrFY5BqYqlpSXWrFmDNm3aYNasWdi2bRv8/Pzw4sULREREoF27djh79iwePHggt127du1Qp04dnD17FhUqVEBgYCCsra3x77//4vHjxxg9erTCpnAiIiIibWIN5f9LTk4GAFhbW4vLZDIZVq1ahbVr16JRo0Y4e/YsNm/ejKSkJISGhiI6OhqtW7fO1+sFBgbi33//RefOnfHq1Sts3boVsbGx+N///ocNGzYo3MbExAQREREYO3YsXFxccOjQIURERMDHxwcnT55UOmMPERGRoZDl4490jzWUyBpmJz4+HoDioXy6d++O7t27q52fss42H6pWrRq2bNkiKQ8bGxv8/PPP+PnnnxWuV9TDOygoSGnPb1XbERERFTap83N/5DMe6g0GlADmzJkDQRBQunRp1KhRQ9fFISIi+mjpW6ccUs9HG1Bmd4i5du0aoqOjAQBTp05VOhI+aS49U0B6Rt41oVIuDkbG0i4l2vwlayzxsmZkon56qRXIaRmZeSfKQUqTkbmptP3MlFh4N3srtdOamUi7a2fxZ7lnilJGarnvL+4mKb2kGpgva0vMW9o5EqD+vkqZY1nfSDmllmbGeSfKwcpM2neHlGMu9XxKYWmq/n5aSEirEUaUBumjjZ5evnyJtWvXolixYggKCsLw4cMVzrtNREREhUcfBzanvH20AWWtWrV43yARERFRAfhoA0oiIiLSP+yUY5gYUBIREZHe4C2UhokBJREREekPRpQGiQElERER6Q12yjFMDCiJiIhIf0i8h5LxpH7g1IuFTBAEODg4wMjICC9evJBbFxUVBZlMBplMhj/++CPXtmXLloVMJsPdu3cBAA8ePMC0adMQHBwMd3d3mJubw87ODo0aNcLixYuRmal4LMKzZ8+iW7ducHV1hZmZGYoVK4ayZcuiS5cu2L59e8HvNBERERVprKEsZDKZDMHBwdi0aRMOHTqEzz77TFx38OBBuedff/21+P/du3dx7949eHl5oWzZsgCA1atXY9y4cfDy8kKFChXQsGFDxMXF4eTJkzhx4gT279+PzZs3yw2Ke+jQIbRu3RppaWmoWbMm6tevj4yMDDx69Ai7du1CRkYGOnTooPb+xMbGqlwfFxendl5ERES8hdIwMaDUgWbNmmHTpk04ePBgroDSzMwMZcuWRXh4ODIyMmBsbCyuy942W8uWLdGxY0dUq1ZNLv/Hjx+jTZs2+Pvvv7F582Z8+umn4rqff/4ZaWlp+Ouvv9CjRw+57d68eYNr165J2hc3NzdJ6YmIiPLEKNHgsMlbB7KDwpw1ksnJyYiMjET9+vXRrl07vH79GmfOnBHXKwoo/fz8cgWTAFCmTBlMnz4dALBp0ya5dU+fPgUAtGnTJtd2xYsXR7169fK7W0RERBqT5eOPdI81lDpQtmxZeHl54d69e7hz5w68vb1x7NgxvH//Hs2bN4efnx9mzJiBgwcPom7duhAEAYcPH4ZMJkPTpk3l8nr//j3279+P06dP49mzZ3j//j0EQUBCQgIA4MaNG3Lp/f39cfXqVfTo0QNjx45FvXr1NJq/PCYmRuX6uLg4+Pv75zt/IiL6uHBgc8PEgFJHmjVrhqVLl+LgwYPw9vYWayCbN2+O6tWrw9zcHAcPHsSPP/6I6OhovHjxAj4+PrC3txfzOHXqFLp164aHDx8qfZ23b9/K/f/rr7/i4sWL2LNnD/bs2QNLS0vUrl0bQUFB6NGjBypXrixpP1xdXSWlJyIioqKHTd46kt10feDAAQBZTdolSpSAr68vLC0t0aBBA0RGRiIpKUlhc3dSUhI6duyIhw8fonfv3oiKisLLly+Rnp4OQRDEmskP5yt3cnLCmTNnEB4ejh9//BF169bFuXPn8PPPP6Nq1aqYNm1aYew+ERGRQrJ8PEj3GFDqSNOmTSGTyRAeHo5nz57h/PnzaNKkCYyMsk5Js2bNkJqaiqNHjyoMKI8ePYqnT5+idu3aWL58Ofz8/FCiRAmxE8+tW7eUvrZMJkNQUBCmTJmC8PBwvHz5EgsXLoRMJsPYsWNx584dLe45ERGRCowoDRIDSh2xt7dHrVq18PLlS8yYMQOCIKB58+bi+uzgcefOnTh+/DjMzc0REBAgrn/58iUAwN3dXWH+f/31l9plsbCwwKBBg1CjRg1kZmbi4sWL+dklIiIijbFTjmFiQKlD2UHjggULAEAuoPT19YWdnR2WLVuG5ORkNGjQAJaWluL67HsdDx06hKtXr8rlu2TJEmzYsEHha86cOVPhPZfXr18XazU9PDw02CsiIqL8y+6UI+VBuseAUoeyA8qUlBR4eXnB29tbXGdkZIQmTZogJSVFLm02Hx8fdOjQAQkJCfDx8UHLli3RvXt3VK5cGYMGDcLYsWMVvuaUKVPg4eGBypUro3PnzujRoweaNGmC6tWrIzExEb169ULt2rW1tMdERESqscXbMDGg1KGAgACYm5sDyB0wfrhM0fpNmzZhxowZqFixIo4fP479+/fD3d0d+/btQ79+/RS+5u+//47evXvDxMQER44cwZYtW3Dv3j00b94cW7duRVhYWMHsHBEREX00ZMKH3YCJClBsbKw4m87FG/fh4pL3MEOmxur/3jQxlvabSIAW3+4Ss5aSXOqnNC1D8TzuyhhJaDMykvgzVGrZ0zPU38DMRFphMjLVzztTYsFNJb4XJY2zJylnyE23qg4pnwsp7xV9I+WUSr1WSL2PT0r+mdI+zjCSUJSUNPUzf/QoFjUqegLIGoO4IIeNy/ldceTcTTiVcVF72yePHyGwdgWtlIvUx3EoiYiISG9kNWOrHxUb7k+cooUBJRUiQc1f5epfHrRdwZ4hIX8pNWuAtFrEhOR0SXm/SU6TlL6ktZnaad+nS6sueZWYKim9lFpHC1NjSXkff/hc7bQN3Rwk5f3sXYqk9JWdi6md1lxiTaw2axFNtfitoe32MimHRUptNgBIrKBGhoTrRYrEz1ymhLKbSGgRktwMk19SO9owotQLDCiJiIhIb0jtaMN4Uj8woCQiIiL9wYjSILGXNxERERFphDWUWpDdw5Id6ImIiKSROvsNZ8rRDwwoiYiISG9Inf3GgEeyKlIYUBIREZHe4C2UhokBJREREekPRpQGiZ1ytGzp0qWoU6cOrK2tYWdnhzZt2uDUqVNK0798+RJjx45F1apVYWVlBVtbW9SpUwfTp09HcnJyrvQRERGQyWQICgpCUlISxo8fj8qVK8PKygqenp4AgLCwMMhkMoSGhiIxMRE//PADypUrB3Nzczg5OSEkJASPHj3S1iEgIiJSmywff6R7rKHUou+++w5z585Fw4YN0aFDB1y6dAl79uzBgQMHsHHjRnTq1Eku/d27dxEcHIwHDx6gVKlSaNOmDdLS0hAeHo7Ro0djw4YNOHjwIEqUKJHrtVJSUhAUFISrV6+icePGqFmzJl68eCGX5s2bN2jQoAEePnyIgIAAVKtWDSdPnsSqVatw5MgRXLhwAcWLF5e0j7GxsSrXx8XFScqPiIiIDA8DSi1atGgRDh48iODgYHHZjBkzMGrUKPTu3RsNGzaEo6OjuO6LL77AgwcP0L59e6xduxbW1tYAgPj4eLRq1Qrnzp3DkCFDsGbNmlyv9e+//6JGjRq4ffs2nJycFJZn27ZtaNmyJY4dO4ZixbJm6Xj16hWCg4Nx/vx5/PHHH/jhhx8k7WP23KtEREQFQQbtznVP2sEmby0aOHCgXDAJACNHjoSvry/evHmDP//8U1x+/Phx/Pvvv7CyssKSJUvEYBIASpUqhSVLlgAA1q9fr7RWcMGCBUqDSQCwtrbGihUrxGASAEqUKIExY8YAAA4ePCh9J4mIiAqQLB8P0j0GlFoUEhKicHmvXr0AZN3/mC37eatWrVC6dOlc29SpUwc1a9ZEZmYmjhw5kmu9o6MjAgICVJbH19cXzs7OuZZXrlwZAPJ1H2VMTIzKR1RUlOQ8iYjo45U9bJCUB+kem7y1yMvLS+XynDWN2cGcsm0AwNvbGxcuXFAY+GV3wFHF3d1d4fLsGsuUlJQ88/iQq6ur5G2IiIiUYzdvQ8SAUocKciYdS0vLPNMYGbFCmoiI9JzUWkfGk3qBEYYW3bt3T+Hy+/fvA5Cv3XNxcQGQ1dNbmex12WmJiIiI9AEDSi1avXq1yuVBQUHisuzne/fuxdOnT3NtEx0djfPnz8PIyAiNGzcu8LISERHpA3bKMUwMKLVo4cKFch1vAGDOnDmIioqCra0t+vbtKy5v1KgR6tati+TkZAwcOBBJSUniuufPn2PgwIEAgM8//5xD9RARUZHFTjmGifdQalH2sEEBAQFwcXHB5cuXcenSJRgbG2P58uW5hvhZu3YtgoODsX37dnh5eaFx48biwOZv375F7dq1sWDBAh3tDRERkfZJnf2GM+XoB9ZQatGcOXPwxx9/4O3bt9i2bRsePHiAVq1a4ejRo+jatWuu9GXLlsW5c+fwww8/wN7eHjt37sSBAwfg7e2NqVOn4vjx4wpnySEiIioy2OZtkGRCQXY1JvpAbGys2ER/8cY9lHHJe5ghM2P1f+cYG2n3SpIh4eORniHto5SWkal22oTkdEl5v0lOk5S+pLWZ2mnfp6tfbgB4lZgqKb2Zifrn38LUWFLexx8+VzttQzcHSXk/eydt2K3KzsXyTvT/zCUcEwAw0mIboKmJ9vLW9reRlMOSkSmtMFKvRRkSrhcpEj9zmRLKbmKsfrkfP4pFjYpZQ9vFxMQU6LBxOb8rzl29q9Z3Rc5y1a5SVivlIvWxhpKIiIiINMJ7KKnQpGcIatXiZWaq/2vc1lLaW/jpm/eS0pcqpn7NndTalfgE9WsRk95nSMr7ZOxLSekbutmrndbKXFqtoKmEGmdAWg1l6eLmkvKuXVr9W0ak1CADgKWxtOOizcp1qbVl2uzUIOleOInlkFqLKIW278tLl1D292nS3ovJqepfLxyLqf8ZMpYVTh2U1I427JSjHxhQEhERkd5gpxzDxICSiIiI9AdnXjRIDCiJiIhIbzCeNEzslFPEeHp6QiaTidM7EhERGRIObG6YGFASERERkUbY5F3EHDp0CGlpaXBxcdF1UYiIiPJBWqccNnrrBwaURYy3t7eui0BERJRvHDbIMLHJu4hRdA9lUFAQZDIZIiIicP78eXTu3BkODg4wNzdHlSpVMGvWLHDCJCIiIsov1lB+RPbt24fZs2fD29sbzZs3R1xcHI4fP44RI0YgJiYGc+fOlZxnbGysyvVxcXH5LC0REX2MZJBYQ6m1kpAUDCg/IlOnTsWiRYswcOBAcdnhw4fRrFkzLFiwACNGjJA8B2r23KtEREQFgQObGyY2eX9EOnfuLBdMAkBwcDBatmyJjIwMhIeH66hkREREZMhYQ/kRadeuncLllStXxt69e/Ho0SPJecbExKhcHxcXB39/f8n5EhHRx4mdcgwTA8qPiLu7u8LlxYoVAwCkpKRIzlNqEzkREZEq+j5TzoMHD/Dbb79h165diImJgbm5Oby9vfHZZ59h8ODBsLKyKuQS6QcGlB8RIyPe4UBERHpOjyPKHTt2oGfPnnj79q24LCkpCWfOnMGZM2fw559/YteuXShXrlzhFUpPMMIgIiIivSHLx19hiI6ORrdu3fD27VvY2Njg559/RmRkJA4dOoT+/fsDAG7evIlPPvkECQkJhVImfcIaSiIiItIb+noP5bBhw5CcnAwTExPs378f9evXF9cFBwejfPnyGDVqFG7evIlZs2Zh4sSJhVMwPcEaSiIiIiIVoqKicOzYMQBA37595YLJbN9//z0qV64MAJg3bx7S0tIKtYy6xoCSiIiI9IYsHw9t27Ztm/i8d+/eCtMYGRmhV69eAIDXr19/dEPxMaAkIiIi/aGHEeXx48cBANbW1qhTp47SdIGBgeLzEydOaL1c+oT3UBYxOefwzhYREaFym4kTJ2rtXo/09HTx+dMn6k3DaCThhhhrC2lv4ecJ7yWlT31rpnba9Axp86HHSyhLcmqGpLxfPXslKf1TE/WHjLI0M5aU97uU9LwT5WBmov7vXCHRXFLeT58nq53WxFjat9R7iefIIl39oUVMjKX99jeVOKKDdu9X0963fWamtM+clLIL0rKG1EE03qdnqp02+b2091ZKmvrp096q/xl6kuManvPaXtCePHkCKe+brPRZ1JnuNz/D3V27dg0AUK5cOZiYKP/eqVSpUq5tPhYMKEmr4uPjxeetgxvqsCRERFRQ4uPj4enpqZW8GzfI/2QY6kykIUj8tZCSkoLnz58DyDsYLVGiBKytrZGYmJjnxB9FDZu8iYiIiJTIOQSQjY1Nnumtra0BAO/evdNamfQRayhJq6pXr46oqCgAQKlSpWBiYiI3HWNUVBScnZ11WUSt+lj2lftZtHA/SZH09HSx1al69eoFmreTk5PGNXpxcXHi90xByjmLnJlZ3rdBmZtn3UaQnKz+LTZFAQNK0ioLCwv4+fkpXe/s7PzRTN/4sewr97No4X5STtpq5jYxMdH4+Gvr/FlYWIjPU1NT80z//n3W/fGWlpZaKY++YpM3ERERkRK2trbic3WasRMTEwGo1zxelDCgJCIiIlLCwsIC9vb2AIDY2FiVaV+9eiUGlG5ublovmz5hQElERESkQpUqVQAAt2/fVjlk0vXr18Xn2bPmfCwYUBIRERGp0KhRIwBZzdlnz55Vmu7IkSPi84YNP66h8hhQEhEREanQsWNH8fmKFSsUpsnMzMSqVasAAHZ2dmjSpElhFE1vMKAkIiIiUsHf3x8BAQEAgGXLluHkyZO50syaNUucHWfYsGEwNTUt1DLqGocNIiIiIsrDvHnz0LBhQyQnJ6NFixYYO3YsmjRpguTkZKxfvx5LliwBAFSoUAHff/+9jktb+GSC1DmIiIiIiD5CO3bsQM+ePfH27VuF6ytUqIBdu3ahXLlyhVwy3WNASURERKSmBw8eYN68edi1axdiY2NhZmaGcuXK4dNPP8WQIUNgZWWl6yLqBANKIiIiItIIO+UQERERkUYYUBIRERGRRhhQEhEREZFGGFASERERkUYYUBIRERGRRhhQEhEREZFGGFASERERkUYYUBIRERGRRhhQUqF68OABvv/+e1SqVAnW1tYoWbIk/Pz8MGPGDCQlJem6eBqTyWRqPYKCgnRdVKWePXuGnTt3Yvz48WjdujUcHBzEcoeGhkrOb8+ePejUqRNcXV1hbm4OV1dXdOrUCXv27Cn4wktQEPsZFham9jkPCwvT6v4oc+bMGfzvf/9DixYtxHNgY2ODChUqoHfv3jh+/Lik/PT1fBbEfhrC+STSWwJRIfnnn3+EYsWKCQAUPipUqCDcunVL18XUiLJ9+/ARGBio66IqparcISEhaueTkZEh9O3bV2V+/fr1EzIyMrS3MyoUxH6uWLFC7XO+YsUKre6PIgEBAWqVrVevXsL79+9V5qXP57Og9lPfzyeRPjORGoAS5Ud0dDS6deuG5ORk2NjY4IcffkCTJk2QnJyM9evXY+nSpbh58yY++eQTnDlzBra2trouska++uorfP3110rXW1tbF2Jp8s/d3R2VKlXC/v37JW/7448/YtmyZQAAHx8fjBo1Ct7e3rhz5w6mT5+O6Oho/PnnnyhVqhR++eWXgi66JJrsZ7Z9+/ahTJkySte7urrmO+/8evz4MQCgTJky+PTTTxEQEAB3d3dkZGTg5MmTmDVrFh49eoRVq1YhLS0Na9euVZqXPp/PgtzPbPp4Pon0mq4jWvo4ZNcgmJiYCJGRkbnWT58+XfzlP2HChMIvYAEpCvswfvx4YceOHcKTJ08EQRCEe/fuSa65u3HjhmBiYiIAEHx9fYWkpCS59YmJiYKvr6/4ntBFzXRB7GfOGq179+5pr7D59MknnwgbNmwQ0tPTFa6Pj48XKlSoIO7DkSNHFKbT9/NZUPup7+eTSJ/xHkrSuqioKBw7dgwA0LdvX9SvXz9Xmu+//x6VK1cGAMybNw9paWmFWkb6z6RJk9C2bVuULl0633nMnTsX6enpAID58+fD0tJSbr2VlRXmz58PAEhPT8ecOXPyX+B8Koj91Hc7d+7EZ599BmNjY4XrHRwcMGvWLPH/zZs3K0yn7+ezoPaTiPKPASVp3bZt28TnvXv3VpjGyMgIvXr1AgC8fv0a4eHhhVE00gJBELB9+3YAQKVKlVCvXj2F6erVq4eKFSsCALZv3w5BEAqtjPSfJk2aiM/v3LmTa31ROZ957ScRaYYBJWlddu9Ka2tr1KlTR2m6wMBA8fmJEye0Xi7Sjnv37on3tOU8p4pkr3/06BHu37+v7aKRAu/fvxefK6rhKyrnM6/9JCLNMKAkrbt27RoAoFy5cjAxUd4PrFKlSrm2MVSbNm1ClSpVYGVlBVtbW5QvXx4hISEfRc3r1atXxec5z6kiRemc9+7dG2XKlIGZmRkcHBxQr149/PTTT3j06JGui6bSkSNHxOfZt53kVFTOZ177+SFDPZ9EusKAkrQqJSUFz58/B5B3r8gSJUqIvZ9jYmK0XjZtunr1Kq5du4bk5GS8e/cOt2/fxqpVqxAcHIxOnTrhzZs3ui6i1sTGxorP8zrnbm5u4nNDP+cRERGIi4tDWloaXrx4gX///Rc///wzypUrh8WLF+u6eAplZmZi6tSp4v+fffZZrjRF4Xyqs58fMsTzSaRLHDaItCohIUF8bmNjk2d6a2trJCYm4t27d9osltZYWVmhffv2aNq0KSpVqgQbGxvEx8fjyJEjWLRoEV68eIFt27ahQ4cOOHDgAExNTXVd5AIn5ZznHD7JUM952bJl0blzZ9SvX18MqO7evYstW7Zg8+bNSElJwaBBgyCTyTBgwAAdl1benDlzEBUVBQDo3LmzwltSisL5VGc/sxny+STSJQaUpFUpKSniczMzszzTm5ubAwCSk5O1ViZtevToEezs7HItb968OYYOHYrWrVsjOjoaR44cwcKFC/HNN98UfiG1TMo5zz7fgGGe806dOiEkJAQymUxuuZ+fH7p164adO3eic+fOSEtLw/Dhw9G+fXs4OTnpqLTyjhw5gjFjxgAAHB0dsXDhQoXpDP18qrufgGGfTyJdY5M3aZWFhYX4PDU1Nc/02TfOfzgsiaFQFExmK126NDZv3izWSmYPs1LUSDnnOTtKGOI5L168eK7gI6e2bdti/PjxAICkpCRxYHBdu3LlCjp16oT09HRYWFhg06ZNcHR0VJjWkM+nlP0EDPd8EukDBpSkVTlnvFGnCSwxMRGAes3jhqhs2bJo3rw5AOD27dti79miRMo5zz7fQNE95wMGDBCDlJwdQ3Tl3r17aNGiBV69egVjY2OsX78ejRs3VpreUM+n1P1Ul76dTyJ9wYCStMrCwgL29vYA5G/uV+TVq1fiF1LOm/uLmipVqojPi2KP0ZwdN/I65zk7bhTVc+7o6Ch+BnR9vh8/foxmzZrh8ePHkMlkWL58OTp06KByG0M8n/nZT3Xp0/kk0icMKEnrsgOo27dvi7NtKHL9+nXxuTrDehgqVU1qRUHOgDnnOVWE57zwPH/+HM2bN8fdu3cBZN1ykT2ZgCqGdj7zu59S6MP5JNI3DChJ6xo1agQgqzns7NmzStPlbD5q2LCh1sulKznH9StTpowOS6IdXl5e4n7l1SR49OhRAICLiws8PT21XTSdiI+PF4fO0tX5fvPmDVq2bCm+96ZOnYrBgwerta0hnU9N9lNd+nA+ifQRA0rSuo4dO4rPV6xYoTBNZmYmVq1aBSCrY0vOadKKknv37uHAgQMAAG9vb7i4uOi4RAVPJpOJzYvXr1/HqVOnFKY7deqUWKPVoUOHIlvrs2TJEnEawrxmmtGGpKQkfPLJJzh37hwA4Mcff8To0aPV3t5Qzqem+6kuXZ9PIr0lEBWCgIAAAYBgYmIiREZG5lo/ffp0AYAAQJgwYULhF7AA/PPPP0JaWprS9U+ePBF8fHzE/Zw1a1Yhli7/7t27J5Y5JCRErW1u3LghGBsbCwAEX19fISkpSW59UlKS4OvrK74nbt68qYWSSyN1P+/duyecO3dOZZodO3YIZmZmAgDB0tJSiI2NLaDSquf9+/dCixYtxP0aNmxYvvLR9/NZEPtpCOeTSJ9xHEoqFPPmzUPDhg2RnJyMFi1aYOzYsWjSpAmSk5Oxfv16LFmyBABQoUIFfP/99zoubf4MHToUaWlp6NKlC+rXrw9PT09YWlri+fPniIiIwOLFi8WmskaNGhV4U1xBOX78OG7fvi3+n11mIOs+2LCwMLn0oaGhufKoUKECRo4cialTp+LMmTNo2LAhRo8eDW9vb9y5cwfTpk1DdHQ0AGDkyJEoX768VvZFFU338/79+2jSpAnq16+Pdu3aoWbNmuKQNHfv3sXmzZuxefNmsTZr5syZhV4j3b17d+zfvx8AEBwcjL59++Ly5ctK05uZmaFChQq5luv7+SyI/TSE80mk13Qd0dLH459//hGKFSsm1iJ8+KhQoYJw69YtXRcz3zw8PJTuW85Hly5dhFevXum6uEqFhISotR/ZD2UyMjKEPn36qNy2b9++QkZGRiHu3X803c/w8HC1trOyshIWL16sgz0UJO0fAMHDw0NpXvp8PgtiPw3hfBLpM9ZQUqFp164dLl68iHnz5mHXrl2IjY2FmZkZypUrh08//RRDhgyBlZWVrouZbytXrsSRI0dw8uRJ3L17F8+fP8fbt29hY2MDNzc3NGjQACEhIahfv76ui1oojIyMsGzZMnTp0gVLlizB6dOn8fz5czg4OMDPzw8DBw5E69atdV3MfKtTpw7++usvnDx5EmfOnEFcXByeP3+O9PR0lChRAlWrVkXTpk3Rr18/lYNpGwqez6J1PokKmkwQ/r/+noiIiIgoH9jLm4iIiIg0woCSiIiIiDTCgJKIiIiINMKAkoiIiIg0woCSiIiIiDTCgJKIiIiINMKAkoiIiIg0woCSiIiIiDTCgJKIiIiINMKAkoiIiIg0woCSiIiIiDTCgJKIiIiINMKAkoiIiIg0woCSiIiIiDTCgJKIiIiINMKAkoiIiIg0woCSiIiIiDTCgJKI9FZERARkMhlkMhkmTpyo6+IQEZESDCiJiIiISCMMKImIiIhIIwwoiYiIiEgjDCiJiIiISCMMKImoSEpMTMSGDRvQr18/1KpVC8WLF4epqSlKlSqFwMBAzJw5E+/evVO47W+//SZ2Bjp16lSer9WlSxfIZDKULFkSKSkpCtNs27YNn376Kdzd3WFhYQE7Ozv4+vpi0qRJePXqldK8Q0NDIZPJ4OnpCQCIi4vD6NGjUbVqVdja2kImkyEiIkJMn5GRgbCwMLRs2RJOTk4wMzND8eLFUb58eTRt2hS//PILrl69muc+ERFJIhAR6anw8HABgABAmDBhgqRtAwMDxW2VPby8vIRr167l2vbFixeCubm5AEAYOHCgyteJj48XTE1NBQDC4MGDc61/+fKlEBwcrLIcjo6OwsmTJxXmHxISIgAQPDw8hJMnTwoODg65tg8PDxcEQRASEhKEgICAPPe7S5cuko4lEVFeTLQfshIRFb709HRUr14d7du3h6+vL8qUKQNBEPDgwQNs3boVGzduxL1799CxY0ecP38eFhYW4rYlS5ZE586dsW7dOqxfvx5z5syBpaWlwtdZs2YN0tLSAAB9+vSRW/f+/Xs0a9YM586dg7GxMb744gu0adMGXl5eSEtLw9GjRzF79mw8e/YMbdq0QXR0NDw8PBS+zrt379ClSxekpKTgxx9/RPPmzWFlZYVLly7B2dkZADBx4kQcO3YMANC2bVv06NFDrBF99uwZoqOjsXPnTshkMo2PLxGRHF1HtEREymhSQ3nz5k2V6w8cOCAYGRkJAIQ///wz1/pDhw6Jr71mzRql+dSsWVMAINSsWTPXurFjxwoABDs7O+HMmTMKt79//77g7OwsABC++OKLXOuzaygBCDY2NsL58+eVlsXNzU0AIHTt2lVpGkHIqoElIipIvIeSiIqk8uXLq1zfrFkztG/fHkDW/Y0fatKkCby9vQEAK1asUJjHuXPncOHCBQC5ayffvXuH33//HQAwefJk1KlTR2EeHh4eGDduHABg06ZNSExMVFrmUaNGoWbNmkrXP3nyBAAQEBCgNA2QVQNLRFSQGFAS0UchPj4et27dwuXLl8VHqVKlAEAMCnOSyWRikHj48GE8fPgwV5rsQNPMzAw9evSQW3fkyBG8efMGANC1a1eVZWvcuDEAIC0tDWfPnlWa7sPX+FB20/eGDRuQlJSkMi0RUUFiQElERdaJEyfQrVs32Nvbw9HRERUqVED16tXFx9KlSwEAz58/V7h9aGgojI2NkZmZiZUrV8qte//+PdauXQsA6NChA+zt7eXWnzlzRnzu7Ows9hpX9KhWrZqYNruW8UM2NjYoW7asyv0NCQkBAERGRsLLywtDhgzB1q1bER8fr3I7IiJNMaAkoiJp4sSJaNSoETZu3IiXL1+qTJucnKxweZkyZdCmTRsAQFhYGARBENdt375dzPfD5m4AePbsWb7Kraxm0c7OLs9tx40bhz59+kAmk+HZs2f4/fff0blzZzg6OqJatWqYMGECnj59mq9yERGpwl7eRFTkHDp0CJMmTQIAlC1bFiNGjECjRo3g7u4Oa2trmJhkXfrGjx+PyZMnq8yrX79+2LFjB+7evYujR48iMDAQwH/N3a6urmjRokWu7TIyMsTn586dg6mpqVpld3V1Vbjc2Ng4z21NTU2xbNkyfP/991i3bh0OHz6MM2fOIDU1FVeuXMGVK1cwe/Zs/PXXX+jQoYNa5SEiUgcDSiIqcrKbskuUKIFTp06J90p+KK+aSwD45JNP4OzsjLi4OKxYsQKBgYF49OgR9u/fDyCrmdnIKHdjT84m8FKlSikNFLWhSpUqmDx5MiZPnoyUlBQcP34ca9euxapVq/Du3Tt0794dd+7cEe+5JCLSFJu8iajIuXLlCoCsntrKgklA/j5HZYyNjREaGgoA+L/27uYV2jYO4/gxmRphJiMbtzRkoZSwECXvspbIxhR52RDZWVjYSvEPCJOy8jYRi7FQxoiNZqFGLGbDkDIbkdI8z2Iy4R5juO5yP57vZ3XV9XbO7pjfeZ2/c3l5WXd3d3K5XIpEIjKZTOrp6Yl7X3l5eex4f3//E6P/s1JTU9Xc3Ky5uTlNTU1Jik7xb25uftuYAPw8BEoAP87T05MkJWzBc3x8rMPDw6Se19vbK5PJFNvOcWFhQVJ0dfZza6G3mpublZaWJim6lePL7y+/S1NTU+z4vYVIAPAVBEoAP85zD0qv16vz8/Pfzt/c3MjpdCb9vMLCQtXX10uSxsfHdXZ2Jin+YpxnmZmZGhoakhRddT06OqpIJPLu9dfX15qdnU16TG/d3t5qY2MjYXB9nqaXpIKCgi+/CwDeMv3zN/xtBoA4dnd31dDQICnamqe1tfXDexobG3V0dKSOjg5J0ZXaY2NjscbiPp9P09PTurq6UlVVlQ4ODiTpwwri0tLSqz6QNptNoVAoVoWM5/HxUXV1dbFKaGlpqfr7+1VWVqb09HSFw2GdnJxoZ2dH29vbKikp+W0avru7Wy6XSw6HQ8Fg8N13BYNBFRQUKD8/X21tbaqsrJTD4ZDZbFYoFNLGxoZmZ2cViUSUm5urQCCgjIyMhL8ZAJLFohwA/wlut1tut/vD69bW1tTe3q6enh7Nz8/r8vJSw8PDr65JSUnRzMyMwuFwLFB+pK2tTXa7XeFwWJLU2dmZMExKksVikcfjUXd3t1ZXV+X3+2NVy3hsNltSY0kkGAxqenr63fM5OTlyu92ESQB/FFPeAH6kubk5LS4uqqamRlarVRaLRQ6HQ06nUz6fTyMjI596XmpqaqzqKSWe7n7JarVqZWVFe3t76uvrU1FRkaxWq8xms7KyslRRUaHBwUFtbW3J4/F8akwvORwOHR0daWJiQi0tLSoqKlJmZqbMZrOys7NVW1urqakpBQKBd7eBBICvYsobAJJUXV0tn8+n4uLi2EpyAAAVSgBIyunpqXw+n6Tkq5MA8H9BoASAJExOTkqKTn0/96UEAESxKAcA4nh4eNDFxYXu7++1vr4e6z05MDDwahccAADfUAJAXC9bFj3Ly8uT3++X3W7/plEBwN+JKW8ASMBkMunXr1/q6uqS1+slTAJAHFQoAQAAYAgVSgAAABhCoAQAAIAhBEoAAAAYQqAEAACAIQRKAAAAGEKgBAAAgCEESgAAABhCoAQAAIAhBEoAAAAYQqAEAACAIQRKAAAAGEKgBAAAgCEESgAAABhCoAQAAIAhBEoAAAAYQqAEAACAIQRKAAAAGEKgBAAAgCEESgAAABjyL5/twlfFJ1MhAAAAAElFTkSuQmCC", 125 | "text/plain": [ 126 | "
" 127 | ] 128 | }, 129 | "metadata": {}, 130 | "output_type": "display_data" 131 | } 132 | ], 133 | "source": [ 134 | "from viz_tools import plot_attention_flow\n", 135 | "flow_matrix = data.all_token_contrib_norms[:, 1:data.num_prompt_tokens].T\n", 136 | "# Get token labels \n", 137 | "token_labels = data.token_labels[1:data.num_prompt_tokens]\n", 138 | "fig = plot_attention_flow(flow_matrix, token_labels, topk_prefix=14)\n", 139 | "fig.savefig(f\"./assets/sample_{item['player_name'].lower().replace(' ', '_')}.png\", bbox_inches=\"tight\")\n" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": null, 145 | "id": "37fd0d8c-4472-4869-b92e-00c8fdbc77c3", 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [] 149 | } 150 | ], 151 | "metadata": { 152 | "kernelspec": { 153 | "display_name": "Python 3 (ipykernel)", 154 | "language": "python", 155 | "name": "python3" 156 | }, 157 | "language_info": { 158 | "codemirror_mode": { 159 | "name": "ipython", 160 | "version": 3 161 | }, 162 | "file_extension": ".py", 163 | "mimetype": "text/x-python", 164 | "name": "python", 165 | "nbconvert_exporter": "python", 166 | "pygments_lexer": "ipython3", 167 | "version": "3.11.8" 168 | } 169 | }, 170 | "nbformat": 4, 171 | "nbformat_minor": 5 172 | } 173 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MICROSOFT RESEARCH LICENSE TERMS 2 | 3 | IF YOU LIVE IN THE UNITED STATES, PLEASE READ THE “BINDING ARBITRATION AND CLASS 4 | ACTION WAIVER” SECTION BELOW. IT AFFECTS HOW DISPUTES ARE RESOLVED. 5 | These license terms are an agreement between you and Microsoft Corporation (or one of its affiliates). They 6 | apply to the source code, object code, machine learning models, or data (collectively “Materials”) that 7 | accompany this license. IF YOU COMPLY WITH THESE LICENSE TERMS, YOU HAVE THE RIGHTS BELOW. BY 8 | USING THE MATERIALS, YOU ACCEPT THESE TERMS. 9 | 10 | 1) INSTALLATION AND USE RIGHTS TO THE MATERIALS. 11 | Subject to the terms of this agreement, you have the below rights, if applicable, to use the Materials solely for 12 | non-commercial, non-revenue generating, research purposes: 13 | a) Source Code. If source code is included, you may use and modify the source code, but you may not 14 | distribute the source code. 15 | b) Object Code. If object code is included, you may use the object code, but you may not distribute the 16 | object code. 17 | c) Models. If machine learning model(s) are included, you may use the model(s), but you may not 18 | distribute the models. 19 | d) Data. If data is included, you may use and modify the data, but your use and modification must be 20 | consistent with the consent under which the data was provided and/or gathered and you may not 21 | distribute the data or your modifications to the data. 22 | 2) SCOPE OF LICENSE. The Materials are licensed, not sold. Microsoft reserves all other rights. Unless 23 | applicable law gives you more rights despite this limitation, you will not (and have no right to): 24 | a) work around any technical limitations in the Materials that only allow you to use it in certain ways; 25 | b) reverse engineer, decompile or disassemble the Materials; 26 | c) remove, minimize, block, or modify any notices of Microsoft or its suppliers in the Materials; 27 | d) use the Materials in any way that is against the law or to create or propagate malware; or 28 | e) share, publish, distribute or lend the Materials, provide the Materials as a stand-alone hosted solution 29 | for others to use, or transfer the Materials or this agreement to any third party. 30 | 31 | 3) PERSONAL DATA. If the data (set forth in Section 1(c) above) includes or is found to include any data 32 | that enables any ability to identify an individual (“Personal Data”), you will not use such Personal Data for 33 | any purpose other than was authorized and consented to by the data subject/research participant. You 34 | will not use Personal Data to contact any person. You will keep Personal Data in strict confidence. You 35 | will not share any Personal Data that is collected or in your possession with any third party for any reason 36 | and as required under the original consent agreement. Further, you will destroy the Personal Data and 37 | any backup or copies, immediately upon the completion of your research. 38 | 39 | 4) LICENSE TO MICROSOFT. Notwithstanding the limitations in Section 1, you may distribute your 40 | modifications back to Microsoft, and if you do provide Microsoft with modifications of the Materials, you 41 | hereby grant Microsoft, without any restrictions or limitations, a non-exclusive, perpetual, irrevocable, 42 | royalty-free, assignable and sub-licensable license, to reproduce, publicly perform or display, install, use, 43 | modify, post, distribute, make and have made, sell and transfer such modifications and derivatives for any 44 | purpose. 45 | 46 | 5) PUBLICATION. You may publish (or present papers or articles) on your results from using the Materials 47 | provided that no material or substantial portion of the Materials is included in any such publication or 48 | presentation. 49 | 50 | 6) FEEDBACK. Any feedback about the Materials provided by you to us is voluntarily given, and Microsoft 51 | shall be free to use the feedback as it sees fit without obligation or restriction of any kind, even if the 52 | feedback is designated by you as confidential. Such feedback shall be considered a contribution and 53 | licensed to Microsoft under the terms of Section 4 above. 54 | 55 | 7) COMPLIANCE WITH TRADE LAWS. You acknowledge that the Materials may be subject to applicable 56 | trade laws in one or more countries. You will comply with all relevant laws and regulations applicable to 57 | the import or export of the Materials, including but not limited to, trade laws such as the U.S. Export 58 | Administration Regulations or other end-user, end use, and destination restrictions by the U.S. and other 59 | governments, as well as sanctions regulations administered by the U.S. Office of Foreign Assets Control. 60 | Microsoft may suspend or terminate the agreement immediately to the extent that Microsoft reasonably 61 | concludes that continued performance would violate trade laws or put it at risk of becoming subject to 62 | sanctions or penalties under trade laws. For additional information, see www.microsoft.com/exporting. 63 | 64 | 8) SUPPORT SERVICES. Microsoft is not obligated under this agreement to provide any support services for 65 | the Materials. Any support provided is “as is”, “with all faults”, and without warranty of any kind. 66 | 67 | 9) BINDING ARBITRATION AND CLASS ACTION WAIVER. This Section applies if you live in (or, if 68 | a business, your principal place of business is in) the United States. If you and Microsoft have a 69 | dispute, you and Microsoft agree to try for 60 days to resolve it informally. If you and Microsoft can’t, you 70 | and Microsoft agree to binding individual arbitration before the American Arbitration Association 71 | under the Federal Arbitration Act (“FAA”), and not to sue in court in front of a judge or jury. Instead, 72 | a neutral arbitrator will decide. Class action lawsuits, class-wide arbitrations, private attorneygeneral actions, and any other proceeding where someone acts in a representative capacity are not 73 | allowed; nor is combining individual proceedings without the consent of all parties. The complete 74 | Arbitration Agreement contains more terms and is at aka.ms/arb-agreement-1. You and Microsoft agree to 75 | these terms. 76 | 77 | 10)ENTIRE AGREEMENT. This agreement, and any other terms Microsoft may provide for supplements, 78 | updates, or third-party applications, is the entire agreement for the Materials. 79 | 80 | 11)APPLICABLE LAW AND PLACE TO RESOLVE DISPUTES. If you acquired the Materials in the United 81 | States or Canada, the laws of the state or province where you live (or, if a business, where your principal 82 | place of business is located) govern the interpretation of this agreement, claims for its breach, and all 83 | other claims (including consumer protection, unfair competition, and tort claims), regardless of conflict of 84 | laws principles, except that the FAA governs everything related to arbitration. If you acquired the Materials 85 | in any other country, its laws apply, except that the FAA governs everything related to arbitration. If U.S. 86 | federal jurisdiction exists, you and Microsoft consent to exclusive jurisdiction and venue in the federal court 87 | in King County, Washington for all disputes heard in court (excluding arbitration). If not, you and Microsoft 88 | consent to exclusive jurisdiction and venue in the Superior Court of King County, Washington for all 89 | disputes heard in court (excluding arbitration). 90 | 91 | 12)CONSUMER RIGHTS; REGIONAL VARIATIONS. This agreement describes certain legal rights. You 92 | may have other rights, including consumer rights, under the laws of your state, province, or country. 93 | Separate and apart from your relationship with Microsoft, you may also have rights with respect to the 94 | party from which you acquired the Materials. This agreement does not change those other rights if the 95 | laws of your state, province, or country do not permit it to do so. For example, if you acquired the 96 | Materials in one of the below regions, or mandatory country law applies, then the following provisions 97 | apply to you: 98 | a) Australia. You have statutory guarantees under the Australian Consumer Law and nothing in this 99 | agreement is intended to affect those rights. 100 | b) Canada. If you acquired this software in Canada, you may stop receiving updates by turning off the 101 | automatic update feature, disconnecting your device from the Internet (if and when you re-connect to 102 | the Internet, however, the Materials will resume checking for and installing updates), or uninstalling 103 | the Materials. The product documentation, if any, may also specify how to turn off updates for your 104 | specific device or software. 105 | c) Germany and Austria. 106 | i. Warranty. The properly licensed software will perform substantially as described in any 107 | Microsoft materials that accompany the Materials. However, Microsoft gives no contractual 108 | guarantee in relation to the licensed software. 109 | ii. Limitation of Liability. In case of intentional conduct, gross negligence, claims based on 110 | the Product Liability Act, as well as, in case of death or personal or physical injury, Microsoft is liable 111 | according to the statutory law. 112 | Subject to the foregoing clause (ii), Microsoft will only be liable for slight negligence if Microsoft is in 113 | breach of such material contractual obligations, the fulfillment of which facilitate the due performance 114 | of this agreement, the breach of which would endanger the purpose of this agreement and the 115 | compliance with which a party may constantly trust in (so-called "cardinal obligations"). In other cases 116 | of slight negligence, Microsoft will not be liable for slight negligence. 117 | 118 | 13)DISCLAIMER OF WARRANTY. THE MATERIALS ARE LICENSED “AS IS.” YOU BEAR THE RISK 119 | OF USING THEM. MICROSOFT GIVES NO EXPRESS WARRANTIES, GUARANTEES, OR 120 | CONDITIONS. TO THE EXTENT PERMITTED UNDER APPLICABLE LAWS, MICROSOFT 121 | EXCLUDES ALL IMPLIED WARRANTIES, INCLUDING MERCHANTABILITY, FITNESS FOR A 122 | PARTICULAR PURPOSE, AND NON-INFRINGEMENT. 123 | 124 | 14)LIMITATION ON AND EXCLUSION OF DAMAGES. IF YOU HAVE ANY BASIS FOR RECOVERING 125 | DAMAGES DESPITE THE PRECEDING DISCLAIMER OF WARRANTY, YOU CAN RECOVER FROM 126 | MICROSOFT AND ITS SUPPLIERS ONLY DIRECT DAMAGES UP TO U.S. $5.00. YOU CANNOT 127 | RECOVER ANY OTHER DAMAGES, INCLUDING CONSEQUENTIAL, LOST PROFITS, SPECIAL, 128 | INDIRECT OR INCIDENTAL DAMAGES. 129 | This limitation applies to (a) anything related to the Materials, services, content (including 130 | code) on third party Internet sites, or third party applications; and (b) claims for breach of 131 | contract, warranty, guarantee, or condition; strict liability, negligence, or other tort; or any 132 | other claim; in each case to the extent permitted by applicable law. 133 | It also applies even if Microsoft knew or should have known about the possibility of the 134 | damages. The above limitation or exclusion may not apply to you because your state, 135 | province, or country may not allow the exclusion or limitation of incidental, consequential, or 136 | other damages. 137 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | Source code for the ICLR 2024 paper: Attention Satisfies: A Constraint-Satisfaction Lens on Factual Errors of Language Models; Mert Yuksekgonul, Varun Chandrasekaran, Erik Jones, Suriya Gunasekar, Ranjita Naik, Hamid Palangi, Ece Kamar, Besmira Nushi. 3 | 4 | **Installation**:
5 | The code is written in Python 3.11, and you can use `requirements.txt` to install the required packages. 6 | ```bash 7 | conda create -n satprobe python=3.11 8 | conda activate satprobe 9 | pip install -r requirements.txt 10 | ``` 11 | 12 | ## Datasets 13 | We provide the datasets used in the paper in the `factual_queries` folder. It is as simple to load as: 14 | ```python 15 | from factual_queries import load_constraint_data 16 | items = load_constraint_data('basketball_players') 17 | print(items[0]) 18 | # {'player_name': 'Michael Jordan', 'label': 1963, 'prompt': 'User: Tell me the year the basketball player Michael Jordan was born in.\nAssistant: The player was born in', ... 19 | ``` 20 | 21 | ## Attention Tools 22 | `model_lib/attention_tools.py` contains the code for collecting attention-based metrics from the Llama-2 family of models. It can be used as follows: 23 | ```python 24 | from model_lib import HF_Llama2_Wrapper, run_attention_monitor 25 | tokenizer = transformers.AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf") 26 | model = transformers.AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf", torch_dtype=torch.bfloat16, device_map="cuda") 27 | 28 | model_wrapped = HF_Llama2_Wrapper(model, tokenizer, device="cuda") 29 | prompt_info = {"prompt": "The great Michael Jordan was born in", 30 | "constraints": [" Michael Jordan"]} 31 | data = run_attention_monitor(prompt_info, 32 | model_wrapped) 33 | ``` 34 | This `data` object will contain the attention flow information for the given prompt and constraints. You can use this object to visualize the attention flow as follows: 35 | 36 | ```python 37 | from viz_tools import plot_attention_flow 38 | # Collect the attention contribution data 39 | flow_matrix = data.all_token_contrib_norms[:, 1:data.num_prompt_tokens].T 40 | # Get token labels 41 | token_labels = data.token_labels[1:data.num_prompt_tokens] 42 | fig = plot_attention_flow(flow_matrix, token_labels) 43 | fig 44 | ``` 45 | which would produce the following visualization: 46 | ![attention_flow](./assets/sample_jordan.png) 47 | 48 | Here is another example to see the contrast with a different prompt where the model is incorrect (notice low attention on the constraint tokens): 49 | ![attention_flow2](./assets/sample_bill_behr.png) 50 | 51 | Here is a multi-constraint example: 52 | ```python 53 | prompt_info = {'prompt': "User: Is there a person who was born in the city of Portland OR and who is a Nobel Prize Winner\nAssistant: Yes, the person's name is", 54 | 'constraints': [' who is a Nobel Prize Winner', 55 | ' who was born in the city of Portland OR']} 56 | data = run_attention_monitor(prompt_info, 57 | model_wrapped) 58 | from viz_tools import plot_attention_flow 59 | # Collect the attention contribution data 60 | flow_matrix = data.all_token_contrib_norms[:, 1:data.num_prompt_tokens].T 61 | # Get token labels 62 | token_labels = data.token_labels[1:data.num_prompt_tokens] 63 | fig = plot_attention_flow(flow_matrix, token_labels, topk_prefix=24, figsize=(3, 4), title=f"Model answer: {data['completion'].split('.')[0]}") 64 | ``` 65 | 66 | And here is the visualization: 67 | 68 | ![attention_flow3](./assets/sample_mc.png) 69 | 70 | ### Detecting Factual Errors 71 | Our probing experiments have 2 main steps: 72 | - Collect attention-based metrics for a given dataset. This is done using the `main_flow_collection.py`. 73 | - Train a simple linear probe on the collected metrics. This is done using the `main_probe.py`. 74 | 75 | An example of how to use these scripts is as follows: 76 | ```bash 77 | python main_flow_collection.py --dataset_name basketball_players --model_name meta-llama/Llama-2-7b-hf --output_dir ./outputs 78 | python main_probe.py --dataset_name basketball_players --model_name meta-llama/Llama-2-7b-hf --output_dir ./outputs 79 | ``` 80 | which would save the resulting figures and probe results in the `./outputs` folder. 81 | 82 | # Contact 83 | Mert Yuksekgonul (merty@stanford.edu) 84 | Besmira Nushi (besmira.nushi@microsoft.com) 85 | 86 | # Citation 87 | If you find this repository or the ideas therein useful, please consider citing our paper: 88 | ``` 89 | @inproceedings{ 90 | yuksekgonul2024attention, 91 | title={Attention Satisfies: A Constraint-Satisfaction Lens on Factual Errors of Language Models}, 92 | author={Mert Yuksekgonul and Varun Chandrasekaran and Erik Jones and Suriya Gunasekar and Ranjita Naik and Hamid Palangi and Ece Kamar and Besmira Nushi}, 93 | booktitle={The Twelfth International Conference on Learning Representations}, 94 | year={2024}, 95 | url={https://openreview.net/forum?id=gfFVATffPd} 96 | } 97 | ``` 98 | 99 | # Contributing 100 | This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 101 | 102 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. 103 | 104 | This project has adopted the Microsoft Open Source Code of Conduct. For more information see the Code of Conduct FAQ or contact opencode@microsoft.com with any additional questions or comments. 105 | 106 | ## Trademarks 107 | 108 | This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 109 | trademarks or logos is subject to and must follow 110 | [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). 111 | Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. 112 | Any use of third-party trademarks or logos are subject to those third-party's policies. -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # TODO: The maintainer of this repo has not yet edited this file 2 | 3 | **REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? 4 | 5 | - **No CSS support:** Fill out this template with information about how to file issues and get help. 6 | - **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps. 7 | - **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide. 8 | 9 | *Then remove this first heading from this SUPPORT.MD file before publishing your repo.* 10 | 11 | # Support 12 | 13 | ## How to file issues and get help 14 | 15 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 16 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 17 | feature request as a new Issue. 18 | 19 | For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE 20 | FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER 21 | CHANNEL. WHERE WILL YOU HELP PEOPLE?**. 22 | 23 | ## Microsoft Support Policy 24 | 25 | Support for this **PROJECT or PRODUCT** is limited to the resources listed above. 26 | -------------------------------------------------------------------------------- /assets/sample_bill_behr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mechanistic-error-probe/3a1b7a5050aff2b5ccb42e3500f9ce694755c165/assets/sample_bill_behr.png -------------------------------------------------------------------------------- /assets/sample_jordan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mechanistic-error-probe/3a1b7a5050aff2b5ccb42e3500f9ce694755c165/assets/sample_jordan.png -------------------------------------------------------------------------------- /assets/sample_mc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mechanistic-error-probe/3a1b7a5050aff2b5ccb42e3500f9ce694755c165/assets/sample_mc.png -------------------------------------------------------------------------------- /factual_queries/__init__.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from .single_constraint import * 3 | from .multi_constraint import * 4 | 5 | def load_constraint_dataset(dataset_name, subsample_count=None): 6 | 7 | if dataset_name == "basketball_players": 8 | items = load_basketball_players() 9 | elif dataset_name == "football_teams": 10 | items = load_football_teams() 11 | elif dataset_name == "songs": 12 | items = load_songs() 13 | elif dataset_name == "movies": 14 | items = load_movies() 15 | elif "counterfact_" in dataset_name: 16 | items = load_counterfact_subset(dataset_name) 17 | elif dataset_name == "nobel": 18 | items = load_nobel_city() 19 | elif dataset_name == "words": 20 | items = load_word_startend() 21 | elif dataset_name == "books": 22 | items = load_books() 23 | else: 24 | raise ValueError(f"Unknown dataset {dataset_name}") 25 | 26 | if subsample_count is not None: 27 | items = np.random.choice(items, subsample_count, replace=False) 28 | 29 | return items -------------------------------------------------------------------------------- /factual_queries/data/basketball_players.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mechanistic-error-probe/3a1b7a5050aff2b5ccb42e3500f9ce694755c165/factual_queries/data/basketball_players.pkl -------------------------------------------------------------------------------- /factual_queries/data/books_multiconstraint.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mechanistic-error-probe/3a1b7a5050aff2b5ccb42e3500f9ce694755c165/factual_queries/data/books_multiconstraint.pkl -------------------------------------------------------------------------------- /factual_queries/data/counterfact_citizenship.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mechanistic-error-probe/3a1b7a5050aff2b5ccb42e3500f9ce694755c165/factual_queries/data/counterfact_citizenship.pkl -------------------------------------------------------------------------------- /factual_queries/data/counterfact_headquarter_location.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mechanistic-error-probe/3a1b7a5050aff2b5ccb42e3500f9ce694755c165/factual_queries/data/counterfact_headquarter_location.pkl -------------------------------------------------------------------------------- /factual_queries/data/counterfact_mother_tongue.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mechanistic-error-probe/3a1b7a5050aff2b5ccb42e3500f9ce694755c165/factual_queries/data/counterfact_mother_tongue.pkl -------------------------------------------------------------------------------- /factual_queries/data/football_teams.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mechanistic-error-probe/3a1b7a5050aff2b5ccb42e3500f9ce694755c165/factual_queries/data/football_teams.pkl -------------------------------------------------------------------------------- /factual_queries/data/movies.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mechanistic-error-probe/3a1b7a5050aff2b5ccb42e3500f9ce694755c165/factual_queries/data/movies.pkl -------------------------------------------------------------------------------- /factual_queries/data/nobel_multiconstraint.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mechanistic-error-probe/3a1b7a5050aff2b5ccb42e3500f9ce694755c165/factual_queries/data/nobel_multiconstraint.pkl -------------------------------------------------------------------------------- /factual_queries/data/schools.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mechanistic-error-probe/3a1b7a5050aff2b5ccb42e3500f9ce694755c165/factual_queries/data/schools.pkl -------------------------------------------------------------------------------- /factual_queries/data/songs.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mechanistic-error-probe/3a1b7a5050aff2b5ccb42e3500f9ce694755c165/factual_queries/data/songs.pkl -------------------------------------------------------------------------------- /factual_queries/data/word_multiconstraint.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mechanistic-error-probe/3a1b7a5050aff2b5ccb42e3500f9ce694755c165/factual_queries/data/word_multiconstraint.pkl -------------------------------------------------------------------------------- /factual_queries/multi_constraint.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import numpy as np 3 | 4 | ### Multi-constraint datasets 5 | 6 | def load_nobel_city(): 7 | filename = "./factual_queries/data/nobel_multiconstraint.pkl" 8 | items = pickle.load(open(filename, "rb")) 9 | return items 10 | 11 | def load_word_startend(): 12 | filename = "./factual_queries/data/word_multiconstraint.pkl" 13 | items = pickle.load(open(filename, "rb")) 14 | return items 15 | 16 | def load_books(): 17 | filename = "./factual_queries/data/books_multiconstraint.pkl" 18 | items = pickle.load(open(filename, "rb")) 19 | return items 20 | -------------------------------------------------------------------------------- /factual_queries/single_constraint.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import numpy as np 3 | 4 | def load_basketball_players(): 5 | with open("./factual_queries/data/basketball_players.pkl", "rb") as f: 6 | items = pickle.load(f) 7 | 8 | prompt_template = "Tell me the year the basketball player {} was born in." 9 | prompt_fn = lambda prompt: f"User: {prompt}\nAssistant: The player was born in" 10 | for item in items: 11 | item["constraint"] = item["player_name"] 12 | item["prompt"] = prompt_fn(prompt_template.format(item["constraint"])) 13 | item["label"] = item["birth_year"] 14 | item["popularity"] = item["popularity"] 15 | return items 16 | 17 | 18 | def load_football_teams(): 19 | with open("./factual_queries/data/football_teams.pkl", "rb") as f: 20 | items = pickle.load(f) 21 | 22 | prompt_template = "Tell me the year the football team {} was founded in." 23 | prompt_fn = lambda prompt: f"User: {prompt}\nAssistant: The team was founded in" 24 | for item in items: 25 | item["constraint"] = item["team_name"] 26 | item["prompt"] = prompt_fn(prompt_template.format(item["constraint"])) 27 | item["label"] = item["founding_year"] 28 | item["popularity"] = item["popularity"] 29 | return items 30 | 31 | 32 | def load_songs(): 33 | with open("./factual_queries/data/songs.pkl", "rb") as f: 34 | items = pickle.load(f) 35 | 36 | prompt_template = "Tell me the performer of the song {}" 37 | prompt_fn = lambda prompt: f"User: {prompt}\nAssistant: The performer is" 38 | for item in items: 39 | item["constraint"] = item["song_name"] 40 | item["prompt"] = prompt_fn(prompt_template.format(item["constraint"])) 41 | item["label"] = item["artist_name"] 42 | item["popularity"] = item["popularity"] 43 | return items 44 | 45 | 46 | def load_movies(): 47 | with open("./factual_queries/data/movies.pkl", "rb") as f: 48 | items = pickle.load(f) 49 | prompt_template = "Tell me the director of the movie {}." 50 | prompt_fn = lambda prompt: f"User: {prompt}\nAssistant: The director is" 51 | for item in items: 52 | item["constraint"] = item["movie_name"] 53 | item["prompt"] = prompt_fn(prompt_template.format(item["constraint"])) 54 | item["label"] = item["director_name"] 55 | return items 56 | 57 | def load_counterfact_subset(subset): 58 | filename = f"./factual_queries/data/{subset}.pkl" 59 | items = pickle.load(open(filename, "rb")) 60 | return items -------------------------------------------------------------------------------- /factual_queries/verifiers/.ipynb_checkpoints/Verifier Books -checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "d08d7f60-209f-4464-a8a6-0507b36d2bf1", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import requests\n", 11 | "import os\n", 12 | "import openai\n", 13 | "import pandas as pd\n", 14 | "from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor\n", 15 | "from tenacity import retry, stop_after_attempt, wait_fixed\n", 16 | "\n", 17 | "openai.api_type = \"azure\"\n", 18 | "openai.api_version = \"2023-07-01-preview\"\n", 19 | "engine = \"GPT35\"\n", 20 | "\n", 21 | "from tenacity import (\n", 22 | " retry,\n", 23 | " stop_after_attempt,\n", 24 | " wait_chain,\n", 25 | " wait_fixed\n", 26 | ") \n", 27 | "\n", 28 | "@retry(wait=wait_chain(*[wait_fixed(3) for i in range(3)] +\n", 29 | " [wait_fixed(5) for i in range(2)] +\n", 30 | " [wait_fixed(10)]))\n", 31 | "def completion_with_backoff(**kwargs):\n", 32 | " return openai.ChatCompletion.create(**kwargs)\n", 33 | "\n", 34 | "\n", 35 | "def verify_entity_exists(response, list_in_txt):\n", 36 | " system_prompt = \"You are an AI assistant that helps the user verify whether an entity is captured in a list.\"\n", 37 | " prompt = \"\"\"I will give you one entity and one list, and I want you to respond with \\\"YES\\\" if the entity is within the list, \\\"NO\\\" if it is not in the list. \n", 38 | " \n", 39 | " List: \n", 40 | " {}\n", 41 | " Entity: {}\n", 42 | " Give your response in the following format: \n", 43 | " `Reference in the list: {{item in the list if exists, None otherwise}}\n", 44 | " Answer: {{YES or NO}}` and say nothing else.\n", 45 | " \"\"\"\n", 46 | " response = completion_with_backoff(\n", 47 | " engine=engine,\n", 48 | " messages = [{\"role\":\"system\",\"content\":system_prompt},\n", 49 | " {\"role\":\"user\",\"content\":prompt.format(list_in_txt, response)},],\n", 50 | " temperature=0,\n", 51 | " max_tokens=25,\n", 52 | " top_p=0.95,\n", 53 | " frequency_penalty=0,\n", 54 | " presence_penalty=0,\n", 55 | " stop=None)\n", 56 | " \n", 57 | "\n", 58 | " try:\n", 59 | " result = response[\"choices\"][0][\"message\"][\"content\"]\n", 60 | " lines = result.split(\"\\n\")\n", 61 | " reference = lines[0].split(\": \")[1]\n", 62 | " answer = lines[1].split(\": \")[1]\n", 63 | " return reference, answer\n", 64 | " except:\n", 65 | " print(prompt.format(list_in_txt, response))\n", 66 | " print(\"--------------------------------\")\n", 67 | " return None, None\n", 68 | " return result\n" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": null, 74 | "id": "32fcae4b", 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "import pickle\n", 79 | "\n", 80 | "def save_output(records, correctness, output_file):\n", 81 | " records_final = []\n", 82 | "\n", 83 | " for i in range(len(records['prompt'])):\n", 84 | " prompt = records['prompt'][i]\n", 85 | " completion = records['completion'][i]\n", 86 | " record = (prompt, completion, [correctness[i][0], correctness[i][1]])\n", 87 | " records_final.append(record)\n", 88 | "\n", 89 | " # Save records_final as a pkl file\n", 90 | " with open(output_file, 'wb') as file:\n", 91 | " pickle.dump(records_final, file)\n" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "id": "d958a125", 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "import requests\n", 102 | "import requests\n", 103 | "from datetime import datetime\n", 104 | "\n", 105 | "def query_wikidata(book_name):\n", 106 | " book_name = book_name.replace('\"', '')\n", 107 | " url = 'https://query.wikidata.org/sparql' \n", 108 | " query = '''\n", 109 | " SELECT ?bookLabel ?authorLabel (YEAR(?publicationDate) as ?publishing_year) WHERE {{\n", 110 | " ?book rdfs:label \"{0}\"@en .\n", 111 | " ?book wdt:P50 ?author .\n", 112 | " ?book wdt:P577 ?publicationDate .\n", 113 | " \n", 114 | " SERVICE wikibase:label {{\n", 115 | " bd:serviceParam wikibase:language \"[AUTO_LANGUAGE],en\" .\n", 116 | " }}\n", 117 | " }}\n", 118 | " '''.format(book_name)\n", 119 | " \n", 120 | " # Pass the query as a URL-encoded string in the params parameter\n", 121 | " params = {\n", 122 | " 'format': 'json',\n", 123 | " 'query': query\n", 124 | " }\n", 125 | " \n", 126 | " # Make the HTTP GET request\n", 127 | " response = requests.get(url, params=params)\n", 128 | " \n", 129 | " # Check if the request was successful (status code 200)\n", 130 | " if response.status_code == 200:\n", 131 | " data = response.json()\n", 132 | " return data\n", 133 | " else:\n", 134 | " print(f\"Error: {response.status_code}\")\n", 135 | " return None\n", 136 | " \n", 137 | "def organize_data(data):\n", 138 | " organized_data = []\n", 139 | " \n", 140 | " try:\n", 141 | " if data is not None:\n", 142 | " for item in data['results']['bindings']:\n", 143 | " person = item['authorLabel']['value']\n", 144 | " publishing_year = item['publishing_year']['value']\n", 145 | " book = item['bookLabel']['value']\n", 146 | "\n", 147 | " organized_data.append({\n", 148 | " \"Author\": person,\n", 149 | " \"Publishing_Year\" : publishing_year,\n", 150 | " \"Book\": book,\n", 151 | " })\n", 152 | " except:\n", 153 | " print(\"wiki extraction failed\")\n", 154 | " \n", 155 | " return organized_data\n", 156 | "\n", 157 | "def get_book_info(book_name):\n", 158 | " book_data = query_wikidata(book_name)\n", 159 | " organized_data = organize_data(book_data)\n", 160 | "\n", 161 | " matching_authors = []\n", 162 | " matching_publishing_years = []\n", 163 | " for entry in organized_data:\n", 164 | "\n", 165 | " if entry['Book'].lower().strip() == book_name.lower().strip():\n", 166 | " if entry['Author'] not in matching_authors:\n", 167 | " matching_authors.append(entry['Author'])\n", 168 | " \n", 169 | " if entry['Publishing_Year'] not in matching_publishing_years: \n", 170 | " matching_publishing_years.append(entry['Publishing_Year'])\n", 171 | " \n", 172 | " return matching_authors, matching_publishing_years\n", 173 | "\n", 174 | " " 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": null, 180 | "id": "07974b0a", 181 | "metadata": {}, 182 | "outputs": [], 183 | "source": [ 184 | "import time\n", 185 | "def remove_delimiter(text):\n", 186 | " text = text.replace('\"','')\n", 187 | " delimiters = ['.', ',']\n", 188 | " for delimiter in delimiters:\n", 189 | " if text.endswith(delimiter):\n", 190 | " return text[:-len(delimiter)] \n", 191 | " return text\n", 192 | "\n", 193 | "def verify_books(records, debug_file):\n", 194 | " \n", 195 | " correctness = np.zeros((len(records[\"prompt\"]), 2))\n", 196 | " with open(debug_file,\"w\") as fd:\n", 197 | " for i in range(len(records[\"prompt\"])):\n", 198 | " time.sleep(5)\n", 199 | "\n", 200 | " constraints = records[\"name\"][i]\n", 201 | " completion = records[\"completion\"][i]\n", 202 | " book_name = remove_delimiter(completion.split(\"\\n\")[0].strip())\n", 203 | " fd.write(f\"Book name from the completion : {book_name}\"+ \"\\n\")\n", 204 | " matching_authors, matching_publishing_years = get_book_info(book_name) \n", 205 | "\n", 206 | " fd.write(\"wiki data:\" + \"\\n\")\n", 207 | " fd.write(f\"author : {matching_authors}\"+ \"\\n\")\n", 208 | " fd.write(f\"publishing year : {matching_publishing_years}\"+ \"\\n\")\n", 209 | " fd.write(\"..................................\\n\")\n", 210 | "\n", 211 | " for constraint in constraints: \n", 212 | " if \"written by\" in constraint and len(matching_authors) > 0 :\n", 213 | " state_reference, state_answer = verify_entity_exists(constraint, matching_authors)\n", 214 | "\n", 215 | " if state_answer is not None:\n", 216 | " correctness[i][0] = (1 if state_answer.lower() == \"yes\" else 0)\n", 217 | " fd.write(f\"author constraint : {constraint} \\n\")\n", 218 | " fd.write(f\"Turbo author reference : {state_reference} \\n\")\n", 219 | " fd.write(f\"Turbo author answer : {state_answer} \\n\")\n", 220 | " fd.write(\".................................\\n\")\n", 221 | "\n", 222 | " else:\n", 223 | " if len(matching_publishing_years) > 0 :\n", 224 | " #correctness[i][0] - holds the awards constraint outcome\n", 225 | " state_reference, state_answer = verify_entity_exists(constraint, matching_publishing_years)\n", 226 | " if state_answer is not None:\n", 227 | " correctness[i][1] = (1 if state_answer.lower() == \"yes\" else 0)\n", 228 | " fd.write(f\"publishing year constraint: {constraint}\"+ \"\\n\")\n", 229 | " fd.write(f\"Turbo publishing year reference : {state_reference}\"+ \"\\n\")\n", 230 | " fd.write(f\"Turbo publishing year answer : {state_answer} \\n\")\n", 231 | " fd.write(\"..............................\\n\")\n", 232 | " fd.write(\"===============================================\\n\")\n", 233 | " return correctness" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": null, 239 | "id": "f145aa6d-5ec8-4cca-98ec-adcc61ca82a8", 240 | "metadata": {}, 241 | "outputs": [], 242 | "source": [ 243 | "import pickle\n", 244 | "import os\n", 245 | "import numpy as np\n", 246 | "from easydict import EasyDict as edict\n", 247 | "\n", 248 | "data_pretty = {\n", 249 | " \"books\": \"Books\", \n", 250 | "}\n", 251 | "result_records = []\n", 252 | "for model_size in [\"7b\", \"13b\", \"70b\"]:\n", 253 | " for data_name in data_pretty:\n", 254 | " filename = f\"./outputs/Llama-2-{model_size}-hf_{data_name}_localized_track.pkl\"\n", 255 | " output_file = f\"./outputs/Llama-2-{model_size}-hf_{data_name}_localized_track.pkl\"\n", 256 | " debug_file = f\".outputs/Llama-2-{model_size}-hf_{data_name}_localized_track.debug.txt\"\n", 257 | "\n", 258 | " \n", 259 | " if not os.path.exists(filename):\n", 260 | " print(filename)\n", 261 | " continue\n", 262 | " records_to_save = edict(pickle.load(open(filename, \"rb\")))\n", 263 | " records = records_to_save\n", 264 | " correctness = verify_books(records, debug_file) \n", 265 | " save_output(records, correctness, output_file )\n" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": null, 271 | "id": "15c6a16e-f8a8-464d-ade1-03f4cdbbee06", 272 | "metadata": {}, 273 | "outputs": [], 274 | "source": [ 275 | "## Finally \n", 276 | "## records = [(prompt, completion, [0, 0])]" 277 | ] 278 | } 279 | ], 280 | "metadata": { 281 | "kernelspec": { 282 | "display_name": "Python 3 (ipykernel)", 283 | "language": "python", 284 | "name": "python3" 285 | }, 286 | "language_info": { 287 | "codemirror_mode": { 288 | "name": "ipython", 289 | "version": 3 290 | }, 291 | "file_extension": ".py", 292 | "mimetype": "text/x-python", 293 | "name": "python", 294 | "nbconvert_exporter": "python", 295 | "pygments_lexer": "ipython3", 296 | "version": "3.11.8" 297 | } 298 | }, 299 | "nbformat": 4, 300 | "nbformat_minor": 5 301 | } 302 | -------------------------------------------------------------------------------- /factual_queries/verifiers/.ipynb_checkpoints/Verifier Nobel Winner-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "d08d7f60-209f-4464-a8a6-0507b36d2bf1", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import requests\n", 11 | "import os\n", 12 | "import openai\n", 13 | "import pandas as pd\n", 14 | "from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor\n", 15 | "from tenacity import retry, stop_after_attempt, wait_fixed\n", 16 | "\n", 17 | "\n", 18 | "openai.api_type = \"azure\"\n", 19 | "openai.api_version = \"2023-07-01-preview\"\n", 20 | "openai.api_key = \"YOURAPIKEY\" \n", 21 | "engine = \"GPT35\"\n", 22 | "\n", 23 | "from tenacity import (\n", 24 | " retry,\n", 25 | " stop_after_attempt,\n", 26 | " wait_chain,\n", 27 | " wait_fixed\n", 28 | ") \n", 29 | "\n", 30 | "@retry(wait=wait_chain(*[wait_fixed(3) for i in range(3)] +\n", 31 | " [wait_fixed(5) for i in range(2)] +\n", 32 | " [wait_fixed(10)]))\n", 33 | "def completion_with_backoff(**kwargs):\n", 34 | " return openai.ChatCompletion.create(**kwargs)\n", 35 | "\n", 36 | "\n", 37 | "def verify_entity_exists(response, list_in_txt):\n", 38 | " system_prompt = \"You are an AI assistant that helps the user verify whether an entity is captured in a list.\"\n", 39 | " prompt = \"\"\"I will give you one entity and one list, and I want you to respond with \\\"YES\\\" if the entity is within the list, \\\"NO\\\" if it is not in the list. \n", 40 | " \n", 41 | " List: \n", 42 | " {}\n", 43 | " Entity: {}\n", 44 | " Give your response in the following format: \n", 45 | " `Reference in the list: {{item in the list if exists, None otherwise}}\n", 46 | " Answer: {{YES or NO}}` and say nothing else.\n", 47 | " \"\"\"\n", 48 | " response = completion_with_backoff(\n", 49 | " engine=engine,\n", 50 | " messages = [{\"role\":\"system\",\"content\":system_prompt},\n", 51 | " {\"role\":\"user\",\"content\":prompt.format(list_in_txt, response)},],\n", 52 | " temperature=0,\n", 53 | " max_tokens=25,\n", 54 | " top_p=0.95,\n", 55 | " frequency_penalty=0,\n", 56 | " presence_penalty=0,\n", 57 | " stop=None)\n", 58 | " result = response[\"choices\"][0][\"message\"][\"content\"]\n", 59 | "\n", 60 | " try:\n", 61 | " lines = result.split(\"\\n\")\n", 62 | " reference = lines[0].split(\": \")[1]\n", 63 | " answer = lines[1].split(\": \")[1]\n", 64 | " return reference, answer\n", 65 | " except:\n", 66 | " return None, None\n", 67 | " return result\n" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": null, 73 | "id": "32fcae4b", 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "import pickle\n", 78 | "\n", 79 | "def save_output(records, correctness, output_file):\n", 80 | " records_final = []\n", 81 | "\n", 82 | " for i in range(len(records['prompt'])):\n", 83 | " prompt = records['prompt'][i]\n", 84 | " completion = records['completion'][i]\n", 85 | " record = (prompt, completion, [correctness[i][0], correctness[i][1]])\n", 86 | " records_final.append(record)\n", 87 | "\n", 88 | " # Save records_final as a pkl file\n", 89 | " with open(output_file, 'wb') as file:\n", 90 | " pickle.dump(records_final, file)\n" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "id": "23fbc1d8", 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | "def remove_delimiter(name):\n", 101 | " delimiters = ['.', ',']\n", 102 | " for delimiter in delimiters:\n", 103 | " if name.endswith(delimiter):\n", 104 | " return name[:-len(delimiter)]\n", 105 | " return name\n" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "id": "8eec6f57", 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "def query_wikidata(person_name):\n", 116 | " url = 'https://query.wikidata.org/sparql'\n", 117 | " query = '''\n", 118 | " SELECT DISTINCT ?personLabel ?awardLabel ?cityLabel WHERE {{\n", 119 | " ?person rdfs:label \"{0}\"@en .\n", 120 | " ?person wdt:P166 ?award .\n", 121 | " ?person wdt:P19 ?city .\n", 122 | " \n", 123 | " SERVICE wikibase:label {{\n", 124 | " bd:serviceParam wikibase:language \"[AUTO_LANGUAGE],en\" .\n", 125 | " }}\n", 126 | " }}\n", 127 | " '''.format(person_name)\n", 128 | " \n", 129 | " try:\n", 130 | " \n", 131 | " response = requests.get(url, params={'format': 'json', 'query': query})\n", 132 | " data = response.json()\n", 133 | " \n", 134 | " except:\n", 135 | " print(\"wiki query failed\")\n", 136 | " return None\n", 137 | " return data\n", 138 | "\n", 139 | "\n", 140 | "def organize_data(data):\n", 141 | " organized_data = []\n", 142 | " \n", 143 | " if data is not None:\n", 144 | " for item in data['results']['bindings']:\n", 145 | " person = item['personLabel']['value']\n", 146 | " award = item['awardLabel']['value']\n", 147 | " city = item['cityLabel']['value']\n", 148 | "\n", 149 | " organized_data.append({\n", 150 | " \"Person\": person,\n", 151 | " \"Award\" : award,\n", 152 | " \"City\": city,\n", 153 | " })\n", 154 | " return organized_data\n", 155 | "\n", 156 | "def get_nobel_info(person):\n", 157 | "\n", 158 | " nobel_winner_data = query_wikidata(person)\n", 159 | " nobel_winner_organized_data = organize_data(nobel_winner_data)\n", 160 | "\n", 161 | " matching_awards = []\n", 162 | " matching_cities = []\n", 163 | " for entry in nobel_winner_organized_data:\n", 164 | " if entry['Person'].lower() == person.lower():\n", 165 | " if entry['Award'] not in matching_awards:\n", 166 | " matching_awards.append(entry['Award'])\n", 167 | " \n", 168 | " if entry['City'] not in matching_cities: \n", 169 | " matching_cities.append(entry['City'])\n", 170 | " \n", 171 | " return matching_awards, matching_cities\n", 172 | " " 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "id": "aeaa2113", 179 | "metadata": {}, 180 | "outputs": [], 181 | "source": [ 182 | "import time\n", 183 | "\n", 184 | "def verify_nobel_winner(records, debug_file):\n", 185 | " \n", 186 | " correctness = np.zeros((len(records[\"prompt\"]), 2))\n", 187 | " \n", 188 | " with open(debug_file, \"w\") as fd:\n", 189 | " for i in range(len(records[\"prompt\"])):\n", 190 | " time.sleep(5)\n", 191 | " completion = records[\"completion\"][i]\n", 192 | " completion = completion.split(\"\\n\")[0]\n", 193 | " completion = remove_delimiter(completion).strip() \n", 194 | " awards, cities = get_nobel_info(completion)\n", 195 | " fd.write(f\"Completion : {completion}\\n\")\n", 196 | " fd.write(f\"Awards retrieved from wiki : {awards}\\n\")\n", 197 | " fd.write(f\"City retrieved from wiki : {cities}\\n\")\n", 198 | "\n", 199 | " for constraint in records[\"name\"][i]:\n", 200 | " if \"was born\" in constraint.lower(): \n", 201 | " city_reference, city_answer = verify_entity_exists(constraint, cities) \n", 202 | "\n", 203 | " fd.write(f\"City constraint : {constraint}\\n\")\n", 204 | " fd.write(f\"Turbo City refernce : {city_reference}\\n\")\n", 205 | " fd.write(f\"Turbo City answer : {city_answer}\\n\")\n", 206 | " fd.write(f\"=====================================\\n\")\n", 207 | " if city_answer is not None:\n", 208 | " correctness[i][1] = (1 if city_answer.lower() == \"yes\" else 0)\n", 209 | " else:\n", 210 | " award_reference, award_answer = verify_entity_exists(constraint, awards) \n", 211 | "\n", 212 | " fd.write(f\"award constraint : {constraint}\\n\")\n", 213 | " fd.write(f\"Turbo award refernce : {award_reference}\\n\")\n", 214 | " fd.write(f\"Turbo award answer : {award_answer}\\n\")\n", 215 | " fd.write(f\"=====================================\\n\")\n", 216 | " if award_answer is not None:\n", 217 | " correctness[i][0] = (1 if award_answer.lower() == \"yes\" else 0) \n", 218 | "\n", 219 | " return correctness" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": null, 225 | "id": "dab44708", 226 | "metadata": {}, 227 | "outputs": [], 228 | "source": [ 229 | "import pickle\n", 230 | "import os\n", 231 | "import numpy as np\n", 232 | "from easydict import EasyDict as edict\n", 233 | "\n", 234 | "data_pretty = {\n", 235 | " \"nobel_city\": \"Nobel Winners\"\n", 236 | "}\n", 237 | "result_records = []\n", 238 | "for model_size in [\"7b\", \"13b\", \"70b\"]:\n", 239 | " for data_name in data_pretty:\n", 240 | " filename = f\"./outputs/Llama-2-{model_size}-hf_{data_name}_localized_track.pkl\"\n", 241 | " output_file = f\"./outputs/Llama-2-{model_size}-hf_{data_name}_localized_track.pkl\"\n", 242 | " debug_file = f\"./outputs/Llama-2-{model_size}-hf_{data_name}_localized_track.debug\"\n", 243 | "\n", 244 | " if not os.path.exists(filename):\n", 245 | " print(filename)\n", 246 | " continue\n", 247 | " records_to_save = edict(pickle.load(open(filename, \"rb\")))\n", 248 | " records = records_to_save\n", 249 | " correctness = verify_nobel_winner(records, debug_file) \n", 250 | " save_output(records, correctness, output_file )" 251 | ] 252 | } 253 | ], 254 | "metadata": { 255 | "kernelspec": { 256 | "display_name": "Python 3 (ipykernel)", 257 | "language": "python", 258 | "name": "python3" 259 | }, 260 | "language_info": { 261 | "codemirror_mode": { 262 | "name": "ipython", 263 | "version": 3 264 | }, 265 | "file_extension": ".py", 266 | "mimetype": "text/x-python", 267 | "name": "python", 268 | "nbconvert_exporter": "python", 269 | "pygments_lexer": "ipython3", 270 | "version": "3.11.8" 271 | } 272 | }, 273 | "nbformat": 4, 274 | "nbformat_minor": 5 275 | } 276 | -------------------------------------------------------------------------------- /factual_queries/verifiers/Verifier Books .ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "d08d7f60-209f-4464-a8a6-0507b36d2bf1", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import requests\n", 11 | "import os\n", 12 | "import openai\n", 13 | "import pandas as pd\n", 14 | "from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor\n", 15 | "from tenacity import retry, stop_after_attempt, wait_fixed\n", 16 | "\n", 17 | "openai.api_type = \"azure\"\n", 18 | "openai.api_version = \"2023-07-01-preview\"\n", 19 | "engine = \"GPT35\"\n", 20 | "\n", 21 | "from tenacity import (\n", 22 | " retry,\n", 23 | " stop_after_attempt,\n", 24 | " wait_chain,\n", 25 | " wait_fixed\n", 26 | ") \n", 27 | "\n", 28 | "@retry(wait=wait_chain(*[wait_fixed(3) for i in range(3)] +\n", 29 | " [wait_fixed(5) for i in range(2)] +\n", 30 | " [wait_fixed(10)]))\n", 31 | "def completion_with_backoff(**kwargs):\n", 32 | " return openai.ChatCompletion.create(**kwargs)\n", 33 | "\n", 34 | "\n", 35 | "def verify_entity_exists(response, list_in_txt):\n", 36 | " system_prompt = \"You are an AI assistant that helps the user verify whether an entity is captured in a list.\"\n", 37 | " prompt = \"\"\"I will give you one entity and one list, and I want you to respond with \\\"YES\\\" if the entity is within the list, \\\"NO\\\" if it is not in the list. \n", 38 | " \n", 39 | " List: \n", 40 | " {}\n", 41 | " Entity: {}\n", 42 | " Give your response in the following format: \n", 43 | " `Reference in the list: {{item in the list if exists, None otherwise}}\n", 44 | " Answer: {{YES or NO}}` and say nothing else.\n", 45 | " \"\"\"\n", 46 | " response = completion_with_backoff(\n", 47 | " engine=engine,\n", 48 | " messages = [{\"role\":\"system\",\"content\":system_prompt},\n", 49 | " {\"role\":\"user\",\"content\":prompt.format(list_in_txt, response)},],\n", 50 | " temperature=0,\n", 51 | " max_tokens=25,\n", 52 | " top_p=0.95,\n", 53 | " frequency_penalty=0,\n", 54 | " presence_penalty=0,\n", 55 | " stop=None)\n", 56 | " \n", 57 | "\n", 58 | " try:\n", 59 | " result = response[\"choices\"][0][\"message\"][\"content\"]\n", 60 | " lines = result.split(\"\\n\")\n", 61 | " reference = lines[0].split(\": \")[1]\n", 62 | " answer = lines[1].split(\": \")[1]\n", 63 | " return reference, answer\n", 64 | " except:\n", 65 | " print(prompt.format(list_in_txt, response))\n", 66 | " print(\"--------------------------------\")\n", 67 | " return None, None\n", 68 | " return result\n" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": null, 74 | "id": "32fcae4b", 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "import pickle\n", 79 | "\n", 80 | "def save_output(records, correctness, output_file):\n", 81 | " records_final = []\n", 82 | "\n", 83 | " for i in range(len(records['prompt'])):\n", 84 | " prompt = records['prompt'][i]\n", 85 | " completion = records['completion'][i]\n", 86 | " record = (prompt, completion, [correctness[i][0], correctness[i][1]])\n", 87 | " records_final.append(record)\n", 88 | "\n", 89 | " # Save records_final as a pkl file\n", 90 | " with open(output_file, 'wb') as file:\n", 91 | " pickle.dump(records_final, file)\n" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "id": "d958a125", 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "import requests\n", 102 | "import requests\n", 103 | "from datetime import datetime\n", 104 | "\n", 105 | "def query_wikidata(book_name):\n", 106 | " book_name = book_name.replace('\"', '')\n", 107 | " url = 'https://query.wikidata.org/sparql' \n", 108 | " query = '''\n", 109 | " SELECT ?bookLabel ?authorLabel (YEAR(?publicationDate) as ?publishing_year) WHERE {{\n", 110 | " ?book rdfs:label \"{0}\"@en .\n", 111 | " ?book wdt:P50 ?author .\n", 112 | " ?book wdt:P577 ?publicationDate .\n", 113 | " \n", 114 | " SERVICE wikibase:label {{\n", 115 | " bd:serviceParam wikibase:language \"[AUTO_LANGUAGE],en\" .\n", 116 | " }}\n", 117 | " }}\n", 118 | " '''.format(book_name)\n", 119 | " \n", 120 | " # Pass the query as a URL-encoded string in the params parameter\n", 121 | " params = {\n", 122 | " 'format': 'json',\n", 123 | " 'query': query\n", 124 | " }\n", 125 | " \n", 126 | " # Make the HTTP GET request\n", 127 | " response = requests.get(url, params=params)\n", 128 | " \n", 129 | " # Check if the request was successful (status code 200)\n", 130 | " if response.status_code == 200:\n", 131 | " data = response.json()\n", 132 | " return data\n", 133 | " else:\n", 134 | " print(f\"Error: {response.status_code}\")\n", 135 | " return None\n", 136 | " \n", 137 | "def organize_data(data):\n", 138 | " organized_data = []\n", 139 | " \n", 140 | " try:\n", 141 | " if data is not None:\n", 142 | " for item in data['results']['bindings']:\n", 143 | " person = item['authorLabel']['value']\n", 144 | " publishing_year = item['publishing_year']['value']\n", 145 | " book = item['bookLabel']['value']\n", 146 | "\n", 147 | " organized_data.append({\n", 148 | " \"Author\": person,\n", 149 | " \"Publishing_Year\" : publishing_year,\n", 150 | " \"Book\": book,\n", 151 | " })\n", 152 | " except:\n", 153 | " print(\"wiki extraction failed\")\n", 154 | " \n", 155 | " return organized_data\n", 156 | "\n", 157 | "def get_book_info(book_name):\n", 158 | " book_data = query_wikidata(book_name)\n", 159 | " organized_data = organize_data(book_data)\n", 160 | "\n", 161 | " matching_authors = []\n", 162 | " matching_publishing_years = []\n", 163 | " for entry in organized_data:\n", 164 | "\n", 165 | " if entry['Book'].lower().strip() == book_name.lower().strip():\n", 166 | " if entry['Author'] not in matching_authors:\n", 167 | " matching_authors.append(entry['Author'])\n", 168 | " \n", 169 | " if entry['Publishing_Year'] not in matching_publishing_years: \n", 170 | " matching_publishing_years.append(entry['Publishing_Year'])\n", 171 | " \n", 172 | " return matching_authors, matching_publishing_years\n", 173 | "\n", 174 | " " 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": null, 180 | "id": "07974b0a", 181 | "metadata": {}, 182 | "outputs": [], 183 | "source": [ 184 | "import time\n", 185 | "def remove_delimiter(text):\n", 186 | " text = text.replace('\"','')\n", 187 | " delimiters = ['.', ',']\n", 188 | " for delimiter in delimiters:\n", 189 | " if text.endswith(delimiter):\n", 190 | " return text[:-len(delimiter)] \n", 191 | " return text\n", 192 | "\n", 193 | "def verify_books(records, debug_file):\n", 194 | " \n", 195 | " correctness = np.zeros((len(records[\"prompt\"]), 2))\n", 196 | " with open(debug_file,\"w\") as fd:\n", 197 | " for i in range(len(records[\"prompt\"])):\n", 198 | " time.sleep(5)\n", 199 | "\n", 200 | " constraints = records[\"name\"][i]\n", 201 | " completion = records[\"completion\"][i]\n", 202 | " book_name = remove_delimiter(completion.split(\"\\n\")[0].strip())\n", 203 | " fd.write(f\"Book name from the completion : {book_name}\"+ \"\\n\")\n", 204 | " matching_authors, matching_publishing_years = get_book_info(book_name) \n", 205 | "\n", 206 | " fd.write(\"wiki data:\" + \"\\n\")\n", 207 | " fd.write(f\"author : {matching_authors}\"+ \"\\n\")\n", 208 | " fd.write(f\"publishing year : {matching_publishing_years}\"+ \"\\n\")\n", 209 | " fd.write(\"..................................\\n\")\n", 210 | "\n", 211 | " for constraint in constraints: \n", 212 | " if \"written by\" in constraint and len(matching_authors) > 0 :\n", 213 | " state_reference, state_answer = verify_entity_exists(constraint, matching_authors)\n", 214 | "\n", 215 | " if state_answer is not None:\n", 216 | " correctness[i][0] = (1 if state_answer.lower() == \"yes\" else 0)\n", 217 | " fd.write(f\"author constraint : {constraint} \\n\")\n", 218 | " fd.write(f\"Turbo author reference : {state_reference} \\n\")\n", 219 | " fd.write(f\"Turbo author answer : {state_answer} \\n\")\n", 220 | " fd.write(\".................................\\n\")\n", 221 | "\n", 222 | " else:\n", 223 | " if len(matching_publishing_years) > 0 :\n", 224 | " #correctness[i][0] - holds the awards constraint outcome\n", 225 | " state_reference, state_answer = verify_entity_exists(constraint, matching_publishing_years)\n", 226 | " if state_answer is not None:\n", 227 | " correctness[i][1] = (1 if state_answer.lower() == \"yes\" else 0)\n", 228 | " fd.write(f\"publishing year constraint: {constraint}\"+ \"\\n\")\n", 229 | " fd.write(f\"Turbo publishing year reference : {state_reference}\"+ \"\\n\")\n", 230 | " fd.write(f\"Turbo publishing year answer : {state_answer} \\n\")\n", 231 | " fd.write(\"..............................\\n\")\n", 232 | " fd.write(\"===============================================\\n\")\n", 233 | " return correctness" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": null, 239 | "id": "f145aa6d-5ec8-4cca-98ec-adcc61ca82a8", 240 | "metadata": {}, 241 | "outputs": [], 242 | "source": [ 243 | "import pickle\n", 244 | "import os\n", 245 | "import numpy as np\n", 246 | "from easydict import EasyDict as edict\n", 247 | "\n", 248 | "data_pretty = {\n", 249 | " \"books\": \"Books\", \n", 250 | "}\n", 251 | "result_records = []\n", 252 | "for model_size in [\"7b\", \"13b\", \"70b\"]:\n", 253 | " for data_name in data_pretty:\n", 254 | " filename = f\"./outputs/Llama-2-{model_size}-hf_{data_name}_localized_track.pkl\"\n", 255 | " output_file = f\"./outputs/Llama-2-{model_size}-hf_{data_name}_localized_track.pkl\"\n", 256 | " debug_file = f\".outputs/Llama-2-{model_size}-hf_{data_name}_localized_track.debug.txt\"\n", 257 | "\n", 258 | " \n", 259 | " if not os.path.exists(filename):\n", 260 | " print(filename)\n", 261 | " continue\n", 262 | " records_to_save = edict(pickle.load(open(filename, \"rb\")))\n", 263 | " records = records_to_save\n", 264 | " correctness = verify_books(records, debug_file) \n", 265 | " save_output(records, correctness, output_file )\n" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": null, 271 | "id": "15c6a16e-f8a8-464d-ade1-03f4cdbbee06", 272 | "metadata": {}, 273 | "outputs": [], 274 | "source": [ 275 | "## Finally \n", 276 | "## records = [(prompt, completion, [0, 0])]" 277 | ] 278 | } 279 | ], 280 | "metadata": { 281 | "kernelspec": { 282 | "display_name": "Python 3 (ipykernel)", 283 | "language": "python", 284 | "name": "python3" 285 | }, 286 | "language_info": { 287 | "codemirror_mode": { 288 | "name": "ipython", 289 | "version": 3 290 | }, 291 | "file_extension": ".py", 292 | "mimetype": "text/x-python", 293 | "name": "python", 294 | "nbconvert_exporter": "python", 295 | "pygments_lexer": "ipython3", 296 | "version": "3.11.8" 297 | } 298 | }, 299 | "nbformat": 4, 300 | "nbformat_minor": 5 301 | } 302 | -------------------------------------------------------------------------------- /factual_queries/verifiers/Verifier Nobel Winner.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "d08d7f60-209f-4464-a8a6-0507b36d2bf1", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import requests\n", 11 | "import os\n", 12 | "import openai\n", 13 | "import pandas as pd\n", 14 | "from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor\n", 15 | "from tenacity import retry, stop_after_attempt, wait_fixed\n", 16 | "\n", 17 | "\n", 18 | "openai.api_type = \"azure\"\n", 19 | "openai.api_version = \"2023-07-01-preview\"\n", 20 | "openai.api_key = \"YOURAPIKEY\" \n", 21 | "engine = \"GPT35\"\n", 22 | "\n", 23 | "from tenacity import (\n", 24 | " retry,\n", 25 | " stop_after_attempt,\n", 26 | " wait_chain,\n", 27 | " wait_fixed\n", 28 | ") \n", 29 | "\n", 30 | "@retry(wait=wait_chain(*[wait_fixed(3) for i in range(3)] +\n", 31 | " [wait_fixed(5) for i in range(2)] +\n", 32 | " [wait_fixed(10)]))\n", 33 | "def completion_with_backoff(**kwargs):\n", 34 | " return openai.ChatCompletion.create(**kwargs)\n", 35 | "\n", 36 | "\n", 37 | "def verify_entity_exists(response, list_in_txt):\n", 38 | " system_prompt = \"You are an AI assistant that helps the user verify whether an entity is captured in a list.\"\n", 39 | " prompt = \"\"\"I will give you one entity and one list, and I want you to respond with \\\"YES\\\" if the entity is within the list, \\\"NO\\\" if it is not in the list. \n", 40 | " \n", 41 | " List: \n", 42 | " {}\n", 43 | " Entity: {}\n", 44 | " Give your response in the following format: \n", 45 | " `Reference in the list: {{item in the list if exists, None otherwise}}\n", 46 | " Answer: {{YES or NO}}` and say nothing else.\n", 47 | " \"\"\"\n", 48 | " response = completion_with_backoff(\n", 49 | " engine=engine,\n", 50 | " messages = [{\"role\":\"system\",\"content\":system_prompt},\n", 51 | " {\"role\":\"user\",\"content\":prompt.format(list_in_txt, response)},],\n", 52 | " temperature=0,\n", 53 | " max_tokens=25,\n", 54 | " top_p=0.95,\n", 55 | " frequency_penalty=0,\n", 56 | " presence_penalty=0,\n", 57 | " stop=None)\n", 58 | " result = response[\"choices\"][0][\"message\"][\"content\"]\n", 59 | "\n", 60 | " try:\n", 61 | " lines = result.split(\"\\n\")\n", 62 | " reference = lines[0].split(\": \")[1]\n", 63 | " answer = lines[1].split(\": \")[1]\n", 64 | " return reference, answer\n", 65 | " except:\n", 66 | " return None, None\n", 67 | " return result\n" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": null, 73 | "id": "32fcae4b", 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "import pickle\n", 78 | "\n", 79 | "def save_output(records, correctness, output_file):\n", 80 | " records_final = []\n", 81 | "\n", 82 | " for i in range(len(records['prompt'])):\n", 83 | " prompt = records['prompt'][i]\n", 84 | " completion = records['completion'][i]\n", 85 | " record = (prompt, completion, [correctness[i][0], correctness[i][1]])\n", 86 | " records_final.append(record)\n", 87 | "\n", 88 | " # Save records_final as a pkl file\n", 89 | " with open(output_file, 'wb') as file:\n", 90 | " pickle.dump(records_final, file)\n" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "id": "23fbc1d8", 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | "def remove_delimiter(name):\n", 101 | " delimiters = ['.', ',']\n", 102 | " for delimiter in delimiters:\n", 103 | " if name.endswith(delimiter):\n", 104 | " return name[:-len(delimiter)]\n", 105 | " return name\n" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "id": "8eec6f57", 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "def query_wikidata(person_name):\n", 116 | " url = 'https://query.wikidata.org/sparql'\n", 117 | " query = '''\n", 118 | " SELECT DISTINCT ?personLabel ?awardLabel ?cityLabel WHERE {{\n", 119 | " ?person rdfs:label \"{0}\"@en .\n", 120 | " ?person wdt:P166 ?award .\n", 121 | " ?person wdt:P19 ?city .\n", 122 | " \n", 123 | " SERVICE wikibase:label {{\n", 124 | " bd:serviceParam wikibase:language \"[AUTO_LANGUAGE],en\" .\n", 125 | " }}\n", 126 | " }}\n", 127 | " '''.format(person_name)\n", 128 | " \n", 129 | " try:\n", 130 | " \n", 131 | " response = requests.get(url, params={'format': 'json', 'query': query})\n", 132 | " data = response.json()\n", 133 | " \n", 134 | " except:\n", 135 | " print(\"wiki query failed\")\n", 136 | " return None\n", 137 | " return data\n", 138 | "\n", 139 | "\n", 140 | "def organize_data(data):\n", 141 | " organized_data = []\n", 142 | " \n", 143 | " if data is not None:\n", 144 | " for item in data['results']['bindings']:\n", 145 | " person = item['personLabel']['value']\n", 146 | " award = item['awardLabel']['value']\n", 147 | " city = item['cityLabel']['value']\n", 148 | "\n", 149 | " organized_data.append({\n", 150 | " \"Person\": person,\n", 151 | " \"Award\" : award,\n", 152 | " \"City\": city,\n", 153 | " })\n", 154 | " return organized_data\n", 155 | "\n", 156 | "def get_nobel_info(person):\n", 157 | "\n", 158 | " nobel_winner_data = query_wikidata(person)\n", 159 | " nobel_winner_organized_data = organize_data(nobel_winner_data)\n", 160 | "\n", 161 | " matching_awards = []\n", 162 | " matching_cities = []\n", 163 | " for entry in nobel_winner_organized_data:\n", 164 | " if entry['Person'].lower() == person.lower():\n", 165 | " if entry['Award'] not in matching_awards:\n", 166 | " matching_awards.append(entry['Award'])\n", 167 | " \n", 168 | " if entry['City'] not in matching_cities: \n", 169 | " matching_cities.append(entry['City'])\n", 170 | " \n", 171 | " return matching_awards, matching_cities\n", 172 | " " 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "id": "aeaa2113", 179 | "metadata": {}, 180 | "outputs": [], 181 | "source": [ 182 | "import time\n", 183 | "\n", 184 | "def verify_nobel_winner(records, debug_file):\n", 185 | " \n", 186 | " correctness = np.zeros((len(records[\"prompt\"]), 2))\n", 187 | " \n", 188 | " with open(debug_file, \"w\") as fd:\n", 189 | " for i in range(len(records[\"prompt\"])):\n", 190 | " time.sleep(5)\n", 191 | " completion = records[\"completion\"][i]\n", 192 | " completion = completion.split(\"\\n\")[0]\n", 193 | " completion = remove_delimiter(completion).strip() \n", 194 | " awards, cities = get_nobel_info(completion)\n", 195 | " fd.write(f\"Completion : {completion}\\n\")\n", 196 | " fd.write(f\"Awards retrieved from wiki : {awards}\\n\")\n", 197 | " fd.write(f\"City retrieved from wiki : {cities}\\n\")\n", 198 | "\n", 199 | " for constraint in records[\"name\"][i]:\n", 200 | " if \"was born\" in constraint.lower(): \n", 201 | " city_reference, city_answer = verify_entity_exists(constraint, cities) \n", 202 | "\n", 203 | " fd.write(f\"City constraint : {constraint}\\n\")\n", 204 | " fd.write(f\"Turbo City refernce : {city_reference}\\n\")\n", 205 | " fd.write(f\"Turbo City answer : {city_answer}\\n\")\n", 206 | " fd.write(f\"=====================================\\n\")\n", 207 | " if city_answer is not None:\n", 208 | " correctness[i][1] = (1 if city_answer.lower() == \"yes\" else 0)\n", 209 | " else:\n", 210 | " award_reference, award_answer = verify_entity_exists(constraint, awards) \n", 211 | "\n", 212 | " fd.write(f\"award constraint : {constraint}\\n\")\n", 213 | " fd.write(f\"Turbo award refernce : {award_reference}\\n\")\n", 214 | " fd.write(f\"Turbo award answer : {award_answer}\\n\")\n", 215 | " fd.write(f\"=====================================\\n\")\n", 216 | " if award_answer is not None:\n", 217 | " correctness[i][0] = (1 if award_answer.lower() == \"yes\" else 0) \n", 218 | "\n", 219 | " return correctness" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": null, 225 | "id": "dab44708", 226 | "metadata": {}, 227 | "outputs": [], 228 | "source": [ 229 | "import pickle\n", 230 | "import os\n", 231 | "import numpy as np\n", 232 | "from easydict import EasyDict as edict\n", 233 | "\n", 234 | "data_pretty = {\n", 235 | " \"nobel_city\": \"Nobel Winners\"\n", 236 | "}\n", 237 | "result_records = []\n", 238 | "for model_size in [\"7b\", \"13b\", \"70b\"]:\n", 239 | " for data_name in data_pretty:\n", 240 | " filename = f\"./outputs/Llama-2-{model_size}-hf_{data_name}_localized_track.pkl\"\n", 241 | " output_file = f\"./outputs/Llama-2-{model_size}-hf_{data_name}_localized_track.pkl\"\n", 242 | " debug_file = f\"./outputs/Llama-2-{model_size}-hf_{data_name}_localized_track.debug\"\n", 243 | "\n", 244 | " if not os.path.exists(filename):\n", 245 | " print(filename)\n", 246 | " continue\n", 247 | " records_to_save = edict(pickle.load(open(filename, \"rb\")))\n", 248 | " records = records_to_save\n", 249 | " correctness = verify_nobel_winner(records, debug_file) \n", 250 | " save_output(records, correctness, output_file )" 251 | ] 252 | } 253 | ], 254 | "metadata": { 255 | "kernelspec": { 256 | "display_name": "Python 3 (ipykernel)", 257 | "language": "python", 258 | "name": "python3" 259 | }, 260 | "language_info": { 261 | "codemirror_mode": { 262 | "name": "ipython", 263 | "version": 3 264 | }, 265 | "file_extension": ".py", 266 | "mimetype": "text/x-python", 267 | "name": "python", 268 | "nbconvert_exporter": "python", 269 | "pygments_lexer": "ipython3", 270 | "version": "3.11.8" 271 | } 272 | }, 273 | "nbformat": 4, 274 | "nbformat_minor": 5 275 | } 276 | -------------------------------------------------------------------------------- /main_flow_collection.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | import torch 4 | import pickle 5 | import transformers 6 | 7 | from tqdm import tqdm 8 | from easydict import EasyDict as edict 9 | 10 | from model_lib.hf_tooling import HF_Llama2_Wrapper 11 | from model_lib.attention_tools import run_attention_monitor 12 | from factual_queries import load_constraint_dataset 13 | 14 | 15 | def config(): 16 | parser = argparse.ArgumentParser() 17 | parser.add_argument("--model_name", type=str, default="meta-llama/Llama-2-7b-hf") 18 | parser.add_argument("--max_new_tokens", type=int, default=15, help="Number of tokens to generate for each prompt.") 19 | parser.add_argument("--dataset-name", type=str, default="basketball_players") 20 | parser.add_argument("--load-in-8bit", action="store_true", help="Whether to load the model in 8-bit mode. We used this only for Llama-2 70B.") 21 | parser.add_argument("--subsample-count", type=int, default=None, help="Number of items to run for, mostly for testing mode.") 22 | parser.add_argument("--output-dir", type=str, default="./outputs", help="Output directory to save the attention flow.") 23 | return parser.parse_args() 24 | 25 | args = config() 26 | 27 | ## Load the model and tokenizer 28 | tokenizer = transformers.AutoTokenizer.from_pretrained(args.model_name) 29 | model = transformers.AutoModelForCausalLM.from_pretrained(args.model_name, 30 | trust_remote_code=True, 31 | torch_dtype=torch.bfloat16, 32 | load_in_8bit=args.load_in_8bit, 33 | device_map="auto") 34 | model_wrapped = HF_Llama2_Wrapper(model, tokenizer, device="cuda") 35 | 36 | items = load_constraint_dataset(args.dataset_name, subsample_count=args.subsample_count) 37 | 38 | print(f"Will run for {len(items)} items") 39 | 40 | records_to_save = edict({"token_contrib_norms_constraints": [], "attention_weights_constraints": [], 41 | "gt_logprob": [], "pred_logprob": [], "constraint": [], "prompt": [], "popularity": [], 42 | }) 43 | 44 | for item in tqdm(items): 45 | prompt_info = {"prompt": item["prompt"]} 46 | if "constraints" in item: 47 | prompt_info["constraints"] = item["constraints"] 48 | else: 49 | prompt_info["constraints"] = [f" {item['constraint']}"] 50 | 51 | data = run_attention_monitor(prompt_info, 52 | model_wrapped, args.max_new_tokens) 53 | 54 | print(item, data["completion"]) 55 | # Completing the likelihoods of the completion vs the ground truth 56 | if "label" in item: 57 | # Multi-constraint data do not have pre-defined labels. We run different verifiers. 58 | ground_truth = " " + str(item["label"]) 59 | completion_len_gt = len(model_wrapped.tokenizer(ground_truth)["input_ids"][1:]) # Offset by 1 to account for 60 | completion_tokenized = model_wrapped.tokenizer(data["completion"]) 61 | completion_cut = tokenizer.decode(completion_tokenized["input_ids"][1:1+completion_len_gt]) 62 | 63 | prompts_logprob = [item["prompt"] + ground_truth, item["prompt"] + completion_cut] 64 | completion_offset = torch.tensor([completion_len_gt, completion_len_gt]) 65 | _, loglikelihoods = model_wrapped.get_conditional_loglikelihood_batch(texts=prompts_logprob, 66 | completion_offset=completion_offset) 67 | records_to_save.gt_logprob.append(loglikelihoods[0]) 68 | records_to_save.pred_logprob.append(loglikelihoods[1]) 69 | 70 | # Saving these records for later probing analysis. 71 | records_to_save.attention_weights_constraints.append(data.attention_weights_constraints) 72 | records_to_save.token_contrib_norms_constraints.append(data.token_contrib_norms_constraints) 73 | records_to_save.popularity.append(item["popularity"]) 74 | 75 | records_to_save.prompt.append(item["prompt"]) 76 | records_to_save.constraint.append(prompt_info["constraints"]) 77 | 78 | os.makedirs(args.output_dir, exist_ok=True) 79 | output_file = os.path.join(args.output_dir, f"{args.model_name.split('/')[-1]}_{args.dataset_name}.pkl") 80 | with open(output_file, "wb") as f: 81 | pickle.dump(records_to_save, f) 82 | -------------------------------------------------------------------------------- /main_probe.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import argparse 4 | import numpy as np 5 | import pandas as pd 6 | import seaborn as sns 7 | import matplotlib.pyplot as plt 8 | 9 | from collections import defaultdict 10 | from sklearn.metrics import roc_auc_score 11 | from sklearn.model_selection import train_test_split 12 | from sklearn.preprocessing import StandardScaler 13 | from sklearn.linear_model import LogisticRegression 14 | 15 | 16 | def config(): 17 | parser = argparse.ArgumentParser() 18 | parser.add_argument("--model_name", type=str, default="meta-llama/Llama-2-7b-hf") 19 | parser.add_argument("--dataset-name", type=str, default="basketball_players") 20 | parser.add_argument("--output-dir", type=str, default="./outputs", help="Output directory where the attention flow is saved.") 21 | return parser.parse_args() 22 | 23 | args = config() 24 | 25 | def extract_predictors(records): 26 | token_contribs = records.token_contrib_norms_constraints 27 | attention_weights = records.attention_weights_constraints 28 | indices = np.arange(len(attention_weights)) 29 | corr = [] 30 | predictors = defaultdict(list) 31 | 32 | for token_head_norms, idx in zip(token_contribs, indices): 33 | for constraint_idx in range(len(records["constraint"][idx])): 34 | constraint_contributions = token_head_norms[constraint_idx] 35 | constraint_weights = attention_weights[idx][constraint_idx] 36 | corr.append(int(records["gt_logprob"][idx]==records["pred_logprob"][idx])) 37 | predictors[r"$||a_{C,T}^{\ell, [h]}||$"].append(constraint_contributions.max(axis=2).reshape((1, -1))) 38 | predictors[r"$||A_{C,T}^{\ell, [h]}||$"].append(constraint_weights.max(axis=2).reshape((1, -1))) 39 | 40 | predictors = {k: np.array(v)[:, 0] for k,v in predictors.items()} 41 | predictors[r"$\hat{P}(\hat{Y}|X)$"] = np.array(records["pred_logprob"]) 42 | y = np.array(corr) 43 | predictors["Majority"] = np.ones(y.shape[0]) if np.mean(y) > 0.5 else np.zeros(y.shape[0]) 44 | predictors["Popularity"] = np.array(records.popularity) 45 | predictors["Combined"] = np.concatenate([predictors[r"$||A_{C,T}^{\ell, [h]}||$"], predictors[r"$\hat{P}(\hat{Y}|X)$"].reshape(-1, 1)], axis=1) 46 | return predictors, y 47 | 48 | def get_metrics(y, score): 49 | roc_auc = roc_auc_score(y, score) 50 | bottom20_idx = np.argsort(score)[:int(score.shape[0]*0.2)] 51 | top20_idx = np.argsort(-score)[:int(score.shape[0]*0.2)] 52 | risk_at_top20 = 1-y[top20_idx].mean() 53 | risk_at_bottom20 = 1-y[bottom20_idx].mean() 54 | 55 | return {r"AUROC$\textcolor{Green}{\mathbf{(\Uparrow)}}$": roc_auc, 56 | r"$\text{Risk}_{\textrm{Top 20\%}}(\textcolor{Blue}{\mathbf{\Downarrow}})$": risk_at_top20, 57 | r"$\text{Risk}_{\textrm{Bottom 20\%}}(\textcolor{Green}{\mathbf{\Uparrow}})$":risk_at_bottom20} 58 | 59 | 60 | output_file = os.path.join(args.output_dir, f"{args.model_name.split('/')[-1]}_{args.dataset_name}.pkl") 61 | records = pickle.load(open(output_file, "rb")) 62 | predictors, y = extract_predictors(records) 63 | performance_records = [] 64 | seed = 0 65 | train_idx, test_idx = train_test_split(np.arange(y.shape[0]), test_size=0.5, stratify=y, random_state=seed) 66 | 67 | for predictor, X in predictors.items(): 68 | X_train, y_train, X_test, y_test = X[train_idx], y[train_idx], X[test_idx], y[test_idx] 69 | 70 | if predictor == '$\\hat{P}(\\hat{Y}|X)$': 71 | performance = get_metrics(y_test, X_test) 72 | elif predictor in ['Majority', 'Popularity']: 73 | performance = get_metrics(y_test, X_test) 74 | else: 75 | ss = StandardScaler() 76 | X_train = ss.fit_transform(X_train) 77 | X_test = ss.transform(X_test) 78 | 79 | performance = get_metrics(y_test, LogisticRegression(penalty="l1", solver="liblinear", C=0.05, max_iter=1000).fit(X_train, y_train).predict_proba(X_test)[:, 1]) 80 | for metric, val in performance.items(): 81 | performance_records.append({"Data": args.dataset_name, 82 | "BaseRate": y_test.mean(), 83 | "Predictor": predictor, 84 | "seed": seed, 85 | "Metric": metric, 86 | "Value": val 87 | }) 88 | 89 | # Rendering figures 90 | df_results = pd.DataFrame(performance_records) 91 | metric_to_plot = r"AUROC$\textcolor{Green}{\mathbf{(\Uparrow)}}$" 92 | metrics_fig = [metric_to_plot] 93 | df_results = df_results[df_results.Metric.isin(metrics_fig)] 94 | predictors_fig = ['$||A_{C,T}^{\ell, [h]}||$', 'Majority', r"$\hat{P}(\hat{Y}|X)$", "Popularity", "Combined"] 95 | df_results = df_results[df_results.Predictor.isin(predictors_fig)] 96 | 97 | name_map = { 98 | r"$\hat{P}(\hat{Y}|X)$": r"$\textsc{Confidence}$", 99 | '$||A_{C,T}^{\ell, [h]}||$': r"$\textsc{SAT-Probe}$", 100 | "Majority": r"$\textsc{Constant}$", 101 | "Popularity": r"$\textsc{Popularity}$", 102 | "Combined": r"$\textsc{Combined}$" 103 | } 104 | df_results['Predictor'] = df_results['Predictor'].apply(lambda x: name_map[x]) 105 | name_map_rev= {v: k for k,v in name_map.items()} 106 | color_map = { 107 | r"$\hat{P}(\hat{Y}|X)$": 'firebrick', # Lighter green 108 | 'Majority': 'darkorange', # Lighter orange 109 | '$||A_{C,T}^{\ell, [h]}||$': 'royalblue', # Lighter blue, 110 | "Popularity": '#BEBEBE', 111 | "Combined": "turquoise" 112 | } 113 | name_to_color = {k: color_map[name_map_rev[k]] for k, v in name_map_rev.items()} 114 | 115 | plt.rcParams['text.usetex'] = True 116 | 117 | custom_order = [r"$\textsc{SAT-Probe}$", r"$\textsc{Confidence}$", r"$\textsc{Popularity}$", r"$\textsc{Constant}$", r"$\textsc{Combined}$"] 118 | 119 | fig, ax = plt.subplots(figsize=(5, 3), dpi=200) 120 | 121 | sns.barplot(data=df_results, y="Data", x="Value", hue="Predictor", ax=ax, palette=name_to_color, hue_order=custom_order) 122 | ax.legend(loc="lower right", fontsize="medium", framealpha=0.9) 123 | ax.set_xlabel("AUROC") 124 | ax.set_title(args.model_name) 125 | fig.tight_layout() 126 | figure_path = os.path.join(args.output_dir, f"{args.model_name.split('/')[-1]}_{args.dataset_name}_performance.png") 127 | fig.savefig(figure_path) -------------------------------------------------------------------------------- /model_lib/__init__.py: -------------------------------------------------------------------------------- 1 | from .attention_tools import run_attention_monitor 2 | from .hf_tooling import HF_Llama2_Wrapper 3 | from .hooks import TraceDict 4 | from .misc_utils import repeat_kv, find_within_text -------------------------------------------------------------------------------- /model_lib/attention_tools.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from tqdm import tqdm 4 | from easydict import EasyDict as edict 5 | 6 | from .hooks import TraceDict 7 | from .misc_utils import repeat_kv, find_within_text 8 | 9 | @torch.no_grad() 10 | def run_attention_monitor(prompt_info, 11 | model_wrapped, 12 | max_new_tokens=15): 13 | """ 14 | Args: 15 | prompt_info: should have a "prompt" and "constraints" field. 16 | model_wrapped : A model object using our wrapper: model_lib.hf_tooling.HF_LM 17 | max_new_tokens (int, optional): Maximum number of tokens to sample. Defaults to 15. 18 | 19 | Returns: 20 | A list of dictionaries, each containing the attention weights, hidden states, and other relevant information. 21 | """ 22 | o_proj_matrices = model_wrapped.get_output_projection_matrices() 23 | 24 | model_config = model_wrapped.model.config 25 | num_key_value_groups = model_config.num_attention_heads // model_config.num_key_value_heads 26 | attention_layers = [(i, f"model.layers.{i}.self_attn") for i in range(model_wrapped.model.config.num_hidden_layers)] 27 | # mlp_layers = [(i, f"model.layers.{i}.mlp") for i in range(model_wrapped.model.config.num_hidden_layers)] 28 | all_layers = attention_layers # + mlp_layers 29 | 30 | prompt = prompt_info["prompt"] 31 | with torch.no_grad(): 32 | # Sample the model's completion 33 | completion = model_wrapped.sample([prompt], max_new_tokens=max_new_tokens)[0] 34 | 35 | with torch.no_grad(), TraceDict(model_wrapped.model, [l[1] for l in all_layers]) as ret: 36 | # Run the full text through the model. 37 | inputs = model_wrapped.tokenizer(prompt+completion, return_tensors="pt").to(model_wrapped.device) 38 | outs = model_wrapped.model(input_ids=inputs.input_ids, 39 | attention_mask=inputs.attention_mask, 40 | output_hidden_states=True, 41 | output_attentions=True) 42 | num_prompt_tokens = len(model_wrapped.tokenizer.encode(prompt)) 43 | 44 | att_weights, proj_contributions, token_contribs = [], [], [] 45 | for (l, layername) in all_layers: 46 | if "self_attn" in layername: 47 | o_proj = o_proj_matrices[l] 48 | rep_layer = ret[layername].output 49 | # A^{i,j}_{l} will be H x T x T 50 | att_weights.append(rep_layer[1].cpu().float().numpy()[0]) 51 | # past key value states 52 | pkv = rep_layer[2][1][0] 53 | # compute |A_{i, j}*h_{j}*W_{v}*W_{o}| 54 | # Which will be H x T x D 55 | pkv = repeat_kv(pkv, num_key_value_groups)[0] 56 | proj_contributions.append(torch.einsum("HDd, HTd->HTD", o_proj, pkv).detach().cpu().float().numpy()) 57 | token_contribs.append((att_weights[-1][:, num_prompt_tokens-1, :, np.newaxis]*proj_contributions[-1][:, :, :]).sum(axis=0)) 58 | 59 | #elif "mlp" in layername: 60 | # We played around with MLP contributions as well, but it's not included in the final version. 61 | # mlp_contribs = ret[layername].output 62 | # mlps.append(mlp_contribs.cpu().float().numpy()[0]) 63 | 64 | data = edict({ 65 | "completion": completion, 66 | "full_prompt": prompt+completion, 67 | "num_prompt_tokens": num_prompt_tokens, 68 | "prompt_tokens": model_wrapped.tokenizer.encode(prompt), 69 | **prompt_info 70 | }) 71 | att_weights = np.stack(att_weights) 72 | proj_contribs = np.stack(proj_contributions) 73 | token_contribs = np.stack(token_contribs) 74 | 75 | constraint_indices = find_within_text(data.prompt, data.constraints, model_wrapped.tokenizer) 76 | # Get the locations of the filler tokens and template tokens. 77 | generation_start = num_prompt_tokens-1 78 | generation_end = att_weights.shape[-1]-1 79 | 80 | data.token_labels = [model_wrapped.tokenizer.decode(t) for t in data.prompt_tokens] 81 | data.all_max_attention_weights = np.max(att_weights[:, :, generation_start], axis=1) 82 | data.all_token_contrib_norms = np.linalg.norm(token_contribs, axis=-1) 83 | 84 | data.token_contrib_norms_constraints = [] 85 | data.attention_weights_constraints = [] 86 | for (constraint_start, constraint_end) in constraint_indices: 87 | attention_weights_constraints = att_weights[:, :, generation_start, constraint_start:constraint_end+1] 88 | 89 | token_contribs_constraints = (attention_weights_constraints[:, :, :, np.newaxis]*proj_contribs[:, :, constraint_start:constraint_end+1]) 90 | token_contrib_norms_constraints = np.linalg.norm(token_contribs_constraints, axis=-1) 91 | 92 | data.token_contrib_norms_constraints.append(token_contrib_norms_constraints) 93 | data.attention_weights_constraints.append(attention_weights_constraints) 94 | 95 | return data -------------------------------------------------------------------------------- /model_lib/hf_tooling.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from tqdm import tqdm 4 | import torch.nn.functional as F 5 | 6 | class HF_Llama2_Wrapper: 7 | def __init__(self, model, tokenizer, device="cuda", load_in_8bit=False): 8 | """A wrapper around HuggingFace language models to compute likelihoods and generate text. 9 | Args: 10 | model: a Llama model from HuggingFace. 11 | tokenizer : a tokenizer from HuggingFace. 12 | device (str): Defaults to "cuda". Not extensively tested for CPUs. 13 | """ 14 | self.tokenizer = tokenizer 15 | self.model = model.eval() 16 | self.device = device 17 | # Careful: Below are tested for Llama-2 family. 18 | self.tokenizer.pad_token_id = self.tokenizer.eos_token_id 19 | self.tokenizer.padding_side = "left" 20 | self.load_in_8bit = load_in_8bit 21 | 22 | 23 | @torch.no_grad() 24 | def get_batch_loglikelihood(self, texts): 25 | """ 26 | Compute the loglikelihood of the given set of texts. 27 | """ 28 | perplexities = [] 29 | total_likelihoods = [] 30 | for text in texts: 31 | tokenized = self.tokenizer([text], return_tensors="pt", padding=True).to(self.device) 32 | labels = tokenized.input_ids 33 | outputs = self.model(input_ids=tokenized["input_ids"], 34 | attention_mask=tokenized["attention_mask"], 35 | labels=labels) 36 | logits = outputs.logits.cpu() 37 | labels = labels.cpu() 38 | 39 | shift_logits = logits[..., :-1, :].contiguous() 40 | shift_labels = labels[..., 1:].contiguous() 41 | shift_labels[shift_labels == self.tokenizer.pad_token_id] = -100 42 | loss = F.cross_entropy(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1), reduction="none").detach() 43 | ll_per_sample = -loss.view(shift_logits.shape[0], shift_logits.shape[1]) 44 | nonpad_per_row = (shift_labels != -100).sum(dim=1) 45 | ll_mean = ll_per_sample.sum(dim=1)/nonpad_per_row 46 | 47 | ll_per_sample[(shift_labels == -100)] = 0 48 | ll_total = ll_per_sample.sum(dim=1) 49 | perplexities.append(ll_mean.cpu().numpy()) 50 | total_likelihoods.append(ll_total.cpu().numpy()) 51 | perplexities = np.concatenate(perplexities, axis=0) 52 | total_likelihoods = np.concatenate(total_likelihoods, axis=0) 53 | return perplexities, total_likelihoods 54 | 55 | 56 | 57 | @torch.no_grad() 58 | def get_conditional_loglikelihood(self, texts: list, 59 | completion_offset: torch.Tensor): 60 | tokenized = self.tokenizer( 61 | texts, return_tensors="pt", truncation=True, return_token_type_ids=False, padding=True, 62 | ).to(self.device) 63 | labels = tokenized.input_ids 64 | outputs = self.model(**tokenized) 65 | logits = outputs["logits"].detach().to(device="cpu", dtype=torch.float32) 66 | labels = labels.cpu() 67 | 68 | shift_logits = logits[:, :-1, :].contiguous() 69 | shift_labels = labels[:, 1:].contiguous() 70 | shift_labels[shift_labels == self.tokenizer.pad_token_id] = -100 71 | 72 | # Ignore the tokens prior to the completion offset. 73 | mask = (torch.arange(shift_labels.size(1)).unsqueeze(0) < (shift_labels.shape[1]-completion_offset.unsqueeze(1))) 74 | # Choose which tokens to mask from LL computation. 75 | shift_labels[(mask | (shift_labels == self.tokenizer.pad_token_id))] = -100 76 | 77 | loss = F.cross_entropy(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1), reduction="none").detach() 78 | ll_per_sample = -loss.view(shift_logits.shape[0], shift_logits.shape[1]) 79 | #nonpad_per_row = (shift_labels != -100).sum(dim=1) 80 | #ll_mean = ll_per_sample.sum(dim=1)/nonpad_per_row 81 | 82 | ll_per_sample[(shift_labels == -100)] = 0 83 | ll_total = ll_per_sample.sum(dim=1) 84 | torch.cuda.empty_cache() 85 | return ll_per_sample.float().cpu().numpy(), ll_total.float().cpu().numpy() 86 | 87 | @torch.no_grad() 88 | def get_conditional_loglikelihood_batch(self, texts: list, 89 | completion_offset: torch.Tensor): 90 | """Padding messes with the likelihood computation. Will do 1 by 1 for simplicity, but this is not a great way.""" 91 | ll_mean_all = [] 92 | ll_total_all = [] 93 | for i in tqdm(range(len(texts))): 94 | llm, llt = self.get_conditional_loglikelihood(texts[i:i+1], completion_offset[i:i+1]) 95 | ll_mean_all.append(llm) 96 | ll_total_all.append(llt) 97 | return ll_mean_all, np.concatenate(ll_total_all, axis=0) 98 | 99 | @torch.no_grad() 100 | def generate(self, texts, *args, **kwargs): 101 | tokenized = self.tokenizer(texts, return_tensors="pt", padding=True).to(self.device) 102 | generated = self.model.generate(input_ids=tokenized["input_ids"], attention_mask=tokenized["attention_mask"], 103 | pad_token_id=self.tokenizer.pad_token_id, *args, **kwargs) 104 | torch.cuda.empty_cache() 105 | return generated 106 | 107 | @torch.no_grad() 108 | def _sample_api_single(self, text, *args, **kwargs): 109 | if "max_tokens" in kwargs: 110 | kwargs["max_new_tokens"] = kwargs["max_tokens"] 111 | kwargs.pop("max_tokens") 112 | generated = self.generate([text], do_sample=False, temperature=None, 113 | output_scores=True, return_dict_in_generate=True, top_p=None, *args, **kwargs) 114 | decoded = self.tokenizer.batch_decode(generated.sequences, skip_special_tokens=True)[0] 115 | target_completion = decoded.replace(text, "") 116 | return target_completion 117 | 118 | @torch.no_grad() 119 | def sample(self, texts, *args, **kwargs): 120 | if "max_tokens" in kwargs: 121 | kwargs["max_new_tokens"] = kwargs["max_tokens"] 122 | kwargs.pop("max_tokens") 123 | generated = self.generate(texts, do_sample=False, num_beams=1, 124 | output_scores=True, return_dict_in_generate=True, temperature=None, top_p=None, *args, **kwargs) 125 | decoded = self.tokenizer.batch_decode(generated.sequences, skip_special_tokens=True) 126 | target_completions = [d.replace(t, "") for d, t in zip(decoded, texts)] 127 | return target_completions 128 | 129 | def get_output_projection_matrices(self): 130 | """Get the output projection matrices for the model.""" 131 | 132 | if self.load_in_8bit: 133 | # If the model is loaded in 8bit, we need to dequantize the output projection matrices for the downstream usage. 134 | o_proj_matrices = [] 135 | for nm, p in self.model.named_modules(): 136 | if "o_proj" not in nm: 137 | continue 138 | weight = p.weight 139 | dequantized = (weight.CB * weight.SCB) / 127 140 | o_proj_matrices.append(torch.stack(dequantized.split(self.model.config.hidden_size // self.model.config.num_attention_heads, dim=1)).detach().bfloat16()) 141 | 142 | else: 143 | o_proj_matrices = [torch.stack(p.weight.split(self.model.config.hidden_size // self.model.config.num_attention_heads, dim=1)).detach() for nm, p in self.model.named_modules() if "o_proj" in nm] 144 | return o_proj_matrices -------------------------------------------------------------------------------- /model_lib/hooks.py: -------------------------------------------------------------------------------- 1 | """ 2 | The below file is taken from https://github.com/kmeng01/rome/blob/main/util/nethook.py and slightly modified. 3 | It belongs to the above repository's authors under the MIT License. 4 | 5 | Utilities for instrumenting a torch model. 6 | 7 | Trace will hook one layer at a time. 8 | TraceDict will hook multiple layers at once. 9 | subsequence slices intervals from Sequential modules. 10 | get_module, replace_module, get_parameter resolve dotted names. 11 | set_requires_grad recursively sets requires_grad in module parameters. 12 | """ 13 | 14 | import contextlib 15 | import copy 16 | import inspect 17 | from collections import OrderedDict 18 | 19 | import torch 20 | 21 | 22 | class Trace(contextlib.AbstractContextManager): 23 | """ 24 | To retain the output of the named layer during the computation of 25 | the given network: 26 | 27 | with Trace(net, 'layer.name') as ret: 28 | _ = net(inp) 29 | representation = ret.output 30 | 31 | A layer module can be passed directly without a layer name, and 32 | its output will be retained. By default, a direct reference to 33 | the output object is returned, but options can control this: 34 | 35 | clone=True - retains a copy of the output, which can be 36 | useful if you want to see the output before it might 37 | be modified by the network in-place later. 38 | detach=True - retains a detached reference or copy. (By 39 | default the value would be left attached to the graph.) 40 | retain_grad=True - request gradient to be retained on the 41 | output. After backward(), ret.output.grad is populated. 42 | 43 | retain_input=True - also retains the input. 44 | retain_output=False - can disable retaining the output. 45 | edit_output=fn - calls the function to modify the output 46 | of the layer before passing it the rest of the model. 47 | fn can optionally accept (output, layer) arguments 48 | for the original output and the layer name. 49 | stop=True - throws a StopForward exception after the layer 50 | is run, which allows running just a portion of a model. 51 | """ 52 | 53 | def __init__( 54 | self, 55 | module, 56 | layer=None, 57 | retain_output=True, 58 | retain_input=False, 59 | clone=False, 60 | detach=False, 61 | retain_grad=False, 62 | edit_output=None, 63 | stop=False, 64 | ): 65 | """ 66 | Method to replace a forward method with a closure that 67 | intercepts the call, and tracks the hook so that it can be reverted. 68 | """ 69 | retainer = self 70 | self.layer = layer 71 | if layer is not None: 72 | module = get_module(module, layer) 73 | 74 | def retain_hook(m, inputs, output): 75 | if retain_input: 76 | retainer.input = recursive_copy( 77 | inputs[0] if len(inputs) == 1 else inputs, 78 | clone=clone, 79 | detach=detach, 80 | retain_grad=False, 81 | ) # retain_grad applies to output only. 82 | if edit_output: 83 | output = invoke_with_optional_args( 84 | edit_output, output=output, layer=self.layer 85 | ) 86 | if retain_output: 87 | retainer.output = recursive_copy( 88 | output, clone=clone, detach=detach, retain_grad=retain_grad 89 | ) 90 | # When retain_grad is set, also insert a trivial 91 | # copy operation. That allows in-place operations 92 | # to follow without error. 93 | if retain_grad: 94 | output = recursive_copy(retainer.output, clone=True, detach=False) 95 | if stop: 96 | raise StopForward() 97 | return output 98 | 99 | self.registered_hook = module.register_forward_hook(retain_hook) 100 | self.stop = stop 101 | 102 | def __enter__(self): 103 | return self 104 | 105 | def __exit__(self, type, value, traceback): 106 | self.close() 107 | if self.stop and issubclass(type, StopForward): 108 | return True 109 | 110 | def close(self): 111 | self.registered_hook.remove() 112 | 113 | 114 | class TraceDict(OrderedDict, contextlib.AbstractContextManager): 115 | """ 116 | To retain the output of multiple named layers during the computation 117 | of the given network: 118 | 119 | with TraceDict(net, ['layer1.name1', 'layer2.name2']) as ret: 120 | _ = net(inp) 121 | representation = ret['layer1.name1'].output 122 | 123 | If edit_output is provided, it should be a function that takes 124 | two arguments: output, and the layer name; and then it returns the 125 | modified output. 126 | 127 | Other arguments are the same as Trace. If stop is True, then the 128 | execution of the network will be stopped after the last layer 129 | listed (even if it would not have been the last to be executed). 130 | """ 131 | 132 | def __init__( 133 | self, 134 | module, 135 | layers=None, 136 | retain_output=True, 137 | retain_input=False, 138 | clone=False, 139 | detach=False, 140 | retain_grad=False, 141 | edit_output=None, 142 | stop=False, 143 | ): 144 | self.stop = stop 145 | 146 | def flag_last_unseen(it): 147 | try: 148 | it = iter(it) 149 | prev = next(it) 150 | seen = set([prev]) 151 | except StopIteration: 152 | return 153 | for item in it: 154 | if item not in seen: 155 | yield False, prev 156 | seen.add(item) 157 | prev = item 158 | yield True, prev 159 | 160 | for is_last, layer in flag_last_unseen(layers): 161 | self[layer] = Trace( 162 | module=module, 163 | layer=layer, 164 | retain_output=retain_output, 165 | retain_input=retain_input, 166 | clone=clone, 167 | detach=detach, 168 | retain_grad=retain_grad, 169 | edit_output=edit_output, 170 | stop=stop and is_last, 171 | ) 172 | 173 | def __enter__(self): 174 | return self 175 | 176 | def __exit__(self, type, value, traceback): 177 | self.close() 178 | if self.stop and issubclass(type, StopForward): 179 | return True 180 | 181 | def close(self): 182 | for layer, trace in reversed(self.items()): 183 | trace.close() 184 | 185 | 186 | class StopForward(Exception): 187 | """ 188 | If the only output needed from running a network is the retained 189 | submodule then Trace(submodule, stop=True) will stop execution 190 | immediately after the retained submodule by raising the StopForward() 191 | exception. When Trace is used as context manager, it catches that 192 | exception and can be used as follows: 193 | 194 | with Trace(net, layername, stop=True) as tr: 195 | net(inp) # Only runs the network up to layername 196 | print(tr.output) 197 | """ 198 | 199 | pass 200 | 201 | 202 | def recursive_copy(x, clone=None, detach=None, retain_grad=None): 203 | """ 204 | Copies a reference to a tensor, or an object that contains tensors, 205 | optionally detaching and cloning the tensor(s). If retain_grad is 206 | true, the original tensors are marked to have grads retained. 207 | """ 208 | if not clone and not detach and not retain_grad: 209 | return x 210 | if isinstance(x, torch.Tensor): 211 | if retain_grad: 212 | if not x.requires_grad: 213 | x.requires_grad = True 214 | x.retain_grad() 215 | elif detach: 216 | x = x.detach() 217 | if clone: 218 | x = x.clone() 219 | return x 220 | # Only dicts, lists, and tuples (and subclasses) can be copied. 221 | if isinstance(x, dict): 222 | # We changed this part of the code where we also pass retain_grad. 223 | return type(x)({k: recursive_copy(v, retain_grad=retain_grad) for k, v in x.items()}) 224 | elif isinstance(x, (list, tuple)): 225 | return type(x)([recursive_copy(v, retain_grad=retain_grad) for v in x]) 226 | else: 227 | assert False, f"Unknown type {type(x)} cannot be broken into tensors." 228 | 229 | 230 | def subsequence( 231 | sequential, 232 | first_layer=None, 233 | last_layer=None, 234 | after_layer=None, 235 | upto_layer=None, 236 | single_layer=None, 237 | share_weights=False, 238 | ): 239 | """ 240 | Creates a subsequence of a pytorch Sequential model, copying over 241 | modules together with parameters for the subsequence. Only 242 | modules from first_layer to last_layer (inclusive) are included, 243 | or modules between after_layer and upto_layer (exclusive). 244 | Handles descent into dotted layer names as long as all references 245 | are within nested Sequential models. 246 | 247 | If share_weights is True, then references the original modules 248 | and their parameters without copying them. Otherwise, by default, 249 | makes a separate brand-new copy. 250 | """ 251 | assert (single_layer is None) or ( 252 | first_layer is last_layer is after_layer is upto_layer is None 253 | ) 254 | if single_layer is not None: 255 | first_layer = single_layer 256 | last_layer = single_layer 257 | first, last, after, upto = [ 258 | None if d is None else d.split(".") 259 | for d in [first_layer, last_layer, after_layer, upto_layer] 260 | ] 261 | return hierarchical_subsequence( 262 | sequential, 263 | first=first, 264 | last=last, 265 | after=after, 266 | upto=upto, 267 | share_weights=share_weights, 268 | ) 269 | 270 | 271 | def hierarchical_subsequence( 272 | sequential, first, last, after, upto, share_weights=False, depth=0 273 | ): 274 | """ 275 | Recursive helper for subsequence() to support descent into dotted 276 | layer names. In this helper, first, last, after, and upto are 277 | arrays of names resulting from splitting on dots. Can only 278 | descend into nested Sequentials. 279 | """ 280 | assert (last is None) or (upto is None) 281 | assert (first is None) or (after is None) 282 | if first is last is after is upto is None: 283 | return sequential if share_weights else copy.deepcopy(sequential) 284 | assert isinstance(sequential, torch.nn.Sequential), ( 285 | ".".join((first or last or after or upto)[:depth] or "arg") + " not Sequential" 286 | ) 287 | including_children = (first is None) and (after is None) 288 | included_children = OrderedDict() 289 | # A = current level short name of A. 290 | # AN = full name for recursive descent if not innermost. 291 | (F, FN), (L, LN), (A, AN), (U, UN) = [ 292 | (d[depth], (None if len(d) == depth + 1 else d)) 293 | if d is not None 294 | else (None, None) 295 | for d in [first, last, after, upto] 296 | ] 297 | for name, layer in sequential._modules.items(): 298 | if name == F: 299 | first = None 300 | including_children = True 301 | if name == A and AN is not None: # just like F if not a leaf. 302 | after = None 303 | including_children = True 304 | if name == U and UN is None: 305 | upto = None 306 | including_children = False 307 | if including_children: 308 | # AR = full name for recursive descent if name matches. 309 | FR, LR, AR, UR = [ 310 | n if n is None or n[depth] == name else None for n in [FN, LN, AN, UN] 311 | ] 312 | chosen = hierarchical_subsequence( 313 | layer, 314 | first=FR, 315 | last=LR, 316 | after=AR, 317 | upto=UR, 318 | share_weights=share_weights, 319 | depth=depth + 1, 320 | ) 321 | if chosen is not None: 322 | included_children[name] = chosen 323 | if name == L: 324 | last = None 325 | including_children = False 326 | if name == U and UN is not None: # just like L if not a leaf. 327 | upto = None 328 | including_children = False 329 | if name == A and AN is None: 330 | after = None 331 | including_children = True 332 | for name in [first, last, after, upto]: 333 | if name is not None: 334 | raise ValueError("Layer %s not found" % ".".join(name)) 335 | # Omit empty subsequences except at the outermost level, 336 | # where we should not return None. 337 | if not len(included_children) and depth > 0: 338 | return None 339 | result = torch.nn.Sequential(included_children) 340 | result.training = sequential.training 341 | return result 342 | 343 | 344 | def set_requires_grad(requires_grad, *models): 345 | """ 346 | Sets requires_grad true or false for all parameters within the 347 | models passed. 348 | """ 349 | for model in models: 350 | if isinstance(model, torch.nn.Module): 351 | for param in model.parameters(): 352 | param.requires_grad = requires_grad 353 | elif isinstance(model, (torch.nn.Parameter, torch.Tensor)): 354 | model.requires_grad = requires_grad 355 | else: 356 | assert False, "unknown type %r" % type(model) 357 | 358 | 359 | def get_module(model, name): 360 | """ 361 | Finds the named module within the given model. 362 | """ 363 | for n, m in model.named_modules(): 364 | if n == name: 365 | return m 366 | raise LookupError(name) 367 | 368 | 369 | def get_parameter(model, name): 370 | """ 371 | Finds the named parameter within the given model. 372 | """ 373 | for n, p in model.named_parameters(): 374 | if n == name: 375 | return p 376 | raise LookupError(name) 377 | 378 | 379 | def replace_module(model, name, new_module): 380 | """ 381 | Replaces the named module within the given model. 382 | """ 383 | if "." in name: 384 | parent_name, attr_name = name.rsplit(".", 1) 385 | model = get_module(model, parent_name) 386 | # original_module = getattr(model, attr_name) 387 | setattr(model, attr_name, new_module) 388 | 389 | 390 | def invoke_with_optional_args(fn, *args, **kwargs): 391 | """ 392 | Invokes a function with only the arguments that it 393 | is written to accept, giving priority to arguments 394 | that match by-name, using the following rules. 395 | (1) arguments with matching names are passed by name. 396 | (2) remaining non-name-matched args are passed by order. 397 | (3) extra caller arguments that the function cannot 398 | accept are not passed. 399 | (4) extra required function arguments that the caller 400 | cannot provide cause a TypeError to be raised. 401 | Ordinary python calling conventions are helpful for 402 | supporting a function that might be revised to accept 403 | extra arguments in a newer version, without requiring the 404 | caller to pass those new arguments. This function helps 405 | support function callers that might be revised to supply 406 | extra arguments, without requiring the callee to accept 407 | those new arguments. 408 | """ 409 | argspec = inspect.getfullargspec(fn) 410 | pass_args = [] 411 | used_kw = set() 412 | unmatched_pos = [] 413 | used_pos = 0 414 | defaulted_pos = len(argspec.args) - ( 415 | 0 if not argspec.defaults else len(argspec.defaults) 416 | ) 417 | # Pass positional args that match name first, then by position. 418 | for i, n in enumerate(argspec.args): 419 | if n in kwargs: 420 | pass_args.append(kwargs[n]) 421 | used_kw.add(n) 422 | elif used_pos < len(args): 423 | pass_args.append(args[used_pos]) 424 | used_pos += 1 425 | else: 426 | unmatched_pos.append(len(pass_args)) 427 | pass_args.append( 428 | None if i < defaulted_pos else argspec.defaults[i - defaulted_pos] 429 | ) 430 | # Fill unmatched positional args with unmatched keyword args in order. 431 | if len(unmatched_pos): 432 | for k, v in kwargs.items(): 433 | if k in used_kw or k in argspec.kwonlyargs: 434 | continue 435 | pass_args[unmatched_pos[0]] = v 436 | used_kw.add(k) 437 | unmatched_pos = unmatched_pos[1:] 438 | if len(unmatched_pos) == 0: 439 | break 440 | else: 441 | if unmatched_pos[0] < defaulted_pos: 442 | unpassed = ", ".join( 443 | argspec.args[u] for u in unmatched_pos if u < defaulted_pos 444 | ) 445 | raise TypeError(f"{fn.__name__}() cannot be passed {unpassed}.") 446 | # Pass remaining kw args if they can be accepted. 447 | pass_kw = { 448 | k: v 449 | for k, v in kwargs.items() 450 | if k not in used_kw and (k in argspec.kwonlyargs or argspec.varargs is not None) 451 | } 452 | # Pass remaining positional args if they can be accepted. 453 | if argspec.varargs is not None: 454 | pass_args += list(args[used_pos:]) 455 | return fn(*pass_args, **pass_kw) -------------------------------------------------------------------------------- /model_lib/misc_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | 4 | class AverageMeter: 5 | def __init__(self): 6 | self.reset() 7 | 8 | def reset(self): 9 | self.val = 0 10 | self.avg = 0 11 | self.sum = 0 12 | self.count = 0 13 | 14 | def update(self, val, n=1): 15 | self.val = val 16 | self.sum += val*n 17 | self.count += n 18 | self.avg = self.sum/self.count 19 | 20 | 21 | def find_sub_list(sl,l, offset=0): 22 | sll=len(sl) 23 | for ind in (i for i,e in enumerate(l) if e==sl[0]): 24 | if ind < offset: 25 | continue 26 | if l[ind:ind+sll]==sl: 27 | return ind,ind+sll-1 28 | 29 | def cosine(A, B): 30 | return np.dot(A,B.T)/(np.linalg.norm(A)*np.linalg.norm(B)) 31 | 32 | def find_within_text(prompt, parts, tokenizer): 33 | """ 34 | A function that identifies the indices of tokens of a part of the prompt. 35 | By default we use the first occurence. 36 | """ 37 | prompt_tokens = tokenizer.encode(prompt) 38 | part_tokens = [tokenizer.encode(p)[2:] for p in parts] 39 | part_token_indices = [find_sub_list(pt, prompt_tokens) for pt in part_tokens] 40 | return part_token_indices 41 | 42 | 43 | def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: 44 | """ 45 | From https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L225 46 | This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, 47 | num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) 48 | """ 49 | batch, num_key_value_heads, slen, head_dim = hidden_states.shape 50 | if n_rep == 1: 51 | return hidden_states 52 | hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) 53 | return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) 54 | 55 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate==0.27.2 2 | aiohttp==3.9.3 3 | aiosignal==1.3.1 4 | anyio==4.3.0 5 | argon2-cffi==23.1.0 6 | argon2-cffi-bindings==21.2.0 7 | arrow==1.3.0 8 | asttokens==2.4.1 9 | async-lru==2.0.4 10 | attrs==23.2.0 11 | Babel==2.14.0 12 | beautifulsoup4==4.12.3 13 | bitsandbytes==0.43.0 14 | bleach==6.1.0 15 | certifi==2024.2.2 16 | cffi==1.16.0 17 | charset-normalizer==3.3.2 18 | comm==0.2.1 19 | contourpy==1.2.0 20 | cycler==0.12.1 21 | datasets==2.18.0 22 | debugpy==1.8.1 23 | decorator==5.1.1 24 | defusedxml==0.7.1 25 | dill==0.3.8 26 | easydict==1.13 27 | executing==2.0.1 28 | fastjsonschema==2.19.1 29 | filelock==3.13.1 30 | fonttools==4.49.0 31 | fqdn==1.5.1 32 | frozenlist==1.4.1 33 | fsspec==2024.2.0 34 | h11==0.14.0 35 | httpcore==1.0.4 36 | httpx==0.27.0 37 | huggingface-hub==0.21.4 38 | idna==3.6 39 | ipykernel==6.29.3 40 | ipython==8.22.2 41 | isoduration==20.11.0 42 | jedi==0.19.1 43 | Jinja2==3.1.3 44 | joblib==1.3.2 45 | json5==0.9.22 46 | jsonpointer==2.4 47 | jsonschema==4.21.1 48 | jsonschema-specifications==2023.12.1 49 | jupyter-events==0.9.0 50 | jupyter-lsp==2.2.4 51 | jupyter_client==8.6.0 52 | jupyter_core==5.7.1 53 | jupyter_server==2.13.0 54 | jupyter_server_terminals==0.5.2 55 | jupyterlab==4.1.4 56 | jupyterlab_pygments==0.3.0 57 | jupyterlab_server==2.25.3 58 | kiwisolver==1.4.5 59 | MarkupSafe==2.1.5 60 | matplotlib==3.8.3 61 | matplotlib-inline==0.1.6 62 | mistune==3.0.2 63 | mpmath==1.3.0 64 | multidict==6.0.5 65 | multiprocess==0.70.16 66 | nbclient==0.9.0 67 | nbconvert==7.16.2 68 | nbformat==5.9.2 69 | nest-asyncio==1.6.0 70 | networkx==3.2.1 71 | notebook==7.1.1 72 | notebook_shim==0.2.4 73 | numpy==1.26.4 74 | nvidia-cublas-cu12==12.1.3.1 75 | nvidia-cuda-cupti-cu12==12.1.105 76 | nvidia-cuda-nvrtc-cu12==12.1.105 77 | nvidia-cuda-runtime-cu12==12.1.105 78 | nvidia-cudnn-cu12==8.9.2.26 79 | nvidia-cufft-cu12==11.0.2.54 80 | nvidia-curand-cu12==10.3.2.106 81 | nvidia-cusolver-cu12==11.4.5.107 82 | nvidia-cusparse-cu12==12.1.0.106 83 | nvidia-nccl-cu12==2.19.3 84 | nvidia-nvjitlink-cu12==12.4.99 85 | nvidia-nvtx-cu12==12.1.105 86 | overrides==7.7.0 87 | packaging==23.2 88 | pandas==2.2.1 89 | pandocfilters==1.5.1 90 | parso==0.8.3 91 | pexpect==4.9.0 92 | pillow==10.3.0 93 | platformdirs==4.2.0 94 | prometheus_client==0.20.0 95 | prompt-toolkit==3.0.43 96 | psutil==5.9.8 97 | ptyprocess==0.7.0 98 | pure-eval==0.2.2 99 | pyarrow==15.0.1 100 | pyarrow-hotfix==0.6 101 | pycparser==2.21 102 | Pygments==2.17.2 103 | pyparsing==3.1.2 104 | python-dateutil==2.9.0.post0 105 | python-json-logger==2.0.7 106 | pytz==2024.1 107 | PyYAML==6.0.1 108 | pyzmq==25.1.2 109 | referencing==0.33.0 110 | regex==2023.12.25 111 | requests==2.31.0 112 | rfc3339-validator==0.1.4 113 | rfc3986-validator==0.1.1 114 | rpds-py==0.18.0 115 | safetensors==0.4.2 116 | scikit-learn==1.4.1.post1 117 | scipy==1.12.0 118 | seaborn==0.13.2 119 | Send2Trash==1.8.2 120 | sentencepiece==0.2.0 121 | six==1.16.0 122 | sniffio==1.3.1 123 | soupsieve==2.5 124 | stack-data==0.6.3 125 | sympy==1.12 126 | terminado==0.18.0 127 | threadpoolctl==3.3.0 128 | tiktoken==0.6.0 129 | tinycss2==1.2.1 130 | tokenizers==0.15.2 131 | torch==2.2.1 132 | torchvision==0.17.1 133 | tornado==6.4 134 | tqdm==4.66.2 135 | traitlets==5.14.1 136 | transformers==4.38.2 137 | triton==2.2.0 138 | types-python-dateutil==2.8.19.20240106 139 | typing_extensions==4.10.0 140 | tzdata==2024.1 141 | uri-template==1.3.0 142 | urllib3==2.2.1 143 | wcwidth==0.2.13 144 | webcolors==1.13 145 | webencodings==0.5.1 146 | websocket-client==1.7.0 147 | xxhash==3.4.1 148 | yarl==1.9.4 149 | -------------------------------------------------------------------------------- /viz_tools/__init__.py: -------------------------------------------------------------------------------- 1 | from .attention_viz import plot_attention_flow -------------------------------------------------------------------------------- /viz_tools/attention_viz.py: -------------------------------------------------------------------------------- 1 | import os 2 | import matplotlib.pyplot as plt 3 | 4 | def plot_attention_flow(flow_matrix, token_labels, topk_prefix=15, savepdf=None, 5 | cbar_text=None, 6 | title=None, 7 | figsize=(3,2)): 8 | flow_matrix = flow_matrix[:topk_prefix] 9 | token_labels = token_labels[:topk_prefix] 10 | fig, ax = plt.subplots(figsize=figsize, dpi=200) 11 | h = ax.pcolor( 12 | flow_matrix, 13 | cmap="Blues", 14 | vmin=0, 15 | ) 16 | ax.invert_yaxis() 17 | ax.set_yticks([0.5 + i for i in range(len(flow_matrix))]) 18 | ax.set_xticks([0.5 + i for i in range(0, flow_matrix.shape[1] - 6, 5)]) 19 | ax.set_xticklabels(list(range(0, flow_matrix.shape[1] - 6, 5))) 20 | ax.set_yticklabels(token_labels, fontsize=8) 21 | cb = plt.colorbar(h) 22 | ax.set_xlabel(f"Layers") 23 | if title: 24 | ax.set_title(title) 25 | else: 26 | ax.set_title("Attention contribution to generation") 27 | if cbar_text: 28 | cb.ax.set_title(cbar_text, y=-0.16, fontsize=8) 29 | 30 | if savepdf: 31 | os.makedirs(os.path.dirname(savepdf), exist_ok=True) 32 | plt.savefig(savepdf, bbox_inches="tight") 33 | plt.close() 34 | return fig --------------------------------------------------------------------------------