├── README.md └── lm_truthfulness_gpt-j_sparse.ipynb /README.md: -------------------------------------------------------------------------------- 1 | Code for https://arxiv.org/abs/2312.03729. 2 | 3 | Expects subdirectories named : 4 | 5 | ``` 6 | boolean-questions 7 | creak 8 | sciq 9 | ``` 10 | 11 | containing the BoolQ, CREAK, and SciQ datasets respectively. 12 | -------------------------------------------------------------------------------- /lm_truthfulness_gpt-j_sparse.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "MODEL = \"gpt-j\"\n", 10 | "N_DEV = 500" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 2, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "import csv\n", 20 | "import jsonlines\n", 21 | "import json\n", 22 | "import torch\n", 23 | "from torch import nn, optim\n", 24 | "from transformers import AutoTokenizer, GPT2LMHeadModel, GPT2Tokenizer, GPTJForCausalLM\n", 25 | "from tqdm import tqdm\n", 26 | "import numpy as np\n", 27 | "import os\n", 28 | "\n", 29 | "from matplotlib import pyplot as plt\n", 30 | "from matplotlib.patches import Rectangle\n", 31 | "import seaborn as sns\n", 32 | "import pandas as pd\n", 33 | "import scipy.stats\n", 34 | "\n", 35 | "torch.manual_seed(0)\n", 36 | "np.random.seed(0)" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 3, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "sns.set()\n", 46 | "sns.set_style(\"white\")\n", 47 | "sns.set_style({'font.family':'serif', 'font.serif':'Times New Roman'})" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 4, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "def load_boolq(split, max_examples=None):\n", 57 | " data = jsonlines.open(f\"boolean-questions/{split}.jsonl\")\n", 58 | " questions = []\n", 59 | " for line in data:\n", 60 | " q = line[\"question\"]\n", 61 | " q = f\"{q}?\"\n", 62 | " \n", 63 | " a_good = str(line[\"answer\"]).lower()\n", 64 | " a_bad = \"false\" if a_good == \"true\" else \"true\"\n", 65 | " \n", 66 | " questions.append((q, a_good, a_bad))\n", 67 | " if max_examples is not None and len(questions) == max_examples:\n", 68 | " break\n", 69 | " np.random.shuffle(questions)\n", 70 | " return questions\n", 71 | "\n", 72 | "def load_sciq(split, max_examples=None):\n", 73 | " questions = []\n", 74 | " with open(f\"sciq/{split}.json\") as reader:\n", 75 | " data = json.load(reader)\n", 76 | " \n", 77 | " for line in data:\n", 78 | " q = line[\"question\"]\n", 79 | " \n", 80 | " a_good = line[\"correct_answer\"]\n", 81 | " a_bad = line[\"distractor1\"]\n", 82 | " \n", 83 | " questions.append((q, a_good, a_bad))\n", 84 | " if max_examples is not None and len(questions) == max_examples:\n", 85 | " break\n", 86 | " np.random.shuffle(questions)\n", 87 | " return questions\n", 88 | "\n", 89 | "def load_creak(split, max_examples=None):\n", 90 | " questions = []\n", 91 | " reader = jsonlines.open(f\"creak/data/creak/{split}.json\")\n", 92 | " for line in reader:\n", 93 | " q = line[\"sentence\"]\n", 94 | " a_good = line[\"label\"]\n", 95 | " a_bad = \"true\" if line[\"label\"] == \"false\" else \"false\"\n", 96 | " questions.append((q, a_good, a_bad))\n", 97 | " np.random.shuffle(questions)\n", 98 | " return questions\n", 99 | " \n", 100 | " " 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 5, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "if MODEL.startswith(\"gpt2\"):\n", 110 | " #lm = GPT2LMHeadModel.from_pretrained(MODEL).eval().cuda()\n", 111 | " tokenizer = GPT2Tokenizer.from_pretrained(MODEL)\n", 112 | "elif MODEL == \"gpt-j\":\n", 113 | " #lm = GPTJForCausalLM.from_pretrained(\n", 114 | " # \"EleutherAI/gpt-j-6B\",\n", 115 | " # revision=\"float16\",\n", 116 | " # torch_dtype=torch.float16\n", 117 | " #).eval().cuda()\n", 118 | " tokenizer = AutoTokenizer.from_pretrained(\"EleutherAI/gpt-j-6B\")\n", 119 | "\n", 120 | "probe_layer = -1\n", 121 | "probe_reduction = lambda tensor: tensor[:, -1, :]" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": 6, 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [ 130 | "def lm_pred(toks):\n", 131 | " results = lm(toks, labels=toks, output_hidden_states=True)\n", 132 | " assert toks.shape[0] == 1\n", 133 | " results.logprob = -results[\"loss\"] * toks.shape[1]\n", 134 | " return results\n", 135 | "\n", 136 | "@torch.no_grad()\n", 137 | "def get_preds_and_reprs(data):\n", 138 | " ratios = []\n", 139 | " corrects = []\n", 140 | " qa_good_reprs = []\n", 141 | " qa_bad_reprs = []\n", 142 | " for q, a_good, a_bad in tqdm(data):\n", 143 | " qa_good_tok = tokenizer(f\"{q} {a_good}\", return_tensors=\"pt\").input_ids.cuda()\n", 144 | " qa_good_pred = lm_pred(qa_good_tok)\n", 145 | " qa_good_repr = probe_reduction(qa_good_pred.hidden_states[probe_layer])\n", 146 | "\n", 147 | " qa_bad_tok = tokenizer(f\"{q} {a_bad}\", return_tensors=\"pt\").input_ids.cuda()\n", 148 | " qa_bad_pred = lm_pred(qa_bad_tok)\n", 149 | " qa_bad_repr = probe_reduction(qa_bad_pred.hidden_states[probe_layer])\n", 150 | " \n", 151 | " ratios.append(qa_good_pred.logprob - qa_bad_pred.logprob)\n", 152 | " corrects.append(qa_good_pred.logprob > qa_bad_pred.logprob)\n", 153 | " qa_good_reprs.append(qa_good_repr)\n", 154 | " qa_bad_reprs.append(qa_bad_repr)\n", 155 | " return {\n", 156 | " \"ratio\": torch.tensor(ratios),\n", 157 | " \"correct\": torch.tensor(corrects),\n", 158 | " \"good_repr\": torch.cat(qa_good_reprs, dim=0),\n", 159 | " \"bad_repr\": torch.cat(qa_bad_reprs, dim=0)\n", 160 | " }" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 7, 166 | "metadata": {}, 167 | "outputs": [], 168 | "source": [ 169 | "data = {\n", 170 | " dataset_name: {\n", 171 | " split_name: loader(split_name) for split_name in [\"train\", \"valid\"]\n", 172 | " }\n", 173 | " for dataset_name, loader in [(\"boolq\", load_boolq), (\"sciq\", load_sciq), (\"creak\", load_creak)]\n", 174 | "}\n", 175 | "\n", 176 | "def process_data(data, suffix):\n", 177 | " data_path = f\"cache/data_processed_{MODEL}{suffix}.pt\"\n", 178 | " if os.path.exists(data_path):\n", 179 | " data_processed = torch.load(data_path)\n", 180 | " else:\n", 181 | " data_processed = {}\n", 182 | "\n", 183 | " changed = False\n", 184 | " for dataset_name, dataset in data.items():\n", 185 | " if dataset_name not in data_processed:\n", 186 | " data_processed[dataset_name] = {}\n", 187 | " for split_name, split_data in dataset.items():\n", 188 | " if split_name not in data_processed[dataset_name]:\n", 189 | " changed = True\n", 190 | " data_processed[dataset_name][split_name] = get_preds_and_reprs(split_data)\n", 191 | "\n", 192 | " if changed:\n", 193 | " torch.save(data_processed, data_path)\n", 194 | " \n", 195 | " return data_processed\n", 196 | "\n", 197 | "data_processed = process_data(data, \"\")" 198 | ] 199 | }, 200 | { 201 | "cell_type": "markdown", 202 | "metadata": {}, 203 | "source": [ 204 | "## Probes are more accurate than models" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": 8, 210 | "metadata": {}, 211 | "outputs": [ 212 | { 213 | "name": "stdout", 214 | "output_type": "stream", 215 | "text": [ 216 | "boolq\n", 217 | "probe acc 0.6238532066345215\n", 218 | "model acc 0.6149846911430359\n", 219 | "\n", 220 | "sciq\n", 221 | "probe acc 0.8860000371932983\n", 222 | "model acc 0.8379999995231628\n", 223 | "\n", 224 | "creak\n", 225 | "probe acc 0.7089715600013733\n", 226 | "model acc 0.50255286693573\n", 227 | "\n" 228 | ] 229 | } 230 | ], 231 | "source": [ 232 | "class Probe(nn.Module):\n", 233 | " def __init__(self, model_dim):\n", 234 | " super().__init__()\n", 235 | " self.layer = nn.Linear(model_dim, 2)\n", 236 | " \n", 237 | " def forward(self, reprs):\n", 238 | " return self.layer(reprs)\n", 239 | " \n", 240 | " def predict(self, good_reprs, bad_reprs):\n", 241 | " return (self(good_reprs)[:, 1] > self(bad_reprs)[:, 1]).float()\n", 242 | " \n", 243 | " def l1(self):\n", 244 | " return torch.norm(self.layer.weight, 1)\n", 245 | " \n", 246 | "@torch.no_grad()\n", 247 | "def eval_probe(probe, data_processed):\n", 248 | " return probe.predict(\n", 249 | " data_processed[\"good_repr\"],\n", 250 | " data_processed[\"bad_repr\"]\n", 251 | " ).mean().item()\n", 252 | "\n", 253 | "def train_probe(train_data_processed, val_data_processed, l1=0):\n", 254 | " n_train, model_dim = train_data_processed[\"good_repr\"].shape\n", 255 | " probe = Probe(model_dim).float().cuda()\n", 256 | " objective = nn.CrossEntropyLoss()\n", 257 | " opt = optim.Adam(probe.parameters(), lr=0.003)\n", 258 | " for i in range(200):\n", 259 | " loss_good = objective(probe(train_data_processed[\"good_repr\"][:-N_DEV].float()), torch.ones(n_train-N_DEV).long().cuda())\n", 260 | " loss_bad = objective(probe(train_data_processed[\"bad_repr\"][:-N_DEV].float()), torch.zeros(n_train-N_DEV).long().cuda())\n", 261 | " loss = loss_good + loss_bad + l1 * probe.l1()\n", 262 | " opt.zero_grad()\n", 263 | " loss.backward()\n", 264 | " opt.step()\n", 265 | " if False:\n", 266 | " if (i+1) % 10 == 0:\n", 267 | " print(loss.item())\n", 268 | " return probe.half()\n", 269 | "\n", 270 | "#def plot_accuracy(model_preds, probe_preds):\n", 271 | "# df = pd.DataFrame({\"model\": model_preds > 0.5, \"probe\": probe_preds > 0.5})\n", 272 | "# sns.barplot(data=df)\n", 273 | "# plt.ylim(0.5, 1)\n", 274 | "# plt.show()\n", 275 | " \n", 276 | "\n", 277 | "probes = {}\n", 278 | "probe_preds = {}\n", 279 | "probe_dev_preds = {}\n", 280 | "for dataset in data:\n", 281 | " print(dataset)\n", 282 | " probe = train_probe(data_processed[dataset][\"train\"], data_processed[dataset][\"valid\"])\n", 283 | " probes[dataset] = probe\n", 284 | " probe_preds[dataset] = torch.sigmoid(\n", 285 | " probe(data_processed[dataset][\"valid\"][\"good_repr\"])[:, 1]\n", 286 | " - probe(data_processed[dataset][\"valid\"][\"bad_repr\"])[:, 1]\n", 287 | " ).detach().cpu().numpy()\n", 288 | " probe_dev_preds[dataset] = torch.sigmoid(\n", 289 | " probe(data_processed[dataset][\"train\"][\"good_repr\"][-N_DEV:])[:, 1]\n", 290 | " - probe(data_processed[dataset][\"train\"][\"bad_repr\"][-N_DEV:])[:, 1]\n", 291 | " ).detach().cpu().numpy()\n", 292 | "\n", 293 | " print(\"probe acc\", eval_probe(probe, data_processed[dataset][\"valid\"]))\n", 294 | " print(\"model acc\", data_processed[dataset][\"valid\"][\"correct\"].float().mean().item())\n", 295 | " #plot_accuracy(\n", 296 | " # torch.sigmoid(data_processed[dataset][\"valid\"][\"ratio\"].float()),\n", 297 | " # probe_preds[dataset]\n", 298 | " #)\n", 299 | " print()" 300 | ] 301 | }, 302 | { 303 | "cell_type": "markdown", 304 | "metadata": {}, 305 | "source": [ 306 | "## Probes and predictions are sensitive to formatting!" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": 9, 312 | "metadata": {}, 313 | "outputs": [ 314 | { 315 | "name": "stdout", 316 | "output_type": "stream", 317 | "text": [ 318 | "qa\n", 319 | "boolq\n", 320 | "probe acc 0.6360856294631958\n", 321 | "model acc 0.6140672564506531\n", 322 | "\n", 323 | "sciq\n", 324 | "probe acc 0.9040000438690186\n", 325 | "model acc 0.8740000128746033\n", 326 | "\n", 327 | "creak\n", 328 | "probe acc 0.7381473183631897\n", 329 | "model acc 0.5069292783737183\n", 330 | "\n", 331 | "\n", 332 | "yn\n", 333 | "boolq\n", 334 | "probe acc 0.6174311637878418\n", 335 | "model acc 0.43180426955223083\n", 336 | "\n", 337 | "sciq\n", 338 | "probe acc 0.0\n", 339 | "model acc 0.0\n", 340 | "\n", 341 | "creak\n", 342 | "probe acc 0.7359591722488403\n", 343 | "model acc 0.470459520816803\n", 344 | "\n", 345 | "\n", 346 | "cap\n", 347 | "boolq\n", 348 | "probe acc 0.635168194770813\n", 349 | "model acc 0.621100902557373\n", 350 | "\n", 351 | "sciq\n", 352 | "probe acc 0.8980000615119934\n", 353 | "model acc 0.8370000123977661\n", 354 | "\n", 355 | "creak\n", 356 | "probe acc 0.6965718269348145\n", 357 | "model acc 0.5076586604118347\n", 358 | "\n", 359 | "\n", 360 | "dot\n", 361 | "boolq\n", 362 | "probe acc 0.6348623633384705\n", 363 | "model acc 0.48990824818611145\n", 364 | "\n", 365 | "sciq\n", 366 | "probe acc 0.9000000357627869\n", 367 | "model acc 0.8479999899864197\n", 368 | "\n", 369 | "creak\n", 370 | "probe acc 0.7293946146965027\n", 371 | "model acc 0.50255286693573\n", 372 | "\n", 373 | "\n" 374 | ] 375 | } 376 | ], 377 | "source": [ 378 | "tf_to_yn = lambda a: \"yes\" if a == \"true\" else \"no\"\n", 379 | "\n", 380 | "formatters = {\n", 381 | " \"qa\": lambda q, ag, ab: (f\"Q: {q} A:\", ag, ab),\n", 382 | " \"yn\": lambda q, ag, ab: (q, tf_to_yn(ag), tf_to_yn(ab)),\n", 383 | " \"cap\": lambda q, ag, ab: (q, ag.capitalize(), ab.capitalize()),\n", 384 | " \"dot\": lambda q, ag, ab: (q, ag + \".\", ab + \".\"),\n", 385 | "}\n", 386 | "batch_format = lambda formatter, dataset: [formatter(*datum) for datum in dataset]\n", 387 | "\n", 388 | "for format_name, formatter in formatters.items():\n", 389 | " data_fmt = {\n", 390 | " dataset_name: {\n", 391 | " split_name: batch_format(formatter, loader(split_name)) \n", 392 | " for split_name in [\"train\", \"valid\"]\n", 393 | " }\n", 394 | " for dataset_name, loader in [(\"boolq\", load_boolq), (\"sciq\", load_sciq), (\"creak\", load_creak)]\n", 395 | " }\n", 396 | " data_processed_fmt = process_data(data_fmt, f\"_{format_name}\")\n", 397 | "\n", 398 | " print(format_name)\n", 399 | " for dataset in data_fmt:\n", 400 | " print(dataset)\n", 401 | " n_train, model_dim = data_processed_fmt[dataset][\"train\"][\"good_repr\"].shape\n", 402 | " probe = train_probe(data_processed_fmt[dataset][\"train\"], data_processed_fmt[dataset][\"valid\"])\n", 403 | " print(\"probe acc\", eval_probe(probe, data_processed_fmt[dataset][\"valid\"]))\n", 404 | " print(\"model acc\", data_processed_fmt[dataset][\"valid\"][\"correct\"].float().mean().item())\n", 405 | " print()\n", 406 | " print()" 407 | ] 408 | }, 409 | { 410 | "cell_type": "code", 411 | "execution_count": 35, 412 | "metadata": { 413 | "scrolled": false 414 | }, 415 | "outputs": [ 416 | { 417 | "name": "stdout", 418 | "output_type": "stream", 419 | "text": [ 420 | "boolq\n", 421 | "probe acc 0.6266055107116699\n", 422 | "probe sparse 3841 8192\n", 423 | "model acc 0.6149846911430359\n" 424 | ] 425 | }, 426 | { 427 | "data": { 428 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO9klEQVR4nO3da2wUhb/G8afbgoBoAQGDooSrgXCpQiRGvEBRQKMpkBiM4D1EEyKSGPBEQ4LRoEViuMVYa9S04ZbQcFF4cSLkAEqFhnusioWIi6IFShGqtGznvODQ/2labtOn3U77/bxiZ6Yzv437dWe63d2UIAgCAWiQWLIHAFoCQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMWLECFVWVqpbt27JHgUtXGlpqdq2bauioqIrbhfJkM6fP69EIpHsMdAKXLhwQdfyxz+RDKl79+6SpG+++SbJk6Cly8zMvKbtuEYCDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAgxYTUnV1das8NpqHtGQP4BKLxbR69WqVlpY26XG7deump556qkmPieanxYQkSaWlpfr999+TPQZaoRZzagckEyEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAgT2kM2fO6Pjx4+7dAs1a6JBmzpxZ7/L9+/dr/PjxysnJCT0UEDWhQ/r111/rXT5q1Cht27ZN69evDz0UEDWhQ0pJSbnsuoqKCp0/fz7sroHIua53yC5dulTLli2ruT1w4MDLbjt27NjwUwERc10h3XvvvZKkIAi0atUqTZkype4O09LUs2dPPfroo54JgQi47pAuxXT48GHNmDGjUYYCoib0NdJHH310xfU7duwIu2sgchrtBdns7OzG2jXQ7IT+OK6TJ09qwYIF2rFjh06cOMGHJKJVCx3SW2+9pZKSEo0ZM0adO3dWLPafJ7dLv4wAWovQIf3www/6+uuvddNNN9W7Ph6Phx4KiJrQ10g9e/a8bESS9MEHH4TdNRA5oUMaP368Nm3adNn1zz33XNhdA5ET+tSuuLhYubm5+vjjj9WnTx+1b9++1voff/yxwcMBURE6pA0bNqh79+46e/as9u/fX2d9RUVFgwYDoiR0SP369dPatWsvuz4rKyvsroHICX2N9Pbbb19xPS/IojUJHdKIESOuuH7nzp1hdw1ETuhTu6t9odfKlSs1derUsLsHIiV0SGPGjLnim/uA1iR0SD169NBrr71Wa9m5c+dUUlKinTt3atq0aQ0eDoiK0CFNmDBBEydOrHddSUmJli9fHnooIGpC/7Jh9uzZl13Xt29f7d27N+yugchplPcjHTp0SKdOnWqMXQPNUuhTu2effbbOsiAIVF5ersOHD+vpp59u0GBAlIQO6cCBAxo8eHCtZampqerfv7+mTZt22esnoCUKHVKvXr2Ul5fnnAWIrNDXSF988YVxDCDaQj8jderUSZJUVFSkbdu2qaysTJ07d9YDDzxw1T8fAlqa0CElEgnNnj1bGzduVBAENctzcnL02GOPacGCBbU+xwFoyUKH9Mknn2jXrl2aM2eOMjIy1KlTJ50+fVp79+7V559/rpycHL3yyivOWYFmK3RI69evV15ennr16lVreUZGhkaPHq3p06cTElqN0OdesVisTkSX9OrVS6mpqaGHAqImdEiVlZUqKyurd92pU6f4Whe0KqFDyszM1AsvvKAtW7bo5MmTunDhgk6ePKnNmzfrpZde0iOPPOKcE2jWQl8jzZw5Uy+++KJeffXVOu9LGjZsWJ23WAAtWeiQOnTooPz8fG3YsEHbt2+veR1p1KhReuKJJ5SWFnrXQOSEfrSXlZVp9+7d6tKlixYuXFizfOPGjTpz5oy6dOliGRCIgtDXSPn5+Zo5c6bWrVtXa/n333+vrKws/fLLLw0eDoiK0M9IW7Zs0WeffaaRI0fWWj5v3jyNHj1a2dnZysnJafCAQBSEfkYKgqBORJc8/PDDKi0tDT0UEDWhQyovL2/QeqAlCR1S3759tXjxYiUSiVrLE4mEFi1apD59+jR4OCAqGvQ60tSpU7V69WoNHDhQ6enpKi8vV3Fxsc6ePcunCKFVCR3S4MGDlZeXp+zsbH377beqrq5WLBbT8OHDNXv2bA0aNMg5J9CsNehV0yFDhigvL0///vuvysvLlZ6ernbt2rlmAyLD8ucH7dq1IyC0aryFFTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkNAsBUEQqWPz1eNollJSUvTXX3+pqqqqSY/bpk0bde/e/bp/jpDQbFVVVamysjLZY1wTTu0AA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAipkQXV1a3y2K1NWrIHaOlSYjEdWZ+tf04cbdLjtu96p3o/ObtJj9maEVIT+OfEUf3zZ0myx0Aj4tSuFUsk8dQvmcduDDwjtWKpsZjm/89/62h5WZMe9870zvqvhx5p0mM2NkJq5Y6Wl+mXkyeSPUbkcWoHGKQEQRAke4jrNWTIECUSCfXo0aPW8nPnzimRSDTpLKmpqbrxxhuvuM2FinIFiQtNNNFFKalpSuuQftXtTv/7jy408fVKWiymTu3aX3W7pv5veUlqamrNv//44w+lpqbqwIEDV/yZSJ7a3XDDDaqsrKyz/GoP6GS5lgd0slzLAzpZ/v8DOlnS0tLUtm3bq24XyWckoLnhGgkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQIiIej2vJkiWKx+PJHgX1IKSIOHbsmJYuXapjx44lexTUg5AAAz5FKAJyc3O1atUqHT16VHfeeac6dOigs2fP6tSpU6qoqFCfPn00depUPfPMMyoqKtK8efNUUlKijIwMjRkzRgUFBSopKdGbb76p4uJiHTp0SPF4XMOHD9fcuXN122231RwrCAJ9+eWXWrlypSSpqqpKY8eO1euvv6727ZvvR3clXYBIKCwsDAYMGBAUFhbWLNu3b18wYMCAYPXq1bW23bp1azB9+vSa27/99lswYMCAYOTIkcGePXuCIAiC06dPB1lZWcG4ceOCqqqqmm3fe++9YNiwYcG+ffuCIAiC0tLSYMKECcHLL7/ciPcu+ji1i7ChQ4eqf//+WrNmTa3lBQUFmjx5cp3tMzMzlZGRIUlKT0/XjBkzdOTIEa1du1aSdPToUeXl5WnixIkaOnSoJKlr166aPn26tm7dqqKioka9P1FGSBE3adIk7dmzR4cPH5YklZeXa/fu3Ro9enSdbe+6665aty/FsmfPHknSd999p+rqag0fPrzenyssLLTP31JE8iOL8R9PPvmkFi5cqIKCAr3xxhv66quvNG7cOLVp06bOth07dqx1Oz394kcp//nnn5KksrKLX++yaNEiffrppzXbVVdXq2vXrqqoqGisuxF5hBRxXbt21YMPPqi1a9dq1qxZWrNmjebPn1/vtn///Xet26dPn5Yk3XrrrZKkzp07S5LmzJmjsWPHNt7QLRCndhGRlnbx/3nB//2S9eDBgzpy5IgkafLkySotLVVubq5isVidU7hLfv7551q39+/fL0m6++67JUn333+/YrGYiouL6/zsu+++q127dnnuTAtESBHRs2dPpaSk6Pjx45IuPrD37dsnSXrooYd0yy23aNGiRZo0adJl97Fjxw7t3btX0sVrqWXLlql3797KysqSJN1xxx16/vnnlZ+fr4MHD0q6GO6KFSu0efNmDRo0qPHuYMTxOlKELFmyRAUFBerYsaN69+6tDz/8sOYrR95//30tX75c27dv180331zr5+LxuDIzMzV37lz99NNPOnjwoOLxuO655556X0fKz8/XihUrlEgk1L59e/Xr10+zZs3S7bff3qT3N0oIqYVYt26dtm7dqoULF9ZZdymk+fPnX/EZC+FxatdCbNq0qd7XjtA0CCnCpkyZooqKCpWUlCgej+u+++5L9kitFr/+jrA2bdro8ccfV3p6ut555x2lpKTU2SY3N1cFBQWSpMWLF6uwsFDZ2dlNPWqLxzUSYMCpHWBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYPC/WvFQp8SvArkAAAAASUVORK5CYII=\n", 429 | "text/plain": [ 430 | "
" 431 | ] 432 | }, 433 | "metadata": {}, 434 | "output_type": "display_data" 435 | }, 436 | { 437 | "name": "stdout", 438 | "output_type": "stream", 439 | "text": [ 440 | "\n", 441 | "boolq\n", 442 | "probe acc 0.6214067339897156\n", 443 | "probe sparse 362 8192\n", 444 | "model acc 0.6149846911430359\n" 445 | ] 446 | }, 447 | { 448 | "data": { 449 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO7UlEQVR4nO3da2wUhb/G8afbgoBoAQGDooSrgXCpQiRGvEBRQKMpkBiM4D1EEyKSGPBEQ4LRoEViuMVYa9S04ZbQcFF4cSLkAEqFhnusioWIi6IFShGqtGznvODQ/2labtOn3U77/bxyZ2Znfhv3y850u9uUIAgCAWiQWLIHAFoCQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMWLECFVWVqpbt27JHgUtXGlpqdq2bauioqIrbhfJkM6fP69EIpHsMdAKXLhwQdfyyz+RDKl79+6SpG+++SbJk6Cly8zMvKbtuEYCDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwaDEhVVdXt8pjo3lIS/YALrFYTKtXr1ZpaWmTHrdbt2566qmnmvSYaH5aTEiSVFpaqt9//z3ZY6AVajGndkAyERJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAgT2kM2fO6Pjx4+7dAs1a6JBmzpxZ7/L9+/dr/PjxysnJCT0UEDWhQ/r111/rXT5q1Cht27ZN69evDz0UEDWhQ0pJSbnsuoqKCp0/fz7sroHIua7vbFi6dKmWLVtWc3vgwIGX3Xbs2LHhpwIi5rpCuvfeeyVJQRBo1apVmjJlSt0dpqWpZ8+eevTRRz0TAhFw3SFdiunw4cOaMWNGowwFRE3oa6SPPvroiut37NgRdtdA5DTaG7LZ2dmNtWug2Qn9BZEnT57UggULtGPHDp04cYKv7UWrFjqkt956SyUlJRozZow6d+6sWOw/L26XfhgBtBahQ/rhhx/09ddf66abbqp3fTweDz0UEDWhr5F69ux52Ygk6YMPPgi7ayByQoc0fvx4bdq06bLrn3vuubC7BiIn9KldcXGxcnNz9fHHH6tPnz5q3759rfU//vhjg4cDoiJ0SBs2bFD37t119uxZ7d+/v876ioqKBg0GREnokPr166e1a9dedn1WVlbYXQORE/oa6e23377iet6QRWsSOqQRI0Zccf3OnTvD7hqInNCndlf7E5MrV67U1KlTw+4eiJTQIY0ZM+aKH+4DWpPQIfXo0UOvvfZarWXnzp1TSUmJdu7cqWnTpjV4OCAqQoc0YcIETZw4sd51JSUlWr58eeihgKgJ/cOG2bNnX3Zd3759tXfv3rC7BiKnUT6PdOjQIZ06daoxdg00S6FP7Z599tk6y4IgUHl5uQ4fPqynn366QYMBURI6pAMHDmjw4MG1lqWmpqp///6aNm3aZa+fgJYodEi9evVSXl6ecxYgskJfI33xxRfGMYBoC/2K1KlTJ0lSUVGRtm3bprKyMnXu3FkPPPDAVX99CGhpQoeUSCQ0e/Zsbdy4UUEQ1CzPycnRY489pgULFtT6HgegJQsd0ieffKJdu3Zpzpw5ysjIUKdOnXT69Gnt3btXn3/+uXJycvTKK684ZwWardAhrV+/Xnl5eerVq1et5RkZGRo9erSmT59OSGg1Qp97xWKxOhFd0qtXL6WmpoYeCoia0CFVVlaqrKys3nWnTp3iz7qgVQkdUmZmpl544QVt2bJFJ0+e1IULF3Ty5Elt3rxZL730kh555BHnnECzFvoaaebMmXrxxRf16quv1vlc0rBhw+p8xAJoyUKH1KFDB+Xn52vDhg3avn17zftIo0aN0hNPPKG0tNC7BiIn9LO9rKxMu3fvVpcuXbRw4cKa5Rs3btSZM2fUpUsXy4BAFIS+RsrPz9fMmTO1bt26Wsu///57ZWVl6ZdffmnwcEBUhH5F2rJliz777DONHDmy1vJ58+Zp9OjRys7OVk5OToMHBKIg9CtSEAR1Irrk4YcfVmlpaeihgKgJHVJ5eXmD1gMtSeiQ+vbtq8WLFyuRSNRankgktGjRIvXp06fBwwFR0aD3kaZOnarVq1dr4MCBSk9PV3l5uYqLi3X27Fm+RQitSuiQBg8erLy8PGVnZ+vbb79VdXW1YrGYhg8frtmzZ2vQoEHOOYFmrUHvmg4ZMkR5eXn6999/VV5ervT0dLVr1841GxAZll8/aNeuHQGhVeMjrIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEpqlIAgidWz+PiWapZSUFP3111+qqqpq0uO2adNG3bt3v+77ERKaraqqKlVWViZ7jGvCqR1gQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASI0sqK5ulcdubdKSPUBLlxKL6cj6bP1z4miTHrd91zvV+8nZV9wmUV2t1Fhy/i1N5rEbAyE1gX9OHNU/f5Yke4w6UmMxzf+f/9bR8rImPe6d6Z31Xw890qTHbGwpQRAEyR7ieg0ZMkSJREI9evSotfzcuXNKJBJNOktqaqpuvPHGK25zoaJcQeJCE010UUpqmtI6pF91u9P//qMLTXwKmBaLqVO79lfdrqn/X16Smppa899//PGHUlNTdeDAgSveJ5KvSDfccIMqKyvrLL/aEzpZruUJnSzX8oROlv//hE6WtLQ0tW3b9qrbRfIVCWhuWs7VHpBEhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBBSRMTjcS1ZskTxeDzZo6AehBQRx44d09KlS3Xs2LFkj4J6EBJgwLcIRUBubq5WrVqlo0eP6s4771SHDh109uxZnTp1ShUVFerTp4+mTp2qZ555RkVFRZo3b55KSkqUkZGhMWPGqKCgQCUlJXrzzTdVXFysQ4cOKR6Pa/jw4Zo7d65uu+22mmMFQaAvv/xSK1eulCRVVVVp7Nixev3119W+ffP96q6kCxAJhYWFwYABA4LCwsKaZfv27QsGDBgQrF69uta2W7duDaZPn15z+7fffgsGDBgQjBw5MtizZ08QBEFw+vTpICsrKxg3blxQVVVVs+17770XDBs2LNi3b18QBEFQWloaTJgwIXj55Zcb8dFFH6d2ETZ06FD1799fa9asqbW8oKBAkydPrrN9ZmamMjIyJEnp6emaMWOGjhw5orVr10qSjh49qry8PE2cOFFDhw6VJHXt2lXTp0/X1q1bVVRU1KiPJ8oIKeImTZqkPXv26PDhw5Kk8vJy7d69W6NHj66z7V133VXr9qVY9uzZI0n67rvvVF1dreHDh9d7v8LCQvv8LUUkv7IY//Hkk09q4cKFKigo0BtvvKGvvvpK48aNU5s2beps27Fjx1q309MvfpXyn3/+KUkqK7v4ZfqLFi3Sp59+WrNddXW1unbtqoqKisZ6GJFHSBHXtWtXPfjgg1q7dq1mzZqlNWvWaP78+fVu+/fff9e6ffr0aUnSrbfeKknq3LmzJGnOnDkaO3Zs4w3dAnFqFxFpaRf/zQv+74esBw8e1JEjRyRJkydPVmlpqXJzcxWLxeqcwl3y888/17q9f/9+SdLdd98tSbr//vsVi8VUXFxc577vvvuudu3a5XkwLRAhRUTPnj2VkpKi48ePS7r4xN63b58k6aGHHtItt9yiRYsWadKkSZfdx44dO7R3715JF6+lli1bpt69eysrK0uSdMcdd+j5559Xfn6+Dh48KOliuCtWrNDmzZs1aNCgxnuAEcf7SBGyZMkSFRQUqGPHjurdu7c+/PDDmj858v7772v58uXavn27br755lr3i8fjyszM1Ny5c/XTTz/p4MGDisfjuueee+p9Hyk/P18rVqxQIpFQ+/bt1a9fP82aNUu33357kz7eKCGkFmLdunXaunWrFi5cWGfdpZDmz59/xVcshMepXQuxadOmet87QtMgpAibMmWKKioqVFJSong8rvvuuy/ZI7Va/Pg7wtq0aaPHH39c6enpeuedd5SSklJnm9zcXBUUFEiSFi9erMLCQmVnZzf1qC0e10iAAad2gAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAwf8CtAhLOm3vUEMAAAAASUVORK5CYII=\n", 450 | "text/plain": [ 451 | "
" 452 | ] 453 | }, 454 | "metadata": {}, 455 | "output_type": "display_data" 456 | }, 457 | { 458 | "name": "stdout", 459 | "output_type": "stream", 460 | "text": [ 461 | "\n", 462 | "boolq\n", 463 | "probe acc 0.6217125058174133\n", 464 | "probe sparse 43 8192\n", 465 | "model acc 0.6149846911430359\n" 466 | ] 467 | }, 468 | { 469 | "data": { 470 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO7UlEQVR4nO3da2wUhb/G8afbgoBoAQGDooSrgXCpQiRGvEBRQKMpkBiM4D1EEyKSGPBEQ4LRoEViuMVYa9S04ZbQcFF4cSLkAEqFhnusioWIi6IFShGqtGznvODQ/2labtOn3U77/bxyZ2Znfhv3y850u9uUIAgCAWiQWLIHAFoCQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMWLECFVWVqpbt27JHgUtXGlpqdq2bauioqIrbhfJkM6fP69EIpHsMdAKXLhwQdfyyz+RDKl79+6SpG+++SbJk6Cly8zMvKbtuEYCDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwaDEhVVdXt8pjo3lIS/YALrFYTKtXr1ZpaWmTHrdbt2566qmnmvSYaH5aTEiSVFpaqt9//z3ZY6AVajGndkAyERJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAgT2kM2fO6Pjx4+7dAs1a6JBmzpxZ7/L9+/dr/PjxysnJCT0UEDWhQ/r111/rXT5q1Cht27ZN69evDz0UEDWhQ0pJSbnsuoqKCp0/fz7sroHIua7vbFi6dKmWLVtWc3vgwIGX3Xbs2LHhpwIi5rpCuvfeeyVJQRBo1apVmjJlSt0dpqWpZ8+eevTRRz0TAhFw3SFdiunw4cOaMWNGowwFRE3oa6SPPvroiut37NgRdtdA5DTaG7LZ2dmNtWug2Qn9BZEnT57UggULtGPHDp04cYKv7UWrFjqkt956SyUlJRozZow6d+6sWOw/L26XfhgBtBahQ/rhhx/09ddf66abbqp3fTweDz0UEDWhr5F69ux52Ygk6YMPPgi7ayByQoc0fvx4bdq06bLrn3vuubC7BiIn9KldcXGxcnNz9fHHH6tPnz5q3759rfU//vhjg4cDoiJ0SBs2bFD37t119uxZ7d+/v876ioqKBg0GREnokPr166e1a9dedn1WVlbYXQORE/oa6e23377iet6QRWsSOqQRI0Zccf3OnTvD7hqInNCndlf7E5MrV67U1KlTw+4eiJTQIY0ZM+aKH+4DWpPQIfXo0UOvvfZarWXnzp1TSUmJdu7cqWnTpjV4OCAqQoc0YcIETZw4sd51JSUlWr58eeihgKgJ/cOG2bNnX3Zd3759tXfv3rC7BiKnUT6PdOjQIZ06daoxdg00S6FP7Z599tk6y4IgUHl5uQ4fPqynn366QYMBURI6pAMHDmjw4MG1lqWmpqp///6aNm3aZa+fgJYodEi9evVSXl6ecxYgskJfI33xxRfGMYBoC/2K1KlTJ0lSUVGRtm3bprKyMnXu3FkPPPDAVX99CGhpQoeUSCQ0e/Zsbdy4UUEQ1CzPycnRY489pgULFtT6HgegJQsd0ieffKJdu3Zpzpw5ysjIUKdOnXT69Gnt3btXn3/+uXJycvTKK684ZwWardAhrV+/Xnl5eerVq1et5RkZGRo9erSmT59OSGg1Qp97xWKxOhFd0qtXL6WmpoYeCoia0CFVVlaqrKys3nWnTp3iz7qgVQkdUmZmpl544QVt2bJFJ0+e1IULF3Ty5Elt3rxZL730kh555BHnnECzFvoaaebMmXrxxRf16quv1vlc0rBhw+p8xAJoyUKH1KFDB+Xn52vDhg3avn17zftIo0aN0hNPPKG0tNC7BiIn9LO9rKxMu3fvVpcuXbRw4cKa5Rs3btSZM2fUpUsXy4BAFIS+RsrPz9fMmTO1bt26Wsu///57ZWVl6ZdffmnwcEBUhH5F2rJliz777DONHDmy1vJ58+Zp9OjRys7OVk5OToMHBKIg9CtSEAR1Irrk4YcfVmlpaeihgKgJHVJ5eXmD1gMtSeiQ+vbtq8WLFyuRSNRankgktGjRIvXp06fBwwFR0aD3kaZOnarVq1dr4MCBSk9PV3l5uYqLi3X27Fm+RQitSuiQBg8erLy8PGVnZ+vbb79VdXW1YrGYhg8frtmzZ2vQoEHOOYFmrUHvmg4ZMkR5eXn6999/VV5ervT0dLVr1841GxAZll8/aNeuHQGhVeMjrIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEhGYpCIJIHZs/q4dmKSUlRX/99Zeqqqqa9Lht2rRR9+7dr/t+hIRmq6qqSpWVlcke45pwagcYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASI0sqK5ulcdubdKSPUBLlxKL6cj6bP1z4miTHrd91zvV+8nZV9wmUV2t1Fhy/i1N5rEbAyE1gX9OHNU/f5Yke4w6UmMxzf+f/9bR8rImPe6d6Z31Xw890qTHbGwpQRAEyR7ieg0ZMkSJREI9evSotfzcuXNKJBJNOktqaqpuvPHGK25zoaJcQeJCE010UUpqmtI6pF91u9P//qMLTXwKmBaLqVO79lfdrqn/X16Smppa899//PGHUlNTdeDAgSveJ5KvSDfccIMqKyvrLL/aEzpZruUJnSzX8oROlv//hE6WtLQ0tW3b9qrbRfIVCWhuWs7VHpBEhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBBSRMTjcS1ZskTxeDzZo6AehBQRx44d09KlS3Xs2LFkj4J6EBJgwLcIRUBubq5WrVqlo0eP6s4771SHDh109uxZnTp1ShUVFerTp4+mTp2qZ555RkVFRZo3b55KSkqUkZGhMWPGqKCgQCUlJXrzzTdVXFysQ4cOKR6Pa/jw4Zo7d65uu+22mmMFQaAvv/xSK1eulCRVVVVp7Nixev3119W+ffP96q6kCxAJhYWFwYABA4LCwsKaZfv27QsGDBgQrF69uta2W7duDaZPn15z+7fffgsGDBgQjBw5MtizZ08QBEFw+vTpICsrKxg3blxQVVVVs+17770XDBs2LNi3b18QBEFQWloaTJgwIXj55Zcb8dFFH6d2ETZ06FD1799fa9asqbW8oKBAkydPrrN9ZmamMjIyJEnp6emaMWOGjhw5orVr10qSjh49qry8PE2cOFFDhw6VJHXt2lXTp0/X1q1bVVRU1KiPJ8oIKeImTZqkPXv26PDhw5Kk8vJy7d69W6NHj66z7V133VXr9qVY9uzZI0n67rvvVF1dreHDh9d7v8LCQvv8LUUkv7IY//Hkk09q4cKFKigo0BtvvKGvvvpK48aNU5s2beps27Fjx1q309MvfpXyn3/+KUkqK7v4ZfqLFi3Sp59+WrNddXW1unbtqoqKisZ6GJFHSBHXtWtXPfjgg1q7dq1mzZqlNWvWaP78+fVu+/fff9e6ffr0aUnSrbfeKknq3LmzJGnOnDkaO3Zs4w3dAnFqFxFpaRf/zQv+74esBw8e1JEjRyRJkydPVmlpqXJzcxWLxeqcwl3y888/17q9f/9+SdLdd98tSbr//vsVi8VUXFxc577vvvuudu3a5XkwLRAhRUTPnj2VkpKi48ePS7r4xN63b58k6aGHHtItt9yiRYsWadKkSZfdx44dO7R3715JF6+lli1bpt69eysrK0uSdMcdd+j5559Xfn6+Dh48KOliuCtWrNDmzZs1aNCgxnuAEcf7SBGyZMkSFRQUqGPHjurdu7c+/PDDmj858v7772v58uXavn27br755lr3i8fjyszM1Ny5c/XTTz/p4MGDisfjuueee+p9Hyk/P18rVqxQIpFQ+/bt1a9fP82aNUu33357kz7eKCGkFmLdunXaunWrFi5cWGfdpZDmz59/xVcshMepXQuxadOmet87QtMgpAibMmWKKioqVFJSong8rvvuuy/ZI7Va/Pg7wtq0aaPHH39c6enpeuedd5SSklJnm9zcXBUUFEiSFi9erMLCQmVnZzf1qC0e10iAAad2gAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAwf8CcF1LOlx57hUAAAAASUVORK5CYII=\n", 471 | "text/plain": [ 472 | "
" 473 | ] 474 | }, 475 | "metadata": {}, 476 | "output_type": "display_data" 477 | }, 478 | { 479 | "name": "stdout", 480 | "output_type": "stream", 481 | "text": [ 482 | "\n", 483 | "boolq\n", 484 | "probe acc 0.6217125058174133\n", 485 | "probe sparse 43 8192\n", 486 | "model acc 0.6149846911430359\n" 487 | ] 488 | }, 489 | { 490 | "data": { 491 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO7UlEQVR4nO3da2wUhb/G8afbgoBoAQGDooSrgXCpQiRGvEBRQKMpkBiM4D1EEyKSGPBEQ4LRoEViuMVYa9S04ZbQcFF4cSLkAEqFhnusioWIi6IFShGqtGznvODQ/2labtOn3U77/bxyZ2Znfhv3y850u9uUIAgCAWiQWLIHAFoCQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMWLECFVWVqpbt27JHgUtXGlpqdq2bauioqIrbhfJkM6fP69EIpHsMdAKXLhwQdfyyz+RDKl79+6SpG+++SbJk6Cly8zMvKbtuEYCDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwaDEhVVdXt8pjo3lIS/YALrFYTKtXr1ZpaWmTHrdbt2566qmnmvSYaH5aTEiSVFpaqt9//z3ZY6AVajGndkAyERJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAgT2kM2fO6Pjx4+7dAs1a6JBmzpxZ7/L9+/dr/PjxysnJCT0UEDWhQ/r111/rXT5q1Cht27ZN69evDz0UEDWhQ0pJSbnsuoqKCp0/fz7sroHIua7vbFi6dKmWLVtWc3vgwIGX3Xbs2LHhpwIi5rpCuvfeeyVJQRBo1apVmjJlSt0dpqWpZ8+eevTRRz0TAhFw3SFdiunw4cOaMWNGowwFRE3oa6SPPvroiut37NgRdtdA5DTaG7LZ2dmNtWug2Qn9BZEnT57UggULtGPHDp04cYKv7UWrFjqkt956SyUlJRozZow6d+6sWOw/L26XfhgBtBahQ/rhhx/09ddf66abbqp3fTweDz0UEDWhr5F69ux52Ygk6YMPPgi7ayByQoc0fvx4bdq06bLrn3vuubC7BiIn9KldcXGxcnNz9fHHH6tPnz5q3759rfU//vhjg4cDoiJ0SBs2bFD37t119uxZ7d+/v876ioqKBg0GREnokPr166e1a9dedn1WVlbYXQORE/oa6e23377iet6QRWsSOqQRI0Zccf3OnTvD7hqInNCndlf7E5MrV67U1KlTw+4eiJTQIY0ZM+aKH+4DWpPQIfXo0UOvvfZarWXnzp1TSUmJdu7cqWnTpjV4OCAqQoc0YcIETZw4sd51JSUlWr58eeihgKgJ/cOG2bNnX3Zd3759tXfv3rC7BiKnUT6PdOjQIZ06daoxdg00S6FP7Z599tk6y4IgUHl5uQ4fPqynn366QYMBURI6pAMHDmjw4MG1lqWmpqp///6aNm3aZa+fgJYodEi9evVSXl6ecxYgskJfI33xxRfGMYBoC/2K1KlTJ0lSUVGRtm3bprKyMnXu3FkPPPDAVX99CGhpQoeUSCQ0e/Zsbdy4UUEQ1CzPycnRY489pgULFtT6HgegJQsd0ieffKJdu3Zpzpw5ysjIUKdOnXT69Gnt3btXn3/+uXJycvTKK684ZwWardAhrV+/Xnl5eerVq1et5RkZGRo9erSmT59OSGg1Qp97xWKxOhFd0qtXL6WmpoYeCoia0CFVVlaqrKys3nWnTp3iz7qgVQkdUmZmpl544QVt2bJFJ0+e1IULF3Ty5Elt3rxZL730kh555BHnnECzFvoaaebMmXrxxRf16quv1vlc0rBhw+p8xAJoyUKH1KFDB+Xn52vDhg3avn17zftIo0aN0hNPPKG0tNC7BiIn9LO9rKxMu3fvVpcuXbRw4cKa5Rs3btSZM2fUpUsXy4BAFIS+RsrPz9fMmTO1bt26Wsu///57ZWVl6ZdffmnwcEBUhH5F2rJliz777DONHDmy1vJ58+Zp9OjRys7OVk5OToMHBKIg9CtSEAR1Irrk4YcfVmlpaeihgKgJHVJ5eXmD1gMtSeiQ+vbtq8WLFyuRSNRankgktGjRIvXp06fBwwFR0aD3kaZOnarVq1dr4MCBSk9PV3l5uYqLi3X27Fm+RQitSuiQBg8erLy8PGVnZ+vbb79VdXW1YrGYhg8frtmzZ2vQoEHOOYFmrUHvmg4ZMkR5eXn6999/VV5ervT0dLVr1841GxAZll8/aNeuHQGhVeMjrIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEhGYpCIJIHZs/q4dmKSUlRX/99Zeqqqqa9Lht2rRR9+7dr/t+hIRmq6qqSpWVlcke45pwagcYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASI0sqK5ulcdubdKSPUBLlxKL6cj6bP1z4miTHrd91zvV+8nZV9wmUV2t1Fhy/i1N5rEbAyE1gX9OHNU/f5Yke4w6UmMxzf+f/9bR8rImPe6d6Z31Xw890qTHbGwpQRAEyR7ieg0ZMkSJREI9evSotfzcuXNKJBJNOktqaqpuvPHGK25zoaJcQeJCE010UUpqmtI6pF91u9P//qMLTXwKmBaLqVO79lfdrqn/X16Smppa899//PGHUlNTdeDAgSveJ5KvSDfccIMqKyvrLL/aEzpZruUJnSzX8oROlv//hE6WtLQ0tW3b9qrbRfIVCWhuWs7VHpBEhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBBSRMTjcS1ZskTxeDzZo6AehBQRx44d09KlS3Xs2LFkj4J6EBJgwLcIRUBubq5WrVqlo0eP6s4771SHDh109uxZnTp1ShUVFerTp4+mTp2qZ555RkVFRZo3b55KSkqUkZGhMWPGqKCgQCUlJXrzzTdVXFysQ4cOKR6Pa/jw4Zo7d65uu+22mmMFQaAvv/xSK1eulCRVVVVp7Nixev3119W+ffP96q6kCxAJhYWFwYABA4LCwsKaZfv27QsGDBgQrF69uta2W7duDaZPn15z+7fffgsGDBgQjBw5MtizZ08QBEFw+vTpICsrKxg3blxQVVVVs+17770XDBs2LNi3b18QBEFQWloaTJgwIXj55Zcb8dFFH6d2ETZ06FD1799fa9asqbW8oKBAkydPrrN9ZmamMjIyJEnp6emaMWOGjhw5orVr10qSjh49qry8PE2cOFFDhw6VJHXt2lXTp0/X1q1bVVRU1KiPJ8oIKeImTZqkPXv26PDhw5Kk8vJy7d69W6NHj66z7V133VXr9qVY9uzZI0n67rvvVF1dreHDh9d7v8LCQvv8LUUkv7IY//Hkk09q4cKFKigo0BtvvKGvvvpK48aNU5s2beps27Fjx1q309MvfpXyn3/+KUkqK7v4ZfqLFi3Sp59+WrNddXW1unbtqoqKisZ6GJFHSBHXtWtXPfjgg1q7dq1mzZqlNWvWaP78+fVu+/fff9e6ffr0aUnSrbfeKknq3LmzJGnOnDkaO3Zs4w3dAnFqFxFpaRf/zQv+74esBw8e1JEjRyRJkydPVmlpqXJzcxWLxeqcwl3y888/17q9f/9+SdLdd98tSbr//vsVi8VUXFxc577vvvuudu3a5XkwLRAhRUTPnj2VkpKi48ePS7r4xN63b58k6aGHHtItt9yiRYsWadKkSZfdx44dO7R3715JF6+lli1bpt69eysrK0uSdMcdd+j5559Xfn6+Dh48KOliuCtWrNDmzZs1aNCgxnuAEcf7SBGyZMkSFRQUqGPHjurdu7c+/PDDmj858v7772v58uXavn27br755lr3i8fjyszM1Ny5c/XTTz/p4MGDisfjuueee+p9Hyk/P18rVqxQIpFQ+/bt1a9fP82aNUu33357kz7eKCGkFmLdunXaunWrFi5cWGfdpZDmz59/xVcshMepXQuxadOmet87QtMgpAibMmWKKioqVFJSong8rvvuuy/ZI7Va/Pg7wtq0aaPHH39c6enpeuedd5SSklJnm9zcXBUUFEiSFi9erMLCQmVnZzf1qC0e10iAAad2gAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAwf8CcF1LOlx57hUAAAAASUVORK5CYII=\n", 492 | "text/plain": [ 493 | "
" 494 | ] 495 | }, 496 | "metadata": {}, 497 | "output_type": "display_data" 498 | }, 499 | { 500 | "name": "stdout", 501 | "output_type": "stream", 502 | "text": [ 503 | "\n", 504 | "boolq\n", 505 | "probe acc 0.6217125058174133\n", 506 | "probe sparse 154 8192\n", 507 | "model acc 0.6149846911430359\n" 508 | ] 509 | }, 510 | { 511 | "data": { 512 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO7UlEQVR4nO3da2wUhb/G8afbgoBoAQGDooSrgXCpQiRGvEBRQKMpkBiM4D1EEyKSGPBEQ4LRoEViuMVYa9S04ZbQcFF4cSLkAEqFhnusioWIi6IFShGqtGznvODQ/2labtOn3U77/bxyZ2Znfhv3y850u9uUIAgCAWiQWLIHAFoCQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMWLECFVWVqpbt27JHgUtXGlpqdq2bauioqIrbhfJkM6fP69EIpHsMdAKXLhwQdfyyz+RDKl79+6SpG+++SbJk6Cly8zMvKbtuEYCDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwaDEhVVdXt8pjo3lIS/YALrFYTKtXr1ZpaWmTHrdbt2566qmnmvSYaH5aTEiSVFpaqt9//z3ZY6AVajGndkAyERJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAgT2kM2fO6Pjx4+7dAs1a6JBmzpxZ7/L9+/dr/PjxysnJCT0UEDWhQ/r111/rXT5q1Cht27ZN69evDz0UEDWhQ0pJSbnsuoqKCp0/fz7sroHIua7vbFi6dKmWLVtWc3vgwIGX3Xbs2LHhpwIi5rpCuvfeeyVJQRBo1apVmjJlSt0dpqWpZ8+eevTRRz0TAhFw3SFdiunw4cOaMWNGowwFRE3oa6SPPvroiut37NgRdtdA5DTaG7LZ2dmNtWug2Qn9BZEnT57UggULtGPHDp04cYKv7UWrFjqkt956SyUlJRozZow6d+6sWOw/L26XfhgBtBahQ/rhhx/09ddf66abbqp3fTweDz0UEDWhr5F69ux52Ygk6YMPPgi7ayByQoc0fvx4bdq06bLrn3vuubC7BiIn9KldcXGxcnNz9fHHH6tPnz5q3759rfU//vhjg4cDoiJ0SBs2bFD37t119uxZ7d+/v876ioqKBg0GREnokPr166e1a9dedn1WVlbYXQORE/oa6e23377iet6QRWsSOqQRI0Zccf3OnTvD7hqInNCndlf7E5MrV67U1KlTw+4eiJTQIY0ZM+aKH+4DWpPQIfXo0UOvvfZarWXnzp1TSUmJdu7cqWnTpjV4OCAqQoc0YcIETZw4sd51JSUlWr58eeihgKgJ/cOG2bNnX3Zd3759tXfv3rC7BiKnUT6PdOjQIZ06daoxdg00S6FP7Z599tk6y4IgUHl5uQ4fPqynn366QYMBURI6pAMHDmjw4MG1lqWmpqp///6aNm3aZa+fgJYodEi9evVSXl6ecxYgskJfI33xxRfGMYBoC/2K1KlTJ0lSUVGRtm3bprKyMnXu3FkPPPDAVX99CGhpQoeUSCQ0e/Zsbdy4UUEQ1CzPycnRY489pgULFtT6HgegJQsd0ieffKJdu3Zpzpw5ysjIUKdOnXT69Gnt3btXn3/+uXJycvTKK684ZwWardAhrV+/Xnl5eerVq1et5RkZGRo9erSmT59OSGg1Qp97xWKxOhFd0qtXL6WmpoYeCoia0CFVVlaqrKys3nWnTp3iz7qgVQkdUmZmpl544QVt2bJFJ0+e1IULF3Ty5Elt3rxZL730kh555BHnnECzFvoaaebMmXrxxRf16quv1vlc0rBhw+p8xAJoyUKH1KFDB+Xn52vDhg3avn17zftIo0aN0hNPPKG0tNC7BiIn9LO9rKxMu3fvVpcuXbRw4cKa5Rs3btSZM2fUpUsXy4BAFIS+RsrPz9fMmTO1bt26Wsu///57ZWVl6ZdffmnwcEBUhH5F2rJliz777DONHDmy1vJ58+Zp9OjRys7OVk5OToMHBKIg9CtSEAR1Irrk4YcfVmlpaeihgKgJHVJ5eXmD1gMtSeiQ+vbtq8WLFyuRSNRankgktGjRIvXp06fBwwFR0aD3kaZOnarVq1dr4MCBSk9PV3l5uYqLi3X27Fm+RQitSuiQBg8erLy8PGVnZ+vbb79VdXW1YrGYhg8frtmzZ2vQoEHOOYFmrUHvmg4ZMkR5eXn6999/VV5ervT0dLVr1841GxAZll8/aNeuHQGhVeMjrIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEhGYpCIJIHZs/q4dmKSUlRX/99Zeqqqqa9Lht2rRR9+7dr/t+hIRmq6qqSpWVlcke45pwagcYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASI0sqK5ulcdubdKSPUBLlxKL6cj6bP1z4miTHrd91zvV+8nZV9wmUV2t1Fhy/i1N5rEbAyE1gX9OHNU/f5Yke4w6UmMxzf+f/9bR8rImPe6d6Z31Xw890qTHbGwpQRAEyR7ieg0ZMkSJREI9evSotfzcuXNKJBJNOktqaqpuvPHGK25zoaJcQeJCE010UUpqmtI6pF91u9P//qMLTXwKmBaLqVO79lfdrqn/X16Smppa899//PGHUlNTdeDAgSveJ5KvSDfccIMqKyvrLL/aEzpZruUJnSzX8oROlv//hE6WtLQ0tW3b9qrbRfIVCWhuWs7VHpBEhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBBSRMTjcS1ZskTxeDzZo6AehBQRx44d09KlS3Xs2LFkj4J6EBJgwLcIRUBubq5WrVqlo0eP6s4771SHDh109uxZnTp1ShUVFerTp4+mTp2qZ555RkVFRZo3b55KSkqUkZGhMWPGqKCgQCUlJXrzzTdVXFysQ4cOKR6Pa/jw4Zo7d65uu+22mmMFQaAvv/xSK1eulCRVVVVp7Nixev3119W+ffP96q6kCxAJhYWFwYABA4LCwsKaZfv27QsGDBgQrF69uta2W7duDaZPn15z+7fffgsGDBgQjBw5MtizZ08QBEFw+vTpICsrKxg3blxQVVVVs+17770XDBs2LNi3b18QBEFQWloaTJgwIXj55Zcb8dFFH6d2ETZ06FD1799fa9asqbW8oKBAkydPrrN9ZmamMjIyJEnp6emaMWOGjhw5orVr10qSjh49qry8PE2cOFFDhw6VJHXt2lXTp0/X1q1bVVRU1KiPJ8oIKeImTZqkPXv26PDhw5Kk8vJy7d69W6NHj66z7V133VXr9qVY9uzZI0n67rvvVF1dreHDh9d7v8LCQvv8LUUkv7IY//Hkk09q4cKFKigo0BtvvKGvvvpK48aNU5s2beps27Fjx1q309MvfpXyn3/+KUkqK7v4ZfqLFi3Sp59+WrNddXW1unbtqoqKisZ6GJFHSBHXtWtXPfjgg1q7dq1mzZqlNWvWaP78+fVu+/fff9e6ffr0aUnSrbfeKknq3LmzJGnOnDkaO3Zs4w3dAnFqFxFpaRf/zQv+74esBw8e1JEjRyRJkydPVmlpqXJzcxWLxeqcwl3y888/17q9f/9+SdLdd98tSbr//vsVi8VUXFxc577vvvuudu3a5XkwLRAhRUTPnj2VkpKi48ePS7r4xN63b58k6aGHHtItt9yiRYsWadKkSZfdx44dO7R3715JF6+lli1bpt69eysrK0uSdMcdd+j5559Xfn6+Dh48KOliuCtWrNDmzZs1aNCgxnuAEcf7SBGyZMkSFRQUqGPHjurdu7c+/PDDmj858v7772v58uXavn27br755lr3i8fjyszM1Ny5c/XTTz/p4MGDisfjuueee+p9Hyk/P18rVqxQIpFQ+/bt1a9fP82aNUu33357kz7eKCGkFmLdunXaunWrFi5cWGfdpZDmz59/xVcshMepXQuxadOmet87QtMgpAibMmWKKioqVFJSong8rvvuuy/ZI7Va/Pg7wtq0aaPHH39c6enpeuedd5SSklJnm9zcXBUUFEiSFi9erMLCQmVnZzf1qC0e10iAAad2gAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAwf8CcF1LOlx57hUAAAAASUVORK5CYII=\n", 513 | "text/plain": [ 514 | "
" 515 | ] 516 | }, 517 | "metadata": {}, 518 | "output_type": "display_data" 519 | }, 520 | { 521 | "name": "stdout", 522 | "output_type": "stream", 523 | "text": [ 524 | "\n", 525 | "\n", 526 | "sciq\n", 527 | "probe acc 0.8820000290870667\n", 528 | "probe sparse 3873 8192\n", 529 | "model acc 0.8379999995231628\n" 530 | ] 531 | }, 532 | { 533 | "data": { 534 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO+klEQVR4nO3deWwUBf/H8U+3BQHRUgQMihJOA+GoQiRGPKAooNEUSAxG8A7RhIgkBnyiIcFo0CIxXDHWGjVtuBIaDoU/ngh5AKVCwx2rYiHigmiBUoQqx3Z+f/Cjz9O0XNNPu532/fqLnZmd+W7ctzvT7W5TgiAIBKBeYskeAGgOCAkwICTAgJAAA0ICDAgJMCAkwICQAIO0ZA8QxtChQ3Xu3Dl17tw52aOgmSsrK1Pr1q1VXFx8xe0iGdLZs2eVSCSSPQZagAsXLuhafvknkiF16dJFkvTNN98keRI0d1lZWde0HddIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGDSbkKqqqlrksdE0pCV7AJdYLKYVK1aorKysUY/buXNnPfXUU416TDQ9zSYkSSorK9ORI0eSPQZaoGZzagckEyEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBvaQTp06paNHj7p3CzRpoUOaNm1ancv37NmjMWPGKDc3N/RQQNSEDunXX3+tc/nw4cO1efNmrVmzJvRQQNSEDiklJeWy6yorK3X27NmwuwYiJ+16Nl60aJEWL15cfbtfv36X3XbUqFHhpwIi5rpCuvfeeyVJQRBo+fLlmjhxYu0dpqWpW7duevTRRz0TAhFw3SFdiunAgQOaOnVqgwwFRE3oa6SPPvroiuu3bt0adtdA5DTYG7I5OTkNtWugybmuU7v/dfz4cc2dO1dbt27VsWPHVFVV5ZwLiJTQIb311lsqLS3VyJEjlZGRoVjsvy9ul34YAbQUoUP64Ycf9PXXX+umm26qc308Hg89FBA1oa+RunXrdtmIJOmDDz4Iu2sgckKHNGbMGK1fv/6y65977rmwuwYiJ/SpXUlJifLy8vTxxx+rZ8+eatu2bY31P/74Y72HA6IidEhr165Vly5ddPr0ae3Zs6fW+srKynoNBkRJ6JB69+6tVatWXXZ9dnZ22F0DkRP6Guntt9++4nrekEVLEjqkoUOHXnH9tm3bwu4aiJzQp3ZHjhy54vply5Zp0qRJYXcPRErokEaOHHnFD/cBLUnokLp27arXXnutxrIzZ86otLRU27Zt0+TJk+s9HBAVoUMaO3asxo0bV+e60tJSLVmyJPRQQNSE/mHDjBkzLruuV69e2rVrV9hdA5HTIJ9H2r9/v06cONEQuwaapNCnds8++2ytZUEQqKKiQgcOHNDTTz9dr8GAKAkd0t69ezVgwIAay1JTU9WnTx9Nnjz5stdPQHMUOqTu3bsrPz/fOQsQWaGvkb744gvjGEC0hX5F6tChgySpuLhYmzdvVnl5uTIyMvTAAw9c9deHgOYmdEiJREIzZszQunXrFARB9fLc3Fw99thjmjt3bo3vcQCas9AhffLJJ9q+fbtmzpypzMxMdejQQSdPntSuXbv0+eefKzc3V6+88opzVqDJCh3SmjVrlJ+fr+7du9dYnpmZqREjRmjKlCmEhBYj9LlXLBarFdEl3bt3V2pqauihgKgJHdK5c+dUXl5e57oTJ07wZ13QooQOKSsrSy+88II2btyo48eP68KFCzp+/Lg2bNigl156SY888ohzTqBJC32NNG3aNL344ot69dVXa30uafDgwbU+YgE0Z6FDateunQoKCrR27Vpt2bKl+n2k4cOH64knnlBaWuhdA5ET+tleXl6uHTt2qGPHjpo3b1718nXr1unUqVPq2LGjZUAgCkJfIxUUFGjatGlavXp1jeXff/+9srOz9csvv9R7OCAqQr8ibdy4UZ999pmGDRtWY/ns2bM1YsQI5eTkKDc3t94DAlEQ+hUpCIJaEV3y8MMPq6ysLPRQQNSEDqmioqJe64HmJHRIvXr10oIFC5RIJGosTyQSmj9/vnr27Fnv4YCoqNf7SJMmTdKKFSvUr18/paenq6KiQiUlJTp9+jTfIoQWJXRIAwYMUH5+vnJycvTtt9+qqqpKsVhMQ4YM0YwZM9S/f3/nnECTVq93TQcOHKj8/Hz9888/qqioUHp6utq0aeOaDYgMy68ftGnThoDQovERVsCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQGlhQVdUij93SpCV7gOYuJRbTwTU5+vvYoUY9bttOd6rHkzMa9ZgtGSE1gr+PHdLff5Qmeww0IE7tAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCasESSfy8UjKP3RD4GEULlhqLac5//q1DFeWNetw70zP0r4ceadRjNjRCauEOVZTrl+PHkj1G5HFqBxgQEmBASIABIQEGhAQYEBJgQEhokoIgiNSxeR8JTVJKSor+/PNPnT9/vlGP26pVK3Xp0uW670dIaLLOnz+vc+fOJXuMa8KpHWBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmCQEiTzgx8hDRw4UIlEQl27dq2x/MyZM0okEo06S2pqqm688cYrbnOhskJB4kIjTXRRSmqa0tqlX3W7k//8rQuN/GWNabGYOrRpe9XtGvu/5SWpqanV//7999+VmpqqvXv3XvE+kfwYxQ033FDnr9df7QmdLNfyhE6Wa3lCJ8v/PqGTJS0tTa1bt77qdpF8RQKaGq6RAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCCki4vG4Fi5cqHg8nuxRUAdCiojDhw9r0aJFOnz4cLJHQR0ICTDgW4QiIC8vT8uXL9ehQ4d05513ql27djp9+rROnDihyspK9ezZU5MmTdIzzzyj4uJizZ49W6WlpcrMzNTIkSNVWFio0tJSvfnmmyopKdH+/fsVj8c1ZMgQzZo1S7fddlv1sYIg0Jdffqlly5ZJuviXxUeNGqXXX39dbds23a/uSroAkVBUVBT07ds3KCoqql62e/fuoG/fvsGKFStqbLtp06ZgypQp1bd/++23oG/fvsGwYcOCnTt3BkEQBCdPngyys7OD0aNHB+fPn6/e9r333gsGDx4c7N69OwiCICgrKwvGjh0bvPzyyw346KKPU7sIGzRokPr06aOVK1fWWF5YWKgJEybU2j4rK0uZmZmSpPT0dE2dOlUHDx7UqlWrJEmHDh1Sfn6+xo0bp0GDBkmSOnXqpClTpmjTpk0qLi5u0McTZYQUcePHj9fOnTt14MABSVJFRYV27NihESNG1Nr2rrvuqnH7Uiw7d+6UJH333XeqqqrSkCFD6rxfUVGRff7mIpJfWYz/evLJJzVv3jwVFhbqjTfe0FdffaXRo0erVatWtbZt3759jdvp6Re/SvmPP/6QJJWXl0uS5s+fr08//bR6u6qqKnXq1EmVlZUN9TAij5AirlOnTnrwwQe1atUqTZ8+XStXrtScOXPq3Pavv/6qcfvkyZOSpFtvvVWSlJGRIUmaOXOmRo0a1XBDN0Oc2kVEWtrF/+cF//9D1n379ungwYOSpAkTJqisrEx5eXmKxWK1TuEu+fnnn2vc3rNnjyTp7rvvliTdf//9isViKikpqXXfd999V9u3b/c8mGaIkCKiW7duSklJ0dGjRyVdfGLv3r1bkvTQQw/plltu0fz58zV+/PjL7mPr1q3atWuXpIvXUosXL1aPHj2UnZ0tSbrjjjv0/PPPq6CgQPv27ZN0MdylS5dqw4YN6t+/f8M9wIjjfaQIWbhwoQoLC9W+fXv16NFDH374YfWfHHn//fe1ZMkSbdmyRTfffHON+8XjcWVlZWnWrFn66aeftG/fPsXjcd1zzz11vo9UUFCgpUuXKpFIqG3bturdu7emT5+u22+/vVEfb5QQUjOxevVqbdq0SfPmzau17lJIc+bMueIrFsLj1K6ZWL9+fZ3vHaFxEFKETZw4UZWVlSotLVU8Htd9992X7JFaLH78HWGtWrXS448/rvT0dL3zzjtKSUmptU1eXp4KCwslSQsWLFBRUZFycnIae9Rmj2skwIBTO8CAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwOD/AOAaUKe1jmeTAAAAAElFTkSuQmCC\n", 535 | "text/plain": [ 536 | "
" 537 | ] 538 | }, 539 | "metadata": {}, 540 | "output_type": "display_data" 541 | }, 542 | { 543 | "name": "stdout", 544 | "output_type": "stream", 545 | "text": [ 546 | "\n", 547 | "sciq\n", 548 | "probe acc 0.8840000629425049\n", 549 | "probe sparse 837 8192\n", 550 | "model acc 0.8379999995231628\n" 551 | ] 552 | }, 553 | { 554 | "data": { 555 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO+UlEQVR4nO3da2wUhf7G8afbgoBoAQGDooSrgXCpQiRGvEBRQKMpkBiM4D1EEyKSGPBEQ4LRoEViuMVYa9S04ZbQcFF4cSLkAEqFhnusioWIC6IFShGqULbzf8GfntO03KZPu532+3nFzszO/Dbsl53pdpeUIAgCAaiXWLIHAJoDQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMWzYMJ0/f15dunRJ9iho5kpLS9W6dWsVFRVdcbtIhnTu3DklEolkj4EW4MKFC7qWX/6JZEhdu3aVJH3zzTdJngTNXWZm5jVtxzUSYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGzSakqqqqFnlsNA1pyR7AJRaLaeXKlSotLW3U43bp0kVPPfVUox4TTU+zCUmSSktLdfTo0WSPgRao2ZzaAclESIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAgT2k06dP69ixY+7dAk1a6JCmT59e5/K9e/dq7NixysnJCT0UEDWhQ/r111/rXD5ixAht2bJFa9euDT0UEDWhQ0pJSbnsuoqKCp07dy7sroHISbuejRcvXqwlS5ZU3+7fv/9ltx09enT4qYCIua6Q7r33XklSEARasWKFJk2aVHuHaWnq3r27Hn30Uc+EQARcd0iXYjp48KCmTZvWIEMBURP6Gumjjz664vpt27aF3TUQOQ32hmx2dnZD7Rpocq7r1O5/nThxQvPmzdO2bdt0/PhxVVVVOecCIiV0SG+99ZZKSko0atQodezYUbHYf1/cLv0wAmgpQof0ww8/6Ouvv9ZNN91U5/p4PB56KCBqQl8jde/e/bIRSdIHH3wQdtdA5IQOaezYsdqwYcNl1z/33HNhdw1ETuhTu+LiYuXm5urjjz9Wr1691LZt2xrrf/zxx3oPB0RF6JDWrVunrl276syZM9q7d2+t9RUVFfUaDIiS0CH16dNHq1evvuz6rKyssLsGIif0NdLbb799xfW8IYuWJHRIw4YNu+L67du3h901EDmhT+2OHj16xfXLly/X5MmTw+4eiJTQIY0aNeqKH+4DWpLQIXXr1k2vvfZajWVnz55VSUmJtm/frilTptR7OCAqQoc0btw4jR8/vs51JSUlWrp0aeihgKgJ/cOGmTNnXnZd7969tXv37rC7BiKnQT6PdODAAZ08ebIhdg00SaFP7Z599tlay4IgUHl5uQ4ePKinn366XoMBURI6pH379mngwIE1lqWmpqpv376aMmXKZa+fgOYodEg9evRQXl6ecxYgskJfI33xxRfGMYBoC/2K1KFDB0lSUVGRtmzZorKyMnXs2FEPPPDAVX99CGhuQoeUSCQ0c+ZMrV+/XkEQVC/PycnRY489pnnz5tX4HgegOQsd0ieffKIdO3Zo1qxZysjIUIcOHXTq1Cnt3r1bn3/+uXJycvTKK684ZwWarNAhrV27Vnl5eerRo0eN5RkZGRo5cqSmTp1KSGgxQp97xWKxWhFd0qNHD6WmpoYeCoia0CGdP39eZWVlda47efIk/60LWpTQIWVmZuqFF17Qpk2bdOLECV24cEEnTpzQxo0b9dJLL+mRRx5xzgk0aaGvkaZPn64XX3xRr776aq3PJQ0ZMqTWRyyA5ix0SO3atVN+fr7WrVunrVu3Vr+PNGLECD3xxBNKSwu9ayByQj/by8rKtHPnTnXq1Enz58+vXr5+/XqdPn1anTp1sgwIREHoa6T8/HxNnz5da9asqbH8+++/V1ZWln755Zd6DwdERehXpE2bNumzzz7T8OHDayyfM2eORo4cqezsbOXk5NR7QCAKQr8iBUFQK6JLHn74YZWWloYeCoia0CGVl5fXaz3QnIQOqXfv3lq4cKESiUSN5YlEQgsWLFCvXr3qPRwQFfV6H2ny5MlauXKl+vfvr/T0dJWXl6u4uFhnzpzhW4TQooQOaeDAgcrLy1N2dra+/fZbVVVVKRaLaejQoZo5c6YGDBjgnBNo0ur1rumgQYOUl5enf/75R+Xl5UpPT1ebNm1cswGRYfn1gzZt2hAQWjQ+wgoYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhNTAgqqqFnnsliYt2QM0dymxmA6tzdbfxw836nHbdr5TPZ+c2ajHbMkIqRH8ffyw/v6jJNljoAFxagcYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEgtWCKJH/xL5rEbAp9HasFSYzHN/c+/dbi8rFGPe2d6R/3roUca9ZgNjZBauMPlZfrlxPFkjxF5nNqhSQqCIFLH5hUJTVJKSor+/PNPVVZWNupxW7Vqpa5du173/QgJTVZlZaXOnz+f7DGuCad2gAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBikBMn8KGJIgwYNUiKRULdu3WosP3v2rBKJRKPOkpqaqhtvvPGK21yoKFeQuNBIE12UkpqmtHbpV93u1D9/60IjfxFJWiymDm3aXnW7xv67vCQ1NbX6z7///rtSU1O1b9++K94nkh/su+GGG+r8wNfVntDJci1P6GS5lid0svzvEzpZ0tLS1Lp166tuF8lXJKCp4RoJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkCIiHo9r0aJFisfjyR4FdSCkiDhy5IgWL16sI0eOJHsU1IGQAAO+RSgCcnNztWLFCh0+fFh33nmn2rVrpzNnzujkyZOqqKhQr169NHnyZD3zzDMqKirSnDlzVFJSooyMDI0aNUoFBQUqKSnRm2++qeLiYh04cEDxeFxDhw7V7Nmzddttt1UfKwgCffnll1q+fLkkqbKyUqNHj9brr7+utm2b7ld3JV2ASCgsLAz69esXFBYWVi/bs2dP0K9fv2DlypU1tt28eXMwderU6tu//fZb0K9fv2D48OHBrl27giAIglOnTgVZWVnBmDFjgsrKyupt33vvvWDIkCHBnj17giAIgtLS0mDcuHHByy+/3ICPLvo4tYuwwYMHq2/fvlq1alWN5QUFBZo4cWKt7TMzM5WRkSFJSk9P17Rp03To0CGtXr1aknT48GHl5eVp/PjxGjx4sCSpc+fOmjp1qjZv3qyioqIGfTxRRkgRN2HCBO3atUsHDx6UJJWXl2vnzp0aOXJkrW3vuuuuGrcvxbJr1y5J0nfffaeqqioNHTq0zvsVFhba528uIvmVxfivJ598UvPnz1dBQYHeeOMNffXVVxozZoxatWpVa9v27dvXuJ2efvGrlP/44w9JUllZmSRpwYIF+vTTT6u3q6qqUufOnVVRUdFQDyPyCCniOnfurAcffFCrV6/WjBkztGrVKs2dO7fObf/6668at0+dOiVJuvXWWyVJHTt2lCTNmjVLo0ePbrihmyFO7SIiLe3iv3nB//+Qdf/+/Tp06JAkaeLEiSotLVVubq5isVitU7hLfv755xq39+7dK0m6++67JUn333+/YrGYiouLa9333Xff1Y4dOzwPphkipIjo3r27UlJSdOzYMUkXn9h79uyRJD300EO65ZZbtGDBAk2YMOGy+9i2bZt2794t6eK11JIlS9SzZ09lZWVJku644w49//zzys/P1/79+yVdDHfZsmXauHGjBgwY0HAPMOJ4HylCFi1apIKCArVv3149e/bUhx9+WP1fjrz//vtaunSptm7dqptvvrnG/eLxuDIzMzV79mz99NNP2r9/v+LxuO65554630fKz8/XsmXLlEgk1LZtW/Xp00czZszQ7bff3qiPN0oIqZlYs2aNNm/erPnz59dadymkuXPnXvEVC+FxatdMbNiwoc73jtA4CCnCJk2apIqKCpWUlCgej+u+++5L9kgtFj/+jrBWrVrp8ccfV3p6ut555x2lpKTU2iY3N1cFBQWSpIULF6qwsFDZ2dmNPWqzxzUSYMCpHWBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYPB/pNpQp0PpQe0AAAAASUVORK5CYII=\n", 556 | "text/plain": [ 557 | "
" 558 | ] 559 | }, 560 | "metadata": {}, 561 | "output_type": "display_data" 562 | }, 563 | { 564 | "name": "stdout", 565 | "output_type": "stream", 566 | "text": [ 567 | "\n", 568 | "sciq\n", 569 | "probe acc 0.8700000643730164\n", 570 | "probe sparse 371 8192\n", 571 | "model acc 0.8379999995231628\n" 572 | ] 573 | }, 574 | { 575 | "data": { 576 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO+UlEQVR4nO3daWwVhf7G8aenBQHRAgIGRQmrgbBUIRIjLlAU0GgKJAYjuIdoQkQSA95oSDAatEgMW4y1Rk0btoSGReHFjZALKBUa9lgVCxEPiBYoRahCOZ3/C/703qZlmz7t6bTfzyvOzHTmd8L5cmZ6FlKCIAgEoF5iyR4AaA4ICTAgJMCAkAADQgIMCAkwICTAgJAAg7RkDxDGsGHDdP78eXXp0iXZo6CZKy0tVevWrVVUVHTF7SIZ0rlz55RIJJI9BlqACxcu6Fre/BPJkLp27SpJ+uabb5I8CZq7zMzMa9qOayTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAoNmEVFVV1SKPjaYhLdkDuMRiMa1cuVKlpaWNetwuXbroqaeeatRjoulpNiFJUmlpqY4ePZrsMdACNZtTOyCZCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADe0inT5/WsWPH3LsFmrTQIU2fPr3O5Xv37tXYsWOVk5MTeiggakKH9Ouvv9a5fMSIEdqyZYvWrl0beiggakKHlJKSctl1FRUVOnfuXNhdA5GTdj0bL168WEuWLKm+3b9//8tuO3r06PBTARFzXSHde++9kqQgCLRixQpNmjSp9g7T0tS9e3c9+uijngmBCLjukC7FdPDgQU2bNq1BhgKiJvQ10kcffXTF9du2bQu7ayByGuwF2ezs7IbaNdDkXNep3f86ceKE5s2bp23btun48eOqqqpyzgVESuiQ3nrrLZWUlGjUqFHq2LGjYrH/Prld+mUE0FKEDumHH37Q119/rZtuuqnO9fF4PPRQQNSEvkbq3r37ZSOSpA8++CDsroHICR3S2LFjtWHDhsuuf+6558LuGoic0Kd2xcXFys3N1ccff6xevXqpbdu2Ndb/+OOP9R4OiIrQIa1bt05du3bVmTNntHfv3lrrKyoq6jUYECWhQ+rTp49Wr1592fVZWVlhdw1ETuhrpLfffvuK63lBFi1J6JCGDRt2xfXbt28Pu2sgckKf2h09evSK65cvX67JkyeH3T0QKaFDGjVq1BU/3Ae0JKFD6tatm1577bUay86ePauSkhJt375dU6ZMqfdwQFSEDmncuHEaP358netKSkq0dOnS0EMBURP6lw0zZ8687LrevXtr9+7dYXcNRE6DfB7pwIEDOnnyZEPsGmiSQp/aPfvss7WWBUGg8vJyHTx4UE8//XS9BgOiJHRI+/bt08CBA2ssS01NVd++fTVlypTLXj8BzVHokHr06KG8vDznLEBkhb5G+uKLL4xjANEW+hmpQ4cOkqSioiJt2bJFZWVl6tixox544IGrvn0IaG5Ch5RIJDRz5kytX79eQRBUL8/JydFjjz2mefPm1fgeB6A5Cx3SJ598oh07dmjWrFnKyMhQhw4ddOrUKe3evVuff/65cnJy9MorrzhnBZqs0CGtXbtWeXl56tGjR43lGRkZGjlypKZOnUpIaDFCn3vFYrFaEV3So0cPpaamhh4KiJrQIZ0/f15lZWV1rjt58iT/rQtalNAhZWZm6oUXXtCmTZt04sQJXbhwQSdOnNDGjRv10ksv6ZFHHnHOCTRpoa+Rpk+frhdffFGvvvpqrc8lDRkypNZHLIDmLHRI7dq1U35+vtatW6etW7dWv440YsQIPfHEE0pLC71rIHJCP9rLysq0c+dOderUSfPnz69evn79ep0+fVqdOnWyDAhEQehrpPz8fE2fPl1r1qypsfz7779XVlaWfvnll3oPB0RF6GekTZs26bPPPtPw4cNrLJ8zZ45Gjhyp7Oxs5eTk1HtAIApCPyMFQVAroksefvhhlZaWhh4KiJrQIZWXl9drPdCchA6pd+/eWrhwoRKJRI3liURCCxYsUK9eveo9HBAV9XodafLkyVq5cqX69++v9PR0lZeXq7i4WGfOnOFbhNCihA5p4MCBysvLU3Z2tr799ltVVVUpFotp6NChmjlzpgYMGOCcE2jS6vWq6aBBg5SXl6d//vlH5eXlSk9PV5s2bVyzAZFheftBmzZtCAgtGh9hBQwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQGlhQVdUij93SpCV7gOYuJRbTobXZ+vv44UY9btvOd6rnkzMb9ZgtGSE1gr+PH9bff5Qkeww0IE7tAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0JqwRJJfC9eMo/dEHiLUAuWGotp7n/+rcPlZY163DvTO+pfDz3SqMdsaITUwh0uL9MvJ44ne4zI49QOMCAkwICQAANCAgwICTAgJMCAkNAkBUEQqWPzOhKapJSUFP3555+qrKxs1OO2atVKXbt2ve6fIyQ0WZWVlTp//nyyx7gmnNoBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYJASJPOjiCENGjRIiURC3bp1q7H87NmzSiQSjTpLamqqbrzxxituc6GiXEHiQiNNdFFKaprS2qVfdbtT//ytC4389cFpsZg6tGl71e0a++/yktTU1Oo///7770pNTdW+ffuu+DOR/GDfDTfcUOcHvq72gE6Wa3lAJ8u1PKCT5X8f0MmSlpam1q1bX3W7SD4jAU0N10iAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEFBHxeFyLFi1SPB5P9iioAyFFxJEjR7R48WIdOXIk2aOgDoQEGPAtQhGQm5urFStW6PDhw7rzzjvVrl07nTlzRidPnlRFRYV69eqlyZMn65lnnlFRUZHmzJmjkpISZWRkaNSoUSooKFBJSYnefPNNFRcX68CBA4rH4xo6dKhmz56t2267rfpYQRDoyy+/1PLlyyVJlZWVGj16tF5//XW1bdt0v7or6QJEQmFhYdCvX7+gsLCwetmePXuCfv36BStXrqyx7ebNm4OpU6dW3/7tt9+Cfv36BcOHDw927doVBEEQnDp1KsjKygrGjBkTVFZWVm/73nvvBUOGDAn27NkTBEEQlJaWBuPGjQtefvnlBrx30cepXYQNHjxYffv21apVq2osLygo0MSJE2ttn5mZqYyMDElSenq6pk2bpkOHDmn16tWSpMOHDysvL0/jx4/X4MGDJUmdO3fW1KlTtXnzZhUVFTXo/YkyQoq4CRMmaNeuXTp48KAkqby8XDt37tTIkSNrbXvXXXfVuH0pll27dkmSvvvuO1VVVWno0KF1/lxhYaF9/uYikl9ZjP968sknNX/+fBUUFOiNN97QV199pTFjxqhVq1a1tm3fvn2N2+npF79K+Y8//pAklZWVSZIWLFigTz/9tHq7qqoqde7cWRUVFQ11NyKPkCKuc+fOevDBB7V69WrNmDFDq1at0ty5c+vc9q+//qpx+9SpU5KkW2+9VZLUsWNHSdKsWbM0evTohhu6GeLULiLS0i7+mxf8/y9Z9+/fr0OHDkmSJk6cqNLSUuXm5ioWi9U6hbvk559/rnF77969kqS7775bknT//fcrFoupuLi41s++++672rFjh+fONEOEFBHdu3dXSkqKjh07JuniA3vPnj2SpIceeki33HKLFixYoAkTJlx2H9u2bdPu3bslXbyWWrJkiXr27KmsrCxJ0h133KHnn39e+fn52r9/v6SL4S5btkwbN27UgAEDGu4ORhyvI0XIokWLVFBQoPbt26tnz5768MMPq//Lkffff19Lly7V1q1bdfPNN9f4uXg8rszMTM2ePVs//fST9u/fr3g8rnvuuafO15Hy8/O1bNkyJRIJtW3bVn369NGMGTN0++23N+r9jRJCaibWrFmjzZs3a/78+bXWXQpp7ty5V3zGQnic2jUTGzZsqPO1IzQOQoqwSZMmqaKiQiUlJYrH47rvvvuSPVKLxa+/I6xVq1Z6/PHHlZ6ernfeeUcpKSm1tsnNzVVBQYEkaeHChSosLFR2dnZjj9rscY0EGHBqBxgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGPwfZCtQpxFXcngAAAAASUVORK5CYII=\n", 577 | "text/plain": [ 578 | "
" 579 | ] 580 | }, 581 | "metadata": {}, 582 | "output_type": "display_data" 583 | }, 584 | { 585 | "name": "stdout", 586 | "output_type": "stream", 587 | "text": [ 588 | "\n", 589 | "sciq\n", 590 | "probe acc 0.7780000567436218\n", 591 | "probe sparse 187 8192\n", 592 | "model acc 0.8379999995231628\n" 593 | ] 594 | }, 595 | { 596 | "data": { 597 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO+0lEQVR4nO3deWwUhd/H8U+3BQHRUgQMihJOA+GoQiRGPKAooNEUSAxG8A7RhIgkBnyiIcFo0CIxXDHWGjVtuBIaDoU/fhHyAEqFhjtWxULEBdECpQhVju08f/BQf03LNf2022nfr7/YmdmZ78Z9uzPd7jYlCIJAAOolluwBgOaAkAADQgIMCAkwICTAgJAAA0ICDAgJMEhL9gBhDB06VOfOnVPnzp2TPQqaubKyMrVu3VrFxcVX3C6SIZ09e1aJRCLZY6AFuHDhgq7ll38iGVKXLl0kSd98802SJ0Fzl5WVdU3bcY0EGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBs0mpKqqqhZ5bDQNackewCUWi2nFihUqKytr1ON27txZTz31VKMeE01PswlJksrKynTkyJFkj4EWqNmc2gHJREiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgYA/p1KlTOnr0qHu3QJMWOqRp06bVuXzPnj0aM2aMcnNzQw8FRE3okH799dc6lw8fPlybN2/WmjVrQg8FRE3okFJSUi67rrKyUmfPng27ayBy0q5n40WLFmnx4sXVt/v163fZbUeNGhV+KiBiriuke++9V5IUBIGWL1+uiRMn1t5hWpq6deumRx991DMhEAHXHdKlmA4cOKCpU6c2yFBA1IS+Rvroo4+uuH7r1q1hdw1EToO9IZuTk9NQuwaanOs6tftvx48f19y5c7V161YdO3ZMVVVVzrmASAkd0ltvvaXS0lKNHDlSGRkZisX+fXG79MMIoKUIHdIPP/ygr7/+WjfddFOd6+PxeOihgKgJfY3UrVu3y0YkSR988EHYXQOREzqkMWPGaP369Zdd/9xzz4XdNRA5oU/tSkpKlJeXp48//lg9e/ZU27Zta6z/8ccf6z0cEBWhQ1q7dq26dOmi06dPa8+ePbXWV1ZW1mswIEpCh9S7d2+tWrXqsuuzs7PD7hqInNDXSG+//fYV1/OGLFqS0CENHTr0iuu3bdsWdtdA5IQ+tTty5MgV1y9btkyTJk0Ku3sgUkKHNHLkyCt+uA9oSUKH1LVrV7322ms1lp05c0alpaXatm2bJk+eXO/hgKgIHdLYsWM1bty4OteVlpZqyZIloYcCoib0DxtmzJhx2XW9evXSrl27wu4aiJwG+TzS/v37deLEiYbYNdAkhT61e/bZZ2stC4JAFRUVOnDggJ5++ul6DQZESeiQ9u7dqwEDBtRYlpqaqj59+mjy5MmXvX4CmqPQIXXv3l35+fnOWYDICn2N9MUXXxjHAKIt9CtShw4dJEnFxcXavHmzysvLlZGRoQceeOCqvz4ENDehQ0okEpoxY4bWrVunIAiql+fm5uqxxx7T3Llza3yPA9CchQ7pk08+0fbt2zVz5kxlZmaqQ4cOOnnypHbt2qXPP/9cubm5euWVV5yzAk1W6JDWrFmj/Px8de/evcbyzMxMjRgxQlOmTCEktBihz71isVitiC7p3r27UlNTQw8FRE3okM6dO6fy8vI61504cYI/64IWJXRIWVlZeuGFF7Rx40YdP35cFy5c0PHjx7Vhwwa99NJLeuSRR5xzAk1a6GukadOm6cUXX9Srr75a63NJgwcPrvURC6A5Cx1Su3btVFBQoLVr12rLli3V7yMNHz5cTzzxhNLSQu8aiJzQz/by8nLt2LFDHTt21Lx586qXr1u3TqdOnVLHjh0tAwJREPoaqaCgQNOmTdPq1atrLP/++++VnZ2tX375pd7DAVER+hVp48aN+uyzzzRs2LAay2fPnq0RI0YoJydHubm59R4QiILQr0hBENSK6JKHH35YZWVloYcCoiZ0SBUVFfVaDzQnoUPq1auXFixYoEQiUWN5IpHQ/Pnz1bNnz3oPB0RFvd5HmjRpklasWKF+/fopPT1dFRUVKikp0enTp/kWIbQooUMaMGCA8vPzlZOTo2+//VZVVVWKxWIaMmSIZsyYof79+zvnBJq0er1rOnDgQOXn5+uff/5RRUWF0tPT1aZNG9dsQGRYfv2gTZs2BIQWjY+wAgaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhtWCJqqoWeeyGkJbsAZA8qbGY5vzvf3SoorxRj3tneob+56FHGvWYDY2QWrhDFeX65fixZI8ReZzaAQaEBBgQEmBASIABIQEGhAQYEBJgQEgNLEjiO/jJPHZLwxuyDSwlFtPBNTn6+9ihRj1u2053qseTMxr1mC0ZITWCv48d0t9/lCZ7DDQgTu0AA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQkKTFARBpI7N13GhSUpJSdGff/6p8+fPN+pxW7VqpS5dulz3/QgJTdb58+d17ty5ZI9xTTi1AwwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgxSgmR+8COkgQMHKpFIqGvXrjWWnzlzRolEolFnSU1N1Y033njFbS5UVihIXGikiS5KSU1TWrv0q2538p+/daGqqhEm+ldaLKYObdpedbvG/m95SWpqavW/f//9d6Wmpmrv3r1XvE8kP0Zxww031Pnr9Vd7QifLtTyhk+VantDJ8t9P6GRJS0tT69atr7pdJF+RgKaGayTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCioh4PK6FCxcqHo8nexTUgZAi4vDhw1q0aJEOHz6c7FFQB0ICDPgWoQjIy8vT8uXLdejQId15551q166dTp8+rRMnTqiyslI9e/bUpEmT9Mwzz6i4uFizZ89WaWmpMjMzNXLkSBUWFqq0tFRvvvmmSkpKtH//fsXjcQ0ZMkSzZs3SbbfdVn2sIAj05ZdfatmyZZIu/mXxUaNG6fXXX1fbtk33q7uSLkAkFBUVBX379g2Kioqql+3evTvo27dvsGLFihrbbtq0KZgyZUr17d9++y3o27dvMGzYsGDnzp1BEATByZMng+zs7GD06NHB+fPnq7d97733gsGDBwe7d+8OgiAIysrKgrFjxwYvv/xyAz666OPULsIGDRqkPn36aOXKlTWWFxYWasKECbW2z8rKUmZmpiQpPT1dU6dO1cGDB7Vq1SpJ0qFDh5Sfn69x48Zp0KBBkqROnTppypQp2rRpk4qLixv08UQZIUXc+PHjtXPnTh04cECSVFFRoR07dmjEiBG1tr3rrrtq3L4Uy86dOyVJ3333naqqqjRkyJA671dUVGSfv7mI5FcW419PPvmk5s2bp8LCQr3xxhv66quvNHr0aLVq1arWtu3bt69xOz394lcp//HHH5Kk8vJySdL8+fP16aefVm9XVVWlTp06qbKysqEeRuQRUsR16tRJDz74oFatWqXp06dr5cqVmjNnTp3b/vXXXzVunzx5UpJ06623SpIyMjIkSTNnztSoUaMabuhmiFO7iEhLu/j/vOD/f8i6b98+HTx4UJI0YcIElZWVKS8vT7FYrNYp3CU///xzjdt79uyRJN19992SpPvvv1+xWEwlJSW17vvuu+9q+/btngfTDBFSRHTr1k0pKSk6evSopItP7N27d0uSHnroId1yyy2aP3++xo8ff9l9bN26Vbt27ZJ08Vpq8eLF6tGjh7KzsyVJd9xxh55//nkVFBRo3759ki6Gu3TpUm3YsEH9+/dvuAcYcbyPFCELFy5UYWGh2rdvrx49eujDDz+s/pMj77//vpYsWaItW7bo5ptvrnG/eDyurKwszZo1Sz/99JP27duneDyue+65p873kQoKCrR06VIlEgm1bdtWvXv31vTp03X77bc36uONEkJqJlavXq1NmzZp3rx5tdZdCmnOnDlXfMVCeJzaNRPr16+v870jNA5CirCJEyeqsrJSpaWlisfjuu+++5I9UovFj78jrFWrVnr88ceVnp6ud955RykpKbW2ycvLU2FhoSRpwYIFKioqUk5OTmOP2uxxjQQYcGoHGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQY/B9SbFCn64VxagAAAABJRU5ErkJggg==\n", 598 | "text/plain": [ 599 | "
" 600 | ] 601 | }, 602 | "metadata": {}, 603 | "output_type": "display_data" 604 | }, 605 | { 606 | "name": "stdout", 607 | "output_type": "stream", 608 | "text": [ 609 | "\n", 610 | "sciq\n", 611 | "probe acc 0.7610000371932983\n", 612 | "probe sparse 201 8192\n", 613 | "model acc 0.8379999995231628\n" 614 | ] 615 | }, 616 | { 617 | "data": { 618 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO9UlEQVR4nO3deWwUhd/H8U+3BQHRUgQMihJOA+GoQiRGPKAooNEUSAxG8A7RhIgkBnyiIcFo0CIxXDHWGjVtuBIaDoU/fhHyAEqFhjtWxULEBdECpQhVju08f/BQf03LNf2022nfr7/YmenMd+O+3Znu0ZQgCAIBqJdYsgcAmgNCAgwICTAgJMCAkAADQgIMCAkwICTAIC3ZA4QxdOhQnTt3Tp07d072KGjmysrK1Lp1axUXF19xu0iGdPbsWSUSiWSPgRbgwoULupY3/0QypC5dukiSvvnmmyRPguYuKyvrmrbjGgkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDJpNSFVVVS3y2Gga0pI9gEssFtOKFStUVlbWqMft3LmznnrqqUY9JpqeZhOSJJWVlenIkSPJHgMtULM5tQOSiZAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMDAHtKpU6d09OhR926BJi10SNOmTatz+Z49ezRmzBjl5uaGHgqImtAh/frrr3UuHz58uDZv3qw1a9aEHgqImtAhpaSkXHZdZWWlzp49G3bXQOSkXc/GixYt0uLFi6tv9+vX77Lbjho1KvxUQMRcV0j33nuvJCkIAi1fvlwTJ06svcO0NHXr1k2PPvqoZ0IgAq47pEsxHThwQFOnTm2QoYCoCX2N9NFHH11x/datW8PuGoicBntBNicnp6F2DTQ513Vq99+OHz+uuXPnauvWrTp27JiqqqqccwGREjqkt956S6WlpRo5cqQyMjIUi/375HbplxFASxE6pB9++EFff/21brrppjrXx+Px0EMBURP6Gqlbt26XjUiSPvjgg7C7BiIndEhjxozR+vXrL7v+ueeeC7trIHJCn9qVlJQoLy9PH3/8sXr27Km2bdvWWP/jjz/WezggKkKHtHbtWnXp0kWnT5/Wnj17aq2vrKys12BAlIQOqXfv3lq1atVl12dnZ4fdNRA5oa+R3n777Suu5wVZtCShQxo6dOgV12/bti3sroHICX1qd+TIkSuuX7ZsmSZNmhR290CkhA5p5MiRV/xwH9CShA6pa9eueu2112osO3PmjEpLS7Vt2zZNnjy53sMBURE6pLFjx2rcuHF1ristLdWSJUtCDwVETehfNsyYMeOy63r16qVdu3aF3TUQOQ3yeaT9+/frxIkTDbFroEkKfWr37LPP1loWBIEqKip04MABPf300/UaDIiS0CHt3btXAwYMqLEsNTVVffr00eTJky97/QQ0R6FD6t69u/Lz852zAJEV+hrpiy++MI4BRFvoZ6QOHTpIkoqLi7V582aVl5crIyNDDzzwwFXfPgQ0N6FDSiQSmjFjhtatW6cgCKqX5+bm6rHHHtPcuXNrfI8D0JyFDumTTz7R9u3bNXPmTGVmZqpDhw46efKkdu3apc8//1y5ubl65ZVXnLMCTVbokNasWaP8/Hx17969xvLMzEyNGDFCU6ZMISS0GKHPvWKxWK2ILunevbtSU1NDDwVETeiQzp07p/Ly8jrXnThxgj/rghYldEhZWVl64YUXtHHjRh0/flwXLlzQ8ePHtWHDBr300kt65JFHnHMCTVroa6Rp06bpxRdf1Kuvvlrrc0mDBw+u9RELoDkLHVK7du1UUFCgtWvXasuWLdWvIw0fPlxPPPGE0tJC7xqInNCP9vLycu3YsUMdO3bUvHnzqpevW7dOp06dUseOHS0DAlEQ+hqpoKBA06ZN0+rVq2ss//7775Wdna1ffvml3sMBURH6GWnjxo367LPPNGzYsBrLZ8+erREjRignJ0e5ubn1HhCIgtDPSEEQ1IrokocfflhlZWWhhwKiJnRIFRUV9VoPNCehQ+rVq5cWLFigRCJRY3kikdD8+fPVs2fPeg8HREW9XkeaNGmSVqxYoX79+ik9PV0VFRUqKSnR6dOn+RYhtCihQxowYIDy8/OVk5Ojb7/9VlVVVYrFYhoyZIhmzJih/v37O+cEmrR6vWo6cOBA5efn659//lFFRYXS09PVpk0b12xAZFjeftCmTRsCQovGR1gBA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkBpYUFXVZI+dSOJsyTx2Q0hL9gDNXUospoNrcvT3sUONety2ne5UjydnXHGb1FhMc/73PzpUUd5IU110Z3qG/uehRxr1mA2NkBrB38cO6e8/SpM9Rp0OVZTrl+PHkj1G5HFqBxgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEpqkIAgidWy+shhNUkpKiv7880+dP3++UY/bqlUrdenS5bp/jpDQZJ0/f17nzp1L9hjXhFM7wICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDFKCZL7NNqSBAwcqkUioa9euNZafOXNGiUSiUWdJTU3VjTfeeMVtLlRWKEhcaKSJLkpJTVNau/Srbnfyn791oaqqESb6V1ospg5t2l51u8b+b3lJampq9b9///13paamau/evVf8mUi+afWGG26o882MV3tAJ8u1PKCT5Voe0Mny3w/oZElLS1Pr1q2vul0kn5GApoZrJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0KKiHg8roULFyoejyd7FNSBkCLi8OHDWrRokQ4fPpzsUVAHQgIM+BahCMjLy9Py5ct16NAh3XnnnWrXrp1Onz6tEydOqLKyUj179tSkSZP0zDPPqLi4WLNnz1ZpaakyMzM1cuRIFRYWqrS0VG+++aZKSkq0f/9+xeNxDRkyRLNmzdJtt91WfawgCPTll19q2bJlki7+HddRo0bp9ddfV9u2Tferu5IuQCQUFRUFffv2DYqKiqqX7d69O+jbt2+wYsWKGttu2rQpmDJlSvXt3377Lejbt28wbNiwYOfOnUEQBMHJkyeD7OzsYPTo0cH58+ert33vvfeCwYMHB7t37w6CIAjKysqCsWPHBi+//HID3rvo49QuwgYNGqQ+ffpo5cqVNZYXFhZqwoQJtbbPyspSZmamJCk9PV1Tp07VwYMHtWrVKknSoUOHlJ+fr3HjxmnQoEGSpE6dOmnKlCnatGmTiouLG/T+RBkhRdz48eO1c+dOHThwQJJUUVGhHTt2aMSIEbW2veuuu2rcvhTLzp07JUnfffedqqqqNGTIkDp/rqioyD5/cxHJryzGv5588knNmzdPhYWFeuONN/TVV19p9OjRatWqVa1t27dvX+N2evrFr1L+448/JEnl5eWSpPnz5+vTTz+t3q6qqkqdOnVSZWVlQ92NyCOkiOvUqZMefPBBrVq1StOnT9fKlSs1Z86cOrf966+/atw+efKkJOnWW2+VJGVkZEiSZs6cqVGjRjXc0M0Qp3YRkZZ28f95wf//knXfvn06ePCgJGnChAkqKytTXl6eYrFYrVO4S37++ecat/fs2SNJuvvuuyVJ999/v2KxmEpKSmr97Lvvvqvt27d77kwzREgR0a1bN6WkpOjo0aOSLj6wd+/eLUl66KGHdMstt2j+/PkaP378ZfexdetW7dq1S9LFa6nFixerR48eys7OliTdcccdev7551VQUKB9+/ZJuhju0qVLtWHDBvXv37/h7mDE8TpShCxcuFCFhYVq3769evTooQ8//LD6T468//77WrJkibZs2aKbb765xs/F43FlZWVp1qxZ+umnn7Rv3z7F43Hdc889db6OVFBQoKVLlyqRSKht27bq3bu3pk+frttvv71R72+UEFIzsXr1am3atEnz5s2rte5SSHPmzLniMxbC49SumVi/fn2drx2hcRBShE2cOFGVlZUqLS1VPB7Xfffdl+yRWix+/R1hrVq10uOPP6709HS98847SklJqbVNXl6eCgsLJUkLFixQUVGRcnJyGnvUZo9rJMCAUzvAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMDg/wAVL1Cj3T9ghQAAAABJRU5ErkJggg==\n", 619 | "text/plain": [ 620 | "
" 621 | ] 622 | }, 623 | "metadata": {}, 624 | "output_type": "display_data" 625 | }, 626 | { 627 | "name": "stdout", 628 | "output_type": "stream", 629 | "text": [ 630 | "\n", 631 | "\n", 632 | "creak\n", 633 | "probe acc 0.7024070024490356\n", 634 | "probe sparse 3853 8192\n", 635 | "model acc 0.50255286693573\n" 636 | ] 637 | }, 638 | { 639 | "data": { 640 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO/klEQVR4nO3da2wUhf7G8afbgoBoAQGDooSrgXCpQiRGvEBRQKMpkBiM4D1EEyKSGPBEQ4LRoEViuMVYa9S04ZbQcFF4cSLkAEqFhnusioWIC6IFShGqtGzn/4I/PadpQZk+7Xba7+cVOzOd+W3crzvT7e6mBEEQCECDxJI9ANASEBJgQEiAASEBBoQEGBASYEBIgAEhAQZpyR4gjBEjRqiyslLdunVL9iho4UpLS9W2bVsVFRVddbtIhnThwgUlEolkj4FW4OLFi/onf/wTyZC6d+8uSfrqq6+SPAlauszMzH+0HddIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBgD+ns2bM6ceKEe7dAsxY6pJkzZ9a7fP/+/Ro/frxycnJCDwVETeiQfv7553qXjxo1Stu2bdP69etDDwVETeiQUlJSrriuoqJCFy5cCLtrIHLSrmXjpUuXatmyZTW3Bw4ceMVtx44dG34qIGKuKaS7775bkhQEgVatWqUpU6bU3WFamnr27KmHH37YMyEQAdcc0uWYDh8+rBkzZjTKUEDUhL5G+uCDD666fseOHWF3DUROo70gm52d3Vi7Bpqdazq1+1+nTp3SggULtGPHDp08eVLV1dXOuYBICR3SG2+8oZKSEo0ZM0adO3dWLPbfJ7fLv4wAWovQIX333Xf68ssvdcMNN9S7Ph6Phx4KiJrQ10g9e/a8YkSS9N5774XdNRA5oUMaP368Nm3adMX1zzzzTNhdA5ET+tSuuLhYubm5+vDDD9WnTx+1b9++1vrvv/++wcMBURE6pA0bNqh79+46d+6c9u/fX2d9RUVFgwYDoiR0SP369dPatWuvuD4rKyvsroHICX2N9Oabb151PS/IojUJHdKIESOuun7nzp1hdw1ETuhTu+PHj191/cqVKzV16tSwuwciJXRIY8aMueqb+4DWJHRIPXr00CuvvFJr2fnz51VSUqKdO3dq2rRpDR4OiIrQIU2YMEETJ06sd11JSYmWL18eeiggakL/smH27NlXXNe3b1/t3bs37K6ByGmU9yMdOnRIp0+fboxdA81S6FO7p59+us6yIAhUXl6uw4cP68knn2zQYECUhA7pwIEDGjx4cK1lqamp6t+/v6ZNm3bF6yegJQodUq9evZSXl+ecBYis0NdIn332mXEMINpCPyN16tRJklRUVKRt27aprKxMnTt31n333fe3fz4EtDShQ0okEpo9e7Y2btyoIAhqlufk5OiRRx7RggULan2OA9CShQ7po48+0q5duzRnzhxlZGSoU6dOOnPmjPbu3atPP/1UOTk5eumll5yzAs1W6JDWr1+vvLw89erVq9byjIwMjR49WtOnTycktBqhz71isVidiC7r1auXUlNTQw8FRE3okCorK1VWVlbvutOnT/O1LmhVQoeUmZmp5557Tlu2bNGpU6d08eJFnTp1Sps3b9YLL7yghx56yDkn0KyFvkaaOXOmnn/+eb388st13pc0bNiwOm+xAFqy0CF16NBB+fn52rBhg7Zv317zOtKoUaP02GOPKS0t9K6ByAn9aC8rK9Pu3bvVpUsXLVy4sGb5xo0bdfbsWXXp0sUyIBAFoa+R8vPzNXPmTK1bt67W8m+//VZZWVn66aefGjwcEBWhn5G2bNmiTz75RCNHjqy1fN68eRo9erSys7OVk5PT4AGBKAj9jBQEQZ2ILnvwwQdVWloaeiggakKHVF5e3qD1QEsSOqS+fftq8eLFSiQStZYnEgktWrRIffr0afBwQFQ06HWkqVOnavXq1Ro4cKDS09NVXl6u4uJinTt3jk8RQqsSOqTBgwcrLy9P2dnZ+vrrr1VdXa1YLKbhw4dr9uzZGjRokHNOoFlr0KumQ4YMUV5env766y+Vl5crPT1d7dq1c80GRIblzw/atWtHQGjVeAsrYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGLSYkKqrq1vlsdE8tJgPn4vFYlq9enWTf1ZEt27d9MQTTzTpMdH8tJiQJKm0tFTHjx9P9hhohVrMqR2QTIQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQUiMLkvglZMk8dmvTor4fqTlKicV0ZH22/jx5tEmP277r7er9+OwmPWZrRkhN4M+TR/XnbyXJHgONiFM7wICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAnNUhAEkTp2WiPMATRYSkqKfv/9d1VVVTXpcdu0aaPu3btf888REpqtqqoqVVZWJnuMf4RTO8CAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQmrFEtXVrfLYjYHP/m7FUmMxzf/Pv3W0vKxJj3t7emf964GHmvSYjY2QWrmj5WX66dTJZI8ReZzaAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYJASBEGQ7CGu1ZAhQ5RIJNSjR49ay8+fP69EItGks6Smpur666+/6jYXK8oVJC420USXpKSmKa1D+t9ud+avP3Wxib9hPC0WU6d27f92u6b+b3lZampqzb9//fVXpaam6sCBA1f9mUh+GfN1112nysrKOsv/7gGdLP/kAZ0s/+QBnSz/+4BOlrS0NLVt2/Zvt4vkMxLQ3HCNBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEgREY/HtWTJEsXj8WSPgnoQUkQcO3ZMS5cu1bFjx5I9CupBSIABnyIUAbm5uVq1apWOHj2q22+/XR06dNC5c+d0+vRpVVRUqE+fPpo6daqeeuopFRUVad68eSopKVFGRobGjBmjgoIClZSU6PXXX1dxcbEOHTqkeDyu4cOHa+7cubrllltqjhUEgT7//HOtXLlSklRVVaWxY8fq1VdfVfv2zfeju5IuQCQUFhYGAwYMCAoLC2uW7du3LxgwYECwevXqWttu3bo1mD59es3tX375JRgwYEAwcuTIYM+ePUEQBMGZM2eCrKysYNy4cUFVVVXNtu+8804wbNiwYN++fUEQBEFpaWkwYcKE4MUXX2zEexd9nNpF2NChQ9W/f3+tWbOm1vKCggJNnjy5zvaZmZnKyMiQJKWnp2vGjBk6cuSI1q5dK0k6evSo8vLyNHHiRA0dOlSS1LVrV02fPl1bt25VUVFRo96fKCOkiJs0aZL27Nmjw4cPS5LKy8u1e/dujR49us62d9xxR63bl2PZs2ePJOmbb75RdXW1hg8fXu/PFRYW2udvKSL5kcX4r8cff1wLFy5UQUGBXnvtNX3xxRcaN26c2rRpU2fbjh071rqdnn7po5R/++03SVJZWZkkadGiRfr4449rtquurlbXrl1VUVHRWHcj8ggp4rp27ar7779fa9eu1axZs7RmzRrNnz+/3m3/+OOPWrfPnDkjSbr55pslSZ07d5YkzZkzR2PHjm28oVsgTu0iIi3t0v/zgv//JevBgwd15MgRSdLkyZNVWlqq3NxcxWKxOqdwl/3444+1bu/fv1+SdOedd0qS7r33XsViMRUXF9f52bffflu7du3y3JkWiJAiomfPnkpJSdGJEyckXXpg79u3T5L0wAMP6KabbtKiRYs0adKkK+5jx44d2rt3r6RL11LLli1T7969lZWVJUm67bbb9Oyzzyo/P18HDx6UdCncFStWaPPmzRo0aFDj3cGI43WkCFmyZIkKCgrUsWNH9e7dW++//37NV468++67Wr58ubZv364bb7yx1s/F43FlZmZq7ty5+uGHH3Tw4EHF43Hddddd9b6OlJ+frxUrViiRSKh9+/bq16+fZs2apVtvvbVJ72+UEFILsW7dOm3dulULFy6ss+5ySPPnz7/qMxbC49Suhdi0aVO9rx2haRBShE2ZMkUVFRUqKSlRPB7XPffck+yRWi1+/R1hbdq00aOPPqr09HS99dZbSklJqbNNbm6uCgoKJEmLFy9WYWGhsrOzm3rUFo9rJMCAUzvAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMDg/wCDllCnRfwWvwAAAABJRU5ErkJggg==\n", 641 | "text/plain": [ 642 | "
" 643 | ] 644 | }, 645 | "metadata": {}, 646 | "output_type": "display_data" 647 | }, 648 | { 649 | "name": "stdout", 650 | "output_type": "stream", 651 | "text": [ 652 | "\n", 653 | "creak\n", 654 | "probe acc 0.6710430383682251\n", 655 | "probe sparse 790 8192\n", 656 | "model acc 0.50255286693573\n" 657 | ] 658 | }, 659 | { 660 | "data": { 661 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO/0lEQVR4nO3da2wUBd+G8bvbgoBoAQGDooSjgXCoQiRGPEBRQKMpkBiM4DlEEyKSGPCNhgSjQYvEcIqx1qhpwymh4aDw4YmQF1AqNJxjVSxEXBAtUIpQpWU77wde6tO0oEzvdjvt9fvEzszO/Dfu5c50u9uUIAgCAWiQWLIHAFoCQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMWLECFVWVqpbt27JHgUtXGlpqdq2bauioqKrbhfJkC5cuKBEIpHsMdAKXLx4Uf/ml38iGVL37t0lSV999VWSJ0FLl5mZ+a+24xoJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgzsIZ09e1YnTpxw7xZo1kKHNHPmzHqX79+/X+PHj1dOTk7ooYCoCR3Szz//XO/yUaNGadu2bVq/fn3ooYCoCR1SSkrKFddVVFTowoULYXcNRE7atWy8dOlSLVu2rOb2wIEDr7jt2LFjw08FRMw1hXT33XdLkoIg0KpVqzRlypS6O0xLU8+ePfXwww97JgQi4JpDuhzT4cOHNWPGjEYZCoia0NdIH3zwwVXX79ixI+yugchptDdks7OzG2vXQLNzTad2/+3UqVNasGCBduzYoZMnT6q6uto5FxApoUN64403VFJSojFjxqhz586Kxf5+cbv8wwigtQgd0nfffacvv/xSN9xwQ73r4/F46KGAqAl9jdSzZ88rRiRJ7733XthdA5ETOqTx48dr06ZNV1z/zDPPhN01EDmhT+2Ki4uVm5urDz/8UH369FH79u1rrf/+++8bPBwQFaFD2rBhg7p3765z585p//79ddZXVFQ0aDAgSkKH1K9fP61du/aK67OyssLuGoic0NdIb7755lXX84YsWpPQIY0YMeKq63fu3Bl210DkhD61O378+FXXr1y5UlOnTg27eyBSQoc0ZsyYq364D2hNQofUo0cPvfLKK7WWnT9/XiUlJdq5c6emTZvW4OGAqAgd0oQJEzRx4sR615WUlGj58uWhhwKiJvQPG2bPnn3FdX379tXevXvD7hqInEb5PNKhQ4d0+vTpxtg10CyFPrV7+umn6ywLgkDl5eU6fPiwnnzyyQYNBkRJ6JAOHDigwYMH11qWmpqq/v37a9q0aVe8fgJaotAh9erVS3l5ec5ZgMgKfY302WefGccAoi30K1KnTp0kSUVFRdq2bZvKysrUuXNn3Xffff/460NASxM6pEQiodmzZ2vjxo0KgqBmeU5Ojh555BEtWLCg1vc4AC1Z6JA++ugj7dq1S3PmzFFGRoY6deqkM2fOaO/evfr000+Vk5Ojl156yTkr0GyFDmn9+vXKy8tTr169ai3PyMjQ6NGjNX36dEJCqxH63CsWi9WJ6LJevXopNTU19FBA1IQOqbKyUmVlZfWuO336NH/WBa1K6JAyMzP13HPPacuWLTp16pQuXryoU6dOafPmzXrhhRf00EMPOecEmrXQ10gzZ87U888/r5dffrnO55KGDRtW5yMWQEsWOqQOHTooPz9fGzZs0Pbt22veRxo1apQee+wxpaWF3jUQOaGf7WVlZdq9e7e6dOmihQsX1izfuHGjzp49qy5dulgGBKIg9DVSfn6+Zs6cqXXr1tVa/u233yorK0s//fRTg4cDoiL0K9KWLVv0ySefaOTIkbWWz5s3T6NHj1Z2drZycnIaPCAQBaFfkYIgqBPRZQ8++KBKS0tDDwVETeiQysvLG7QeaElCh9S3b18tXrxYiUSi1vJEIqFFixapT58+DR4OiIoGvY80depUrV69WgMHDlR6errKy8tVXFysc+fO8S1CaFVChzR48GDl5eUpOztbX3/9taqrqxWLxTR8+HDNnj1bgwYNcs4JNGsNetd0yJAhysvL019//aXy8nKlp6erXbt2rtmAyLD8+kG7du0ICK0aH2EFDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkACDFhNSdXV1qzw2mgfLX+xrDmKxmFavXq3S0tImPW63bt30xBNPNOkx0fy0mJAkqbS0VMePH0/2GGiFWsypHZBMhNTIgiRePyXz2K1Nizq1a45SYjEdWZ+tP08ebdLjtu96u3o/PrtJj9maEVIT+PPkUf35W0myx0Aj4tQOMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCCkVixRXd1sjx0EQRNN4jl2WiPMgYhIjcU0/3//o6PlZU163NvTO+t/HnjoqtukpKTo999/V1VVVRNNdUmbNm3UvXv3a74fIbVyR8vL9NOpk8keo15VVVWqrKxM9hj/Cqd2gAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYpARBECR7iGs1ZMgQJRIJ9ejRo9by8+fPK5FINOksqampuv7666+6zcWKcgWJi0000SUpqWlK65D+j9ud+etPXayuboKJ/pYWi6lTu/b/uF1T/7e8LDU1tebfv/76q1JTU3XgwIGr3ietsYdqDNddd50qKyvrLP+nJ3Sy/JsndLL8myd0svz3EzpZ0tLS1LZt23/cLpKvSEBzwzUSYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIUVEPB7XkiVLFI/Hkz0K6kFIEXHs2DEtXbpUx44dS/YoqAchAQZ8i1AE5ObmatWqVTp69Khuv/12dejQQefOndPp06dVUVGhPn36aOrUqXrqqadUVFSkefPmqaSkRBkZGRozZowKCgpUUlKi119/XcXFxTp06JDi8biGDx+uuXPn6pZbbqk5VhAE+vzzz7Vy5UpJUlVVlcaOHatXX31V7ds336/uSroAkVBYWBgMGDAgKCwsrFm2b9++YMCAAcHq1atrbbt169Zg+vTpNbd/+eWXYMCAAcHIkSODPXv2BEEQBGfOnAmysrKCcePGBVVVVTXbvvPOO8GwYcOCffv2BUEQBKWlpcGECROCF198sREfXfRxahdhQ4cOVf/+/bVmzZpaywsKCjR58uQ622dmZiojI0OSlJ6erhkzZujIkSNau3atJOno0aPKy8vTxIkTNXToUElS165dNX36dG3dulVFRUWN+niijJAibtKkSdqzZ48OHz4sSSovL9fu3bs1evToOtvecccdtW5fjmXPnj2SpG+++UbV1dUaPnx4vfcrLCy0z99SRPIri/G3xx9/XAsXLlRBQYFee+01ffHFFxo3bpzatGlTZ9uOHTvWup2efumrlH/77TdJUllZmSRp0aJF+vjjj2u2q66uVteuXVVRUdFYDyPyCCniunbtqvvvv19r167VrFmztGbNGs2fP7/ebf/4449at8+cOSNJuvnmmyVJnTt3liTNmTNHY8eObbyhWyBO7SIiLe3S//OC//8h68GDB3XkyBFJ0uTJk1VaWqrc3FzFYrE6p3CX/fjjj7Vu79+/X5J05513SpLuvfdexWIxFRcX17nv22+/rV27dnkeTAtESBHRs2dPpaSk6MSJE5IuPbH37dsnSXrggQd00003adGiRZo0adIV97Fjxw7t3btX0qVrqWXLlql3797KysqSJN1222169tlnlZ+fr4MHD0q6FO6KFSu0efNmDRo0qPEeYMTxPlKELFmyRAUFBerYsaN69+6t999/v+ZPjrz77rtavny5tm/frhtvvLHW/eLxuDIzMzV37lz98MMPOnjwoOLxuO66665630fKz8/XihUrlEgk1L59e/Xr10+zZs3Srbfe2qSPN0oIqYVYt26dtm7dqoULF9ZZdzmk+fPnX/UVC+FxatdCbNq0qd73jtA0CCnCpkyZooqKCpWUlCgej+uee+5J9kitFj/+jrA2bdro0UcfVXp6ut566y2lpKTU2SY3N1cFBQWSpMWLF6uwsFDZ2dlNPWqLxzUSYMCpHWBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYPB//uVQo8EaQwsAAAAASUVORK5CYII=\n", 662 | "text/plain": [ 663 | "
" 664 | ] 665 | }, 666 | "metadata": {}, 667 | "output_type": "display_data" 668 | }, 669 | { 670 | "name": "stdout", 671 | "output_type": "stream", 672 | "text": [ 673 | "\n", 674 | "creak\n", 675 | "probe acc 0.5973742008209229\n", 676 | "probe sparse 89 8192\n", 677 | "model acc 0.50255286693573\n" 678 | ] 679 | }, 680 | { 681 | "data": { 682 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO/klEQVR4nO3deWwUhd/H8U+3BQHRAgIGRQmngXBUIRIjHlAU0GgKJAYjeIdoQkQSAz7RkGA0aJEYrhhrjZo2XAkNh8Ifvwh5AKVCwx2rYiHigmiBUoQqLbvz/MFD/TUtKNNPj2nfr7/YmenMd+O+3Znd7jYlCIJAAOol1tQDAC0BIQEGhAQYEBJgQEiAASEBBoQEGBASYJDW1AOEMWLECFVWVqpbt25NPQpauNLSUrVt21ZFRUVX3S6SIV24cEGJRKKpx0ArcPHiRf2bX/6JZEjdu3eXJH311VdNPAlauszMzH+1HddIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBgD+ns2bM6ceKEe7dAsxY6pJkzZ9a5fP/+/Ro/frxycnJCDwVETeiQfv755zqXjxo1Stu2bdP69etDDwVETeiQUlJSrriuoqJCFy5cCLtrIHLSrmXjpUuXatmyZdW3Bw4ceMVtx44dG34qIGKuKaS7775bkhQEgVatWqUpU6bU3mFamnr27KmHH37YMyEQAdcc0uWYDh8+rBkzZjTIUEDUhL5G+uCDD666fseOHWF3DUROg70hm52d3VC7Bpqdazq1+2+nTp3SggULtGPHDp08eVLJZNI5FxApoUN64403VFJSojFjxqhz586Kxf5+crv8YgTQWoQO6bvvvtOXX36pG264oc718Xg89FBA1IS+RurZs+cVI5Kk9957L+yugcgJHdL48eO1adOmK65/5plnwu4aiJzQp3bFxcXKzc3Vhx9+qD59+qh9+/Y11n///ff1Hg6IitAhbdiwQd27d9e5c+e0f//+WusrKirqNRgQJaFD6tevn9auXXvF9VlZWWF3DURO6GukN99886rreUMWrUnokEaMGHHV9Tt37gy7ayByQp/aHT9+/KrrV65cqalTp4bdPRApoUMaM2bMVT/cB7QmoUPq0aOHXnnllRrLzp8/r5KSEu3cuVPTpk2r93BAVIQOacKECZo4cWKd60pKSrR8+fLQQwFRE/rFhtmzZ19xXd++fbV3796wuwYip0E+j3To0CGdPn26IXYNNEuhT+2efvrpWsuCIFB5ebkOHz6sJ598sl6DAVESOqQDBw5o8ODBNZalpqaqf//+mjZt2hWvn4CWKHRIvXr1Ul5ennMWILJCXyN99tlnxjGAaAv9jNSpUydJUlFRkbZt26aysjJ17txZ99133z/++hDQ0oQOKZFIaPbs2dq4caOCIKhenpOTo0ceeUQLFiyo8T0OQEsWOqSPPvpIu3bt0pw5c5SRkaFOnTrpzJkz2rt3rz799FPl5OTopZdecs4KNFuhQ1q/fr3y8vLUq1evGsszMjI0evRoTZ8+nZDQaoQ+94rFYrUiuqxXr15KTU0NPRQQNaFDqqysVFlZWZ3rTp8+zZ91QasSOqTMzEw999xz2rJli06dOqWLFy/q1KlT2rx5s1544QU99NBDzjmBZi30NdLMmTP1/PPP6+WXX671uaRhw4bV+ogF0JKFDqlDhw7Kz8/Xhg0btH379ur3kUaNGqXHHntMaWmhdw1ETuhHe1lZmXbv3q0uXbpo4cKF1cs3btyos2fPqkuXLpYBgSgIfY2Un5+vmTNnat26dTWWf/vtt8rKytJPP/1U7+GAqAj9jLRlyxZ98sknGjlyZI3l8+bN0+jRo5Wdna2cnJx6DwhEQehnpCAIakV02YMPPqjS0tLQQwFREzqk8vLyeq0HWpLQIfXt21eLFy9WIpGosTyRSGjRokXq06dPvYcDoqJe7yNNnTpVq1ev1sCBA5Wenq7y8nIVFxfr3LlzfIsQWpXQIQ0ePFh5eXnKzs7W119/rWQyqVgspuHDh2v27NkaNGiQc06gWavXu6ZDhgxRXl6e/vrrL5WXlys9PV3t2rVzzQZEhuXXD9q1a0dAaNX4CCtgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBi0mJCSyWSrPDaahxbzR4xisZhWr17d6N853q1bNz3xxBONekw0Py0mJEkqLS3V8ePHm3oMtEIt5tQOaEqEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBKapSAIInXstAaYA6i3lJQU/f7776qqqmrU47Zp00bdu3e/5p8jJDRbVVVVqqysbOox/hVO7QADQgIMCAkwIKQGFiSTrfLYrQ0vNjSwlFhMR9Zn68+TRxv1uO273q7ej89u1GO2ZoTUCP48eVR//lbS1GOgAXFqBxgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASK1YIplslcduCGlNPQCaTmospvn/+x8dLS9r1OPent5Z//PAQ416zIZGSK3c0fIy/XTqZFOPEXmc2gEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYJASBEHQ1ENcqyFDhiiRSKhHjx41lp8/f16JRKJRZ0lNTdX1119/1W0uVpQrSFxspIkuSUlNU1qH9H/c7sxff+piMtkIE/0tLRZTp3bt/3G7xv5veVlqamr1v3/99VelpqbqwIEDV/2ZtIYeqiFcd911qqysrLX8nx7QTeXfPKCbyr95QDeV/35AN5W0tDS1bdv2H7eL5DMS0NxwjQQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIERGPx7VkyRLF4/GmHgV1IKSIOHbsmJYuXapjx4419SioAyEBBnyLUATk5uZq1apVOnr0qG6//XZ16NBB586d0+nTp1VRUaE+ffpo6tSpeuqpp1RUVKR58+appKREGRkZGjNmjAoKClRSUqLXX39dxcXFOnTokOLxuIYPH665c+fqlltuqT5WEAT6/PPPtXLlSklSVVWVxo4dq1dffVXt2zffr+5qcgEiobCwMBgwYEBQWFhYvWzfvn3BgAEDgtWrV9fYduvWrcH06dOrb//yyy/BgAEDgpEjRwZ79uwJgiAIzpw5E2RlZQXjxo0Lqqqqqrd95513gmHDhgX79u0LgiAISktLgwkTJgQvvvhiA9676OPULsKGDh2q/v37a82aNTWWFxQUaPLkybW2z8zMVEZGhiQpPT1dM2bM0JEjR7R27VpJ0tGjR5WXl6eJEydq6NChkqSuXbtq+vTp2rp1q4qKihr0/kQZIUXcpEmTtGfPHh0+fFiSVF5ert27d2v06NG1tr3jjjtq3L4cy549eyRJ33zzjZLJpIYPH17nzxUWFtrnbyki+ZXF+Nvjjz+uhQsXqqCgQK+99pq++OILjRs3Tm3atKm1bceOHWvcTk+/9FXKv/32mySprKxMkrRo0SJ9/PHH1dslk0l17dpVFRUVDXU3Io+QIq5r1666//77tXbtWs2aNUtr1qzR/Pnz69z2jz/+qHH7zJkzkqSbb75ZktS5c2dJ0pw5czR27NiGG7oF4tQuItLSLv0/L/j/F1kPHjyoI0eOSJImT56s0tJS5ebmKhaL1TqFu+zHH3+scXv//v2SpDvvvFOSdO+99yoWi6m4uLjWz7799tvatWuX5860QIQUET179lRKSopOnDgh6dIDe9++fZKkBx54QDfddJMWLVqkSZMmXXEfO3bs0N69eyVdupZatmyZevfuraysLEnSbbfdpmeffVb5+fk6ePCgpEvhrlixQps3b9agQYMa7g5GHO8jRciSJUtUUFCgjh07qnfv3nr//fer/+TIu+++q+XLl2v79u268cYba/xcPB5XZmam5s6dqx9++EEHDx5UPB7XXXfdVef7SPn5+VqxYoUSiYTat2+vfv36adasWbr11lsb9f5GCSG1EOvWrdPWrVu1cOHCWusuhzR//vyrPmMhPE7tWohNmzbV+d4RGgchRdiUKVNUUVGhkpISxeNx3XPPPU09UqvFy98R1qZNGz366KNKT0/XW2+9pZSUlFrb5ObmqqCgQJK0ePFiFRYWKjs7u7FHbfG4RgIMOLUDDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIM/g+GpFCnNhkBXwAAAABJRU5ErkJggg==\n", 683 | "text/plain": [ 684 | "
" 685 | ] 686 | }, 687 | "metadata": {}, 688 | "output_type": "display_data" 689 | }, 690 | { 691 | "name": "stdout", 692 | "output_type": "stream", 693 | "text": [ 694 | "\n", 695 | "creak\n", 696 | "probe acc 0.4945295453071594\n", 697 | "probe sparse 38 8192\n", 698 | "model acc 0.50255286693573\n" 699 | ] 700 | }, 701 | { 702 | "data": { 703 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO+ElEQVR4nO3deWwUhd/H8U+3BQHRUgQMihJOA+GoQiRGPKAooNEUSAxG8A7RhIgkBnyiIcFo0CIxXDHWGjVtuBIaDoU/fhHyAEqFhjtWxULEBdECpQhVju08f/BQf03LNf2022nfr7/cmdnZ78Z9szO73d2UIAgCAaiXWLIHAJoDQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMXToUJ07d06dO3dO9iho5srKytS6dWsVFxdfcbtIhnT27FklEolkj4EW4MKFC7qWP/6JZEhdunSRJH3zzTdJngTNXVZW1jVtxzkSYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBjYQzp16pSOHj3q3i3QpIUOadq0aXUu37Nnj8aMGaPc3NzQQwFREzqkX3/9tc7lw4cP1+bNm7VmzZrQQwFREzqklJSUy66rrKzU2bNnw+4aiJy069l40aJFWrx4cfXlfv36XXbbUaNGhZ8KiJjrCunee++VJAVBoOXLl2vixIm1d5iWpm7duunRRx/1TAhEwHWHdCmmAwcOaOrUqQ0yFBA1oc+RPvrooyuu37p1a9hdA5HTYG/I5uTkNNSugSbnug7t/tvx48c1d+5cbd26VceOHVNVVZVzLiBSQof01ltvqbS0VCNHjlRGRoZisX+f3C69GAG0FKFD+uGHH/T111/rpptuqnN9PB4PPRQQNaHPkbp163bZiCTpgw8+CLtrIHJChzRmzBitX7/+suufe+65sLsGIif0oV1JSYny8vL08ccfq2fPnmrbtm2N9T/++GO9hwOiInRIa9euVZcuXXT69Gnt2bOn1vrKysp6DQZESeiQevfurVWrVl12fXZ2dthdA5ET+hzp7bffvuJ63pBFSxI6pKFDh15x/bZt28LuGoic0Id2R44cueL6ZcuWadKkSWF3D0RK6JBGjhx5xQ/3AS1J6JC6du2q1157rcayM2fOqLS0VNu2bdPkyZPrPRwQFaFDGjt2rMaNG1fnutLSUi1ZsiT0UEDUhH6xYcaMGZdd16tXL+3atSvsroHIaZDPI+3fv18nTpxoiF0DTVLoQ7tnn3221rIgCFRRUaEDBw7o6aefrtdgQJSEDmnv3r0aMGBAjWWpqanq06ePJk+efNnzJ6A5Ch1S9+7dlZ+f75wFiKzQ50hffPGFcYzmK0jiR/CTedstTehnpA4dOkiSiouLtXnzZpWXlysjI0MPPPDAVf98qCVJicV0cE2O/j52qFFvt22nO9Xjycu/sgqv0CElEgnNmDFD69atUxAE1ctzc3P12GOPae7cuTW+x6El+/vYIf39R2myx0ADCh3SJ598ou3bt2vmzJnKzMxUhw4ddPLkSe3atUuff/65cnNz9corrzhnBZqs0CGtWbNG+fn56t69e43lmZmZGjFihKZMmUJIaDFCH3vFYrFaEV3SvXt3paamhh4KiJrQIZ07d07l5eV1rjtx4gQ/64IWJXRIWVlZeuGFF7Rx40YdP35cFy5c0PHjx7Vhwwa99NJLeuSRR5xzAk1a6HOkadOm6cUXX9Srr75a63NJgwcPrvURC6A5Cx1Su3btVFBQoLVr12rLli3V7yMNHz5cTzzxhNLSQu8aiJzQj/by8nLt2LFDHTt21Lx586qXr1u3TqdOnVLHjh0tAwJREPocqaCgQNOmTdPq1atrLP/++++VnZ2tX375pd7DAVER+hlp48aN+uyzzzRs2LAay2fPnq0RI0YoJydHubm59R4QiILQz0hBENSK6JKHH35YZWVloYcCoiZ0SBUVFfVaDzQnoUPq1auXFixYoEQiUWN5IpHQ/Pnz1bNnz3oPB0RFvd5HmjRpklasWKF+/fopPT1dFRUVKikp0enTp/kWIbQooUMaMGCA8vPzlZOTo2+//VZVVVWKxWIaMmSIZsyYof79+zvnBJq0er1rOnDgQOXn5+uff/5RRUWF0tPT1aZNG9dsQGRY/vygTZs2BIQWjY+wAgaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQUguWSOIv+iXzthsCX4fagqXGYprzv//RoYq6fwyhodyZnqH/eah5fTc8IbVwhyrK9cvxY8keI/I4tAMMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDJpNSFVVVS3yttE0pCV7AJdYLKYVK1aorKysUW+3c+fOeuqppxr1NtH0NJuQJKmsrExHjhxJ9hhogZrNoR2QTIQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBKapCAIInXbzeo3ZNF8pKSk6M8//9T58+cb9XZbtWqlLl26XPf1CAlN1vnz53Xu3Llkj3FNUoJkPoeGNHDgQCUSCXXt2rXG8jNnziiRSDTqLKmpqbrxxhuvuM2FygoFiQuNNNFFKalpSmuXftXtTv7zty5UVTXCRP9Ki8XUoU3bq27X2P8vL0lNTa3+799//12pqanau3fvFa8TyWekG264oc5/qa72gE6Wa3lAJ8u1PKCT5b8f0MmSlpam1q1bX3W7SD4jAU0Nr9oBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQUkTE43EtXLhQ8Xg82aOgDoQUEYcPH9aiRYt0+PDhZI+COhASYMC3CEVAXl6eli9frkOHDunOO+9Uu3btdPr0aZ04cUKVlZXq2bOnJk2apGeeeUbFxcWaPXu2SktLlZmZqZEjR6qwsFClpaV68803VVJSov379ysej2vIkCGaNWuWbrvtturbCoJAX375pZYtWybp4pc0jho1Sq+//rratm26X92VdAEioaioKOjbt29QVFRUvWz37t1B3759gxUrVtTYdtOmTcGUKVOqL//2229B3759g2HDhgU7d+4MgiAITp48GWRnZwejR48Ozp8/X73te++9FwwePDjYvXt3EARBUFZWFowdOzZ4+eWXG/DeRR+HdhE2aNAg9enTRytXrqyxvLCwUBMmTKi1fVZWljIzMyVJ6enpmjp1qg4ePKhVq1ZJkg4dOqT8/HyNGzdOgwYNkiR16tRJU6ZM0aZNm1RcXNyg9yfKCCnixo8fr507d+rAgQOSpIqKCu3YsUMjRoyote1dd91V4/KlWHbu3ClJ+u6771RVVaUhQ4bUeb2ioiL7/M1FJL+yGP968sknNW/ePBUWFuqNN97QV199pdGjR6tVq1a1tm3fvn2Ny+npF79K+Y8//pAklZeXS5Lmz5+vTz/9tHq7qqoqderUSZWVlQ11NyKPkCKuU6dOevDBB7Vq1SpNnz5dK1eu1Jw5c+rc9q+//qpx+eTJk5KkW2+9VZKUkZEhSZo5c6ZGjRrVcEM3QxzaRURa2sV/84L/f5F13759OnjwoCRpwoQJKisrU15enmKxWK1DuEt+/vnnGpf37NkjSbr77rslSffff79isZhKSkpqXffdd9/V9u3bPXemGSKkiOjWrZtSUlJ09OhRSRcf2Lt375YkPfTQQ7rllls0f/58jR8//rL72Lp1q3bt2iXp4rnU4sWL1aNHD2VnZ0uS7rjjDj3//PMqKCjQvn37JF0Md+nSpdqwYYP69+/fcHcw4ngfKUIWLlyowsJCtW/fXj169NCHH35Y/ZMj77//vpYsWaItW7bo5ptvrnG9eDyurKwszZo1Sz/99JP27duneDyue+65p873kQoKCrR06VIlEgm1bdtWvXv31vTp03X77bc36v2NEkJqJlavXq1NmzZp3rx5tdZdCmnOnDlXfMZCeBzaNRPr16+v870jNA5CirCJEyeqsrJSpaWlisfjuu+++5I9UovFy98R1qpVKz3++ONKT0/XO++8o5SUlFrb5OXlqbCwUJK0YMECFRUVKScnp7FHbfY4RwIMOLQDDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIM/g+Yu1CjRssAugAAAABJRU5ErkJggg==\n", 704 | "text/plain": [ 705 | "
" 706 | ] 707 | }, 708 | "metadata": {}, 709 | "output_type": "display_data" 710 | }, 711 | { 712 | "name": "stdout", 713 | "output_type": "stream", 714 | "text": [ 715 | "\n", 716 | "creak\n", 717 | "probe acc 0.5193289518356323\n", 718 | "probe sparse 90 8192\n", 719 | "model acc 0.50255286693573\n" 720 | ] 721 | }, 722 | { 723 | "data": { 724 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO7UlEQVR4nO3daWwVBd+G8bsLSBEtxYJBEcJqICxViMSICxQFNJoCicEI7iGaEJHEgE80JBgNWiSGLcZao6YNW0LDovDhjZAXUCo07LEqFiIeFC1bEaq0nM7zgZc+T9+yTu/2dNrr94kzM535n3guz5w9KQiCQAAaJDnRAwAtASEBBoQEGBASYEBIgAEhAQaEBBgQEmCQmugBwhg2bJiqqqrUuXPnRI+CFq68vFxt27ZVSUnJFbeLZEjnzp1TPB5P9BhoBc6fP69refNPJEPq0qWLJOnrr79O8CRo6bKzs69pOx4jAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASICBPaTTp0/r6NGj7t0CzVrokKZPn37J5Xv37tXYsWOVl5cXeiggakKH9Msvv1xy+YgRI7RlyxatXbs29FBA1IQOKSkp6bLrKisrde7cubC7BiIn9Xo2Xrx4sZYsWVJ7uX///pfddvTo0eGnAiLmukK65557JElBEGjFihWaNGlS/R2mpqpbt2565JFHPBMCEXDdIV2M6eDBg5o2bVqjDAVETejHSB9++OEV12/bti3sroHIabQXZHNzcxtr10Czc12ndv/t+PHjmjdvnrZt26Zjx46ppqbGORcQKaFDevPNN1VWVqZRo0YpIyNDycn/uXO7+GQE0FqEDun777/XV199pZtuuumS62OxWOihgKgJ/RipW7dul41Ikt5///2wuwYiJ3RIY8eO1YYNGy67/tlnnw27ayByQp/alZaWKj8/Xx999JF69eqltLS0Out/+OGHBg8HREXokNatW6cuXbrozJkz2rt3b731lZWVDRoMiJLQIfXp00erV6++7PqcnJywuwYiJ/RjpLfeeuuK63lBFq1J6JCGDRt2xfXbt28Pu2sgckKf2v32229XXL98+XJNnjw57O6BSAkd0qhRo6744T6gNQkdUteuXfXqq6/WWXb27FmVlZVp+/btmjJlSoOHA6IidEjjxo3T+PHjL7murKxMS5cuDT0UEDWhn2yYOXPmZdf17t1bu3fvDrtrIHIa5fNIBw4c0IkTJxpj10CzFPrU7plnnqm3LAgCVVRU6ODBg3rqqacaNBgQJaFD2rdvnwYOHFhnWUpKivr27aspU6Zc9vET0BKFDqlHjx4qKChwzgJEVujHSJ9//rlxDCDaQt8jdezYUZJUUlKiLVu26OTJk8rIyND9999/1bcPAS1N6JDi8bhmzpyp9evXKwiC2uV5eXl69NFHNW/evDrf4wC0ZKFD+vjjj7Vjxw7NmjVLWVlZ6tixo06dOqXdu3frs88+U15enl5++WXnrECzFTqktWvXqqCgQD169KizPCsrSyNHjtTUqVObNKSampqE3QMm8thoHkKHlJycXC+ii3r06KGUlJTQQ4WdZ+XKlSovL2/S43bu3FlPPvlkkx4TzU/okKqqqmqfYPj/Tpw4kZCfdSkvL7/qxzuAxhD6fCQ7O1vPP/+8Nm3apOPHj+v8+fM6fvy4Nm7cqBdffFEPP/ywc06gWQt9jzR9+nS98MILeuWVV+p9LmnIkCH1PmIBtGShQ2rfvr0KCwu1bt06bd26tfY0b8SIEXr88ceVmhp610DkhL61nzx5Ujt37lSnTp00f/782uXr16/X6dOn1alTJ8uAQBSEfoxUWFio6dOna82aNXWWf/fdd8rJydHPP//c4OGAqAh9j7Rp0yZ9+umnGj58eJ3lc+bM0ciRI5Wbm6u8vLwGDwhEQeh7pCAI6kV00UMPPdTkr+cAiRQ6pIqKigatB1qS0CH17t1bCxcuVDwer7M8Ho9rwYIF6tWrV4OHA6KiQa8jTZ48WStXrlT//v2Vnp6uiooKlZaW6syZM3yLEFqV0CENHDhQBQUFys3N1TfffFP7xs2hQ4dq5syZGjBggHNOoFlr0KumgwYNUkFBgf755x9VVFQoPT1d7dq1c80GRIbl7Qft2rUjILRqfIgGMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICQ0S0EQROrY/BoYmqWkpCT9+eefqq6ubtLjtmnTRl26dLnuvyMkNFvV1dWqqqpK9BjXhFM7wICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQmpkQU1Nqzx2a5Oa6AFauqTkZB1am6u/jx1u0uOmZXZXzydmNukxWzNCagJ/Hzusv/8oS/QYaESc2gEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIrVg8gb/ol8hjNwZ+aKwVS0lO1tz//R8drjjZpMftnp6hfz34cJMes7ERUit3uOKkfj5+LNFjRB6ndoABIQEGhAQYEBJgQEiAASEBBoQEGCQFQRAkeojrNWjQIMXjcXXt2rXO8rNnzyoejzfpLCkpKbrxxhuvuM35ygoF8fNNNNEFSSmpSm2fftXtTv3zt8438bsMUpOT1bFd2lW3a+r/lhelpKTU/vv3339XSkqK9u3bd8W/ieQLsjfccIOqqqrqLb/aDTpRruUGnSjXcoNOlP++QSdKamqq2rZte9XtInmPBDQ3PEYCDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgpIiIxWJatGiRYrFYokfBJRBSRBw5ckSLFy/WkSNHEj0KLoGQAAO+RSgC8vPztWLFCh0+fFjdu3dX+/btdebMGZ04cUKVlZXq1auXJk+erKefflolJSWaM2eOysrKlJWVpVGjRqmoqEhlZWV64403VFpaqgMHDigWi2no0KGaPXu2brvtttpjBUGgL774QsuXL5ckVVdXa/To0XrttdeUltZ8v7or4QJEQnFxcdCvX7+guLi4dtmePXuCfv36BStXrqyz7ebNm4OpU6fWXv7111+Dfv36BcOHDw927doVBEEQnDp1KsjJyQnGjBkTVFdX12777rvvBkOGDAn27NkTBEEQlJeXB+PGjQteeumlRrx20cepXYQNHjxYffv21apVq+osLyoq0sSJE+ttn52draysLElSenq6pk2bpkOHDmn16tWSpMOHD6ugoEDjx4/X4MGDJUmZmZmaOnWqNm/erJKSkka9PlFGSBE3YcIE7dq1SwcPHpQkVVRUaOfOnRo5cmS9be+88846ly/GsmvXLknSt99+q5qaGg0dOvSSf1dcXGyfv6WI5FcW4z+eeOIJzZ8/X0VFRXr99df15ZdfasyYMWrTpk29bTt06FDncnr6ha9S/uOPPyRJJ09e+FHmBQsW6JNPPqndrqamRpmZmaqsrGysqxF5hBRxmZmZeuCBB7R69WrNmDFDq1at0ty5cy+57V9//VXn8qlTpyRJt956qyQpIyNDkjRr1iyNHj268YZugTi1i4jU1Av/zwv+70nW/fv369ChQ5KkiRMnqry8XPn5+UpOTq53CnfRTz/9VOfy3r17JUl33XWXJOm+++5TcnKySktL6/3tO++8ox07dniuTAtESBHRrVs3JSUl6ejRo5Iu3LD37NkjSXrwwQd1yy23aMGCBZowYcJl97Ft2zbt3r1b0oXHUkuWLFHPnj2Vk5MjSbrjjjv03HPPqbCwUPv375d0Idxly5Zp48aNGjBgQONdwYjjdaQIWbRokYqKitShQwf17NlTH3zwQe1Pjrz33ntaunSptm7dqptvvrnO38ViMWVnZ2v27Nn68ccftX//fsViMd19992XfB2psLBQy5YtUzweV1pamvr06aMZM2bo9ttvb9LrGyWE1EKsWbNGmzdv1vz58+utuxjS3Llzr3iPhfA4tWshNmzYcMnXjtA0CCnCJk2apMrKSpWVlSkWi+nee+9N9EitFk9/R1ibNm302GOPKT09XW+//baSkpLqbZOfn6+ioiJJ0sKFC1VcXKzc3NymHrXF4zESYMCpHWBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYPBvm9VQ5hV6pDwAAAAASUVORK5CYII=\n", 725 | "text/plain": [ 726 | "
" 727 | ] 728 | }, 729 | "metadata": {}, 730 | "output_type": "display_data" 731 | }, 732 | { 733 | "name": "stdout", 734 | "output_type": "stream", 735 | "text": [ 736 | "\n", 737 | "\n" 738 | ] 739 | } 740 | ], 741 | "source": [ 742 | "for dataset in data:\n", 743 | " for l1 in [0, 0.01, 0.03, 0.1, 0.3]:\n", 744 | " n_train, model_dim = data_processed[dataset][\"train\"][\"good_repr\"].shape\n", 745 | " probe = train_probe(data_processed[dataset][\"train\"], data_processed[dataset][\"valid\"], l1=l1)\n", 746 | " probe.layer.weight.data[probe.layer.weight.data < 1e-3] = 0\n", 747 | " probe_preds = torch.sigmoid(\n", 748 | " probe(data_processed[dataset][\"valid\"][\"good_repr\"])[:, 1]\n", 749 | " - probe(data_processed[dataset][\"valid\"][\"bad_repr\"])[:, 1]\n", 750 | " ).detach().cpu().numpy()\n", 751 | " lm_preds = torch.sigmoid(data_processed[dataset][\"valid\"][\"ratio\"].float()).detach().cpu().numpy()\n", 752 | " lm_correct = lm_preds > 0.499\n", 753 | " probe_correct = probe_preds > 0.499\n", 754 | " \n", 755 | " lm_probe = lm_correct & probe_correct\n", 756 | " lm_noprobe = lm_correct & ~probe_correct\n", 757 | " nolm_probe = ~lm_correct & probe_correct\n", 758 | " nolm_noprobe = ~lm_correct & ~probe_correct\n", 759 | " cat_labels = 0 * lm_probe + 1 * nolm_probe + 2 * lm_noprobe + 3 * nolm_noprobe\n", 760 | " type_order = [\"both right\", \"probe correct\", \"pred correct\", \"both wrong\"]\n", 761 | " #palette = dict(zip(type_order, sns.husl_palette(5, h=0.1)))\n", 762 | " names = dict(enumerate(type_order))\n", 763 | " base_palette = sns.husl_palette(5, h=0.1)\n", 764 | " neutral = (.48, .48, .48)\n", 765 | " neutral_light = (0.87, 0.87, 0.87)\n", 766 | " palette = {\"both right\": neutral, \"probe correct\": base_palette[0], \"pred correct\": base_palette[2], \"both wrong\": neutral_light}\n", 767 | " named_labels = [names[c] for c in cat_labels]\n", 768 | " df = pd.DataFrame({\n", 769 | " 'lm': lm_preds,\n", 770 | " 'probe': probe_preds,\n", 771 | " 'type': named_labels,\n", 772 | " }).sort_values(by=\"type\")\n", 773 | " \n", 774 | " print(dataset)\n", 775 | " print(\"probe acc\", eval_probe(probe, data_processed[dataset][\"valid\"]))\n", 776 | " print(\"probe sparse\", (probe.layer.weight.abs() != 0).int().sum().item(), probe.layer.weight.shape[0] * probe.layer.weight.shape[1])\n", 777 | " print(\"model acc\", data_processed[dataset][\"valid\"][\"correct\"].float().mean().item())\n", 778 | "\n", 779 | " fig, ax = plt.subplots(figsize=(2, 4.8))\n", 780 | " sns.countplot(ax=ax, data=df, x=\"type\", order=type_order, palette=palette)\n", 781 | " ax.xaxis.set_ticklabels([])\n", 782 | " ax.yaxis.set_ticklabels([])\n", 783 | " plt.ylim(0, len(df))\n", 784 | " plt.show()\n", 785 | "\n", 786 | "\n", 787 | " \n", 788 | " print()\n", 789 | " print()" 790 | ] 791 | }, 792 | { 793 | "cell_type": "markdown", 794 | "metadata": {}, 795 | "source": [ 796 | "## Probe superiority seems mainly attributable to probe calibration" 797 | ] 798 | }, 799 | { 800 | "cell_type": "code", 801 | "execution_count": null, 802 | "metadata": { 803 | "scrolled": false 804 | }, 805 | "outputs": [], 806 | "source": [ 807 | "def plot_calibration(probe_preds):\n", 808 | " p_true = probe_preds\n", 809 | " p_false = 1 - p_true\n", 810 | " calib_range = []\n", 811 | " calib_acc = []\n", 812 | " calib_size = []\n", 813 | " step = 0.1\n", 814 | " for start in np.arange(0, 1 + step, step):\n", 815 | " end = start + step\n", 816 | " pred_true = ((start <= p_true) & (p_true < end)).sum()\n", 817 | " pred_false = ((start <= p_false) & (p_false < end)).sum()\n", 818 | " if pred_true + pred_false == 0:\n", 819 | " continue\n", 820 | " frac_true = pred_true / (pred_true + pred_false)\n", 821 | " calib_range.append(start)\n", 822 | " calib_acc.append(frac_true)\n", 823 | " calib_size.append(pred_true + pred_false)\n", 824 | "\n", 825 | " data = pd.DataFrame({\"range\": calib_range, \"acc\": calib_acc, \"size\": calib_size})\n", 826 | " fig, ax = plt.subplots(figsize=(3, 3))\n", 827 | " sns.scatterplot(ax=ax, data=data, x=\"range\", y=\"acc\", size=\"size\")\n", 828 | " ax.get_legend().remove()\n", 829 | "\n", 830 | " plt.xlim(0, 1)\n", 831 | " plt.ylim(0, 1)\n", 832 | " plt.show()\n", 833 | "\n", 834 | "for dataset in data:\n", 835 | " print(dataset)\n", 836 | " plot_calibration(probe_preds[dataset])\n", 837 | " plot_calibration(torch.sigmoid(data_processed[dataset][\"valid\"][\"ratio\"].float()).detach().cpu().numpy())" 838 | ] 839 | }, 840 | { 841 | "cell_type": "markdown", 842 | "metadata": {}, 843 | "source": [ 844 | "## Models are not strictly better than probes" 845 | ] 846 | }, 847 | { 848 | "cell_type": "code", 849 | "execution_count": null, 850 | "metadata": { 851 | "scrolled": false 852 | }, 853 | "outputs": [], 854 | "source": [ 855 | "for dataset in data:\n", 856 | " lm_dev_preds = torch.sigmoid(data_processed[dataset][\"train\"][\"ratio\"][-N_DEV:].float()).detach().cpu().numpy()\n", 857 | " lm_preds = torch.sigmoid(data_processed[dataset][\"valid\"][\"ratio\"].float()).detach().cpu().numpy()\n", 858 | " lm_correct = lm_preds > 0.499\n", 859 | " probe_correct = probe_preds[dataset] > 0.499\n", 860 | " \n", 861 | " ensemble_scores = []\n", 862 | " for d in np.arange(0, 1, 0.05):\n", 863 | " dev_acc = ((d * lm_dev_preds + (1-d) * probe_dev_preds[dataset]) > 0.5).mean()\n", 864 | " acc = ((d * lm_preds + (1-d) * probe_preds[dataset]) > 0.5).mean()\n", 865 | " ensemble_scores.append((dev_acc, acc))\n", 866 | " ensemble_corrects = max(ensemble_scores)[1]\n", 867 | " \n", 868 | " lm_probe = lm_correct & probe_correct\n", 869 | " lm_noprobe = lm_correct & ~probe_correct\n", 870 | " nolm_probe = ~lm_correct & probe_correct\n", 871 | " nolm_noprobe = ~lm_correct & ~probe_correct\n", 872 | " cat_labels = 0 * lm_probe + 1 * nolm_probe + 2 * lm_noprobe + 3 * nolm_noprobe\n", 873 | " type_order = [\"both right\", \"probe correct\", \"pred correct\", \"both wrong\"]\n", 874 | " #palette = dict(zip(type_order, sns.husl_palette(5, h=0.1)))\n", 875 | " names = dict(enumerate(type_order))\n", 876 | " base_palette = sns.husl_palette(5, h=0.1)\n", 877 | " neutral = (.48, .48, .48)\n", 878 | " neutral_light = (0.87, 0.87, 0.87)\n", 879 | " palette = {\"both right\": neutral, \"probe correct\": base_palette[0], \"pred correct\": base_palette[2], \"both wrong\": neutral_light}\n", 880 | "\n", 881 | " #names = {0: \"both right\", 1: \"pred correct\", 2: \"probe_correct\", 3: \"both wrong\"}\n", 882 | " named_labels = [names[c] for c in cat_labels]\n", 883 | " \n", 884 | " print(dataset)\n", 885 | " \n", 886 | " df = pd.DataFrame({\n", 887 | " 'lm': lm_preds,\n", 888 | " 'probe': probe_preds[dataset],\n", 889 | " 'type': named_labels,\n", 890 | " }).sort_values(by=\"type\")\n", 891 | " \n", 892 | " g = sns.JointGrid(data=df, x=\"lm\", y=\"probe\", hue=\"type\", xlim=(0, 1), ylim=(0, 1), space=0, palette=palette)\n", 893 | " g.plot_joint(\n", 894 | " sns.histplot, #discrete=(True, False),\n", 895 | " binrange=[0, 1],\n", 896 | " bins=20,\n", 897 | " #cmap=\"gist_gray_r\",\n", 898 | " )\n", 899 | " g.plot_marginals(\n", 900 | " sns.histplot, multiple=\"stack\",\n", 901 | " binrange=[0, 1],\n", 902 | " bins=20,\n", 903 | " #color=\"gray\",\n", 904 | "\n", 905 | " )\n", 906 | " g.figure.axes[0].set_xticks([0, 0.5, 1])\n", 907 | " g.figure.axes[0].set_yticks([0, 0.5, 1])\n", 908 | " g.figure.axes[0].set_axisbelow(False)\n", 909 | " g.figure.axes[0].grid(color=\"black\")\n", 910 | " g.figure.axes[0].get_legend().remove()\n", 911 | " #g.plot_joint(sns.scatterplot, sns.histplot)\n", 912 | " #g = sns.jointplot(data=df, x=\"lm\", y=\"probe\")\n", 913 | " plt.show()\n", 914 | " \n", 915 | " #sns.relplot(data=df, x='lm', y='probe', hue='type')\n", 916 | " #plt.xlim(0, 1)\n", 917 | " #plt.ylim(0, 1)\n", 918 | " #plt.show()\n", 919 | " \n", 920 | " \n", 921 | " fig, ax = plt.subplots(figsize=(2, 4.8))\n", 922 | " sns.countplot(ax=ax, data=df, x=\"type\", order=type_order, palette=palette)\n", 923 | " ax.xaxis.set_ticklabels([])\n", 924 | " ax.yaxis.set_ticklabels([])\n", 925 | " plt.ylim(0, len(df))\n", 926 | " plt.show()\n", 927 | " \n", 928 | " #print(\"lm & !probe\", (lm_correct & (~probe_correct)).mean())\n", 929 | " #print(\"!lm & probe\", ((~lm_correct) & probe_correct).mean())\n", 930 | " #print()\n", 931 | " print(\"lm acc \", lm_correct.mean())\n", 932 | " print(\"probe acc \", probe_correct.mean())\n", 933 | " print(\"ensemble acc\", ensemble_corrects)\n" 934 | ] 935 | }, 936 | { 937 | "cell_type": "markdown", 938 | "metadata": {}, 939 | "source": [ 940 | "## When probe and model disagree, probe is just uncertain" 941 | ] 942 | }, 943 | { 944 | "cell_type": "code", 945 | "execution_count": null, 946 | "metadata": {}, 947 | "outputs": [], 948 | "source": [ 949 | "for dataset in data:\n", 950 | " print(dataset)\n", 951 | " print()\n", 952 | " lm_preds = torch.sigmoid(data_processed[dataset][\"valid\"][\"ratio\"].float()).detach().cpu().numpy()\n", 953 | " disagreements = (lm_preds > 0.5) ^ (probe_preds[dataset] > 0.5)\n", 954 | " disagree_prob_lm = abs(lm_preds[disagreements]-.5)\n", 955 | " disagree_prob_probe = abs(probe_preds[dataset][disagreements]-.5)\n", 956 | " disagree_acc_lm = (lm_preds[disagreements] > 0.5)\n", 957 | " disagree_acc_probe = (probe_preds[dataset][disagreements] > 0.5)\n", 958 | " agree_prob_lm = abs(lm_preds[~disagreements]-.5)\n", 959 | " agree_prob_probe = abs(probe_preds[dataset][~disagreements]-.5)\n", 960 | " print(\"lm disagree conf\", disagree_prob_lm.mean(), disagree_prob_lm.std())\n", 961 | " print(\"probe disagree conf\", disagree_prob_probe.mean(), disagree_prob_probe.std())\n", 962 | " print(\"lm agree conf\", agree_prob_lm.mean(), agree_prob_lm.std())\n", 963 | " print(\"probe agree conf\", agree_prob_probe.mean(), agree_prob_probe.std())\n", 964 | " print()\n", 965 | " \n", 966 | " print(\"lm disagree acc\", disagree_acc_lm.mean())\n", 967 | " print(\"probe disagree acc\", disagree_acc_probe.mean())\n", 968 | " print()\n", 969 | " \n", 970 | " print()" 971 | ] 972 | }, 973 | { 974 | "cell_type": "code", 975 | "execution_count": null, 976 | "metadata": {}, 977 | "outputs": [], 978 | "source": [ 979 | "labels = [\"deception\", \"heterogeneity (probe)\", \"agreement (correct)\", \"confabulation\", \"agreement (uncertain)\", \"heterogeneity (query)\", \"agreement (incorrect)\", \"probe error (query uncertain)\", \"probe error (query correct)\"]\n", 980 | "dfs = []\n", 981 | "for dataset in data:\n", 982 | " lm_preds = torch.sigmoid(data_processed[dataset][\"valid\"][\"ratio\"].float()).detach().cpu().numpy()\n", 983 | " \n", 984 | " lm_lc = lm_preds < 0.333\n", 985 | " lm_mc = (lm_preds >= 0.333) & (lm_preds < 0.667) #(0.333 <= lm_preds < 0.667)\n", 986 | " lm_hc = 0.667 <= lm_preds\n", 987 | " probe_lc = probe_preds[dataset] < 0.333\n", 988 | " probe_mc = (probe_preds[dataset] >= 0.333) & (probe_preds[dataset] < 0.667) # 0.333 <= probe_preds[dataset] < 0.667\n", 989 | " probe_hc = 0.667 <= probe_preds[dataset]\n", 990 | " \n", 991 | " deception = lm_lc & probe_hc\n", 992 | " het_p = lm_mc & probe_hc\n", 993 | " agr_c = lm_hc & probe_hc\n", 994 | " confab = lm_lc & probe_mc\n", 995 | " agr_u = lm_mc & probe_mc\n", 996 | " het_l = lm_hc & probe_mc\n", 997 | " agr_i = lm_lc & probe_lc\n", 998 | " probe_err_u = lm_mc & probe_lc\n", 999 | " probe_err_c = lm_hc & probe_lc\n", 1000 | " \n", 1001 | " cat_labels = (\n", 1002 | " 0 * deception\n", 1003 | " + 1 * het_p\n", 1004 | " + 2 * agr_c\n", 1005 | " + 3 * confab\n", 1006 | " + 4 * agr_u\n", 1007 | " + 5 * het_l\n", 1008 | " + 6 * agr_i\n", 1009 | " + 7 * probe_err_u\n", 1010 | " + 8 * probe_err_c\n", 1011 | " )\n", 1012 | " named_labels = [labels[c] for c in cat_labels]\n", 1013 | " \n", 1014 | " df = pd.DataFrame({\n", 1015 | " 'lm': [\"gpt2-xl\"] * len(lm_preds),\n", 1016 | " 'dataset': [dataset] * len(lm_preds),\n", 1017 | " 'type': named_labels,\n", 1018 | " }).groupby(\"dataset\")[\"type\"].value_counts(normalize=True).rename(\"percent\").reset_index()\n", 1019 | " dfs.append(df)\n", 1020 | " \n", 1021 | "palette = {\n", 1022 | " \"deception\": \"#E9A365\",\n", 1023 | " \"heterogeneity (probe)\": \"#E9D165\",\n", 1024 | " \"agreement (correct)\": \"#7A7A7A\",\n", 1025 | " \"confabulation\": \"#F4D1B2\",\n", 1026 | " \"agreement (uncertain)\": \"#ABABAB\",\n", 1027 | " \"heterogeneity (query)\": \"#6EC168\",\n", 1028 | " \"agreement (incorrect)\": \"#DEDEDE\",\n", 1029 | " \"probe error (query uncertain)\": \"#CDEBE8\",\n", 1030 | " \"probe error (query correct)\": \"#68C1B8\",\n", 1031 | "}\n", 1032 | "#palette = sns.color_palette(palette)\n", 1033 | "\n", 1034 | " \n", 1035 | "g = sns.catplot(x=\"type\", y=\"percent\", col=\"dataset\",\n", 1036 | " data=pd.concat(dfs), kind=\"bar\", palette=palette, order=labels)\n", 1037 | "g.set(xticklabels=[])\n", 1038 | "plt.ylim(0, 0.7)\n", 1039 | "plt.show()" 1040 | ] 1041 | }, 1042 | { 1043 | "cell_type": "code", 1044 | "execution_count": null, 1045 | "metadata": {}, 1046 | "outputs": [], 1047 | "source": [] 1048 | } 1049 | ], 1050 | "metadata": { 1051 | "kernelspec": { 1052 | "display_name": "Python 3", 1053 | "language": "python", 1054 | "name": "python3" 1055 | }, 1056 | "language_info": { 1057 | "codemirror_mode": { 1058 | "name": "ipython", 1059 | "version": 3 1060 | }, 1061 | "file_extension": ".py", 1062 | "mimetype": "text/x-python", 1063 | "name": "python", 1064 | "nbconvert_exporter": "python", 1065 | "pygments_lexer": "ipython3", 1066 | "version": "3.7.6" 1067 | } 1068 | }, 1069 | "nbformat": 4, 1070 | "nbformat_minor": 4 1071 | } 1072 | --------------------------------------------------------------------------------