├── MNIST ├── MNIST_read_and_plot_rotation5M.ipynb ├── MNIST_read_and_plot_whitepatch2M.ipynb ├── mnist_get_pretrained.ipynb ├── semitorchMNISTclass.py ├── simu_MNIST_patches.py ├── simu_MNIST_patches_2M.py ├── simudata_MNIST.py ├── submit_simu_MNIST_patches.py └── submit_simu_MNIST_patches_2M.py ├── README.md ├── amazon ├── amazon_review_data_2018_subset_regression.py ├── read_and_preprocess_amazon_review_data_2018_subset.ipynb ├── results_amazon │ └── .gitignore └── submit_amazon_review_data_2018_subset_regression.py ├── mmd.py ├── myrandom.py ├── sem.py ├── semiclass.py ├── semitorchclass.py ├── semitorchstocclass.py ├── sim ├── sim_linearSCM_mean_shift_exp1-7.ipynb ├── sim_linearSCM_var_shift_exp8_box_run.py ├── sim_linearSCM_var_shift_exp8_box_submit.py ├── sim_linearSCM_var_shift_exp8_scat_run.py ├── sim_linearSCM_var_shift_exp8_scat_submit.py ├── sim_linearSCM_var_shift_exp9_scat_run.py ├── sim_linearSCM_var_shift_exp9_scat_submit.py ├── sim_linearSCM_variance_shift_exp8-9.ipynb ├── simu_results │ └── .gitignore └── simudata.py └── util.py /MNIST/MNIST_read_and_plot_rotation5M.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np\n", 10 | "import matplotlib.pyplot as plt\n", 11 | "import seaborn as sns\n", 12 | "\n", 13 | "import pandas as pd\n", 14 | "\n", 15 | "plt.rcParams['axes.facecolor'] = 'lightgray'\n", 16 | "sns.set(style=\"darkgrid\")\n", 17 | "np.set_printoptions(precision=3)" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 2, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "def boxplot_all_methods(plt_handle, res_all, title='', names=[], color=[]):\n", 27 | " res_all_df = pd.DataFrame(res_all.T)\n", 28 | " res_all_df.columns = names\n", 29 | " res_all_df_melt = res_all_df.melt(var_name='methods', value_name='accuracy')\n", 30 | " res_all_mean = np.mean(res_all, axis=1)\n", 31 | "# print(res_all_df_melt)\n", 32 | " print(res_all_df.shape, res_all_mean.shape, res_all_df_melt.shape)\n", 33 | " \n", 34 | "# plt_handle.set_title(title, fontsize=15)\n", 35 | "\n", 36 | " plt_handle.axhline(res_all_mean[2], ls='--', color='b')\n", 37 | " plt_handle.axhline(res_all_mean[1], ls='--', color='r')\n", 38 | " ax = sns.boxplot(x=\"methods\", y=\"accuracy\", data=res_all_df_melt, palette=color, ax=plt_handle)\n", 39 | " ax.set_xticklabels(ax.get_xticklabels(), rotation=-60, ha='left', fontsize=15)\n", 40 | " ax.tick_params(labelsize=15)\n", 41 | " ax.yaxis.grid(False) # Hide the horizontal gridlines\n", 42 | " ax.xaxis.grid(True) # Show the vertical gridlines\n", 43 | " ax.set_xlabel(\"methods\")\n", 44 | " ax.set_ylabel(\"accuracy\")\n", 45 | " \n", 46 | " ax.set_xlabel(\"\")\n", 47 | " ax.set_ylabel(\"Accuracy (%)\", fontsize=15)" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 3, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "# perturb = 'whitepatch'\n", 57 | "perturb = 'rotation'\n", 58 | "M = 5\n", 59 | "subset_prop = 0.2\n", 60 | "lamL2 = 0.\n", 61 | "lamL1 = 0.\n", 62 | "lr = 1e-4\n", 63 | "epochs= 100" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 4, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "names_short = ['Original', \"Tar\", \"Src[1]\",\n", 73 | " 'DIP[1]', 'DIPweigh', 'CIP', 'CIRMweigh',\n", 74 | " 'DIP[1]-MMD', 'DIPweigh-MMD', 'CIP-MMD', 'CIRMweigh-MMD']\n", 75 | "\n", 76 | "prefix_template = 'results_MNIST/report_v8_%s_M%d_subsetprop%s_%s_lamMatch%s_lamCIP%s_lamMatchMMD%s_lamCIPMMD%s_epochs%d_seed%d'\n", 77 | "\n" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 5, 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "repeats = 10\n", 87 | "nb_ba = 3 # Original, Tar, Src[1]\n", 88 | "results_src_ba = np.zeros((M-1, nb_ba, 2, 10))\n", 89 | "results_tar_ba = np.zeros((nb_ba, 2, 10))\n", 90 | "for seed in range(repeats):\n", 91 | " savefilename_prefix = prefix_template % (perturb,\n", 92 | " M, str(subset_prop), 'baseline', 1., 0.1, 1., 0.1, epochs, seed)\n", 93 | " res = np.load(\"%s.npy\" %savefilename_prefix, allow_pickle=True)\n", 94 | "\n", 95 | " results_src_ba[:, :, :, seed] =res.item()['src']\n", 96 | " results_tar_ba[:, :, seed] = res.item()['tar']" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 6, 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | "lamMatches = [10.**(k) for k in (np.arange(10)-5)]" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": 7, 111 | "metadata": {}, 112 | "outputs": [], 113 | "source": [ 114 | "# DAmean methods: DIP, DIPOracle, DIPweigh, CIRMweigh\n", 115 | "nb_methods_damean = 4\n", 116 | "repeats = 10\n", 117 | "results_src_damean = np.zeros((len(lamMatches), M-1, nb_methods_damean, 2, 10))\n", 118 | "results_tar_damean = np.zeros((len(lamMatches), nb_methods_damean, 2, 10))\n", 119 | "for i, lam in enumerate(lamMatches):\n", 120 | " for seed in range(repeats):\n", 121 | " savefilename_prefix = prefix_template % (perturb,\n", 122 | " M, str(subset_prop), 'DAmean', lam, 10., lam, 10., epochs, seed)\n", 123 | " res = np.load(\"%s.npy\" %savefilename_prefix, allow_pickle=True)\n", 124 | "\n", 125 | " results_src_damean[i, :, :, :, seed] =res.item()['src']\n", 126 | " results_tar_damean[i, :, :, seed] = res.item()['tar']" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 8, 132 | "metadata": {}, 133 | "outputs": [ 134 | { 135 | "name": "stdout", 136 | "output_type": "stream", 137 | "text": [ 138 | "7 100.0\n", 139 | "3 0.01\n", 140 | "7 100.0\n", 141 | "4 0.1\n" 142 | ] 143 | } 144 | ], 145 | "source": [ 146 | "# choose lambda based on the source test performance\n", 147 | "lam_index_damean = np.zeros(nb_methods_damean, dtype=int)\n", 148 | "for i in range(nb_methods_damean):\n", 149 | " if i == 0 or i == 1:\n", 150 | " src_test_acc_all = results_src_damean[:, 0, i, 1, :].mean(axis=1)\n", 151 | " else:\n", 152 | " # M-2 for the source environment that is selected by weighting methods\n", 153 | " src_test_acc_all = results_src_damean[:, M-2, i, 1, :].mean(axis=1)\n", 154 | " # choose the largest lambda such that the source performance does not drop too much (5%)\n", 155 | " lam_index = 0\n", 156 | " for k, src_test_acc in enumerate(src_test_acc_all):\n", 157 | " \n", 158 | " if src_test_acc > np.max(src_test_acc_all) * 0.99:\n", 159 | " lam_index = k\n", 160 | " lam_index_damean[i] = lam_index\n", 161 | " print(lam_index, lamMatches[lam_index])" 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": 9, 167 | "metadata": {}, 168 | "outputs": [], 169 | "source": [ 170 | "# DAMMD methods: DIP-MMD, DIPweigh-MMD, CIRMweigh-MMD\n", 171 | "nb_methods_dammd = 3\n", 172 | "repeats = 10\n", 173 | "results_src_dammd = np.zeros((len(lamMatches), M-1, nb_methods_dammd, 2, 10))\n", 174 | "results_tar_dammd = np.zeros((len(lamMatches), nb_methods_dammd, 2, 10))\n", 175 | "for i, lam in enumerate(lamMatches):\n", 176 | " for seed in range(repeats):\n", 177 | " savefilename_prefix = prefix_template % (perturb,\n", 178 | " M, str(subset_prop), 'DAMMD', lam, 10., lam, 10., epochs, seed)\n", 179 | " res = np.load(\"%s.npy\" %savefilename_prefix, allow_pickle=True)\n", 180 | "\n", 181 | " results_src_dammd[i, :, :, :, seed] =res.item()['src']\n", 182 | " results_tar_dammd[i, :, :, seed] = res.item()['tar']" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": 10, 188 | "metadata": {}, 189 | "outputs": [ 190 | { 191 | "name": "stdout", 192 | "output_type": "stream", 193 | "text": [ 194 | "6\n", 195 | "5\n", 196 | "7\n" 197 | ] 198 | } 199 | ], 200 | "source": [ 201 | "# choose lambda based on the source test performance\n", 202 | "lam_index_dammd = np.zeros(nb_methods_dammd, dtype=int)\n", 203 | "for i in range(nb_methods_dammd):\n", 204 | " if i == 0:\n", 205 | " src_test_acc_all = results_src_dammd[:, 0, i, 1, :].mean(axis=1)\n", 206 | " else:\n", 207 | " # M-2 for the source environment that is selected by weighting methods\n", 208 | " src_test_acc_all = results_src_dammd[:, M-2, i, 1, :].mean(axis=1)\n", 209 | " # choose the largest lambda such that the source performance does not drop too much (5%)\n", 210 | " lam_index = 0\n", 211 | " for k, src_test_acc in enumerate(src_test_acc_all):\n", 212 | " if src_test_acc > np.max(src_test_acc_all) * 0.99:\n", 213 | " lam_index = k\n", 214 | " lam_index_dammd[i] = lam_index\n", 215 | " print(lam_index)" 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": 11, 221 | "metadata": {}, 222 | "outputs": [], 223 | "source": [ 224 | "# DACIPmean methods\n", 225 | "nb_methods_dacipmean = 1\n", 226 | "repeats = 10\n", 227 | "results_src_dacipmean = np.zeros((len(lamMatches), M-1, nb_methods_dacipmean, 2, 10))\n", 228 | "results_tar_dacipmean = np.zeros((len(lamMatches), nb_methods_dacipmean, 2, 10))\n", 229 | "for i, lam in enumerate(lamMatches):\n", 230 | " for seed in range(repeats):\n", 231 | " savefilename_prefix = prefix_template % (perturb,\n", 232 | " M, str(subset_prop), 'DACIPmean', 1., lam, 1., lam, 100, seed)\n", 233 | " res = np.load(\"%s.npy\" %savefilename_prefix, allow_pickle=True)\n", 234 | "\n", 235 | " results_src_dacipmean[i, :, :, :, seed] = res.item()['src']\n", 236 | " results_tar_dacipmean[i, :, :, seed] = res.item()['tar']" 237 | ] 238 | }, 239 | { 240 | "cell_type": "code", 241 | "execution_count": 12, 242 | "metadata": {}, 243 | "outputs": [ 244 | { 245 | "name": "stdout", 246 | "output_type": "stream", 247 | "text": [ 248 | "5 1.0\n" 249 | ] 250 | } 251 | ], 252 | "source": [ 253 | "# choose lambda based on the source test performance\n", 254 | "lam_index_dacipmean = np.zeros(nb_methods_dacipmean, dtype=int)\n", 255 | "for i in range(nb_methods_dacipmean):\n", 256 | " src_test_acc_all = results_src_dacipmean[:, :-1, i, 1, :].mean(axis=2).mean(axis=1)\n", 257 | " # choose the largest lambda such that the source performance does not drop too much (5%)\n", 258 | " lam_index = 0\n", 259 | " for k, src_test_acc in enumerate(src_test_acc_all):\n", 260 | " \n", 261 | " if src_test_acc > np.max(src_test_acc_all) * 0.99:\n", 262 | " lam_index = k\n", 263 | " lam_index_dacipmean[i] = lam_index\n", 264 | " print(lam_index, lamMatches[lam_index])" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": 13, 270 | "metadata": {}, 271 | "outputs": [], 272 | "source": [ 273 | "# DACIPMMD methods\n", 274 | "nb_methods_dacipmmd = 1\n", 275 | "repeats = 10\n", 276 | "results_src_dacipmmd = np.zeros((len(lamMatches), M-1, nb_methods_dacipmmd, 2, 10))\n", 277 | "results_tar_dacipmmd = np.zeros((len(lamMatches), nb_methods_dacipmmd, 2, 10))\n", 278 | "for i, lam in enumerate(lamMatches):\n", 279 | " for seed in range(repeats):\n", 280 | " savefilename_prefix = prefix_template % (perturb,\n", 281 | " M, str(subset_prop), 'DACIPMMD', 1., lam, 1., lam, 100, seed)\n", 282 | " res = np.load(\"%s.npy\" %savefilename_prefix, allow_pickle=True)\n", 283 | "\n", 284 | " results_src_dacipmmd[i, :, :, :, seed] = res.item()['src']\n", 285 | " results_tar_dacipmmd[i, :, :, seed] = res.item()['tar']" 286 | ] 287 | }, 288 | { 289 | "cell_type": "code", 290 | "execution_count": 14, 291 | "metadata": {}, 292 | "outputs": [ 293 | { 294 | "name": "stdout", 295 | "output_type": "stream", 296 | "text": [ 297 | "6 10.0\n" 298 | ] 299 | } 300 | ], 301 | "source": [ 302 | "# choose lambda based on the source test performance\n", 303 | "lam_index_dacipmmd = np.zeros(nb_methods_dacipmmd, dtype=int)\n", 304 | "for i in range(nb_methods_dacipmmd):\n", 305 | " src_test_acc_all = results_src_dacipmmd[:, :-1, i, 1, :].mean(axis=2).mean(axis=1)\n", 306 | " # choose the largest lambda such that the source performance does not drop too much (5%)\n", 307 | " lam_index = 0\n", 308 | " for k, src_test_acc in enumerate(src_test_acc_all):\n", 309 | " \n", 310 | " if src_test_acc > np.max(src_test_acc_all) * 0.99:\n", 311 | " lam_index = k\n", 312 | " lam_index_dacipmmd[i] = lam_index\n", 313 | " print(lam_index, lamMatches[lam_index])" 314 | ] 315 | }, 316 | { 317 | "cell_type": "code", 318 | "execution_count": 15, 319 | "metadata": {}, 320 | "outputs": [ 321 | { 322 | "data": { 323 | "text/plain": [ 324 | "array([7, 7, 4])" 325 | ] 326 | }, 327 | "execution_count": 15, 328 | "metadata": {}, 329 | "output_type": "execute_result" 330 | } 331 | ], 332 | "source": [ 333 | "lam_index_damean[[0, 2, 3]]" 334 | ] 335 | }, 336 | { 337 | "cell_type": "code", 338 | "execution_count": 16, 339 | "metadata": {}, 340 | "outputs": [], 341 | "source": [ 342 | "test_err_index = 0\n", 343 | "\n", 344 | "results_tar_plot = np.concatenate((results_tar_ba[:, test_err_index, :],\n", 345 | " results_tar_damean[lam_index_damean[0], 0, test_err_index, :].reshape((-1, 10), order='F'),\n", 346 | " results_tar_damean[lam_index_damean[2], 2, test_err_index, :].reshape((-1, 10), order='F'),\n", 347 | " results_tar_dacipmean[lam_index_dacipmean, 0, test_err_index, :].reshape((-1, 10), order='F'),\n", 348 | " results_tar_damean[lam_index_damean[3], 3, test_err_index, :].reshape((-1, 10), order='F'),\n", 349 | " results_tar_dammd[lam_index_dammd[0], 0, test_err_index, :].reshape((-1, 10), order='F'),\n", 350 | " results_tar_dammd[lam_index_dammd[1], 1, test_err_index, :].reshape((-1, 10), order='F'),\n", 351 | " results_tar_dacipmmd[lam_index_dacipmmd, 0, test_err_index, :].reshape((-1, 10), order='F'),\n", 352 | " results_tar_dammd[lam_index_dammd[2], 2, test_err_index, :].reshape((-1, 10), order='F')), axis=0)\n", 353 | " \n" 354 | ] 355 | }, 356 | { 357 | "cell_type": "code", 358 | "execution_count": 17, 359 | "metadata": {}, 360 | "outputs": [ 361 | { 362 | "data": { 363 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnQAAABECAYAAAAIjKhLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAADJklEQVR4nO3bsWucdRzH8U96xYgnTRqtECzYP8A1WNBOjoIl4JTOgohoFhcHJycX26JTZ2eJ0NFBWqHSVQQ3hcqBrSWBPGKEcA6aIUc8Ufn15/d4vZbneH4cfIbngfdwtzSdTgMAQF2neg8AAOC/EXQAAMUJOgCA4gQdAEBxp+ecLSfZSDJJcvho5gAAcIJRkvUkd5MczB7OC7qNJLcajQIA4J+7lOT27M15QTdJkp2dnQzD0GpUV1tbW5m8cLH3jGbWv76TzY++7D2jide+fZgrNzbz6euf9Z7SxJUbm8nVC71ntLP9fQ6u3em9opnldy7m87de7j2jmVc//iLX3v+k94wmLqx9l8vb17Nz9e3eU5q4vH0939z8qveMZp5/5cV8+O6bvWc0cebsWt5474Pkzz6bNS/oDpNkGIbs7+83mPb/cHjvXu8JTU12f+09oYn9n4Zj14W0+0PvBU1N9xbz2Twy3P+x94Sm9h7u9Z7QxHDqwR/X3Qedl7Tz2y+L/e7t/ny/94TWTvwZnD9FAAAUJ+gAAIoTdAAAxQk6AIDiBB0AQHGCDgCgOEEHAFCcoAMAKE7QAQAUJ+gAAIoTdAAAxQk6AIDiBB0AQHGCDgCgOEEHAFCcoAMAKE7QAQAUJ+gAAIoTdAAAxQk6AIDiBB0AQHGCDgCgOEEHAFCcoAMAKE7QAQAUJ+gAAIoTdAAAxQk6AIDiBB0AQHGCDgCgOEEHAFCcoAMAKE7QAQAUJ+gAAIoTdAAAxQk6AIDiBB0AQHGCDgCgOEEHAFCcoAMAKE7QAQAUJ+gAAIoTdAAAxQk6AIDiBB0AQHGCDgCgOEEHAFCcoAMAKE7QAQAUJ+gAAIoTdAAAxZ2eczZKkvF4/Iim9DE6f773hKbWVx/vPaGJJ58ZH7supNXnei9oamllMZ/NI+Nzz/ae0NTK2krvCU2MV58+dl1Ejz2x2O/e6lPnek9o4szZtaOPo5POl6bT6V9996UktxpsAgDg37mU5PbszXlBt5xkI8kkyWG7XQAA/I1RkvUkd5MczB7OCzoAAArwpwgAgOIEHQBAcYIOAKA4QQcAUNzval1qaR6YgroAAAAASUVORK5CYII=\n", 364 | "text/plain": [ 365 | "
" 366 | ] 367 | }, 368 | "metadata": { 369 | "needs_background": "light" 370 | }, 371 | "output_type": "display_data" 372 | } 373 | ], 374 | "source": [ 375 | "COLOR_PALETTE1 = sns.color_palette(\"Set1\", 9, desat=1.)\n", 376 | "COLOR_PALETTE2 = sns.color_palette(\"Set1\", 9, desat=.7)\n", 377 | "COLOR_PALETTE3 = sns.color_palette(\"Set1\", 9, desat=.5)\n", 378 | "COLOR_PALETTE4 = sns.color_palette(\"Set1\", 9, desat=.3)\n", 379 | "# COLOR_PALETTE2 = sns.color_palette(\"Dark2\", 30)\n", 380 | "# COLOR_PALETTE = COLOR_PALETTE1[:8] + COLOR_PALETTE2[:30]\n", 381 | "COLOR_PALETTE = [COLOR_PALETTE1[8], COLOR_PALETTE1[0], COLOR_PALETTE1[1],\n", 382 | " COLOR_PALETTE1[3], COLOR_PALETTE1[4], COLOR_PALETTE1[7], \n", 383 | " COLOR_PALETTE1[6], \n", 384 | " COLOR_PALETTE4[3], COLOR_PALETTE4[4], COLOR_PALETTE4[7], \n", 385 | " COLOR_PALETTE4[6]]\n", 386 | "sns.palplot(COLOR_PALETTE)\n" 387 | ] 388 | }, 389 | { 390 | "cell_type": "code", 391 | "execution_count": 18, 392 | "metadata": {}, 393 | "outputs": [ 394 | { 395 | "name": "stdout", 396 | "output_type": "stream", 397 | "text": [ 398 | "(10, 11) (11,) (110, 2)\n" 399 | ] 400 | }, 401 | { 402 | "data": { 403 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAp4AAAGcCAYAAABwVqAFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeXwcdf3H8dcmaXqlbdKD+2o5Phwil4CCYmmtUBVvFMGfUhQURFR+CgVLf6Ig4I0cYhGLoqjIIaACArXIJVAuFcqHQlMKlqO0WXq3aZrfH99Zut1cm3Z2ZrJ9Px+PPNLMTCaf6e539jPfM9fe3o6IiIiISKXVpB2AiIiIiGwelHiKiIiISCKUeIqIiIhIIpR4ioiIiEgilHiKiIiISCLq0g6ggvoDBwIvA20pxyIiIiLSnVpga+ARYHXKsVRMNSeeBwL3ph2EiIiISC+8C7gv7SAqpZoTz5cBWlqWs25d5eYqHTGigUWLllXs/EnStWRPtVwH6FqyqFquA3QtWVUt15LEddTU5GhqGgxR/lKtqjnxbANYt669ooln4W9UC11L9lTLdYCuJYuq5TpA15JV1XItCV5HVXcP1OAiEREREUmEEk8RERERSYQSTxERERFJhBJPEREREUmEEk8RERERSYQSTxERERFJhBJPEREREUmEEk8RERERSUQ1TyBf1aZPn0Zz89wej8vnW8jnW8o6Zy6Xo7295wlyGxubaGxs6vG40aPHMGnSSWX9bREREal+VZ94vvzzy1nTkn/z5yEHHkTj4eNZt3o1/734Rx2OH3roOxl26LtoW7qUBT+7tMP+xrHjGHLQwbQuXsQrv5jGK/V1tK5Z++b+pvceScO++7HmlZd59ddXd/j94R/4IIP33ItV819g4e+v7bB/5Ec/zsBddmXlc3N4/cbrO+wfdcyxDNhhR+Y+8hD511/vUGX9Rlsba2mnfy5HQ00tK9rXsbYomVzb3k47UJPLUdvJ/1dhf20u12l1+Nr2dhYuX86qBQsYWNPxiEVt4f9iQE0N2734Ii8uXPTmvlx9Pdt99X/DcbfezIrZT2/wu7UNDWxzypcBWHjDH1n1/HMb7K9rGs7WJ34BgNd+/1tWz5+/wf76rbZiy89MAuC5y37GknkvbrC//w47sMUxxwHw8pU/Z23L4g32D9h5F0Z97GgAFlx+CW3LNlwebdAeezLiqA8B8NJPfkj7mjUb7B+8z74MP2IiAC9+74IO/zcb894rfn+VvvdKJfXeW/70Uyz+8y0d9m/5meOp32prlj3xOC1/u73j9Z1xOtCfpQ8/RH7mjA77tzn5VGqHDOGN++9lyf0dlyne9iunU9O/P/m/383SRx7usH/7M84CYPEdt7H8ySc22Bf3e++VVxZsUO6L33uv/no6a155ZYPfz+p7r/D+6u19r1QW3nuMGtLle2+rz59Ev+Ej+sx7r7jc9+a+l8X33trDD6PuwENj+8wtldR7L//Ek7z42z902N/Tfa83771VT/+HEWef0eGYalP1iefmbnCuhsG59T/n29bSBgwABtesTz2Xt68DYE17O+uAgeQY1EliubhtLdWx+JmIiIgkLVdO02oftRPQvGjRsoqurzpq1BAWLlxasfN3Je6m9lWrVgEwYMCAHo/tC03tab0ucauW6wBdSxZVy3WAriWrquVakriOmpocI0Y0AIwG5lX0j6VINZ59VLkJ3cyZdzNjxp09HpfP56mrq6GhYWiPx44bN4GxY8eX9fdFRERECpR4VrmxY8eXnSRWy5OpiIiIZJOmUxIRERGRRCjxFBEREZFEKPEUERERkUQo8RQRERGRRCjxFBEREZFEKPEUERERkUQo8RQRERGRRCjxFBEREZFEKPEUERERkUQo8RQRERGRRCjxFBEREZFEKPEUERERkUQo8RQRERGRRNQl/QfN7MPAtwEDFgCXuPuPSo7JAWcBJwMjgUeA09z9iYTDFREREZGYJFrjaWaHAjcCDwNHAb8ELjKzr5YcOhk4B7goOm4ZcJeZbZVguCIiIiISo6RrPKcC97v756Of/2ZmjcBUM7vc3deY2QBC4nmBu18KYGYPAvOAU4EpCccsIiIiIjFIuo/nvsCdJdv+BjQB74h+PgQYClxXOMDdlwO3AhMTiFFEREREKiDpGs8BwJqSbYWf9wDuAXYH2oA5JcfNBj5Z0ehERESkSzNn3s2MGaX1R53L5/PU1dXQ0DC0x2PHjZvA2LHjNzU86QOSTjyfAw4s2XZQ9H149L0JWObubSXHtQCDzKze3UuTVxEREdlI06dPo7l5bo/H5fMt5PMtZZ1z1apVAAwYMKDHY2+88bqyEtrRo8cwadJJZf19yaakE88rgCvM7ETgekLSeXq0b13CsYiIiAjQ3DyXuc/PYZtR3Y/hHVBTx1bDR5V1zqXLlwEwZHBDWcevWrK02/0LFr5S1nkk25JOPH8J7AP8DJgGrADOBC4BCu+oFqDBzGpLaj2bgBWq7RQREYnfNqO24uSjj087jC797I9Xpx2CxCDRxDNKJE81s3OA7YBmQp9OgH9G358BaoFdAC/69d2jfSIisSu375r6rUk1yudbWPz665lO7ha89grD161NOwzZRIlPIA/g7i2Emk3M7BTgAXcvJJUPAEuAo4HzomMGEebznJZ8tCIi6+Xzi8nlcmUlniIisqFEE08zezvwTuAJwpRJnwKOiLYB4O6rzOxC4BwzayHUcp5OmPrpkiTjFZHNx9ix48uqnZw6dTL19XVMmXJeAlGJJKOxsYnFi16P9Zy97ePZo1yIU/q2pGs8WwlTIn2LMJjoXuBQd/93yXEXEhLNs4ARwCxggru/mlyoItLXlTtStzfmzZtLLpdj6tTJsZ1TI3UlbaNHj4n9nK8sXkgul2PU1lvHcr4xQ4dUJE5JVtJ9PB+l43RKnR3XDpwffYmIbJTm5rnMmzOHnYbFt9puY83A8I/Xuh+BW655b2ikrqSvEg8+ah2QzqTSx1NEJAn5fAu0x3vOxgExNRsWtFP2vIjlqMQgKdBAKQl6M4F8b1oH0nh/aUBhOpR4iohshjRISiqtsXE4dXVJr8wdP5WVeCnxFJGq1djYRH5hvAMm8qvCgInYaj5jHjChQVJSSeW+vwpGjRrCwoXxdEuJm8pKOpR4ikjVqsRAhPy8MGCicYt4BkzstEV5AybiHihViUFSoIFSItI9JZ4iUrWqacBEc/Nc5sz+N4394um0WhutC7fwuX/Fcj6AfGsutnOJSHVS4ikiQvkDDdIcMNHYr53xW2R35Za7X9NHioh0T3cJEZFeqJYBEyLVSt1Ssk2Jp4gIvRs0keUBEyKbu+bmufgzTzGof79YzrdubeiX8mLzs7GcD2DF6tbYztXXKPEUERGRqhHnvLgA/epqYz1fQdxx9hVqLxIRERGRRKjGU0RERKpGY2MTS1sWsud2o9IOpUtPv7Qw1vl7+xLVeIqIiIhIIlTjKSLSB+TzLeRbc5mesijfmqPfZtpvTbJlxepWnn5pYSznao0GF8XZ11ODi0RERESqQNwrlhWmU9p+x9GxnrcSK6v1BUo8RUT6gMbGJlpffzHzE8hvrv3WJDvinhtTa7XHS308RURERCQRSjxFREREJBFqahcR6SPiHFy0KoyXYECMc2PnW3NkdwIbEckCJZ4iIn1ApQZMjIpxwMQoNt8BE9L3zJx5NzNm3Nnjcb1Zq33cuAllL727uVLiKSLSB2jAhEg6GhuHU1ennolxUeIpIiIim52xY8eXXTs5atQQFi5cWuGINg9K4UVEREQkEUo8RURERCQRSjxFREREJBHq4ympmj59Gs3Nc3s8Lp9vId+LNaBzuRzt7e09HtfY2FTWSiujR4+JfXCHiIjI5kaJp6SquXkuT/uz1DSM7Pa49jUraF8T/1KBK1uW88ry7hPUdctej/3vioiIbI6qPvG84ub/0LJk9Zs/H7jHFozbfztWt7bxk+ue7HD8oXtvzTvfujVLV6zh8pv+02H/4ftvy0F7bMniJau48tan6VdfS+uatjf3H3HQDuy760heXrScX9/uHX7/A4fuxF47DWf+q0v53V1zOuz/2Lt3ZpfthvHcS29wwz3Pd9j/qffsyg5bDuGpeYv58/3zOuz/zJHG1iMG88Sc17nj4fkd9p941J4MHzqAh2e/yt8f++8G+/rV13Li+/dgyKB67vvXy9z/75c7/P5XP7EP/fvVMuOxl3hk9msd9p953P4A3P7QfJ58bsOErV+/Gk7/xL4A3HJ/M7PntTB/WSM1/RthXS4cVBMlgYWfI7m6weTqBne5/00bs39d9/tr+jcyf1kjt9zfzAcPDXMe/ui6J2htLf5F2GeXkRx58A4AXPTbxzr86Y157xW/v0rfe6Wy/t4787MHAXT63gM45SNvSfS9V6xhYD++9NG9Abh+5vM8/983NtjfNLQ/Jx21FwDX3vUsr7Ss3KDcbzl8EMdP3B2Aq297hlcXr9jg97ffsoFj37MbANNufWqDexLAztsO4+Njdwbgshv/zbKVrRvs32Onptjfey++toxcLsdFv32s1/e9Ull4740aNWSj7nuQvfdecbkvfe+9+OqyDX4/6++9ww/cgYN2GxnbZ26ppN57Tzz7Gr/56+wO+zflMxc2fO89/cJivnnC2zscU22qPvGUPqC9Hdpaob2ddWvCB36u32ByNf1KjlvHujVLov0N5GpK3r7tbaxbsxTa28jVDiTXb1DJr6+lvTXctGvqh0KupmR/K+2ty6P9wyCXWx+fiIiIbLJcOf3g4mRmxwBnALsBbwB3A5PdfUHRMTngLOBkYCTwCHCauz/Riz+1E9C8aNEy1q2r3DVW09xeaVxLuX08e6OwysSOMa7IklYfT72/sqkarqXaJpCvhtekQNeSPUlcR01NjhEjGgBGA/Mq+sdSlGiNp5l9EPgdcBnwDWBr4DzgL2Z2gLsX6vAnA+dExzwDnA7cZWZvcfdXkoxZKqvcZK7cpc16S8ubiYiIJCfppvZjgcfc/dTCBjNbAtwMGDDbzAYQEs8L3P3S6JgHCdn/qcCUhGOWPkbLm4mIiGRT0olnP0LzerF89L0wmuMQYChwXeEAd19uZrcCE1HiuVnqzdJmUD3NOyK9VW7rQKFLytSpk8s6r1oHRCQOSSeevwT+ZGafAf4EbEVoap/h7oXharsDbUDp8LPZwCeTClREpJqpZUBE0pBo4unufzGz44GrgF9Fmx8APlh0WBOwzN3bSn69BRhkZvXuvqbiwYqI9EG9aR1Qy4CIJC3pwUWHA1cAFwO3AVsC3wJuMrP3dJJspqLcpqp8Pk9dXQ0NDUN7PFbNVCIiIrK5S7qp/YfALe5+ZmGDmT1BGLn+IeBGQs1mg5nVliSiTcCKTantjHt5xlWrVgEwYMCAHo+98cbrykpmtTSjiIiIVKukE8/dCdMpvcnd3cxWAjtHm54BaoFdAC/53Wc25Y8//vijvPzyAurq4rnsXDTBeGtraw9HwsKFr7FwYccVL4qtXbuWfL6FSZNiCU9EREQkU5JOPF8A9i/eYGZ7AANZP1nqA8AS4GjCwCPMbBBwFDBtUwOoq6ujqalpU09TES0tPdeyioiIiPRVSSeeVwA/NrMFrO/jOZWQdP4VwN1XmdmFwDlm1sL6CeRrgEs25Y83Njb1WOvYGytXrgRg4MCBsZ2zsTGbSbGIiIjIpko68fwpsIawFOYXCXN43gec5e7Li467kJBongWMAGYBE9z91U3546NHjynruN728Wxr63lMVGNjU49J5ciRW5Qdo4iIiEhfk/R0Su3Az6Kvno47P/qKTdzLM2pUu4iIiEj5kq7x7BM0D56IiIhI/LRshYiIiIgkQomniIiIiCRCiaeIiIiIJKJXfTzNbGdgK2AAsBh4zt3VwVFEREREetRt4mlmNcBE4DPAeMKylblodzuwzsyeAq4Hfu3u8ysYq4iIiIj0YV02tZvZcYQlK39HSDK/DYwD9gZ2Aw4GPgXcDnwceM7MrjKz7SodtIiIiIj0Pd3VeJ4JnAv80d1Xd3HMLEJt52QzM+CrhGT0+7FGKSIiIiJ9XpeJp7u/tTcncncnrEgkIiIiItKBRrWLiIiISCJ6vXKRmTUAU4HDCQON/g58x92XxBybiIiIiFSRjVky8xdAf+BbQAMwGdgJODq2qERERESk6nSZeJrZh939T53seg+wvbuvjI5bTBhgJCIiIiLSpe76eF5gZneZ2V4l258FTjazgWY2kjDHp1csQhERERGpCt0lnnsDfwFmmtllZjY82n4icBywHHgV2As4oaJRioiIiEif1910SmuBH5vZNcB5wDNmdh5wmbsfYGZDo+M0qEhERKQbM2fezYwZd/Z4XD6fp66uhoaGoT0eO27cBMaOHR9HeCKJ6XE6JXd/3d2/SOjb+SHg32Z2hLsvUdIpIiISn3x+MYsWLUo7DJGK6Wmt9lrC8pj1wLPuPt7MPgpcZmYOfM3dn00gThERkT5r7NjxZdVOTp06mfr6OqZMOS+BqESS191a7QcDzwMPAXcDC8zs0+5+I7AncC/woJn90MyGJRKtiIiIiPRZ3TW1XwncCDS5+0jgK8AvzGyIu69x9wsJA4uGo1HtIiIiItKD7pratwPudPe26OfbCU3uo4ClAO7+CjDJzA6oaJQiIiIi0ud1l3j+AbjczC4FVhLm63zU3eeWHujuj1YoPhERERGpEt0lnl8GPg+MJ9R0/hX4aRJBiYiI9AXTp0+jublDfcxGmzdvLrlcjqlTJ8d2ztGjxzBp0kmxnU9kU/Q0j+cV0ZeIiIiUaG6ey5xnn2Z4Q/9YzlfHWmiHRQuej+V8i5etjuU8InHpbq32Bndf1tsTRoOPlm5aWCIiIn3D8Ib+HLHfdmmH0ak7Hn8p7RBENtBdU/t8M7scmO7u3T56mVl/4AOEke93At+JL0QRERERqQbdJZ4TCAnk2Wb2JPAA8B/gdWA10AiMBg4A3k0YgPQD4NJKBiwiIiIifVN3fTwfBd5nZrsSRrSPB04AijuyzAfuj7bf4u6tFYxVREQkU/L5FhYvXZ3ZJu3FS1dTm29JOwyRN3W7ZCaAu88Bzom+MLMmYACw2N3Va1lERDZra9vWsXhpPB+H69rbAajJ5WI539q2dbGcRyQuPSaepdx9ox+dzGwmoVm+M4e4+4NmlgPOAk4GRgKPAKe5+xMb+3dFREQqYb/9DqCxsSm28xWmU9pxx9GxnXP06DGxnUtkU/U68dxEpwBDS7Z9G9iPkGACTCbUrn4DeAY4HbjLzN4SrZQkIiKSCeXOjzlz5t3MmHFnrH973LgJjB07PtZzilRaoomnuz9d/LOZ1QNvA/7g7mvNbAAh8bzA3S+NjnkQmAecCkxJMl4REZEkNTYOp66uJu0wRCom6RrPUkcCTcDvop8PIdSIXlc4wN2Xm9mtwESUeIqISB80duz4smsnR40awsKFmg5bqlPaiecxwEvAvdHPuwNtwJyS42YDn0wwLhEpQ7nNh/l8nrq6GhoaSnvadKTmQxGR6lVWfb6ZHWVmsdb9m9kg4IPAde7eHm1uApa5e1vJ4S3AoKhpXkT6mHx+MYsWLUo7DBERSVm5NZ5/Al41s2uAq919dgx/+yhgMOub2UWkjym3+XDq1MnU19cxZcp5CUQlIiJZVW4t5s7AlcAngP+Y2YNmdqKZ9dxu1rVjgOfcfVbRthagwcxqS45tAla4+5pN+HsiIiIikqKyEk93n+fu/+fuowlLaT4H/Bh42cyuMbPDe/NHzWwYYbBQaW3nM0AtsEvJ9t2jfSIiIiLSR23MBPIzgBlmtg3we+A44FgzewG4BLjE3df2cJqPEJbeLE08HwCWAEcD58GbfUGPAqb1NlYR2TjTp0+juXlubOcrTIo9derk2M45evSYsudQFBGRbOh14mlm7wYmAR8DWoHLCH1AjwDOBQ4Eju3hNMcAT5b2FXX3VWZ2IXCOmbWwfgL5GkJSKyIJaG6ey7xn/8UODfH0bhlGDbTDugXxDDCav0zjDEVE+qKyEk8z2xH4bPS1EzATOAm4sWi99rujyd5/08O5RgLjidZ+78SFhETzLGAEMAuY4O6vlhOriMRjh4Y1nLXfa2mH0akLHt8i7RBERGQjlFvjORdYAFwN/NLdm7s47ing4e5O5O6vA/262d8OnB99iYiIiEiVKDfx/ABwh7uv6+4gd38W6NVAIxERERHZPJQ7ndJ9wJad7TCzrc2sIb6QRERERKQalVvjeRXwBnBiJ/u+BQwjDBgSkSqQz7fQsrQ+s30pX1haT1O+Je0wRESkl8qt8TwM+EsX+/4a7RcRERER6VK5NZ7DgBVd7FtFWFlIRKpEY2MTQ1c8n+lR7TWNuu2IiPQ15Saec4D3A3/rZN/7gOdji0hEMmH+svia2t9YExpXhtV3Oz6xbPOX1bNTLGcSEZEklZt4XgJcYWZrCFMqvQxsTZjX80vAyRWJTkRSMXr0mFjP90a0clHTNqNjOd9OxB+jiIhUXlmJp7tfaWZbEiZ1P71o1ypgirtfWYngRCQdcS9FOXXqZOrr65gy5bxYzysiIn1L2Utmuvt5ZnYJ8A7CikKLgAfd/Y1KBSciIiIi1aNXa7VHSebtFYpFRERERKpYrxJPM3snsBswoHSfu18eV1AiIiIiUn3KSjyj/p13A3sC7UAu2tVedJgSTxERERHpUrkTyP+QsHLR9oSk82DCwNJzCFMt7VaJ4ERERESkepTb1P5u4CuEaZQAcu4+H/iumdUQajuPqEB8IpJhM2fezYwZd/Z43LxoOqWpUyf3eOy4cRMYO3Z8HOGJiEjGlFvj2QgsdPd1wBKgeFbpB4BD4g5MRKpHY+NwRowYkXYYIiKSsnJrPJsJE8YDPAUcB/w5+vkoYHHMcYlIHzB27PiyaydHjRrCwoVLKxyRiIhkWbmJ51+B9wLXAecBN5vZS0ArsANwZmXCExEREZFqUe7KRZOL/n2bmR0CfAQYCNzp7rdVKD4RERERqRI9Jp5m1h/4OvBnd38SwN1nAbMqHJuIiIiIVJEeBxe5+2rgm4QBRiIiIiIiG6XcUe0PAftXMhARERERqW7lDi46A7jWzFoJA41eZcNVi3D3FTHHJiIiIiJVpNzE86Ho+0+Bi7s4pnbTwxERERGRalVu4nkCJTWcIiIiIiK9Ue50SldXOA4RERERqXLlDi4SEREREdkkZdV4mtlCemhqd/ctutsvIiIiIpu3cvt4XkbHxLMJGA8MBX4ZZ1AiIiIiUn3K7eP5rc62m1mOsH57a4wxiYiIiEgVKrfGs1Pu3m5mvwCmA98t53fMrI6wBOfngB2AhcAf3f1rRcfkgLOAk4GRwCPAae7+xKbEKyIiIiLpiWNw0RigvhfHXw2cBvwAeC8wGVhZcsxk4BzgIuAoYBlwl5lttanBioiIiEg6yh1cdEonm+uBPYDjgD+WeZ4jgU8C+7j7010cM4CQeF7g7pdG2x4E5gGnAlPK+VsiIiIiki3lNrVf2sm21cBLwOXAuWWe5wRgRldJZ+QQwoCl6wob3H25md0KTESJp4iIiEifVO7gorjm+zwYuMXMLgU+E/3924FT3X1BdMzuQBswp+R3ZxNqS0VERESkD0p6AvmtgOOBfYFjgEnAAcBN0YAiCNM0LXP3tpLfbQEGmVlv+pOKiIiISEaU28fzfGCku3+hk31XAAvd/ZwyTpWLvj7k7oui338ZuAcYB9xdbuAiIiIi0reUW+P5KeDeLvbdCxxb5nlagH8Xks7IfcAaYM+iYxrMrLbkd5uAFe6+psy/JSIiIiIZUm7iuQ3w3y72LYj2l2M2ocazVA5YF/37GaAW2KXkmN2jfSIiIiLSB5WbeL4C7N/Fvv0Jk8CX48/A3mY2smjbYUA/4Mno5weAJcDRhQPMbBBhPs/byvw7IiIiIpIx5Sae1wFTzez9xRvN7H2Eid5/X+Z5pgGLgFvN7CgzOxa4BrjL3e8DcPdVwIXA2Wb2JTMbT5gntAa4pMy/IyIiIiIZU+48nlMJI9FvNbNFwMvA1sBw4G+E5LNH7r7EzMYBPyUkq2uAm4GvlRx6ISHRPAsYAcwCJrj7q2XGKyIiIiIZU+48nquA95rZEcDhhGRwEXC3u9/Zmz/o7s8B7+vhmHbg/OhLRERERKpAuTWeALj7HcAdFYpFRERERKpYWX08zewYM/tGF/u+bmafiDcsEREREak25Q4umgys6mLfCkJfTBERERGRLpWbeO4K/KeLfbOj/SIiIiIiXSo38VwBbNfFvu2B1fGEIyIiIiLVqtzE8y7gHDPbonijmY0CvkmYUklEREREpEvljmo/E/gn8LyZ3c76eTyPAPLAGZUJT0RERESqRVk1nu4+H9gHuJTQtD4x+n4JsL+7v1ixCEVERESkKpQ9j6e7L6SL0etm1s/dW2OLSkRERESqTq8mkC9mZjlgHPAp4KOE5TNFRERiMXPm3cyY0fPiePl8nrq6GhoahvZ47LhxExg7dnwc4YnIRuh14mlmbyckm0cDWwKLgd/FHJeIiEhZ8vnF5HK5shJPEUlXWYmnme1NSDaPAXYE1gD1wOnAZe6+tmIRiojIZmns2PFl1U5OnTqZ+vo6pkw5L4GoRGRTdJl4mtkYQrL5KWAPYC1h2qRzgHuA+cDjSjpFREREpBzd1Xg+B7QDDwFfAG5w9xYAMxuWQGwiIiIiUkW6m07pBSAHvAUYCxxiZhs9GElERERENm9dJp7uPho4BLgaGA/cCrxqZldGP7cnEaCIiIiIVIduJ5B393+6+2nAtsB7gT8BHwOujw450czeVtkQRURERKQalLty0Tp3v8vdP0eYQukjwHXR94fMbHYFYxQRERGRKtDrPpvRCkU3Azeb2SDgw4RplkREREREurRJg4XcfQVwbfQlIiIiItKlspraRUREREQ2lRJPEREREUmEEk8RERERSYQmhBcRkURNnz6N5ua5sZ1v3ry55HI5pk6dHNs5R48ew6RJJ8V2PhEJlHiKiEiimpvn8qw/Q8PAhnhO2AbttLNg/kuxnG7ZymWxnEdEOlLiKSIiiWsY2MD+u2Zz/ZHH5sxKOwSRqqU+niIiIiKSCCWeIiIiIpKIRJvazex4YHonu05290Bxs8YAACAASURBVCuiY3LAWcDJwEjgEeA0d38iqThFREREJH5p1XiOA95R9HVj0b7JwDnARcBRwDLgLjPbKukgRURERCQ+aQ0uesTdOwwbNLMBhMTzAne/NNr2IDAPOBWYkmSQIiIiIhKfrPXxPAQYClxX2ODuy4FbgYlpBSUiIiIimy6txPN5M1trZm5mXyjavjvQBswpOX52tE9ERERE+qikE8+XCf03/4fQf/OfwBVm9rVofxOwzN3bSn6vBRhkZvWJRSoiIiIisUq0j6e73wHcUbTptqhf5xQzuzjJWEREREQkWVno43k9MBzYiVCz2WBmtSXHNAEr3H1NwrGJiIiISEyykHi2F31/BqgFdik5Zvdon4iIiIj0UVlIPD8OvA68ADwALAGOLuw0s0GE/qC3pRKdiIiIiMQi6ZWLbgAeBv5FqNn8ZPR1mruvA1aZ2YXAOWbWQqjlPJ2QIF+SZKwiIiIiEq+kJ5B34ARgeyAHPA18xt2vKTrmQkKieRYwApgFTHD3VxOOVURERERilPSo9rOBs3s4ph04P/oSERERkSqR1pKZIiKymcrnW1i6YimPzZmVdiidWrpiKfl8S9phiFSlLAwuEhEREZHNgGo8RUQkUY2NTaxYspz9d31b2qF06rE5s2hsbEo7DJGqpBpPEREREUmEEk8RERERSYQSTxERERFJhBJPEREREUmEEk8RERERSYQSTxERERFJhBJPEREREUmE5vEUEZHELVu5LLaVi9a0rgGgvl99LOdbtnJZLOcRkY6UeIqISKJGjx4T6/nmzZtLLpdjmx22i+2ccccoIoESTxERSdSkSSfFer6pUydTX1/HlCnnxXpeEYmf+niKiIiISCJU4ymSsJkz72bGjDt7PC6fz1NXV0NDw9Aejx03bgJjx46PIzwREZGKUeIpklH5/GJyuVxZiaeIiEhfoMRTJGFjx44vq3ZS/dZERKTaqI+niIiIiCRCNZ4iIpJJ5faHLkynNHXq5B6PVX9okXQp8RQRkT6tsXE4dXVqwBPpC5R4iohIJpXbHxpg1KghLFy4tMIRicim0iOiiIiIiCRCiaeIiIiIJEKJp4iIiIgkQomniIiIiCRCiaeIiIiIJEKJp4iIiIgkQomniIiIiCRCiaeIiIiIJCLVCeTNbFvAgcHAEHdfFm3PAWcBJwMjgUeA09z9ibRiFenJ9OnTaG6eG9v5erMMYG+MHj2GSZNOivWcIiIi5Uh75aLvA8sIiWexycA5wDeAZ4DTgbvM7C3u/kqyIYqUp7l5LnNmO8P6NcVyvpq2WgBee+61WM4H8EZrS2znEhER6a3UEk8zOww4EvguIQEtbB9ASDwvcPdLo20PAvOAU4EpiQcrUqZh/Zo4bNR70g6jS/9YeFfaIYiIyGYslT6eZlYLXAJ8G3i9ZPchwFDgusIGd18O3ApMTCpGEREREYlXWoOLvgj0By7rZN/uQBswp2T77GifiIiIiPRBiTe1m9kI4DvAp9291cxKD2kClrl7W8n2FmCQmdW7+5oEQhXplXy+hXxrS6abs/OtLdTn+6UdhoiIbKbSqPE8H/inu/81hb8tIiIiIilJtMbTzPYCTgAOM7PGaPOg6PswM2sj1Gw2mFltSa1nE7BCtZ2SVY2NTax5vTXzg4saG+MZdS8iItJbSTe17wr0Ax7sZN9LwFXAtUAtsAthjs+C3QlTK4mIiIhIH5R04nkfcHjJtiOBM4H3AXOBF4AlwNHAeQBmNgg4CpiWWKQiIiIiEqtEE093fx2YWbzNzHaK/nlv0cpFFwLnmFkL6yeQryFMwSQiIiIifVDaKxd15UJConkWMAKYBUxw91dTjUqkB2/EOKp9VdtKAAbUDozlfBDi24ItYjufiIhIb6SeeLr71cDVJdvaCaPfz08hJJGNMnr0mFjPV1irfYsd40sUt2CL2OMUEREpV+qJp0i1mDTppFjPN3XqZOrr65gy5bxYzysiIpKWtFYuEhEREZHNjBJPEREREUmEEk8RERERSYQSTxERERFJhBJPEREREUmEEk8RERERSYSmUxJJ2MyZdzNjxp09HleYx3Pq1Mk9Hjtu3ATGjh0fR3giIiIVo8RTJKMaG4dTV6dGCRERqR5KPEUSNnbs+LJrJ0eNGsLChUsrHJGIiEgyVJ0iIiIiIolQ4ikiIiIiiVDiKSIiIiKJUOIpIiIiIolQ4ikiIiIiiVDiKSIiIiKJUOIpIiIiIolQ4ikiIiIiiVDiKSIiIiKJqOaVi2oBampyFf9DSfyNpOhasqdargN0LVlULdcBupasqpZrqfR1FJ2/tqJ/KGW59vb2tGOolHcC96YdhIiIiEgvvAu4L+0gKqWaE8/+wIHAy0BbyrGIiIiIdKcW2Bp4BFidciwVU82Jp4iIiIhkiAYXiYiIiEgilHiKiIiISCKUeIqIiIhIIpR4ioiIiEgilHiKiIiISCKUeIqIiIhIIpR4ioiIiEgilHjKRjOzGjPbKe04REREpG9Q4imb4nzA0w5COjKzOjPr8+U7uo4jzWzrtGOJQzW8JtVGZSWbquE1kc7phZVN8TPgVTM7M+1ANoWZ1ZrZIWZ2kJkNTDueTWFmOTPbA5gOnJ12PBsruo49gR8D1wPXphzSJim8r9x9XdqxSKCykk0qK9VPS2YmzMyOJKwd3w487u6LUg5po0VPpMcDlwMj3H15uhH1npm9HfgzsAZoIty4p7p7c6qBbQQz2xKYBHwG2B4YDOzn7k+mGlgvFV3HZ4GRwB+jf3/e3X+XZmy9ZWbvBL4ENERfjwPTAHf3Pnfzje5fa4C1wL/cPZ9ySBtFZSV7VFY2H3VpB7A5MLNa4FTgDKARGAgsA142s5+6+2VpxtdbZlbr7m3uvs7MHgLqgfOAr6UcWq+Y2RjgFuB24FJgNOGD6BfA+BRD6xUzGwR8CDgZ2JdwTe8n1OJcCRyUXnTl6+Q6bgW+4e4LzOxV4PtAn/gwjfo+Xwq8j9Ad5SXgv8DHgcOA3xJqqDIvun99mXD/GkhICpYS7l8Xuvs1acbXGyor2aOysvlRjWeFmdk+wK+APQk1a/8AHiHUep4AjAWucveL0oqxXGZWU2j+MLM64FvA/wItwFbAvu7+r/Qi7B0zOwm4Ajja3W+Ith1ISEQnufstacbXk+gmdwjhoea9wGxgsrv/I9q/LzADOD7L19LFdZzp7vdG+2uAwkPCJe7+s7RiLYeZfYKQxLwGXAzc4+7/jvZtTaih+gZwWGF7VkXvoV8BBtwE/B14DFhHeNA8gHD/+mFqQZZBZSWbVFY2T+rjWQFmlou+Hwz8CegPfB74krv/xN3vd/d/EpoVfgBMMbPhqQXcg6gPUa4o6fw0MJ8Q/48INQY3EG4gmRRdw+7Rvws1/c9G35dH22sIT9zXA59LPMheMLNdCO+d3xFqab7u7oe4+z+KOuXPifZ/NqUwexS9JhfR8TrujV6zuuh99xpwP/AOM+uXYsjdMrNtCbUdjxPeQ9OKPkj7ufvLwCXA36LvmVN0/zqQUBbagf8BTnP3ae4+y90fA75A6Of9bTMbmVrAPVBZySaVlc2XEs8KKOqPciqh3+AZwG+jglQ8Wm8toYnECW/MzIlqOdvdvT0afPNPwlPd34EPA99198eBKcABZpa5G3dUS3AxUad7d18b7XqDcJM+Ptq+zt2XELoObGlmWyUfbffMbICZTQH+QPiQvBbY3d2vivbXFh4Qoj63bwBNZtaYVsxdiT54/kh4KPstYCXX0V54raLXZQChL3Fr4YafFUVl+iRgN+AMd/+Hu68pHOPurdH3pcBVwFZRi0imFN2/vkxoLvwGcIO7L4T11+ruK4DfEFpwzkkh1G6prKisVFq1lJWkKfGsEDN7CzCRUKhudfe2wg2g6GbXDrwO1AKtqQXbjagf5ygzuwb4JyFZ/iTwVXe/p2hA0RxCP8+9Uwq1S+7eBqwE5pvZNkW7/gM8B+xnZtsXbb8YOM7dX0kwzLK4+ypCE9sY4O3ufoa7r46S68K1Ft/c24GdgSVpxNsdd/8voT/XbHc/093XFGqji6/DwqwDg4Hdgdao9j1TfYSKRuAeBtzo7g9HcXf1ob+C0Mc+kzVSZrYb8B7gbHe/M7oPbHD/iiwDngC2N7OGFELtksqKykoSqqGsJE2JZ+U0EmrOnoT1T6WFnUU3u20Jg1qWJR5hGczsu8DzwDuB0wmDbzp7olsHfBf4Tkqhdqro//lmQt+oMdH2+uip+mnWdwAHwN2fcPfnLWPzyBU+MAnzpw6IthX+/9uKj4tuflsAHyX0m1pX9PupK4rlFOBAMzs6inttUfNVbVQL3UYYSHEA4XXMJAsDPpYAiyC8Jt186L9EeOjJar+1BmAY0Ty9pfevaFsuKkOrANx9WVZq11RWVFYS1KfLShoy9cFaZRqAFwk1hB2erotubt8hPM3dn1KcPXk3oU/RR4DL3X1ucaEqfqJz9zXu/kYKMXap8PTp7g8As4DJZrZXVGMwhHCDbgc63AQ8Y/PIFWrN3f0OwvvlYmBEYX9xTY6FufB+CuxC6Oj+5u8nH3lHUSy1Hqat+inwTWC/aF970TEDzewHhGaqP7j79KjbxyAze1tqF9CJqDmtHRhpXcwHW1T+57n7kYTuKU9lsHl3COGBc4NatVJRrds9hH7qA4FMvCYqKyorCerTZSUNSjwrxN1vJzwFjTezeghvPF8/QGcvwk3kY8D/eXZH7L2f0F3giShZyxVu2kVP21ua2T6FG0KWagsixX2KtgV+b2a/Bv4F7EC4vpbCwWY2Inp9snwtJwBvAb5iUV/Uooeb4wnXNhE41d1virafDdyRdMDdKHQ5OZ0wM8IPzewYM9vZzLY2s1MJLQafI8zndwKAmZ1OaCGYlk7Y3boU+ATwtsIHZ9H34vI/2sz+DNxHqAUZnVK8nXL3ewgfqId1VvMflYucu69199sIg12WAz9PNtJuqayorFRclZSVRGk6pQqICs1aM/scoUbz+4Qn0AUWVpiYQOgn+XbCaL3Ti2tEs1bTVlC4rk62HwJMBha4+xeTj6xnhf/X6Mn/U8AHCc3stwB/dPcl0RPp0YTaEXf3d6UXcdei2o82M5tEaH4bSRj5+QZhwNcOwEPA99z9L2Y2gTCichvC9Z7q7q+nE/2Giq5lb8IAu1MIfbpqCH267gGucPfrzexdhDlWdwJ+SZhJ4Z6oCSszzOxGwkPndHf/TeEao305wkwQXyFMhfNT4C53fz61gEsUvSYnEvptH+vud0f7aoCawn3AzI4gJBA7ElYA+iMZek1UVlRWKqmaykqSlHhWmJmdDxwDDAVWE56M6oAXCNMRzQSmArj7uZbBzuClzOyjwJGEfpG3uPs9ZjaRcIP7irtfV3wDyaLopjbEwwhQzGwc4cbx9uiQCwkzETyVtWspfo9YmCrmW4RJpEcCDxJmSriWcEP/OSHJvpOwwtQ/3f3VFMIui4WVpPYj1PA84u6PWVid5WeEROEuwnU84O6vpRdp18xsFOGB80jgYnf/cbT9RMJ0OOsI88feAPyn8MFjZubunk7UnTOz3xMGqvyJkNS8Em0fTXhN3ktI5K4gg6+JyorKSlL6ellJkhLPCimqYasDdgWOIDQRvAo86u53mNnhhBvh7sCjwKfc/Y2s1npGT3DfBL5I6Ow9mtD08X13vzLqXzTe3fdLMcxesbB60QWEms77CJPHrwAOJDTPjXP3RVl+IIiS6MZCd4Ho528TXqtngZ8AdxbXFJjZLu7+XBrxdqbw/1v83o+u4wLCdGRdXcdHgS09YxNlm1l/QvmYR5jp4SrCIhLXAlcDs4oeenYkvP+OJyzXemPyEW+oqCZnC+ADhPvXl4GFhFVkTiPUQl1CH3lNQGWFDL4uKivZe00qTYlnhXWWsJjZAUSrMQDDgbnAYuDnHi2plcVEx8yGEjrqP0dYqm0todb2dGALwhPdtwkrAWXmRt0ZC527vw2cSJjL83uElaUWRjeRbQnz5T0W9avKrJJanS8SRvLWAJfRsabgAMIEx4cDR3o0t2zWWFhV6iLCAITL6XgdbyN0V5kAvJXwwPP3lMLtVFSbczmhH/e90b//4evn820gDNr7GLAHoVVkoLtnbfAEAGb2EULzbTud10L1hddEZSWbr4vKSsZek0rSWu0V5htOobQXoW/OewnN1FcB17j7s2b2P8DPzGy4u18cPdEO82yNEh8NjAK+U9SM8BPCU940wnQXKwmJdNbtQkiYLyBMiD+3pDl9IWE50MKTduYeBAqi98q7CR+euxEmzJ5Ox5qCTwBHAVuyvp/RkakE3YWoI/4sYB/g19FXV9exLaFf2/OEVbN2SSPmbownlPWvAHe4e2GlLMzsPYQuOIcSatgvJnw43WVmP3b3r6UQb6eiVptHCK/JNYTy0idfE5WVbL4uqKxk8TWpGNV4JsTMDiJ0Jm4hdGa/2t0fLNo/EPg6YQqmXxE6i58PPOvumVmK0sweJ6w/e667z7cwJdGxhD4s7YT+qhcA7VlN1Iq6QYwBXnb3lV3sHwqsc/dMzrFazMIo/f2Bc4H7imoKBhNqCj5OaMZ6ntD88xLhQ+t9hc7waSv6f/8UYZnZO7q4jr0IzXI/dve/mlkTYQ69Ke6eqdG7ZrYH0OxhMnPM7K2ED9EjCXP9XgNc6e4vRfs/Qyg/+3o0V26aNuE1GQ48A5zj7pkavauyorJSCdVYVipFiWeCzOxnhFFsvy/a1q+o+r0/oT9oo7vfZ2GKjCZ3z8yk7FHz0w8IfTtfICwJOo4wSnSyu1/fye9kanBOZ2z9TAQjCdfzP8C7CM3wtwK/cvd/WRcj+9NS1L9oALC9u88p2jeB8HR9OJAnPFVfFV1nDaEJaGd3H59G7KW6qlWOruOThNdjZ0Lfru8W7a8lvCffAhzh2ewfvQ3hQ+fDhJHG/YCV7r5btL/w/juS0HrwOXe/M614CzbxNfkeYZ7ccVl4TVRWVFYqqZrKSqWpqT0BRYnXl3391Ap1QFtR0rkdoVPy54C9zWw7d780taC74O6PWpgm6tuEeTFXEEaDXuTRKL2oIB0MHOPup2U96YQN1m8vXNdDhKfp0YRVm44E9opueJlpdi/830a1BHMALEy38inC+2kosDVwe+FpuujJ/L/AwWa2rYcl+VJV+n9qZvsT+nQdQZgN4i+EWQf2ifYXP9DsCawB2rP0+gCY2YGE99K2wCuElo3HgCfN7Hh3v7ro/XcgYaLzOZ2eLGFlvCZ/ZcPXpPBg1g4YYVRyHeG1SZXKispKJVVTWak0JZ4JKLrhFZ6ei+f2GgaMZf3N7yngY17UtzNrNwfCyPyPE1b5+D93/09hh5ntQOincxKwxMy+5e6LLaMj9eHNEaF1hJGgJwInuvv0ov17A9eZ2QXufhZhlaMsvR5vsjBf4cmEpqlHCU/SOxDi/567z48+SAcREupVwILUAu7esYTX41bgRx6mi9kXmGWh8/7fzGxNdNxhhMF5WXxd5hBaBq4GfuLuqwEsLEd7SdSlYw6hr/QXCTVt8zJY7qHr1+RRM3s/cI+ZrSI0kY4HrnX3TH6Qqqxk7r0FKiuZLCtxU1N7SopqBT9GmB6iHbjQi6ZVyGhhAsDMdi1pqmog3AgmE5YLXQI8Adzt7hdFx2T5ekYSPnx+6+5nRw8Iuahprgb4KmEKj3d5tgZ8bcBCR/wfAN/2aKoRCytn/YlQm3MpoSbhK4QuBV/2jE3lUVTLtCNwsLtfV7J/KmH6m6cI5WYfwrKun8vqjdvMhrj70ujftYS+w+1m9itCbfooQl/CnxPmM8xUv+LuXpPo4fluQuvA80AtYX7J6wivyfI0Yu6JykqfKyu/JlTOqKz0cUo8U2Bhkt9TCMtR7kx4ujvbo0EuZlaf1ZtCqai28APAdwnXciVwI9BMGIV4FqHp6ozUgixD1Hf1JsJKJbd0sv94wujWHxRuDmb2QeBhj0b4Z0VxP9Sifm3DCHOUvgUYTJiB4Hvu/tvouOMJq4K8lFLY3SppKsTCFDITCc1Tvweecfd/Z/zhJkd4mFlX9OFUT3hfbUdIDpZETcGZZ2b93X21hUEhDxJm6fgvYbT43wgTsC8oOj5zr43KSrZej4IuysoAQo30tqis9GlKPFMQFaB5wOOElX6ejbYfSHii24fQD+cvwE3u/qBlbFALQNTs8RfgHYQ5ML9PqDVcHT2h1hCaGz4ETMrak2kxM9uJEPuXgd9HN7ocvDkFS/+iZp+3EWp3jyHU6H4opbC7ZBsuPVf8gbonYf7VR6NrfDfwf4TuHne4+8TUgi6DbThpdvFr8lvgNnf/TaoBbobM7DuElo4J7j6zk/11hKli3unuv0g4vB6prPQ9fTUx6+tlJS5KPBNWdGPb0d1fiLYNBj4NnAAMIowQf47QEXkIsJO7t2axsJnZzwkr/tzSWb9UM2skjEhcnVqQPSh6ov41YRTlV939sWhf8YfS1sCphBredYSn7mOBo9z9L6kE3wsltTujCVNffYCwZv0/CYsavN/db0svyt6xMCjvKkJT6Efd/daUQ9osFJXvowkLLfzI3Sd3ctwowpyFnyfcz77p7hckG23vqaxkj4VZXyYCM909X1qzm1XVXlY2Rk3aAWxufP1AoxeKNh9MeAp6nTAl0Tvd/XjCje41wkTmHUbNZcQpwG9K+z0WYnX3fJaTzhJfAgYAZ1sYJEX0kFBnZl8gzMP6MULT2xnu/mngs4TVpzIr6idVGNxWb2bnAv8ADiL0k/qKu59JmE5mZddnyoaoJh0L8/z9lPD/P77wQVqoqZbKiT5IBxL6Dz4D/LJ4v5kNsjDdzc2EKW92jnZl8R72JpWVTGsjVNAU+kxm+r1UUK1lZVNoVHvKohrBaYSReh8v6udZQ5gn837gGTMb6O4rLWOjw/vCE2dPotrOWndfamGk6wp3nw9gZhMJzep7EDqDX0Voii986FwPtKYRd0+i91B7UY3tpwlrOQ8DbiPEPrOoP/EthKbFTCrUHESv13DCtCu7EpKBfxQdOgBYmcXuKVVmP8JyfwcUdReqI0yQ/XVCa8DNwEcJtTivEFYMyhyVleyXlehh4CfATDM7zN3/0Vmtp5m9E1jq7k+mE2mnqqasxEFN7SmJatTeINzYZgH/6+7XRIMN1hU188wirIP+Z3c/JdqWqeSzWpT0iXoLoVn93YQJpf8M/NrdXyw9Nvr5zT5UWVDSReDthFVa9iXMT3odoY/Xomj/Bl04zGyQu69IIeyyRK/NtdGPEwnz3h1CGKBzCvCCu78vpfA2K2b2YUJi1gZsA0wijAJfQqjheZLwsDYQ+LC7Z245XZWVvlNWohraaYTuZxNK9u1E6AJxMqF71+DkI+xaNZSVuCjxTIGZ7UJ4o10NzCaMAP+8u/+u6JgtgB8CxxFugI8CZ/n6aSYG+2Yy9UKSLEyr9AVCk3odoZntanefFe1/c8BR0e/sQxhw8Ii7P5B0zF2J+qT+H2EFkGZCrc3N7v5c0TGlH6SHE6b3ud3d/55wyD0ys8MItTdjgEsICcLOhGUA30bomP8SYRm9xWnFWe2K3zfRw/KxhA/9HQmvy4Xu/kZUQ7Uz8AV3XxCVn9qs1a6prPSdsmJhVpjDgRuisQ+DCbXTXweWE+7bCwgrM6U+2X+1lZU4KPFMQdTf4zFCQnORhaU09yE031xPmNfzWMJEuncAv3P3u4p+fxBwMXCrdzL1j2ycKNm/hjCadQZhRoE/Fe3vtKbZzHaOjn8VOLb4wyotFpadm0FYjeXnwJ3FSXFXA9XMbDfCTAVLCStPPZtQyGUxs68CPwIKtQH3EzrsjwHeQ+gnPdWjVbSk8ixM13MFYX7I89x9drT9CsK97ESP5sks+b1Gd88nGmwnVFb6blkxs08SkusGwpR+WxOaq6cSukRlKsHp62UlLurjmYKor+bVwGlm9ld3Pzl6er4amEKoar+NMD3RTUW1nO+Ifv9BM3uaMM+cEs/4LCbUBhzi7p8tbCwknKVJp61fhep5MzuN0Gn8CsJNPW2vAX8nrIJ1bmFjUd+v0uXdaghP18+a2amEB6ArCCNgU1f04X8JYZnW/wIPuHuLhcULjib0tb2y9IO0s35gsumKXpOrgDmFWr9ogM7/EvoUTnT3h6PtE4EtCcnBUYT36IfTiL2Eysr63+0TZcXC1IPfJ6zCdD3hWg8ltFadVdx6mAVVVFZioRrPFJnZI4RVGP5BmBR3B0LTx5+B6339dEs7EJao/CRhuqUDgNasPc1VAzMzwpq6H3P3Jzqr5bSiyY2jnwtdJ95D6K/zQXd/PuHQO4iu5SbC1CnPdHMtNUV93N4K/BjYm1Arf4IXTWycJls/LUlxX9y3EmYbeI1QRpYTVgU5gNBq8OEs98Hr6zqrDTSzgwn9CvOEAZJthL7SbxD6sR1GGNTSQFiw4fJEg+6EykrfKCsWpoQ6m7Ak86OEZuqbLCwA8j1Ct4fvR8f2c/fMDPyslrISB9V4pus4wtPnZwh9PX8B/L3oqWcw8BFCf8O3Eka+/8jd1xT6GmqgUbzc3S2sbbwo+rn0w6cw+rPdwiTTXyU8ELQSVgW5PgtJJ7x5LRMJNbmF0fvF/Y0KN+Y2MxtBaLL6JGHKj3MJTY6Z+CCFDaboKkzufxihX95zwDmEQQWHEpoShxMG7e0ZfZcK6OLhdyvCIJaRhEUy/gtMJzT5fpYwTcwOhK4pTyUTafdUVrJfViwsWDIbWAacDtzo7i+Z2aHADYQVgX5VOL6QdJpZk7u3pBDyBqqlrMRBNZ4Z0MWT0HsIc8W9izBf3NXAZd5xhQ0lnhUUNYUcCOTd/Zmi7Z8lTLM0gjD46/fAX7NcCx09yLwfmO3u/y7a/g1Cc88ywnXc5O6PphNleaKk/zbCHLh5Qn/oJ4F7CA9ov/AMzTKwuTGzPQr916KfDyQ8pL2VkNz9DpiW1XuXyko2mdlRwHzgaQ8Di3YEziNMP/TNqFJmJ2B3mIz3LAAAIABJREFUQuvg+wjJ3Y/cfUZKYXerr5eVjaHEMyNs/eo5OxBubAcRnkT/AvzQo9F5FkZdv50wH9i5XZ5QYmNmDwALCc1TBxFGUO5N6A96A2F0ZUt0bKYfBMzsYeBFQk37kcCFhH5EtwJ/AO4tNLd1NagiKyysmlVY+/gaQm3BGo9mezCzYV6ysIFUlpXM+mBhednj/r+9M4/XdK7//3NmzIxlxtj97ELzzhglWUuTkV1lzZJlNMoSQoRCZcuIFMlOREh2ZctYk5HqWwm9QkRk38Yu/P54fe45l+OcmYNzzn3d93k/H495zDnXdd3nfK5z3+/rel/v5fXGZShv48/ZCZXr2RAsH1e7z1naSr0ptbYnAFvh6Xn34DT1Ijh6+AkcTVwQeErSfE1aape0k628V9LxrBlhsd8/4ifSSZJuL9tnwamQTcu/JYD5SsF4rS96rUolqrwOjhhcgyVInsJ1oOepdLB3vojUjS7O5U/AaNzpeh5wraTHy7GdZWMGAXOqJpIrlXMZDozqokFiGZxWnAvXRv0c1379q1WaJ1qdcKf4NsCGOCtwK87YNEbRfgqYWdLkuj2spa20hq1ExEj8fqwFXAf8FzvUl+Es1RLAqngM8i9w427tHLdWtpX3SzqeNaJykVhG0l1l22BsQOthsdlFsYH9EThbRe4nnc++oVKk/0sc+fgZcHKjDrd6TPV7XBB+Cu4cvbnzz20GlXM5H5dxHAqcpYpQcTfnMgrfkI6SdEt/r7unlJvr94G96LgBLYCdhlclrViOS1vpQ0oTyynASKyJebI6RjQOwQ/OP8WlKRO6/UFNJG2l3rZSyRAuCbwm6T9l+yz4/doaR3hvx13u9zZvtd3TDrbyfshZ7TVCHXPcG07nglhC4Qys2/kHrPf5KZzqOSQ8SxjyvewrGn/Xb+AGorvV0fzVmOv8jgtz+f5VfBM6rv+WOkMa57I38BbwfONGWh5wqmmfaRFcWT9uOP4M1pntcNTgVix3s5ukTYHtgTki4nvluFaaT91ySPob7pw+DatDXFHZPQ47Ozdgjcm6krZiamkr6mj+ur80GM1RmkLPwt3tswE7Stqs4XRGxKCw9mptaBNbec9kxLOmlFrOM4B1gL8BX5d0W6djDsS6ZUtLerH/VzkwqESiT8QX5mU1HZH40piwPL5pfQH4rGoy2aRyLscCnwPGq8ylrxxT7eadH6eyvoob3VZTjaYzNYiIOXAW4DfAj4FbcLT54BLdOQY3HKynjnnbSS9T+XzNIumVTvs2xCnPKdgp+Fen/bVKI6at1N9WitP/KZyN2gA/JBwr6aedjvswnhb0FeDTkm7t77V2pp1s5b2SUbKaIukpXGC8h6QVZdH4IeGRWw0Ww/WGMzdlkQOHtwAk7YK17t4xRq/y9Uzh2cjfAi7FY+km1sXpLDTOZQ9gQvVGWo3cRMSspcZtEtYqHAlsWMcbaWE48BpOST0AfA84ICJGyB27Y4D76n4jbXUqWZtXGpHB8lnaCzgaOEnSmqWGcJGI2CUiJpaX1y0KkrZSY0rGaX+cit4U61+vXHU6I2JUREwC/gLsUDZ/rr/X2hVtZivviYx41pCuirojYljjQhARQ/FIsAOAIyQd0IRlDig6P2F28f0iuA53TzyF4ofAQaqRgHGD6Z1LuaF+AuvHboklSY6SVOtUT1hW5UZ8I7oA31xvxAMZbsPpty9J+mXlNcOBsaq5HE6rExGbY03Cx3G38Rs4zTsOGIr1pJcsTlCt+KC2UscayXaylYjYB3+Wdi1p68b2wcC2wOFYlP0E/HkbAuypijRenWhlW3kvpONZc8pT3duVi92WOKI2B67zPEzSC01c4oAmrELwSWB3nGa7AkepHyz7a3fjadBFc8QSOF21DU61nY8L858p+2uZ3unUCLIgrpV6MiLWAK7GGn/HSvphOb7RmPAhLIm1mirNYknvUd6D68q3z+MpbUOx8PrDeK7277DDVotO8K54H7ZSy47wgWArEbEajj6vgms+f4EnHa0FbCHpuum8vGm0i630hJxcVHPUIRi/Cp44sQJwB46oXdFwOsMiwW9KerGuDkKrU56iV5N0c3jm8VhgIn5C/SewlqTJjWPl+e51dTrnw7VRl5Svx+Oxeavjz9d4dch5dDmrvkYMwqmnXfHowm9FxAmSro+IvXFJ0ekAYZ3cXSNiijxqb3UsXZb0DX/BU1juxkMw7sV6ix/HDsGtwPHVG2ndHtZmYCt/AtaUdEc5tqG1WDuns9BWttKpxnZR4BAc6bwNRwqvxXWd8wGbqWNGemMCXZ1oeVvpKRnxbAEiYi2sIXkbnid8mSryEMXgtsdaX98u21ryA1lnSpTjPmA3nLI5ABgGHKgyQ7dV/u6l2F64vmsUjtw8C3xH0gXlmME42t4K59Mo1P8irvf6jaSzK9uH4RGH+wLL4CknRzRzze1O5W8/n4p+ZEnZfhXYBzhR0pFl+zy4Pu835fva2FEPbWW4pNfqGums0o62EhHb4ujmvbhh6sISyd0aOBHX3h4p6eVOzmotPmftYis9JR3PFiEi9gemSLqxfL8Afgq/WdITEfFZXJtzlaQDW+EC2IpExA/wheAt4CRgP3VM/2iZC0CpTzsC31yexILFh1T2t2zUPDxL+3+Sno+ImfG0qYOAzwIXY4fh7maucaASEZvhMYZ/wvJwc2MnZzXcJHm4pJOat8J30wNb+SJOsy/fzetrey1uF1sJ62FOxE7mA/LozI1wWdpZlcDAAjiquzCwBvCYpJ81adnTpRVtpaek41lzKjU21aL2SVhMfghO8Z4h6ZiI+BJ2hpaR9HArOUKtQokufx3/ze8u21ry7xwRC+NI+RmSHi3bWtbhrFKchdH4vdoBp68OlHRlUxc2gImI7bBo+SxYLPtFrEv8IE4lPoqnAj3VrDV2Rze20ohSjcGNIDtIOqeRxo13j0ScFs2qE+1iKxExVKWZM6yBfSbu3r8BmIqju7Phhp3VgVfwZ3GNRkCnLrSyrfSEdDxbjIjYHYsTfx0/CW2A63W+IOmWiLgDOEdS3QWMW552cdKgvc4FpkULLsCduj8ATpP0aonqzAosievd7mpErJO+JSJuwvqWAq7HN8/rVMYCdzq2tg9zlTrOarPRt3B0cJbGMZX6/KVw8+HqwEqyZFFtaEdbiYiv45Q72PEcAdyJ6ySXxZ38awMLATtJOrsZ6+yOdrGV7sjmohai1NyNxc0f50l6OiLuxmmD48IzXRfDF5DGa1ruQ9kqtJOj1k7nUrgTS6hMUsc4veWx0PSWwCLAy8A9EbGHpCk1bThoeSpO2La4OfJKYCaVoRcRMSuO5syNr2XX4hvtq3VLU5fraVfrOR3YLCJWlXRbiYSOAL5Gxwzu0bi7+if9t+Ie0Ta2UnmAPgdrXP8NeBp4DjdV7YBVCMYCNwHflvRwk5b7LtrJVqZHRjxbjIg4BtemfEbS82Xbojg98ndcv7Kl3j3pYLZWeVpNkg9CVw9bYWH/03GzxFnAqbhbdBNgjKQl+32hA5jokPVZEY9vXBNYAngBj5u9XNKuzVzje6GkqwdVyqE2w40h8+NyqE/iNO82jaaQOtCOttL5nCJiThxtnoDfh3uAgyVdXzmmto2U7WYrkI5ny1Cp9RyBncxrgMuBZ3Dtyqa4ZuVY4JRKmmdBPLZxFUnbN2PtSdJMwhqEVwOvA5vonYoQY/CNdWdJd2aGoP8oDsEZuFliNmBxXH+3BVaMOFrSsXWO5HThcH4K2AVYGYuAXwEsgJt29pX0u2attSe0k62UkojVscO8PnbSjpF0aqfjlsbyhJK0V3+vsye0g61UyZGZLUJxOoeUkPva2Mk8Cc/Z3QZ3vW0n6cSS5hkZERvgmp2DgfHhsVuDuvsdSdJOhCd8AayIb/7bSbo3IoZGx+jZN7Ak1srFvt4u0Y+kj6j8fQ/GIt974FTu5yQ9i5tCTgN2Lynd2t5IJb1drs0Ll6bPE3Gk8AI8mvH3eOrMxcDt4bHHtft8tZutRMTiuDlnErARfj9WqDqdETF3RByNeyU+AexRnNDa0E62UiVrPFsIdcx2/UdEvIALpK8HTpB0MUx7ylsZG9vm+D0+uCEnkSQDgbDm6vIRcRGwHPAIpfa50vk6E65jG43r7ubBdW7tVu9aKyp/34/jedTnRsRCwI8j4uelseU1XKM3Atfn1VaWKCK+iqOcQ3F3+zmSro2I9XGt4WmSjm7mGqdHm9rKm/g9uRbYVpURmeVcJmJnbjAWnb8IOBJHdFfr99V2Q7vZSoNaPq0k3VN5ArocOBl3szeczjH4iegnuIj9cuAj6tAwy/c7GSiMwzeXkXgM3VI4EgVMa544DKepXsUC0z+t7J8vIj7WnwseSETEvOXLZ8v/p2Bn5yfhKWxbYxmZaeOAa3wjfQmnpo8EdixO56K4k/1nkvatHhwRy0bE6Bp9vtrOVkrD0CqSNuvkdI7HUejjgV8D6+H0+73YEV06ItZrxpq7o81sBcgaz7ag1HF+FtgKP63dguuJ7ir720oqJ0lmRER8BKs/bCHpyoj4Ps4EPIZrpBbF3bqTgYMatWwRMTuWW9kJl7CMlnRfE06h7YmIi7EQ9haSpkbEhjgleivu2t2xKh9TbrKXAN+Q9JdmrLk7ImJeSU+Wr5cEfgXMDmxc/h+HHbtNcO3nh4H/B8zZaBJtFgPBVsp7cjjOAk4GjgJukzS1cswo4BhgpKTNm7LQbmgnW4F0PFueks7ZHFgXT9U4SNKlZV9tO/WSpK+JiJ/hLtbdsazKLlgEfBF8wZ6kjrFzMwNjgM9jjdzXgO9LOr7/V97eRIfw+lK4vm5/nJ6eGhFXYKfny5X3ZlFghKS7I+IQ4J+SzmnaCXRDo9EIf95+BNyIBcDHAjfjDOMtuCN5Ipb3+bxqINTerrZS6lMPwuf1GO55+I2kxyvHVEdo/hA/FGwp6eUmLPkdtKutZI1n6zMGSyzsW60jKsaUUc5kILMrcBlu7BiC01FTcVfuKY2DSifvGlhuZTXclHCIPHav1p27rUi5kQ6RdF9pyNkc16v/GL83j8C0evX9yr9bcLPOYZJeb9LSZ0hpNLod1w0+AzyPP3/DcIRzT+DLwHm4EenOJi21M21pK2VdE4H/YnWX+zqvsTRJDZeF/UcBH8IlBU2nXW0lHc8WpWLkP8Rj3J4p2wdLeqtuF4Ak6W8kvRwRW+KowJxl85WlG5SImA/L3OwKrIP1Fh/AhfrjsWTZEKB2QtltQOOheBLwV2D28MjDxo10G2AfYC6cUryyXNteL/vr6OS8Xf6fAkxpbI+IefDM8J3xyMOtcLr32boEB9rcVr6Km4dmq0Q2G9qYg4oywWvlfVoXuKp6TPOWPY22s5VMtbc4FQPKOs4k6QGlNm0ZHLXZHrgfX7hvwmnQz5fvl5T0XB0v3O1AV9euiFgBN3msAPwRuBC4StJjTVzq+yI8ZWZ7nOodgoMEvwAeU5n6U+or9wEuknRVk5baLe1iKxFxA26+2VHS41GZ6172r4qj1EsDG0j6a5OW2iXtZivpeCZJMqCIiAlYEWIqcKik48r2xsV9UVzXdpyko5q41AFDRMwBHIejZ4/ilO/lku4p+4cDr9fRqemKiFgXO5wrYq3FY3Ga97Wyf05cd7gOsCpwv6QPN2e13dMutlIko24CzgV+1HDOIiJwY+5EnGI/FDheNRwH2qAdbCVT7UmSDDQEnA/s3uhqrdR4gbtH5waeaNL6BiKLA2thx+BSSbfAtIjbPsDywOsRMQU3VzxaV63CiBiHZ2z/FjuWd6jM2i77t8F1nrPh2kOAJyNioUb6tEa0ha1I+ldEfBfPap8QEZcB8+HmqbG4dGBjSTc3cZk9ZXFa3FYy4pkkyYChcyqwdL2+qY4RsxvhlOhbwOo1dATajkr0bHnchfti2f4V4GjcNf177CgsADwi6dNNW3APCE+N+wPwdCU1Og74Go6C/hunRz+FJwJtWbcUaTvaSpFV2hfXsr6ClWAuknRW2T8IOup160a72Eo6nkmSDEiqdV7h4QuHAJ/GDsEPJN3UzPUNVMKTZfYDDsTzqU/BaeqXIuLTuIFie0nXNHGZXdJVjWNJ8+6GRx2/jlOjJ+Emo/WAIyXdVLeoVJVubGUc1v9sKVspzuXQ8u+VyoNBy/VJtKqtZKo9SZIBiaQ3itDyt4Ftsc7fT3AE5J6mLm5gMz9ORf8CS/U8Xtn3V+AGPC0IqFfXbrVrGjs2++FO6TmA27ECye8i4lu4MWTH4nQOqqvTCd3ayrHAhZLU1MW9D0rH9zu6vlvN6Sy0pK3kCMUkSQYyl+EL9wXAXlgIu1GkP6iZCxtolBo1gI/husFTOt1IwVN0VgKOj4j9YZoO45D+W+mMKTf3WfE4wxdw08pOxencGNgQmCDpzMrxRMQcETFLc1Y9Q95lK5SZ7tA69tKVjmez1vJ+aXVbyYhnkiQDjkpac388qeQySS+UfY0ISMvdkFqViPgosG9EHAG8iCf+DO50zEL4/VoCuAd4tqQa365btLB8vp6LiO2BBytd1KsDJwCXA7+MiOVw1Gop3DDyCrB+ROwn6aSmLL4TM7CVgyLiOUmT8PtVq/ehHWkHW8kazyRJEuqThhqIlJvpdcBWkiZHxE148s+5uPN7DLA3nv5zHHA28JCkV8vrx2DZny9JergJpzBdSjRwMTyxaGUcLVwfi7U/hptBrsbTgJYF/ixpheastudExA7AqcB8kp7qrk4yIoapplN0Wo12sJV0PJMkSZKmUyRu5sYSRCOBLwLfxDfQwcAlwFHAXZJeKq+ZBdflTsA33QmSbuj/1c+Y8IzzU/H4zJewGPsVWFz+beA7WJD9NOxE3F5HZ636gFZE8ifj7unNuuiED1xmMFzShs1ZcfvR6raSjmeSJEnSdCJiQSxB9AQeOTkC2AZHAk/CM6ifqzg9O2IJmceBn+OxiKpDKrEzjUhgQ5IIps3hXhBHp74G/AP4Hj7PZ1sl+h4Rq+Bz2FnS02XbnMABeEzorOXQZbJpr3dodVtJxzNJkiSpBRGxJrAlbop4FddD/hZ4oiLnsw7uqB6NNSQPAi5upe7qiJgN2BHPcH8TR6fOBx6v6GR+HFhB0qlNW2gPiYhRkp6PiMF4CtChwMtYSH9VLLu0X6M2NPngtLKtpOOZJEmS1IqImFfSk522fQg4EethXg/cCDwPfBLXSK4j6X91rtUtjtm6wOHAR3Ba/afAvxpp9YiYF0dAN8Jdy2tLuq45K+45EbEGcATwcVxDeCXwXTyhaWtJf2vi8tqWVrSV7GpPkiRJakXjRlqkX97Cjsz2wL3ALsA1kh4sx/wauArXSH6nCcvtEcXpPBfYHPg1ntX+f5UavEFYrmgC7nT/M06NnobHJNaS4uQcjqNvt+A61buwhukLwCaSHi3HtpxIe91pRVvJiGeSJElSW0pK8VrcCf4r3DDxv8r+IVgo+xFgu0aasU5URh1uhwM+l1Kp44yIz2InYVngIXye5+FGkbuBoyX9qCmL74biSB+B1/0YLhe4oKTcDwB2wpI+5+PmqUHpdPYtrWIr6XgmSZIktaZIyDwgaWqn7fMAXwIOA/aSdHoz1jcjOnWCTxuNGREfweM0x+NO96uAMyU9UPbPhMXavwJ8VNJrzVh/d0TEhdiJOR54sGzeD9d57i7pN+W42SW9ULrclwOu75weTnqHVrCVdDyTJEmSlqJ0h6+KZWQ2xM0re0p6qKkL6yERMQduLvoiFgC/FTuct5X9g2DapJnxwA+Ab0v6bZOW/A4aznNEDMeRzIZG5Dg8VnMK1pocAWyGJaOWBFYEhmPHc82mLH6AUUdbyRrPJEmSpGWIiLH4Bro1bpT4CZ5T3UpRlH3Lv0vxnO1LKxHRhvTSENzx/jCwNO5crgWNiG01AlsklL4HrA58AhgHLAPcBPwL603+GvgCsEpELCbp3/268AFGF7ZyPHBws20lHc8kSZKkJYiIbYFdgQCuAfZtRG5aoXGlssazcP3jryRdUvYNkfRm4xwqGotjsBZm02dsTw9Jz0bEScBt2Nl8Cfg7XvcO+DyWA34GHCrpkWatdSDQha3sX2kyGozHZzbFAR0840OSJEmSpBYMAeYBNpK0paSHyk2Uujud4DUWB1PAOcBeETFf2fdmI8XeICJGYwd1iqQb+33BPaSy7oslHSDpWux0jsdO9t444rmtpJ3T6ewXBvNOW3kwIsZGxGbFVpr2IJM1nkmSJElLUmfNzu6opNJnBf6NBb5P7kKLcUXgx8AKeC73xa1wvhExFFgD2ARYD0sqHS3pzE7HfRT4OrCPpOf6e50DjfKAtjdwJDCzpNc7Nb31W8YgHc8kSZKkJajIEtU+rT49ImKmIuD9ZdzVfi+ObM4FzIm1Pr+M9TD3qnO0s0pEfALreG6MZ4mfCRzWaD4qx8yLz3UHYHbcVDWx/1fb3lRsZaaGpFJEjMLTjf4saedSR/x21ZYiYhNgQUnH99Xa0vFMkiRJkiYREVsBB+IGohdxJ/hDuBbyAjxTuyWc7Ig4GI9l/AVuYrmvsm8Y7uQ/GE/ReQhYGDiyFcaCtgvFsbwQWLza2R4RK+Ga0M8AiwJjJd3dF2vI5qIkSZIk6WcaESlJ50XEVcDKwIexHuatwMt10+3sjkrK9njg1lLjWd2/Nk7xBvAjXP85CY9yvKR/VzvguQJYtdKUtwCeorU28ATQEJwfi4cX9DoZ8UySJEmSJtEKdZvvl9Ic9X1c73k9nnT0Zvn/fknbNHF5A45qiUqpxZ0IbAvMC/wJNxytjeXJ+mxSVna1J0mSJEmT6Ox0du5sb0UiYlREHIOdmTG4XnU9SZNxbeez2PkkIoa2wzm3AhWnc12cbt8Hy17tDRyNR5v+uOF0NhQjepuMeCZJkiRJ0mtExMbARcB38Kz2+4H5gVNxLevGku7s9JqFgeGS7u/n5Q4YImI5YCdcx/k8FvQ/BU+TugZ4GtimyJRNG+3a26TjmSRJkiRJr1Lm0D/QqFONiD3xyMyzgcnAaGADnHpfC1gK1xeOl/Tfpiy6zYmIs7DU1cXAuZJuj4iR2PmcRdJGnY4fVmSXelVFIpuLkiRJkiTpVST9o/F1RKyHm4mGAQvhee6fxFG2IbjG8BU8YnNfYK/+Xm87U3EcDwEuk3RxZffRWCt283LsisDLwEbAQqX56EVcC9orZMQzSZIkSZJep6Il+XHgUFxPeBtO816NndANgOXxfPcLgCM6i+knvU9EzII1ZI/E8lf3A1/Cne1LAf8BHsaz3gGWkXRPb/zujHgmSZIkSdLrNBqnJP1fRGwn6RmAMrVpXeArwMewUP6mkm5v2mIHEOWB4JWirXoPMDOwKo5+PkKH83ksHm5wBn5Y6BXS8UySJEmSpE+o6JU+ExEz4xrDTbHj+RTwTUnndnrNSnis4839v+IBwSBKBzt2LmeT9DhARHwGp+TXBn5ejrlX0su99ssz1Z4kSZIkSV9SnMnPY03PkbjD/YjGOMdyzAJ48tHOZdOw6v6k74iIpbDDuQUeYHAoMEXS1Mox6wNXf9BGo4x4JkmSJEnSZ0TEkngE6Fy4tvO7ncY1zozHNR6AG1m+DUzAk5B2ftcPTHqNiJgL/+33AZ7EY00vB56u6H5+GuuvbgccBez3QX5nRjyTJEmSJOlTiqD8VZJ+22n757GY/BI4Cnoy8E+cir8QN7WktmcfEBFrAL8q354MnAY8LOmNsn8xPE5zHG4M+wweCjBe0ovv9/dmxDNJkiRJkj6hIUQu6Rudto/FDucGeH74rsAfJb1U9t8CXAsciCcfJb3P48DNwOHAXZJegWkR6K/iLvfZcPPX0rjB6LAP4nRCRjyTJEmSJOknImJ+nNbdAzs0RwK/lfR0p+OG4JTuGsBWKbHUu1SkrmaW9Gpl+0bY6QwceT4NmBvLKp0s6YoP+rvT8UySJEmSpM+JiJlwqnZZ7FT+Enik82jGiJhJ0v8i4gAc7Vy2EY1L+oYiHL8TsBoenXkRcCKwHJ50dB4egfoydMx9fz9kqj1JkiRJkj6lTM/5X0T8ClgAuLDaYFQ5blClk/1DOL07FE82SvqAiFgFO5f/xc1f50j6Y0R8EtfdniPpm731+zLimSRJkiRJn9JI7ZavHwXOxN3tb0TEIGBQpYt6CO6gPr0cc2iTlj1giIijgZsaqfTS7X4+jn5uI+nNsu1tXJc7MzAaOLhRl9tTMuKZJEmSJEmfUuoJZyrRzF1w7eAfgEuLQ9pwSpfEAvO7AJOx85n0EZU57vtWHP/F8N99DWBrYLeIWAu/R2sDfwFG4IajUThF32My4pkkSZIkSb8SEafi+sHfYSmfYVhCaW3s8EwGdpOkpi1ygFFpOBoL/Bp4A6ffF8CySy8AfwVWwmoD9+I60BPfi9B/Op5JkiRJkvQLjQhbRMwJbIWlfIYCs+K07lPA4ZLOKcdPS9En/UdErAq8Crwp6W9l2+rAt4Dx+GHhZOCB95pqT8czSZIkSZKmEBGjgWVwdO1NSVdV9g3+IN3TyXunK0c/IsbgcZobAzfgcZp/qup5RsSCkh7tye9IxzNJkiRJktqQDmc9iIh5gT3Lv4dxdPpK4NlKPeg4rMs6v6SVe/Jz0/FMkiRJkiRJADv+uGFob2Au4FisQvBIo5YzIpYCdgPWxCn55XGJxEEz+vnpeCZJkiRJkiTTiIg7gIeAg4F/NqYbRcQI7JRugcdp/h47pYsAZwELz2jKVMopJUmSJEmSJFQkr9YC3pL0QmXf5sAOwOLAPXia0YVF43MEMAWPQJ04vd+REc8kSZIkSZKkSyJiJWB3YEXgCeASPM3oybJ/EDAYmIDT7xtKeri7n5cRzyRJkiRJkuQdlFrPQ4AvAlOBy4GfS/p72T+ocWyJej4HzI5F5dPxTJIkSZIkSXpG0Vt9HfgwsJmkixv7ulEeeBJYAnh9ej93cK+vNEmSJEmSJGlZGtFMSYfgaOdnkPtJAAABOklEQVQC1X2dnc6ImB84CLgTeLBES7skHc8kSZIkSZJkGmV0ZiMrvg/wvYhYsbGvemxEjCzHrAmcLel1PNe9S7K5KEmSJEmSJOmWiJiMI5+nS7oiIoYBI4CVcR3oMsAhkibN6GdlxDNJkiRJkiR5FxExpHy5C/AKcF5EXA38EpgMXIEF5LcCjimvGdTFj5pGRjyTJEmSJEmSLmnMb4+IUcC2wPrAPMCjwNnAHcB/ejrmNB3PJEmSJEmSpMdExEhJU9/PazPVniRJkiRJksyQSrf71Bml1LsjI55JkiRJkiRJv5ARzyRJkiRJkqRfSMczSZIkSZIk6RfS8UySJEmSJEn6hXQ8kyRJkiRJkn4hHc8kSZIkSZKkX0jHM0mSJEmSJOkX0vFMkiRJkiRJ+oV0PJMkSZIkSZJ+4f8D5OTZMGOxzEUAAAAASUVORK5CYII=\n", 404 | "text/plain": [ 405 | "
" 406 | ] 407 | }, 408 | "metadata": { 409 | "needs_background": "light" 410 | }, 411 | "output_type": "display_data" 412 | } 413 | ], 414 | "source": [ 415 | "# hyperparameter choice plot\n", 416 | "fig, axs = plt.subplots(1, 1, figsize=(10,5))\n", 417 | "boxplot_all_methods(axs, results_tar_plot*100,\n", 418 | " title=\"\", names=names_short,\n", 419 | " color=np.array(COLOR_PALETTE)[:len(names_short)])\n", 420 | "\n", 421 | "plt.savefig(\"paper_figures/MNIST_%s_M5_Yintervention.pdf\" %perturb, bbox_inches=\"tight\")\n", 422 | "plt.show()" 423 | ] 424 | }, 425 | { 426 | "cell_type": "code", 427 | "execution_count": null, 428 | "metadata": {}, 429 | "outputs": [], 430 | "source": [] 431 | } 432 | ], 433 | "metadata": { 434 | "kernelspec": { 435 | "display_name": "Python 3", 436 | "language": "python", 437 | "name": "python3" 438 | }, 439 | "language_info": { 440 | "codemirror_mode": { 441 | "name": "ipython", 442 | "version": 3 443 | }, 444 | "file_extension": ".py", 445 | "mimetype": "text/x-python", 446 | "name": "python", 447 | "nbconvert_exporter": "python", 448 | "pygments_lexer": "ipython3", 449 | "version": "3.6.4" 450 | } 451 | }, 452 | "nbformat": 4, 453 | "nbformat_minor": 2 454 | } 455 | -------------------------------------------------------------------------------- /MNIST/MNIST_read_and_plot_whitepatch2M.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np\n", 10 | "import matplotlib.pyplot as plt\n", 11 | "import seaborn as sns\n", 12 | "\n", 13 | "import pandas as pd\n", 14 | "\n", 15 | "plt.rcParams['axes.facecolor'] = 'lightgray'\n", 16 | "sns.set(style=\"darkgrid\")\n", 17 | "np.set_printoptions(precision=3)" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 2, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "def boxplot_all_methods(plt_handle, res_all, title='', names=[], color=[]):\n", 27 | " res_all_df = pd.DataFrame(res_all.T)\n", 28 | " res_all_df.columns = names\n", 29 | " res_all_df_melt = res_all_df.melt(var_name='methods', value_name='accuracy')\n", 30 | " res_all_mean = np.mean(res_all, axis=1)\n", 31 | " \n", 32 | "# plt_handle.set_title(title, fontsize=15)\n", 33 | "\n", 34 | " plt_handle.axhline(res_all_mean[2], ls='--', color='b')\n", 35 | " plt_handle.axhline(res_all_mean[1], ls='--', color='r')\n", 36 | " ax = sns.boxplot(x=\"methods\", y=\"accuracy\", data=res_all_df_melt, palette=color, ax=plt_handle)\n", 37 | " ax.set_xticklabels(ax.get_xticklabels(), rotation=-60, ha='left', fontsize=20)\n", 38 | " ax.tick_params(labelsize=20)\n", 39 | " ax.yaxis.grid(False) # Hide the horizontal gridlines\n", 40 | " ax.xaxis.grid(True) # Show the vertical gridlines\n", 41 | " ax.set_xlabel(\"methods\")\n", 42 | " ax.set_ylabel(\"accuracy\")\n", 43 | " \n", 44 | " ax.set_xlabel(\"\")\n", 45 | " ax.set_ylabel(\"Accuracy (%)\", fontsize=20)" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 3, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "# perturb = 'whitepatch2M'\n", 55 | "# perturb = 'rotation2M'\n", 56 | "perturb = 'rotation2Ma'\n", 57 | "# perturb = 'translation2M'\n", 58 | "M = 2\n", 59 | "subset_prop = 0.2\n", 60 | "lamL2 = 0.\n", 61 | "lamL1 = 0.\n", 62 | "lr = 1e-4\n", 63 | "epochs=100" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 4, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "names_short = [\"Original\", \"Tar\", \"Src[1]\", 'DIP[1]', 'DIP[1]-MMD']\n", 73 | "\n", 74 | "prefix_template = 'results_MNIST/report_v8_%s_M%d_subsetprop%s_%s_lamMatch%s_lamMatchMMD%s_epochs%d_seed%d'" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 5, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "repeats = 10\n", 84 | "nb_ba = 3 # Original, Tar, Src[1]\n", 85 | "results_src_ba = np.zeros((M-1, nb_ba, 2, repeats))\n", 86 | "results_tar_ba = np.zeros((nb_ba, 2, repeats))\n", 87 | "for seed in range(repeats):\n", 88 | " savefilename_prefix = prefix_template % (perturb,\n", 89 | " M, str(subset_prop), 'baseline', 1., 1., epochs, seed)\n", 90 | " res = np.load(\"%s.npy\" %savefilename_prefix, allow_pickle=True)\n", 91 | " \n", 92 | " results_src_ba[:, :, :, seed] =res.item()['src']\n", 93 | " results_tar_ba[:, :, seed] = res.item()['tar']" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 6, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "lamMatches = [10.**(k) for k in (np.arange(10)-5)]" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": 7, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "nb_dip = 3 # DIP DIPOracle DIP-MMD\n", 112 | "results_src_dip = np.zeros((len(lamMatches), M-1, nb_dip, 2, repeats))\n", 113 | "results_tar_dip = np.zeros((len(lamMatches), nb_dip, 2, repeats))\n", 114 | "for i, lam in enumerate(lamMatches):\n", 115 | " for seed in range(repeats):\n", 116 | " savefilename_prefix = prefix_template % (perturb,\n", 117 | " M, str(subset_prop), 'DIP', lam, lam, epochs, seed)\n", 118 | " res = np.load(\"%s.npy\" %savefilename_prefix, allow_pickle=True)\n", 119 | "\n", 120 | " results_src_dip[i, :, :, :, seed] =res.item()['src']\n", 121 | " results_tar_dip[i, :, :, seed] = res.item()['tar']" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": 8, 127 | "metadata": {}, 128 | "outputs": [ 129 | { 130 | "name": "stdout", 131 | "output_type": "stream", 132 | "text": [ 133 | "8\n", 134 | "8\n", 135 | "7\n" 136 | ] 137 | } 138 | ], 139 | "source": [ 140 | "# choose lambda based on the source test performance\n", 141 | "lam_index_dip = np.zeros(nb_dip, dtype=int)\n", 142 | "for i in range(nb_dip):\n", 143 | " src_test_acc_all = results_src_dip[:, 0, i, 1, :].mean(axis=1)\n", 144 | " # choose the largest lambda such that the source performance does not drop too much (5%)\n", 145 | " lam_index = 0\n", 146 | " for k, src_test_acc in enumerate(src_test_acc_all):\n", 147 | " if src_test_acc > src_test_acc_all[0] * 0.95:\n", 148 | " lam_index = k\n", 149 | " lam_index_dip[i] = lam_index\n", 150 | " print(lam_index)" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": 9, 156 | "metadata": {}, 157 | "outputs": [], 158 | "source": [ 159 | "results_tar_plot = np.concatenate((results_tar_ba[:, 0, :],\n", 160 | " results_tar_dip[lam_index_dip[0], 0, 0, :].reshape(1, -1),\n", 161 | " results_tar_dip[lam_index_dip[2], 2, 0, :].reshape(1, -1)), axis=0)" 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": 10, 167 | "metadata": {}, 168 | "outputs": [ 169 | { 170 | "data": { 171 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAABECAYAAADHuCM8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAACHUlEQVR4nO3YMWoUYQCG4S+uEGENBgshkGsELfQQAasELAQv4AG8glhYpbaWHEGSQrH3BIEFi2AgC6YIa6NgFt1uMh/J8zQzzM/AV73Fv7ZYLALQ4s7YAwD+JkpAFVECqogSUOXuirP1JDtJZkkur2cOcAtMkmwl+ZrkYvlwVZR2khwNNArgWZLj5Y+rojRLksPDw8zn86FGjWpvby+zx0/GnjGYrS+fs/v209gzBvH822n2D3bz4dXHsacMYv9gN+/evB97xiA2Njfy8vWL5Hdjlq2K0mWSzOfznJ+fDzCtw+XJydgTBjX78XPsCYM4/z6/8ryJzk7Pxp4wtH9eC7noBqqIElBFlIAqogRUESWgiigBVUQJqCJKQBVRAqqIElBFlIAqogRUESWgiigBVUQJqCJKQBVRAqqIElBFlIAqogRUESWgiigBVUQJqCJKQBVRAqqIElBFlIAqogRUESWgiigBVUQJqCJKQBVRAqqIElBFlIAqogRUESWgiigBVUQJqCJKQBVRAqqIElBFlIAqogRUESWgiigBVUQJqCJKQBVRAqqIElBFlIAqd1ecTZJkOp1e05RxTLa3x54wqK3Ne2NPGMT9R9Mrz5vowcMHY08YxMbmxp/Xyb/O1xaLxf/+fZrkaIBNAEnyLMnx8sdVUVpPspNkluRyuF3ALTNJspXka5KL5cNVUQK4di66gSqiBFQRJaCKKAFVfgGw30VQqzUq5QAAAABJRU5ErkJggg==\n", 172 | "text/plain": [ 173 | "
" 174 | ] 175 | }, 176 | "metadata": { 177 | "needs_background": "light" 178 | }, 179 | "output_type": "display_data" 180 | } 181 | ], 182 | "source": [ 183 | "COLOR_PALETTE1 = sns.color_palette(\"Set1\", 9, desat=1.)\n", 184 | "COLOR_PALETTE2 = sns.color_palette(\"Set1\", 9, desat=.7)\n", 185 | "COLOR_PALETTE3 = sns.color_palette(\"Set1\", 9, desat=.5)\n", 186 | "COLOR_PALETTE4 = sns.color_palette(\"Set1\", 9, desat=.3)\n", 187 | "# COLOR_PALETTE2 = sns.color_palette(\"Dark2\", 30)\n", 188 | "# COLOR_PALETTE = COLOR_PALETTE1[:8] + COLOR_PALETTE2[:30]\n", 189 | "COLOR_PALETTE = [COLOR_PALETTE1[8], COLOR_PALETTE1[0], COLOR_PALETTE1[1], COLOR_PALETTE1[3], COLOR_PALETTE4[3]]\n", 190 | "sns.palplot(COLOR_PALETTE)\n" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": 11, 196 | "metadata": { 197 | "scrolled": false 198 | }, 199 | "outputs": [ 200 | { 201 | "data": { 202 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAGYCAYAAAC6SW7EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd5xcZdn/8c/uZlMgJIHQS4y0S6pYQErooHQEUZAmD0VpKgKx/UTFB6MIUgQBARUwIPiAUgUUCIQmSJV6UUMEAiQhm0JI3f39cd1DJrMlezYzO2dmv+/XK68hc86cvW92cq5zt+tuaGtrQ0REJIvGahdARERqj4KHiIhkpuAhIiKZKXiIiEhmCh4iIpJZv2oXIAcGAJsDk4CFVS6LiNSPJmA14N/A3CqXpewUPCJw3F/tQohI3doWeKDahSg3BY9ocTBt2ge0tvbOmpfhwwczdeqsXvlZ1aD61bZ6rl9v1q2xsYHll18W0j2m3ih4pK6q1ta2XgsehZ9Xz1S/2lbP9atC3eqyO1wD5iIikpmCh4iIZKbgISIimSl4iIhIZgoeIiKSmYKHlFVLyzRGjx5NS0tLtYsiIhWk4CFldeONN/Dcc89x003XV7soIlJBCh5SNi0t07j//nG0tbUxfvy9an2I1DEFDymbG2+8gcLOlG1trWp9iNQxrTDvo8aOvYKJEyd069zp01u61Yr48MPZH/33ggULuPvuf/DQQ12n9Bk2bBhDhw7rVjlGjBjJoYce0a1zRaSyFDySSb+7iHnTFt0gl9t8C4btuDOtc+fy1vnntDt/yDajGLrNtiycOZO3L76w3fFhO+zEclt8jvnvT+Wdyy9d7Ng7/fsxeIddGLzZp5j3ziTeveqKdp9fYa99WHbDjZgz8Q0mX3tNu+Mr7n8Ag9Zdjw9feZkpf23/hL/SQQczcMTH+OD553j/1pvbHX/p3beZMPldBgFDaGh3fAptLASWAVppnxK0OMFD+08v8uGHszs8Xvj8gg9n0zpp8dQ/rcDkdMYQYFC6wmoTJvDftyfRNHgwqx//TQAm3/B/zHn1lcU+32/5FVjtmG8A8N61VzN34sTFjvdfdVVWOfx/AHj3qj8y7513Fjs+YMQIVj7oEAAmXfY7Fkx7f7HjH2yyIcvuvi8Ab190AQtnLZ4raZkNNmT43nH8zfN+Tdu8eYsdX/aTm7HCF3YH4L+/+kW7/zeV/O4BLP/53br87jUfciCssXbFvnurHH4E/VddjVlPPcm0f9zR7viqR3+d5hWGM/PRR2i59552x1c/7kSalluO6Q/ez4wH2z+crPHtk2kcMICWcXcz89+PLnbsnf79WPWk0QC8f+ftfPD0U4sdb+jfnzVPOgWAqbfcxOwXnl/seJbv3tRbbmL4EYe0K1+9UPDoo5YftjzvzprJgNaFNC5on3pnUHMzCxsa6L9wIcyfT2tb62LHFzRGj2djWxuNha4qYH7ROQ0NjTSx6HhHnx/Q0Niu77QBGNS/PwD9FyygsTV+dr/m5uwVFZGKaGjr4B92HzMSeH3q1Fm9ljBtpZWWY/Lkmb3ys3rbFVdczrhx/2SnnXbla187utrFqYh6/v1BfdevN+vW2NjA8OGDAT4OTOiVH9qL1PKQsvriF7/Ee++9zb77HlDtoohIBWm2lZTVsGHLc9ZZZzFsWPcGwUWkNil4iIhIZgoeIiKSmYKHiIhkpuAhIiKZKXiIiEhmCh4iIpKZgoeIiGSm4CEiIpkpeIiISGYKHiIikpmCh4iIZKbgISIimSl4iIhIZgoeIiKSmYKHiIhkpuAhIiKZKXiIiEhmCh4iIpKZgoeIiGSm4CEiIpkpeIiISGYKHiIikpmCh4iIZKbgISIimSl4iIhIZgoeIiKSWb9qF0BEpKceeOA+xo8f161zp09voampkcGDh3T7+ttttyOjRm3f0+LVNQUPEekTWlpaaGxsyBQ8pHO5CB5m1gAcnf5sBDQALwCXA5e6e2sHn9kLOBX4FNAEPAdc5O5X9la5RaS6Ro3avtstgzFjfkpzcxOjR59W4VL1DXkZ8xgLXAqMBP5MBI1lgIuBK0pPNrMTgVuAjdNnLwNWB64ws7N7pcQiIn1Y1YOHme0HHAy8Dmzk7se4+7eBzYBbgcPMbP+i80cCZwPvA5919xPc/TvApsCrwClmtlUvV0NEpE+pevAA9kuvv3b3KYU33X0eUGhfnlh0/pHAAOBCd59QdP40YEz667EVK62IiOQieKyaXl/r4FjhvW3NrH/6753S6x0dnH97yTkiIlIBeQgehdbGxzs4tnZ67Vf035ZeXyo92d0nAR8Aa5rZMuUspIiILJKH4HFbej3ZzFYovGlmzcDpRectn16HptfpnVxvesl5IiJSZnmYqnstcBjwBeB5M7sJmAPsAqwGTARGAO2m64qISHVUveXh7guBvYHvA5OBr6U/LwNbAzPTqe+l1yW1LJbUMhERkaWUh5YH7j4fODP9+YiZDQTWA6a4++uF04EVgfWBh0vOXw1YFnjT3WdXutwiIn1V1VseS3AQ0J9YOFhwT3rdrYPzdy85R0REKiAXwcPM2iWbMbPNgLOAacAviw79EZgLnJgWDBbOXx74YfrrJRUrrIiI5KPbCvinmX0IPEuMcWwA7Al8COzt7m8XTnT3181sNPAb4DEzuw6YBxwArEksNny49AeIiEj55KLlAVwPLAccCpxMpBq5FNjQ3e8rPdndLwD2IZIhHg58HXgHOMLdT+2tQouI9FW5aHm4+1lEF1WWz9xCJEcUEZFelpeWh4iI1BAFDxERyUzBQ0REMlPwEBGRzBQ8REQks1zMthIRKRg79gomTpxQ9uu+8cYEGhsbGDPmp2W/NsCIESM59NAjKnLtPFLwEJFcmThxAi+/5CwzYNmyXrdtQSsLgbfeeLOs1wWYPfeDsl8z7xQ8RCR3lhmwLBuN2Ljaxei25yY+W+0i9DqNeYiISGYKHiIikpmCh4iIZJZ5zMPMPkFsC7sikfX2PeAZd59R5rKJiEhOdSt4mNlOwFHEvuIrdnBKq5k9SWTH/YO7TylfEUVEJG+6DB5mtj/wc2LL1wbgLeAmIv35+8AgYDjwCWAz4LPA6WZ2FfBjd3+3ckUXEZFq6TR4mNl4YBTwAvAD4Fp3n9jF+f2BHYGvEftyHGRmh7n7zeUtsoiIVFtXLY8hwBe7e/N393nAncCdZrYysSWsLX0RRUQkbzoNHu6+WU8v6u7vASf19PMiIpJvmqorIiKZLVV6EjMbCKyT/vqqu89Z+iKJiEje9Sh4mFk/YhbWN4EB6e25ZnY+cJq7LyhT+UREJId62vI4GzgWGAs8DgwE9ga+CzQDp5aldCIikktLWufR4O5tHRw6DDjF3X9b9N65ZnZHOqbgISJSx5Y0YP6QmXU03XYw8HIH77+SjomISB1bUrfVfOApM/sZcKa7t6b3HwbONrNvAE8S4x57EwsEH6pUYUVEJB+6bHm4+3bAaGKF+aNmtmk69C0iLckDwAdEqpKrgBbg2xUrrYiI5MIS13m4+4XApsA04N9mdjrwPLAuMWh+XvpzDLCeuz9fueKKiEgedGu2lbtPAHY1s6OBs4D9gCPd/bIKlk1ERHIq0wpzd78c2Bh4A3jYzM40swFL+JiIiNSZbgcPMxsO4O5vufvewP8ARwJPm9nWFSqfiIjkUJfBw8yaU+tiJvCemc00s1+ZWbO7jwU2Ap4FxpvZ+Wa2TG8UWkREqmtJLY8fEbOtHiXGOh4FTgFOg8ie6+4HAAcBXwGeMbMdK1dcERHJgyUFj0OBf7j7zu7+fXffGbgLOKT4JHe/HtiQWP9xV0VKKiIiubGk4LEi8EzJe/+hg33M3X2aux9KLBYUEZE6tqTg8RRwgJmNADCzNYEvpfc75O5/L1/xREQkj5a0zuMU4G7gVTObDKwEfAgcWOmCiYhIfnUZPNz9MTPbgMiUOwKYCIx197d6o3AiIpJPS1xh7u5vA2f2QllERKRGaA9zERHJrNPgYWYHLM2FzWw1M9tqaa4hIiL51FXL4y9m9riZHZglf5WFc4mNoXZZ6hKKiEjudDXmsTNwLvBnYLqZ3QQ8CDwGTCJStA8k9vX4BLAl8AXgs8QmUucTqdpFRKTOdBo83H2cmX0K+CpwAnA4MeuqMw3EZlDnA79JadxFRKQOLWmqbhtwDXBN2st8F2AUMW13OLHm4z1i1fm9wD3u/mFPCmJmexK7EG6Yrj0JeBw4x90f7uD8rYncW1sCg4g91f8AXODuC3tSBhER6Z5ubQYF4O4OOPDbchfCzM4EvgtMBW4EphA7Fe4LfMnMDk9ZfAvn7wvcAMwBriO2wd2b6GbbBvhyucsoIiKLdDt4VIqZrQqcCrwLbOru7xUd2xG4B/gZMDa9NwS4DFgI7ODuj6X3T0vnHmBmB7n7tb1aERGRPiQP6zw+RpTjkeLAATHuAswk0qIUHJD+fm0hcKRz5xDdWADHVbTEIiJ9XB6Cx8vAPGALM1ssW6+ZbQcsx+Jp3ndKr3d0cK3xwGxga22PKyJSOVUPHu7+PvA9YBXgeTO71Mx+YWZ/Af4B/BP4RtFHLL2+1MG1FgCvE91xa1e04CIifVjVxzwA3P08M5tAzJY6pujQK8AVJd1ZQ9Pr9E4uV3h/WDnLKCIii1S95QFgZt8FrgeuANYBlgU+A7wGXG1mv6pe6UREpFTVWx5mtgORtfdv7n5y0aEnzGw/onvqFDO7xN1fY1HLYigdK7zfUonyiohIhpaHmX26QmXYK72OKz3g7rOBR4lyfqrwdnpdv/R8M+sHfBxYQLRaRESkArJ0Wz1mZo+Y2ZFmtkwZy1CYFbVSJ8cL789Lr/ek1906OHc7YBngIXefW57iiYhIqSzB4zbg08QCvbfN7AIz26QMZbg/vX7dzNYoPmBmuxMrxucAD6W3rydWoB9kZp8tOncgcEb668VlKJeIiHQiS3qSvc1sTeBo4EgiWeLxZvYv4HfAdT182r+eWMexC/CCmf0NeAfYgOjSagC+7+5TUzlmmNkx6XP3mtm1RHqSfYhpvNcTKUtERKRCMs22cvc33f2nwEgi79TfgS2APxKtkXPTnudZrtkK7AF8B3ge2A84hUh4+HfgC+5+fslnbgS2JxYFfgn4JpEG/mTgoJTQUUREKqRHs63SDf8W4JbUGjkKOBb4FvAtM7sfuNDdr+/m9eYTe390e/8Pd3+QCDoiItLLyrHOY0NgUyKNegORGXdb4Lq0E+HIMvwMERHJkR61PMxsZWLc4xiiCwvgbuAi4GYi2eFoIq3IRaiFICJSVzIFDzPbmQgI+wLNxFa05wEXu/srRae+TgymDwC+UqayiohITnQ7eJjZy0SywQZiH/OLiLToc7r42MtEqhEREakjWVoeaxC5py5y98e7+ZmrgXZbyIqISG3LEjxWd/dM+aLc/b/Af7MVSURE8q7bs62yBg4REalfWRIjHmtmr5rZ6p0cXyMdP6p8xRMRkTzKss7jYGCSu7/d0UF3fwt4Ezi0HAUTEZH8yhI8DHh6Cef8B/hEz4sjIiK1IEvwGMqSN1iaASzf8+KIiEgtyBI8JhFpSLqyKTC558UREZFakGWq7jjgMDMb5e4PlB40s22B3YGx5SqciPQ906e3MHvuBzw38dlqF6XbZs/9gOnT+9aE1CzB40zgQOAuM7sIuAN4i1g8uDtwHDA3nSciInUsy2ZQbmZfAa4BTgK+XXS4gRjvONjdXyhvEUWkLxk6dBizWmax0YiNq12Ubntu4rMMHTqs2sXoVZkSI7r7bWa2NnAE8DlgGDGI/i/gysJufyIiUt8yp2RPAeLXFSiLiIjUiHJsBiUiIn1MTzeDWpMYKB/Q0XF3H780hRIRkXzLuhnU54FzWfIq8qYel0hERHIvS2LELYFbiUHyC4kZVuOBy4AX099vAX5W/mKKiEieZBnz+AEwB9jc3QvTdMe5+7HAxsAZwC7A9eUtooiI5E2W4LEVcHNJVt1GAHdvc/cfAy8Ap5exfCIikkNZEyNOLPr7PNrvT/4gsN3SFkpERPItS/B4j8Uz5r4HrFNyTjMwaGkLJSIi+ZYleLzE4sHiX8CuZrY+gJmtCnwJeLl8xRMRkTzKEjzuALY3sxXS388nWhlPmtm/iRlXKwHnlbeIIiKSN1mCx++I8Yz5AO7+IPBl4HVittUk4Dh3v6rchRQRkXzJklV3BvBIyXt/A/5W7kKJiEi+ZVkk+Acz+04lCyMiIrUhS7fVwcDKlSqIiIjUjizBYwIKHiIiQrbgcQ2wu5ktv8QzRUSkrmUJHr8AHgPGmdleZrZKhcokIiI5lyUl+5z02gDcBGBmHZ3X5u492idERERqQ5ab/P1AW6UKIiIitSPLOo8dKlgOERGpIdrDXEREMlPwEBGRzLrdbWVmP+7mqW3u/r89LI+IiNSALAPmP+3iWGEgvSH9t4KHiPTY7Lkf8NzEZ8t6zfkL5gHQ3K9/Wa8LUd6+Jkvw2LGT94cBmwPfAm4DLlnaQolI3zVixMiKXPeNNybQ2NjAGmutWZHrV6rceZVlttV9XRy+ycyuAx4Frl3qUolIn3XooUdU5LpjxvyU5uYmRo8+rSLX72vKtpjP3Z8xs5uAH5IWEXaHmR0B/HEJp7W6e1PJ57YGfgRsSWxK9TLwB+ACd1+YoegiIpJRuVeCTwT2zviZp4DTOzm2LbATcHvxm2a2L3ADser9OuD99HPPBbYhNqkSEZEKKXfw+BzwYZYPuPtTRABpx8weTv95adF7Q4DLgIXADu7+WHr/NOAe4AAzO8jd1X0mIlIhWabqjujiGmsBxwCjgL+UoVyY2SZEl9RbxEB8wQHEXulXFQIHgLvPMbMfAXcDx6GxFxGRisnS8phA17mtGohxh1OXpkBFvp5ef18yhrFTer2jg8+MB2YDW5vZAHefW6ayiIhIkSzB4yo6Dh6twDRiptVN5bhhm9kg4FCia+ry0sPp9aXSz7n7AjN7HdgIWBt4YWnLIiIi7WWZqntEBctR6ivE+pHb3P2/JceGptfpnXy28P6wShRMRETym9uq0GX1u6qWQkREOpRlwHwdYhrsbe4+tYPjKwJ7AA+4+2s9LZCZbQRsDbwJ/L2DUwoti6EdHCt+v6WnZRARka5laXl8H/g1MKOT49OBs4HRS1mmzgbKCzy9rl96wMz6AR8HFgA9DmAiItK1LMFjB+Aud5/f0cH0/j9ZNBsqMzMbCBxGDJT/vpPT7kmvu3VwbDtgGeAhzbQSEamcLMFjDWK6blcmAqv3uDSxMnx54PYOBsoLrgemAAeZ2WcLb6bAc0b668VLUQYREVmCLFN15wFDlnDOcizdPueFLqtLOzvB3WeY2TFEELnXzK4l0pPsQ0zjvZ5IWSIiIhWSpeXxLLCnmTV3dNDM+gN7Ac/3pCBmtgGxQr2zgfKPuPuNwPbEosAvAd8E5gMnAwe5+9IEMBERWYIsLY+xwEXAX8zsOHd/p3DAzFYl9vFYC/hVTwri7i8Qq9S7e/6DxOwuERHpZVmCx6XA/sC+wK5m9h8i79QawKbEQPVdaDMoEZG61+1uK3dvBfYEfkl0EW1JdBltSYyHjAH2TOeJiEgdy5SSPU3H/WHKXvsJIgVIC/CigoaISN/Ro/08UqDo0cC4iIjUvtylJxGR8nnggfsYP35ct8+fPr2FpqZGBg9e0qz8sN12OzJq1PY9LZ7UsDymJxGRKmlpaWHatGnVLobUgCzdVjuwhPQkZrZU6UlEpLxGjdo+U8tgzJif0tzcxOjRp1WwVFIP8paeREREakCW4NEb6UlERKQG5CY9iYiI1I7cpCcREckqy2yyN96YQGNjA2PG/LTb19dsss4pPYn0eVluQJrKWruGDRtGU1Ned96uPd0OHu7eamZ7AqcDxxFpSQpagPOA07XSXOpZS0sLjY0N3Q4eUllZZ5OttNJyTJ48s4Il6jsa2tqyj2+bWSP1k55kJPD61KmzaG3tnbH+ev8C13P98jCVdezYK5g4cUJFrl3o2llrrY+V/dojRozk0EOPKPt1s+jN72ZjYwPDhw+G2Bp7Qq/80F5U1vQkKajs7e43LW3BRKRjEydO4BV/ieWahpb92g2tjbQB777yblmvO3Ph9LJeT6qvR8GjlJl9DDga+B9gNaCpHNcVkY4t1zSUzy23bbWL0W2PzLy/2kWQMutx8DCzJmLw/OvALsS03zZi0FykqirVtdOTGTvdlYduHZHuyhw8zGxt4BjgCGDl9PYU4HfA7939jbKVTqSHJk6cwAsvvwyDVijvhVv7QSu88Ga73KBL58P3u33q9OktzFwwvaae5mcumM7A6QOqXQwpo24FDzPrB+xHtDJ2JFoZ84C/EhtC3eTuP65UIUV6ZNAK8Indq12K7nnx9mqXQCSTLoOHma1HtDK+BqxI7DH+OHAFcI27TzOzWp1lJVKThg4dxpzJc2tuzGPo0GHVLoaU0ZJaHk6MY7wLnANc4e7PVbxUVXDJTc8ybcbcj/6++QYrs9On12Tu/IWc95en252/zSarMWrT1Zg5ex4X/e3Zdsd3/PQabLHBKrw/Yw6X3bL4xLTm/k3stNkabLbeikya+gFX3eHtPr/XNiPZaOQKTHx3Jn++6+V2x7+0/Tqsu+ZQXnlzOjfc92q741/dZT1GrLIcz014n1sfnNDu+OG7GasNX5anXp7CnY9ObHf8mL03ZIUhA3n0hXcZ98Rb7Y4fv9/GLLdMfx74zyQefGZSu/qd8MWNGdDcxD1PvMm/X3iv3ee/d8inAbjjkYk8/cqUxT/f3MjJX9kMgJsffJ0XJiyeInzwoGZO2H8TAK6/91VefWvxmTzLD1nUPdI4rx8NrSULwxpaWThgAQBNc/tB2+LH2xpbae1fON4MbQ3ZjjctpLV5YRyf00w8c3V2vH+8ufIo3mxo5syrn1jid28GqzBz4Us8/MEjzFl9q3bHm1tepXnWm7T2G8ScVbdod7z/tJfo98EkWpsHM2eVzyx2bOGsN+g3620GLZjHwgFDmbvSZu0+P2DKszTNmcrCgcOZu+LG7Y9PfoqmudNZsMzKzFthg7jukFHMbxjEmVc/UdHvHsBJX/lkp9+95v5NnPzlTwKV++59fe+NALjpwdc5cp9N2pWvXnSn26oNuB24oV4Dh9Sf6dNbYPb7tL16K20lN3ca2uIP0NrW0O7mv/jxxvapPrMcLw1cnR1vnc/85deBESOXWLfhw1dkiK3PQvoxqaF/u+NDBw5jOeYznwG828HxYQOHsSytzGMQ75Ucn//eRGicyyrrrslclmVyB59ffs0VGEQ/PmQIUzs4Pnyt4QxgILMZxvvpeDMwcMDAJdZNakeXiwTN7P8BRxEL6dqIlsgVwJ/cfVI6pxW43N2/XunCVshItEiwrPJQv+997yTefncyLFPmAfNKmf0+q6+yEmeeeV5Vi5GHRZCVpEWC5dNlohd3/7m7rw3sDvwNWAf4JTDRzG4zs6/0QhlFMhs6dFhpb1F5zP8w/pRbAxoTkJrSrdlW7n4ncKeZrQwcSSwI3B3YjWiRbGZmn3H3xytWUpEMRnSj+6cnPkrfsebwMl95eEXKnHUP86zrWJT4se/KtM7D3d8jWh6/NLOdiam7+wKfBR5NmXYvd/fflr2kIhlUarFdvXfrKPOsdFePV5i7+93A3Wa2IrFg8Gjgk8BvAAUPkRzImnUW8jFmJfm31Lmt3H0KcDZwtpntQAQRERGpY2VJjFjg7vcC95bzmiIikj9lDR4itaiSW5lqQFnqlYKHSAYaUBYJCh7S52krU5Hs9AglIiKZKXiIiEhmCh4iIpKZgoeIiGSm4CEiIpkpeIiISGYKHiIikpmCh4iIZKbgISIimSl4iIhIZrlKT5I2mDoR2ApYHpgKPAOc7+5/Lzl3a+BHwJbAIOBl4A/ABe6+sDfLLSLS1+QmeJjZr4DRwJvAzcAUYCXgM8AOwN+Lzt0XuAGYA1wHvA/sDZwLbAN8uReLLiLS5+QieJjZMUTguBL4urvPKzneXPTfQ4DLgIXADu7+WHr/NOAe4AAzO8jdr+2t8ouI9DVVH/MwswHAz4GJdBA4ANx9ftFfDyBaJNcWAkc6Zw7RjQVwXOVKLCIieWh57EoEg/OAVjPbE9iY6JJ61N0fLjl/p/R6RwfXGg/MBrY2swHuPrdCZRYR6dPyEDw2T69zgCeJwPERMxsPHODukwtvpdeXSi/k7gvM7HVgI2Bt4IWKlFhEpI+rercVsHJ6HQ20AdsCywGbAv8AtgP+r+j8oel1eifXK7w/rLzFFBGRgjwEj0IZFgD7uPsD7j7L3Z8B9iNmX21vZltVrYQiIrKYPASPlvT6pLtPKD7g7rOBO9Nft0ivhZbFUDpWeL+lk+MiIrKU8hA8PL12drOfll4HlZy/fumJZtYP+DjRinmtXAUUEZHF5SF43E2MdWxoZh2VpzCA/np6vSe97tbBudsBywAPaaaViEjlVD14uPsbwC3ACODbxcfM7PPAF4hWSWFq7vXE6vODzOyzRecOBM5If724wsUWEenT8jBVF+AE4FPAOWmdx5NE99MXiZXkR7v7dAB3n5FWpF8P3Gtm1xLpSfYhpvFeT6QsERGRCql6ywPA3d8kclhdCKxHtEB2IFok27j7DSXn3whsTywK/BLwTWA+cDJwkLu39VrhRUT6oLy0PEiLAL+Z/nTn/AeBPSpaKBER6VAuWh4iIlJbFDxERCQzBQ8REclMwUNERDJT8BARkcwUPEREJDMFDxERyUzBQ0REMlPwEBGRzBQ8REQkMwUPERHJTMFDREQyU/AQEZHMFDxERCQzBQ8REclMwUNERDLLzWZQteyBB+5j/Phx3Tp3+vQWmpoaGTx4SLfO3267HRk1avulKZ6ISNkpePSylpYWGhsbuh08RETySMGjDEaN2r7brYMxY35Kc3MTo0efVuFSiYhUjsY8REQkMwUPERHJTN1WnRg79gomTpxQ9uu+8cYEGhsbGDPmp2W/NsCIESM59NAjKnJtEZECBY9OTJw4gVdffZmhQ4eW9bpNTdHYmzLlvbJeF2D69Ollv6aISEcUPDoxfXoLbW1tZb/uwIEDy37Ngra2NqZPb6nY9UVEChQ8urBw4UJaWsp7M25tbak7DLgAACAASURBVAWgsbH8w00LFy4s+zVFRDqi4NGJTTbZjKFDh3Xr3OnTW7odZObOnQNAc3P/bp0/bNiwbpcDYsxDRKTSFDw6kWXQWSvMRaSvUfAogyyLBAFWWmk5Jk+eWcESiYhUltZ5iIhIZgoeIiKSmYKHiIhkpuAhIiKZKXiIiEhmCh4iIpKZgoeIiGSm4CEiIpkpeIiISGYKHiIikpnSk0ATQGNjQ6/+0N7+eb1N9att9Vy/3qpb0c9p6pUf2MsaKrFnRY0ZBdxf7UKISN3aFnig2oUoNwUPGABsDkwCtCGGiJRLE7Aa8G9gbpXLUnYKHiIikpkGzEVEJDMFDxERyUzBQ0REMlPwEBGRzBQ8REQkMwUPERHJTMFDREQyU/CQXDGzwWbWv9rlqCQzq9/cH4mZrV7tMlRSvdevOxQ8alC93nzM7GPADOCqapelEsysEcDd63Zlbgr+vwfeNLN90nt1c5+p9/ploRXmNcLM+hHBfmVgqrt/mN5vqJebkZkNAX4KfBvY1N2fq26JyiMF+82BQ4l0FROB37r7a1UtWBmlG+jRwCVFb7/m7utWqUhlVe/164k+GTFriZk1mtlewI3Av4AXgDvN7Ldmtm7ReTXfGnH3GcC1wKvAxVUuTjl9ARgLnAh8CfgO8Nd6eXI1s62BFhbdWH8EHA6saWanpXNqto71Xr+eUssjx8xsE+AK4FPprSlEsrU2YAXiCfZX7n5RVQpYAWY2APg6cD7wJXf/W5WLtFTMbCgwDtgM+CXwMLAicBbwIdHCmlaLLcjU738dsE166xrg++7+ppkNAy4E9gPWcvf3a62O9V6/pdXnomWtMLOjgPFE4HgY+BbwVWB9YBNgN2AW8HMz2z99pmb3DSiU3d3nEiny3wV+VdVClceqwCeA54Bz3f0Wd/8j0TU3CDivmoXrCTMbaGbnAG8SN9Z/A6Pc/dB0Y+3n7i3EzXY68JMqFjezeq9fuWgzqBwys88RX8gG4gb6B+B1d5+fjje6+yQz+x/gHOIG9Fd3r7mU8mbW5O4LC2U3s4OB/wesAqxiZt9195oIIma2ITDX3V8tegp9hejymEf8PgtuAj4NHGlmG7n7c3l/ck1dowcCVxN1mQKMdvcri443smhrgweJB5/Pm9la7v7f3i9199V7/cpNLY+cSdNUfwysSTSZL3T3l4oCR4O7twK4+7+JsYHlzezkdLwmfqdpLKexKGhsY2Z3EGMDGxCtj3HA/5rZ8CoWtVvMbAviprN2yaEBRD1GAiulcxvdfRbwD2A2cBjkexaWma0ITCOethuILrgRRTfWfu7elh4E2tL3dDrwEPH/ZJNqlb076r1+lVATN5o+5uNE//iTwHfc/c3ig4UbTNEA+X3AW8TTz6BCYMmz1NpodfdWMxthZhcTweLzwGvAUe6+PXAG0S1wQRWL212NRNDbpujvuPts4AlgeWCPdKzQvfhPokWygZmt0HtFzc7dpxDfsznAHu7+Q3efk2YB4u4LSj5SuLc8DTQDQyC/Dzf1Xr9K6DMVrSGrpT/3u/vszsYx0tNPo7u/BbwDDAfm5XnWVdG4xkIz629mo4n+5G8Qg8dnAFunMQGAR4HLgYPSk32evQU8D2xlZoNTHQvdwuPS6xHpCXa+mTWnQH898F8iiORSUT1OBAYCG5vZMum9DrtKi7pQN0iva6f3c/dwU+/1qxQFj/yZlV5f6uqkQveVma0KrEU82Q7Ic9dHURfV/sS04zOJrpw/A1u6+4/d/b10ToO7fwD8H/A6cGl1St09qb/7QWJwvND6WJgC/BPp2MqkmXOFbkjgPHc/MXVj5ZK7L0i/j3HAbcTkjc+lY+2+b4WbcZqtdFh6+9+9VNzM6r1+laLgkT/zgcksusm0e/IpGVj9HNHV9S+iyZ1bZraBmd1NPG1vRpR5T3c/xN2fKT63qH5PEQsH/57XsY+irooziQBxsJmtmvrIW82smRjbGEL6HRW1wt4q/nuOFep4PDFN/BgzWws+GqcrjGM1FHXxHAl8khhUfqSXy5tVvdev7BQ8csbdnwb+A2xtZpvDohuLlaS3MLMdWDTV87oaaDIPB3YkpkAeD+zi7rd3dnJ6am8jWh9j3H1q7xQzmxQgmtL41DnAl4HvAqSAtxewLbCA6B8vboUNNrNlUzdXbgNIoXyphXUGsCvwv+nYvPTamrpTNzezu4CfEfW9yt1nmFlTXscE6r1+laBFgjlSmLaapuo+BFxJDJpPLzlvVWLw9Xhiuuc/iCmGM/LcbQVgZocB4939jWqXpZyKW4NmdguwJ/AM8D6wBtEn/mt3/146p5no4joYWM/dD6hKwTMoqeMlxGLO8cQssynAXOAQYj0SxDjQKe7+l5LrDAX6u/vk9ICQi4eeeq9fuSl45Ezhy2Zm5xIDyeOA3xCryT8gnmJHEU/ww4FbgJOLb8Z5Xy+QhZmtA0xOT3a5rldR8F8P+BqRhqQ/8bu7k8gGMMHMPg7sAHwF2IUYaN/N3SdVp+TdV/T9XI24kXa0BqcV+D1wsbs/VfL5nYETgGHuvlPFC5xRvdevnBQ8cqboy7ss8EOi+6OJyDY7IP1pI7p+bia+wM+b2dpEmoT78n6T7Uyh3EX/DzYm8gn9Gzjb3d8q3KCrXNRuSd2ObUR3VSHJ465EX/kexMye2cCtwOnu/kL6XM38/sxsV2B3ohXVSOReu4qY8DG76El+U+JmfAAxRvcusKu7P1uNcndXvddvaSh45FBJ83krIpne54iFZs8QU1gfIloljcDOwP+k8z5d+jRUC1JfcX+gMa2NKLz/U6IFdo+7H1Kl4mVSevNPA64bE0GjUId7iN/lW8QDwizgMne/qBYCZEcBLk1RnlXy3hrEGNDBwGeJaeVXEyvsH/eUHTpv6r1+5aDgUSPS9MAh7v5+0XufIrpH9gU+RnRhfb/wBFsr0pz6L6c/mwF3AVe6+7g0iPwTYDSxePCatFaidNFW7qQ1NxsTrY3jiN/RbcTalSfd/e103kjgdGLcaj2v0TQXxf376Xe6N5GGfud0ys1EBoGH8zr5oSv1Xr+sFDxqTBpoXZsIGMcA6xBPrfOJLpA/E7M/nq2FJ1gAM/sG8GsiPUQLUac3gWPd/Z7UffVbYHV3X696Jc0mtTjuIMY3/kUEjbvcfWLROf3SOoNDgD8Bh7j7n6tR3p4qHRQ2s+2Jm+o+xDqe+4nJH3d3NFHCzAZ6rObOZXddvdevpxQ8akhalLQj0Y2zDfAY8BdijvmzxJP78en0LTxWMjcSXUEL8vjltdgA6kUi+B1O3GR3IlKSzHP3DdN5PycWbx3g7ndWqbjdVjRucxzRB34l4IUWU9H4TmGQfVlgGXefXM1yLw0zK8we+wqR/fklYnzgFuBFX7Qwsvj844CF7n5yLxc3s3qvX1bKqlsDUvfHF4AjiKbyZGLh3F+Blwvz0IHLzGwG0fq4gHhybyVmhwAsy6IV7HlR2Av6UXe/L/337Wb2K+C3ZnaGu/+ISOGxDJHrqha0Abj7xWa2TPE4Tnq/MDFgYfr7B8AHtdJaLGVmhwKnEuk6ZgAXEYk9n+xgnGBlYnzuYGKq+SAzu9Pd78zr1NZ6r19PKHjUgHSj2YF44rmEyPz5ZLrhAIumiRJN6CuIwFIIPGsCDxBTRrft1cIv2VtEQBtusSr7nfT+bURZTzWzXwJbE7OWPuj4MvlS3MIrDRyFVkdqmQwgZvOsTUyIeMnM7vO04r6GbjbTgE2JHS//BDzoKdVMQerG24NI6fF5YoHdfcC6xMPO+jmua73XLzN1W+VcUffHQGLg9ZHSL23RuaWzfJbxSK64ObHY6XFg37wN5pnZD4g9PE5z93OL3t+CWADZCgwjprOeXp1Sll/qOz+JuNEMYFHGhznESvWfpC6tmgggFtslu7u/3MGxbYgn8X2IRZOPAn8k1r/sQIwHfd/dz85rfeu9flkpeNSA7oxVlHZ3mNkpxFane7j7HWlm1nNFXVy5Yma3AtsR/8ieIVKzjyae5Ap7m//c3f9bq107xczss0Qr8tPEOpZrgUlEXb9DtLTOcffT8l7frr6faZHnQUSreV2i9Xgh0bX6urt/YGarEE/muxBrlXLVuqz3+vWUgkeNSwPiDb4oV9JuRL6r9YnuoO+6+yVF5+fyRpRW9BqRT2hrFu269zhxE203A8nMhnjKKZTHOnUmDY7fBGxP7NV+JjC1aBroasRmRF8FzN1fr7WnVYs9vvcnZiVtSwTF2UQL60h3vzWdV2hZf4PY2Ozb7p77/VvqvX7d0WeSeNWjNM2zNXVtrGNmtwF/J9YTXA5sWxw4oOMsvTnxDjCTyFL6ITE+831gh9LAYWZrmtkRwFUW+2LktU6d2ZCYUXYv8Bt3n1wUOJo80pRcRbREvgu1tU+EmX2eSN9xFnFjvYuY7PFFIqXOnrYoQ3LhHvQ4sXfGSMt5csF6r193acC8BhWeZtL02/7AL4iuDoh1Bb9199tKz69GWbsrTQqYBwwm+op/Wdq3nKb1bkX8I92X2Od8S2KSQC1ZN73+qnjNBywW3O8npjBvYGYreux0VytWB/YjppJfDdwOTHD3eWZ2NvBNYqfMS9N3eATwbSINT+6/q9R//bpF3VY1LDWFf07sP/Afog/9qtLZPUXn526dRykzW6+DoNFMzHT5EtGV87F06Hqii+7R9I+0Jv5hmtlRxOZWB7r79R2MVxW6On5O3KhO8MVTtvQD1nX3F3u98N2UxtweBp5w9zklx5z4HZ5NdK2OIAabZwEHu/t4W7R4Mpff2XqvX3eo5VGjzOwkYkbO20TL43eFp9iihWefJsYR3gL+4+4t6Xhub7IdBI51iKmsRxEB5HGia24KMUvpCmK/j/+X1zoVFN0oniEWRX7GzG5OT6wf3USK6nGGl+RGMrOtif0mdjCzVTxniwqLvlvne0kKGVuUVuYbRND/YdHhacQU2P/AR7v7rUzccDt8GKqGeq9fFmp51BhbtDK5H5Gq/U/u/nDR8SHEgsJjidXoBS8Q+aI6SjGdO2nQeBsimeDnASdSlNwNvJr+8S1HLJY8lniKvzXPgbGYmd0OrAeclMrd5ROoRf6rHxApaeYQGXmvcvcjeqG4PdZZvcxsXaIuqxBrd8a5+/VFx48nUtb8xd2/1lvlzare69cVBY8a1NkN0iKJ4IXEJjbziX0ibidWlu9B9LXv7+43Ws6TC5rZ6cBpxED6n4iFkS+6+9x0vNkj/cpniTre7u6HV63A3VTUKvw4MWD+NhFAOtzGNAXI44DvEfvU305M692IGEwf5e4P9UbZK8VS7qf03/sQQfJzxABzE7BVZ/9/akG91k/Bo8almRttwHLEDJ19iL0jfg38zdMuhGa2GTHXfCV3/0SVitttFpvqFLrmHnP3mV2c+2l3f6LXCreUisY0vkKsZXmcGNdYWHLe/kTLamOiu+NK4Pq01mVT4BTgLK/BPSMsMh80+aJcX5sAPyLys0F0Rd5APEAMcvd1O7xQTtV7/UDBo25YrFa+jWhtHObunt7vV/QFPpa4Ie/v7s9XrbDdZGZDvWQL3npgi+/XspqX7CBoZp8hgsaexHTdq4GrPfa3L5xTE91zHSmeIGCxZev3iNbVUCKQ/tzdb0zHjyG6985w9xlVKnIm9V6/grqYb9zXdDJPfC8iceAYd3cza0g3qQVpfARi1fZwYGHq4ip0deVSPQYOaJf36qPAYWarm9l5RGbhnYmuuiOA7xUCR+F3X7wupPdKvnQKZS26sR5FrK7/PjFwfArwucKNNbmSmAyR+xtrvdevlGZb1aCiG8c+7n5zCg7LE19QLzqvMHunMLbxGWJ7zEuBl4GjS7tKpPdZ5C07isjvtSoxFnIFcFNRt2NhbU/xvhKNtfD7Kwp4hZvqjkR3zQ7AXGLl9c990eZYHz25e07T6RSr9/p1RsGjBqX+1BOB88zskx4bPz1PLLDbHHjBSxIkEgPmJxMtj6nAJDNbrjCWUMvdILXMzPYEfkz83l4kZtBd6+4T0vEG6HiFeRo3WRcY5u6P9VqhMyj+XpnZ2kS//2HEQPHtwM8Kg8WF8TtffM1LrtdB1Hv9uqIxjxplZocTeZF+4u6/Se+9QbQ8xrj7vWmmzggiG28hcNxOpJW+u4O+9hHACl6De6DXIjM7gNjMay6xLuDmkmnXS5q+uw7RSpkOnJh+57l7CDCzQcTMsOOAlYmNy8a4+7VF53RabossCp8DXnL3d/N2w633+nVGwaNGpa6qV4kB8pPSOMcXiDQIWwD/JLqy1iRyKT1NTHm9rTCYXnK9jYn00SsDy9Zyc7pWpLUb1wDLu/sGRe93KwBYrLw/kJhl9y8i3f7kPN18Uqv3AuB/iGnXFwLnelr8aEtIapm+518GfkX8v/pfd5+VlzrWe/26ogHzGpXGMcYQCwJPSO/dSWzVej6RyuPzxGKys4mFdOeVBo40SwtgArFD4c1AW6G7RCondU1dBphFosdCa6NbLQd3n+/uY4kb1pbE7na54pFW5VGidXSOu49x9w9LB5e7+PwCYk+XF4lpzbnazrXe69cVtTxqnJndTMy0ehR4hRjH2oVIaX4jkfp7XOk6CTPbiEijcCJwtrt/t3gMRHqHRUaAs4D/c/e7unF+8TTfQcTGQ0cSN54mYBN3f66CRc4sdZ/+jdivZWBPutXMbAViV76NgO3d/f68dNHVe/06owHz2ncCMXf8OKK7airRZXUrcJ+7v1l8spmtCBxNrEIfSeS9egDA3WemFkdNzOKpB2mK5jcynF8IHCOJsayjiN/7k0S34/zyl3LppO/VL4kUOasSq+qzaiC6hTYi9jrZJi831nqvX2fU8qgTZrY6kYH1ReLmP6PkeCORkfYEootjHvBjr5FcV/Wu9CnTzD4JfOjuL5XM6FmR2Czra0SLcxqxUvn/gIfyOlaVHkr6e0ovk+Fzw4DjgVOJrYj/Q9T3F56j9Dr1Xr+OKHj0AWa2HdE9dUB660rgFHd/Px0vvXHlurlc78xsVeAWYtbON9MA6nLAJsSWp18l8pXdQWx3epe7T0ufzf1Aa4GZDSa63d5w9zklQbKRGEg+jZjwMZXY5+Vid3+9WmXOot7rp26rOpa6Nr5HDJ6vSOw/8C13fzwdL/4yf5zYte+PChzVk27+75jZy0TrYryZjSOCxhFEKotHiJQltxXWgxR9tiYCR7I9cDox0P+Hou/ilsRNdXeglZjO/BtPCSDTU363JxZUUV3XTy2POpZWut5NjGuMLsw7Lxl0HU5siXoSsUvfPp72X5beZ4uSJg4H3kx/JhPrAF4nWhp/86JEkDUYNICPvnuTgfHEZklziZvtccQYwIPAhe5+XdFnaqZVXO/1U/Coc2Z2uLtfVfT3wn4gg4hVzccTzecG4Dpi+9eni8+tRrn7MluUtn00cCaxt/t1xBPqQ2l6aHcWEa5E7GP/194odxa2aCe97xNTzm8lBv5XJnKw/Q64zBdtYNbleom8qff6gYJH3epsHCMtSlqP2KjmaKA/MJHI3mrEiuVrPfb8qJmnoHpS0jJ8hRhIPdjd/1F6fAnX2ZIIOJe6+xmVLHNWJXV8GViHWCvxe6KL5/l0rJDSo6sguQKwuucoNX291w+0SLBudXDTb0szsk4gUrd/k5hXfjSwnbtvT+wFMgy43GIv8Varoayt9cIX7RQJMQtnBWCPlMaCDK3B14hxrh+m9SS5UVLH76TXce5+qrs/b2ZNhYeXJdxY+xMPQv9JM5dyod7rBwoefclexOrxXwPvEytZv+3uY9MAbZO7P0qkSZiXXpe4QlYqozBN0yN9931E1+LmGS8zn/g3PpDIvZQrRXW8lajjVhaJIgFaO3gAWkz6zs4jUrNMI7r4cqPe66fg0XesQUz1/CWxKO1Cd38NPprdUfgiP0B0Y33KYh9xqZKiVt8xwOXE1N3ufG6AmZ0APEPMtBsHvGUd7wNTVUV1PIHY7/uklOmgq6fxRls8Hf1AYlzomJQ5ITfquX6aqttHuPslZvYM8JS7fwCL9csW8li1AUOAtYi9QepyM6ZakQbNG9z9FeAn3fmMme1GzOjZnMiw/P+IMaxcrh1IdWxKXTmnExMCutpyuHiXvo+zeAr0R3ul0BnUc/00YN4HlQzmFX9ZRxKzQHYFvq/V5/lQNEOuq7TeGxD7ghxIdEv+GRjraS+J4uv0SqEz6M7EjJLv6QBStysxe+l5YkX21RUvbA/Ua/0UPPqoTr6sJwErEckUT87r02pfVhoA0lqCk4jfX39iXOtK4HZ3n9/RZ/Ksg/qVzho8EPgh0QU7ldg869eF6ct5t7T1y9OUXgWPPq7kyzqF2JToHHefU9WCyUcsUpPsDNxR+L2kmTyHEq2NkcSg6hXAX919SjqnZoJGqTRW0Fa0KnsLoq57EONzfwLOcPdXq1fKnstavzwFjQIFjz7KzEYRabz3puMva2FdyLJAMzCzqA9eX5peZGZrESm/n3P3r1lkDvgJkQL8dWAscI177NVSq78ji1xQ65esnl+NeLg5ihg4Hk9smHR3dUrZcz2pX2GSQx7XW2nAvA9Ks6uuIXYZHEdsmXl30fHCCueBwG7E7Kw/EAOvNXdTqgOFtPnfSgvG9gRmEDOw/uTu9xdOrNXAkewFXGNmI919opmdQnTJrUHsVfMrd7+8qiVcOpnqZxk2BqsGtTz6mKLAsB+wlqf9z0vOGezus9J/r0zMUe9H7JN9p6XUC71b8r4tzbwZC3wauIsY17jNF213WstBA/iojv8kckBNI1LPzwAuBs70lMqjVnW3fmkx78DCVPq8UvCQj5jZyUQm17eI2Tr/TAsItyF2u1vB3T+Rzq35m1WtSQF/NeBGd387vVdXKWTM7DhiMdxg4Frg556znRGXRlf1K3qw+xaR7eGH7v5oXv+tqduqj0t9qisSewnsTjwJrUXsjX41cLi7P2hmvwcuMrPR7n4WsTYkd1/oOndz6aBpPQWO5FrgY8R6iJurXZgKaFe/1I1cvFD3YWLiyn5m9qy7z85jAFHLQzCzLwC3A38nZnw8BVwGHAL8yN3PTk3uG4APgM8Xukuk9+XxRiI9UzqLqmiiypXEzKtDPCXEzJvcpSuQqrD0epa7P5GeZs8gdqr7mZktn9Z8zAIGARrvqCIFjtqXWhtd5Y47CVgOODKNO3Z0jWXN7Ltmtn/6e68mMVXwEIjFSBBrPQBIweIyYDZQaHlsSmxOVG9dJSK9qrMHgNTq6OexrfDPiIe1dgHGzPYG7iFy1f0+tUZ7dR2Iuq36sJI0JW8Q0wVHl8xDv4DY+WwiMaVwX3e/oxrlFekLuuqWNLNNiXxXBxCLetuIrBDfdfeze6+UCh59ni3a8WwfYq/lycC5wBPEzmdfT68zgFM7m2dvZp8CTgGO1up0kaVXPJPOzFYhurJOAgYAjxOJL/ckxii/XMgs0FsUPOQj6anmFGAXYFUWZdv9E3C6L0rhXvqlPhU4FlgWOMKLtr0VkZ5LeecOJVobHyN2/DyHyDjwv8Qi3hPd/c+9PW1bU3Xlo2ayu//HzP5LDIw3EH2qvyhJldCW+mX7A0cApxHdWeOJRWy3VqMOIvXGzHYm0tCMAuYQucvOS/9OLwG+Chzj7n+G3p+2reAhpYN3qwFrExsQXV2UiK+hqLWxK7FnxJbAy0QAubZWk9SJ5InFhk+nEgt2IValX+jut6TjJxHdyZcQaYaqslhUwUOAxb58JwA/cPf30vuFVkmbmX2CCBRfJdIrXEzkVvpX1QouUn/OIrqjCuutrihKQ3MQMIZYk3VK0XqrwsSXwjqRiq8F0piHdKhkJtYKxMY0pxCZP28hmtC3e+yxrIVrIkupKD3JhsDRRBfVxKLjmxB77bQB+7v70924ZsX+XSp4SKfSnhEHEl1UaxPbYF4J3FDUMqmr3Eoi1dTBZlGFXSRXJMYUtwZ2cfdH0/GhxH4uTcCOxIZgWwJPu/uPK7kPiIKHdMjMlieyt34KmMCiPSNeTMfV0hCpoKIuqCZiz4/Tie6qB4CNgNXT6zJEosXpwBAWzZLc0N1frNS/VQUP6ZSZXU+sJr+gjvaMEKkpKZXJl4nB8UZgJjAfeI14sHPgEeAzwCeIqfbDiTTvP6hUuRQ8pJ2ivtfBQKsv2j9ZQUOkCsxsCLEg8G0ieLQUrbvqT8yOPIzY7+UtYqHvrZXcE0TBQ5ZI4xoi+ZMe7vYm1lvtSmS8/iMxQ8sLk1kqRcFDRCTnSh/gzGwnoqWxHzHOcSNwIfCIu3/QnWssLQUPEZEaYWYbEOlKvkrMsnoCOJ+YNt8ut5WZbUVM+/2Ou88oZ9ezFgmKiORc6qI6gdiediti7ONHxHbRb5QuDDSzNYls2PsT+/U0AEdSxh1A1fIQEck5M2smAsZwIi3J5cCzpeMaZjaQ6M46DtgMmEes/QBYy93fKleZ1PIQEcmxNPtxvpl9jUjHfpe7z+zgvN2I1sme6a3fAy3AUcSarbLuAKqWh4hIjSnpotqECBoHE4sFHwZ+QKwJ+Quxkdv+xalOykHBQ0SkBqW9zY8hWhYjiS2izyISlq4F3EBkyf6iuz9S7nVaCh4iIjUkrTg/hph1NYpYbX4BkUjxzbTvzlhiq9rjO9v9c2k1VuKiIiJSMSsQg+ajgL8C27j7qSlwNBMp2w8CzigEjpQfq7ChW1koeIiI1Ig0eD4V+CbR8jjQ3R8rOuWLwGgigeJlRZ9ZCIt2G0xTf5eKuq1ERGpEZ+MWqStrd+BS4Gnga+4+xcyWJWbVDiESJzYD2xJTfn/p7s/0dOW5goeISI1L6zsuAw4Bbiay7K4HfJJI2b4+kSG7CfgQGAS86O4b9vRnap2HiEjtG1T031sBexGzr1qIlshVRPB4l1hpvjVgZraTu9/Tkx+oloeISB1Iea+MGMt+DXgRGOjuLWY2AtiDWHm+CfAQcDuxP3qPVp0reIiI1KE0DjIU+ALwLaJFMonIh3WNuz+xpS8UgQAAAulJREFUNNfXbCsRkTpjZssRA+MXE2s+NiF2IjwSGF0aOApTeFPA6RaNeYiI1Im0zmM94HAiFfsKwDhizONGd59ecv5QYDtgTSLQdJu6rURE6oCZrUosDjwS2Bh4HrgauNbdXy85dwCwEbGZ1EHAOsBu7v6P7k7dVfAQEakDZrYNcD8xrnEDMNbdHy05pwFYmxgHOQL4LPAOMZ13krtv0N2fp+AhIlInzOwnwHNEF9WCkmOrEClNDicWFM4B7gD+BKwC/I4YDzmnO60PBQ8RkRpXuNl3dNNPg+efAQ4EvkLMwHqQ6NK60d3fM7MViXxZOwAj3H32krLwasBcRKTGFQJGceBIg+cbAPsSuwuuCziRgffP7u7pvIaUyuQ6YHvge8BPlvQzFTxEROqMmX2cGNc4iJhN9R6RvmSsu99fdF4ji/Y0v5cILjub2QXuPqWrn6F1HiIi9Wd94CIiDcnNxE6DJxUCR2Fdh7u3untb6u6aDPybWKW++pJ+gIKHiEidcfc7gdeJmVS/dvcb3P3D4qBR8pHC4sC3iYy7A6HrRYMKHiIidcTMCsMR3yIW/33GzPqn9zpM517Y7wPYNL2uDNDVgLmCh4hIHSlM0XX324hxjBOJvFadnd8GYGb7EGMkbxLdV11S8BARqTOFbWeBY4k1HMea2WppfKOpeDtaMxtoZrsSOxA2EWMkU5a0Za3WeYiI1KHC9rNm9kPgZKIVcoS7z0rHBwNrAbsCxxOD7LcA33D3d5Z0fU3VFRGpT4VB8TOB5YGTgOfM7HpgFjAC2BDYPJ13BzCmO4ED1PIQEalbRSvPhxCLBX8NrFh0ymxiVtbvgEtKU5p0RcFDRKSPMLMNiX3NNwVeJoLHPe7+XtZrKXiIiNS5JeWp6gnNthIRqXPFgSPLboFdUctDREQyU8tDREQyU/AQEZHMFDxERCQzBQ8REclMwUNERDJT8BARkcwUPEREJDMFDxERyUzBQ0REMlPwEBGRzP4/clRS1bIpi7AAAAAASUVORK5CYII=\n", 203 | "text/plain": [ 204 | "
" 205 | ] 206 | }, 207 | "metadata": { 208 | "needs_background": "light" 209 | }, 210 | "output_type": "display_data" 211 | } 212 | ], 213 | "source": [ 214 | "fig, axs = plt.subplots(1, 1, figsize=(5,5))\n", 215 | "boxplot_all_methods(axs, results_tar_plot*100,\n", 216 | " title=\"MNIST: single source patch intervention\", names=names_short,\n", 217 | " color=np.array(COLOR_PALETTE)[:len(names_short)])\n", 218 | "\n", 219 | "plt.savefig(\"paper_figures/%s\" %\"MNIST_%s_2M.pdf\" %perturb, bbox_inches=\"tight\")\n", 220 | "plt.show()" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": null, 226 | "metadata": {}, 227 | "outputs": [], 228 | "source": [] 229 | } 230 | ], 231 | "metadata": { 232 | "kernelspec": { 233 | "display_name": "Python 3", 234 | "language": "python", 235 | "name": "python3" 236 | }, 237 | "language_info": { 238 | "codemirror_mode": { 239 | "name": "ipython", 240 | "version": 3 241 | }, 242 | "file_extension": ".py", 243 | "mimetype": "text/x-python", 244 | "name": "python", 245 | "nbconvert_exporter": "python", 246 | "pygments_lexer": "ipython3", 247 | "version": "3.6.4" 248 | } 249 | }, 250 | "nbformat": 4, 251 | "nbformat_minor": 2 252 | } 253 | -------------------------------------------------------------------------------- /MNIST/simu_MNIST_patches.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # In[1]: 5 | 6 | 7 | import numpy as np 8 | import pandas as pd 9 | import sys 10 | import argparse 11 | 12 | np.set_printoptions(precision=3) 13 | 14 | import torch 15 | 16 | import torch.nn as nn 17 | import torch.nn.functional as F 18 | import torch.optim as optim 19 | 20 | import torchvision 21 | import time 22 | 23 | import simudata_MNIST 24 | import semitorchMNISTclass 25 | 26 | 27 | # In[2]: 28 | 29 | 30 | # check gpu avail 31 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 32 | 33 | # Assuming that we are on a CUDA machine, this should print a CUDA device: 34 | 35 | # print(device) 36 | 37 | 38 | # In[3]: 39 | 40 | parser = argparse.ArgumentParser() 41 | parser.add_argument("--perturb", type=str, default="whitepatch", help="type of perturbation") 42 | parser.add_argument("--subset_prop", type=float, default=0.2, help="proportion of data points to be used for each env") 43 | parser.add_argument("--seed", type=int, default=0, help="seed") 44 | parser.add_argument("--lamMatch", type=float, default=1., help="DIP matching penalty") 45 | parser.add_argument("--lamMatchMMD", type=float, default=1., help="DIP matching penalty with MMD") 46 | parser.add_argument("--lamCIP", type=float, default=0.1, help="CIP matching penalty") 47 | parser.add_argument("--lamCIPMMD", type=float, default=0.1, help="CIP matching penalty with MMD") 48 | parser.add_argument("--epochs", type=int, default=50, help="number of epochs") 49 | parser.add_argument("--tag_DA", type=str, default="baseline", help="whether to run baseline methods or DA methods") 50 | parser.add_argument("--M", type=int, default=12, help="total number of environments") 51 | myargs = parser.parse_args() 52 | print(myargs) 53 | 54 | M = myargs.M 55 | train_batch_size = 500 56 | test_batch_size = 500 57 | np.random.seed(123456+myargs.seed) 58 | 59 | trainloaders, testloaders = simudata_MNIST.generate_MNIST_envs(perturb=myargs.perturb, subset_prop=myargs.subset_prop, 60 | M=M, interY=True, 61 | train_batch_size=train_batch_size, 62 | test_batch_size=test_batch_size) 63 | 64 | lamL2 = 0. 65 | lamL1 = 0. 66 | 67 | lr = 1e-4 68 | 69 | source = list(np.arange(M)) 70 | target = M-1 71 | source.remove(target) 72 | 73 | savefilename_prefix = 'results_MNIST/report_v8_%s_M%d_subsetprop%s_%s_lamMatch%s_lamCIP%s_lamMatchMMD%s_lamCIPMMD%s_epochs%d_seed%d' % (myargs.perturb, M, 74 | str(myargs.subset_prop), myargs.tag_DA, str(myargs.lamMatch), str(myargs.lamCIP), str(myargs.lamMatchMMD), 75 | str(myargs.lamCIPMMD), myargs.epochs, myargs.seed) 76 | savefilename = '%s.txt' % savefilename_prefix 77 | savefile = open(savefilename, 'w') 78 | 79 | if myargs.tag_DA == 'baseline': 80 | methods = [ 81 | semitorchMNISTclass.Original(), 82 | semitorchMNISTclass.Tar(lamL2=lamL2, lamL1=lamL1, lr=lr, epochs=myargs.epochs), 83 | semitorchMNISTclass.SrcPool(lamL2=lamL2, lamL1=lamL1, lr=lr, epochs=myargs.epochs), 84 | ] 85 | elif myargs.tag_DA == 'DAmean': 86 | methods = [ 87 | semitorchMNISTclass.DIP(lamMatch=myargs.lamMatch, lamL2=0., lamL1=0., 88 | sourceInd = 0, lr=lr, epochs=myargs.epochs, wayMatch='mean'), 89 | semitorchMNISTclass.DIPOracle(lamMatch=myargs.lamMatch, lamL2=0., lamL1=0., 90 | sourceInd = 0, lr=lr, epochs=myargs.epochs, wayMatch='mean'), 91 | semitorchMNISTclass.DIPweigh(lamMatch=myargs.lamMatch, lamL2=0., lamL1=0., 92 | lr=lr, epochs=myargs.epochs, wayMatch='mean'), 93 | semitorchMNISTclass.CIRMweigh(lamMatch=myargs.lamMatch, lamCIP=myargs.lamCIP, lamL2=0., lamL1=0., 94 | lr=lr, epochs=myargs.epochs, wayMatch='mean'), 95 | ] 96 | elif myargs.tag_DA == 'DAMMD': 97 | methods = [ 98 | semitorchMNISTclass.DIP_MMD(lamMatch=myargs.lamMatchMMD, lamL2=0., lamL1=0., 99 | sourceInd = 0, lr=lr, epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1., 10.]), 100 | semitorchMNISTclass.DIPweigh_MMD(lamMatch=myargs.lamMatchMMD, lamL2=0., lamL1=0., 101 | lr=lr, epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1., 10.]), 102 | semitorchMNISTclass.CIRMweigh_MMD(lamMatch=myargs.lamMatchMMD, lamCIP=myargs.lamCIPMMD, lamL2=0., lamL1=0., 103 | lr=lr, epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1., 10.]), 104 | ] 105 | elif myargs.tag_DA == 'DACIPMMD': 106 | methods = [ 107 | semitorchMNISTclass.CIP_MMD(lamCIP=myargs.lamCIPMMD, lamL2=0., lamL1=0., 108 | lr=lr, epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1., 10.]), 109 | ] 110 | elif myargs.tag_DA == 'DACIPmean': 111 | methods = [ 112 | semitorchMNISTclass.CIP(lamCIP=myargs.lamCIP, lamL2=0., lamL1=0., 113 | lr=lr, epochs=myargs.epochs, wayMatch='mean'), 114 | ] 115 | else: 116 | print('tag_DA unrecognized') 117 | 118 | names = [str(m) for m in methods] 119 | print(names, file=savefile) 120 | 121 | 122 | 123 | trained_methods = [None]*len(methods) 124 | results_src_all = np.zeros((M-1, len(methods), 2)) 125 | results_tar_all = np.zeros((len(methods), 2)) 126 | 127 | def compute_accuracy(loader, env, me): 128 | total = 0 129 | correct = 0 130 | with torch.no_grad(): 131 | for data in loader[env]: 132 | images, labels = data[0].to(device), data[1].to(device) 133 | predicted = me.predict(images) 134 | total += labels.size(0) 135 | correct += (predicted == labels).sum().item() 136 | return correct/total 137 | 138 | for i, me in enumerate(methods): 139 | starttime = time.time() 140 | print("fitting %s" %names[i], file=savefile) 141 | me = me.fit(trainloaders, source=source, target=target) 142 | if hasattr(me, 'losses'): 143 | print(me.losses, file=savefile) 144 | if hasattr(me, 'minDiffIndx'): 145 | print("best index="+str(me.minDiffIndx), file=savefile) 146 | trained_methods[i] = me 147 | # evaluate the methods 148 | # target train and test accuracy 149 | results_tar_all[i, 0] = compute_accuracy(trainloaders, target, me) 150 | results_tar_all[i, 1] = compute_accuracy(testloaders, target, me) 151 | # source train and test accuracy 152 | for j, sourcej in enumerate(source): 153 | results_src_all[j, i, 0] = compute_accuracy(trainloaders, sourcej, me) 154 | results_src_all[j, i, 1] = compute_accuracy(testloaders, sourcej, me) 155 | 156 | print('Method %-30s, Target %d, Source accuracy: %.3f %%, Target accuracy: %.3f %%' % (names[i], target, 157 | 100 * results_tar_all[i, 0], 100 * results_tar_all[i, 1]), file=savefile) 158 | endtime = time.time() 159 | print("time elapsed: %.1f s" % (endtime - starttime), file=savefile) 160 | print("\n", file=savefile) 161 | 162 | results_all = {} 163 | results_all['src'] = results_src_all 164 | results_all['tar'] = results_tar_all 165 | np.save("%s.npy" %savefilename_prefix, results_all) 166 | 167 | -------------------------------------------------------------------------------- /MNIST/simu_MNIST_patches_2M.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # In[1]: 5 | 6 | 7 | import numpy as np 8 | import pandas as pd 9 | import sys 10 | import argparse 11 | 12 | np.set_printoptions(precision=3) 13 | 14 | import torch 15 | 16 | import torch.nn as nn 17 | import torch.nn.functional as F 18 | import torch.optim as optim 19 | 20 | import torchvision 21 | import time 22 | 23 | import simudata_MNIST 24 | import semitorchMNISTclass 25 | 26 | 27 | # In[2]: 28 | 29 | 30 | # check gpu avail 31 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 32 | 33 | # Assuming that we are on a CUDA machine, this should print a CUDA device: 34 | 35 | # print(device) 36 | 37 | 38 | # In[3]: 39 | 40 | parser = argparse.ArgumentParser() 41 | parser.add_argument("--perturb", type=str, default="whitepatch2M", help="type of perturbation") 42 | parser.add_argument("--subset_prop", type=float, default=0.2, help="proportion of data points to be used for each env") 43 | parser.add_argument("--seed", type=int, default=0, help="seed") 44 | parser.add_argument("--lamMatch", type=float, default=1., help="DIP matching penalty") 45 | parser.add_argument("--lamMatchMMD", type=float, default=1., help="DIP matching penalty with MMD") 46 | parser.add_argument("--epochs", type=int, default=50, help="number of epochs") 47 | parser.add_argument("--tag_DA", type=str, default="baseline", help="whether to run baseline methods or DA methods") 48 | myargs = parser.parse_args() 49 | print(myargs) 50 | 51 | M = 2 52 | train_batch_size = 64 53 | test_batch_size = 500 54 | save_model = False 55 | np.random.seed(123456+myargs.seed) 56 | 57 | trainloaders, testloaders = simudata_MNIST.generate_MNIST_envs(perturb=myargs.perturb, subset_prop=myargs.subset_prop, 58 | M=M, interY=False, 59 | train_batch_size=train_batch_size, 60 | test_batch_size=test_batch_size) 61 | 62 | lamL2 = 0. 63 | lamL1 = 0. 64 | lr = 1e-4 65 | 66 | source = list(np.arange(M)) 67 | target = M-1 68 | source.remove(target) 69 | 70 | savefilename_prefix = 'results_MNIST/report_v8_%s_M%d_subsetprop%s_%s_lamMatch%s_lamMatchMMD%s_epochs%d_seed%d' % (myargs.perturb, M, 71 | str(myargs.subset_prop), myargs.tag_DA, myargs.lamMatch, myargs.lamMatchMMD, myargs.epochs, myargs.seed) 72 | savefilename = '%s.txt' % savefilename_prefix 73 | savefile = open(savefilename, 'w') 74 | 75 | if myargs.tag_DA == 'baseline': 76 | methods = [ 77 | semitorchMNISTclass.Original(), 78 | semitorchMNISTclass.Tar(lamL2=lamL2, lamL1=lamL1, lr=lr, epochs=myargs.epochs), 79 | semitorchMNISTclass.SrcPool(lamL2=lamL2, lamL1=lamL1, lr=lr, epochs=myargs.epochs), 80 | ] 81 | else: # DIP 82 | methods = [ 83 | semitorchMNISTclass.DIP(lamMatch=myargs.lamMatch, lamL2=0., lamL1=0., 84 | sourceInd = 0, lr=lr, epochs=myargs.epochs, wayMatch='mean'), 85 | semitorchMNISTclass.DIPOracle(lamMatch=myargs.lamMatch, lamL2=0., lamL1=0., 86 | sourceInd = 0, lr=lr, epochs=myargs.epochs, wayMatch='mean'), 87 | semitorchMNISTclass.DIP_MMD(lamMatch=myargs.lamMatchMMD, lamL2=0., lamL1=0., 88 | sourceInd = 0, lr=lr, epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1., 10.]), 89 | ] 90 | 91 | names = [str(m) for m in methods] 92 | print(names, file=savefile) 93 | 94 | 95 | 96 | trained_methods = [None]*len(methods) 97 | results_src_all = np.zeros((M-1, len(methods), 2)) 98 | results_tar_all = np.zeros((len(methods), 2)) 99 | 100 | def compute_accuracy(loader, env, me): 101 | total = 0 102 | correct = 0 103 | with torch.no_grad(): 104 | for data in loader[env]: 105 | images, labels = data[0].to(device), data[1].to(device) 106 | predicted = me.predict(images) 107 | total += labels.size(0) 108 | correct += (predicted == labels).sum().item() 109 | return correct/total 110 | 111 | for i, me in enumerate(methods): 112 | starttime = time.time() 113 | print("fitting %s" %names[i], file=savefile) 114 | me = me.fit(trainloaders, source=source, target=target) 115 | trained_methods[i] = me 116 | # evaluate the methods 117 | # target train and test accuracy 118 | results_tar_all[i, 0] = compute_accuracy(trainloaders, target, me) 119 | results_tar_all[i, 1] = compute_accuracy(testloaders, target, me) 120 | # source train and test accuracy 121 | for j, sourcej in enumerate(source): 122 | results_src_all[j, i, 0] = compute_accuracy(trainloaders, sourcej, me) 123 | results_src_all[j, i, 1] = compute_accuracy(testloaders, sourcej, me) 124 | 125 | 126 | print('Method %-30s, Target %d, Source accuracy: %.3f %%, Target accuracy: %.3f %%' % (names[i], target, 127 | 100 * results_tar_all[i, 0], 100 * results_tar_all[i, 1]), file=savefile) 128 | endtime = time.time() 129 | print("time elapsed: %.1f s" % (endtime - starttime), file=savefile) 130 | print("\n", file=savefile) 131 | 132 | results_all = {} 133 | results_all['src'] = results_src_all 134 | results_all['tar'] = results_tar_all 135 | np.save("%s.npy" %savefilename_prefix, results_all) 136 | 137 | 138 | 139 | -------------------------------------------------------------------------------- /MNIST/simudata_MNIST.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | import torch.optim as optim 7 | 8 | import torchvision 9 | import time 10 | 11 | 12 | # This file generates MNIST perturbed environments 13 | datapath = '/cluster/home/chenyua/Code/causal/data' 14 | 15 | def get_actual_data_idx(all_labels, subset_prop, interY=False): 16 | num_classes = 10 17 | # mask a certain proportion of data 18 | data_idx_mask_prop = torch.rand_like(all_labels.float()) < subset_prop 19 | 20 | if not interY: 21 | data_idx_list = torch.where(data_idx_mask_prop)[0] 22 | else: 23 | # intervention on Y 24 | # the following digits will only have 50% data 25 | mod_digits = [3, 4, 5, 6, 8, 9] 26 | data_idx_mask_mod = torch.rand_like(all_labels.float()) < 0.8 27 | idx_trainset_mod = (all_labels == mod_digits[0]) 28 | for digit in mod_digits: 29 | idx_trainset_mod = idx_trainset_mod | (all_labels == digit) 30 | 31 | data_idx_mask_final = data_idx_mask_prop & (~(data_idx_mask_mod & idx_trainset_mod)) 32 | data_idx_list = torch.where(data_idx_mask_final)[0] 33 | return data_idx_list 34 | 35 | 36 | def generate_MNIST_envs(perturb='noisepatch', subset_prop=0.1, M=12, interY=False, train_batch_size=64, test_batch_size=1000): 37 | trainset_original = torchvision.datasets.MNIST(root=datapath, train=True, 38 | download=False) 39 | testset_original = torchvision.datasets.MNIST(root=datapath, train=False, 40 | download=False) 41 | 42 | # to be commented 43 | # idx_trainsetlist = get_actual_data_idx(trainset_original.targets, subset_prop=subset_prop, interY=interY) 44 | # idx_testsetlist = get_actual_data_idx(testset_original.targets, subset_prop=subset_prop, interY=interY) 45 | 46 | 47 | 48 | trainloaders = {} 49 | testloaders = {} 50 | if perturb == 'whitepatch': 51 | # prepare the noise patces 52 | # noise_patches = [torch.zeros(1, 28, 28)] 53 | noise_patches = [] 54 | offset = 10 55 | initpos = 2 56 | # sqsize 12 works to make CIRM better than CIP 57 | # offset = 4, sqsize = 12, initpos = 6, interY = 0.3 works for CIRM better than CIP, DIP 58 | sqsizesmall = 12 59 | sqsizelarge = 16 60 | if M == 12: 61 | for m in range(M-2): 62 | a = torch.zeros(1, 28, 28) 63 | a[0, (initpos-m+offset):initpos+sqsizelarge-m+offset, (initpos-m+offset):initpos+sqsizelarge-m+offset] = 3.25 64 | noise_patches.append(a) 65 | 66 | for m in [M-2, M-1]: 67 | a = torch.zeros(1, 28, 28) 68 | a[0, (initpos-m+offset):initpos+sqsizesmall-m+offset, (initpos-m+offset):initpos+sqsizesmall-m+offset] = 3.25 69 | noise_patches.append(a) 70 | elif M == 6: 71 | for m in range(M): 72 | a = torch.zeros(1, 28, 28) 73 | a[0, (initpos-2*m+offset):initpos+sqsizelarge-2*m+offset, (initpos-2*m+offset):initpos+sqsizelarge-2*m+offset] = 3.25 74 | noise_patches.append(a) 75 | 76 | 77 | # now transform the data 78 | for m in range(M): 79 | # load MNIST data 80 | transformer = torchvision.transforms.Compose( 81 | [ 82 | torchvision.transforms.ToTensor(), 83 | torchvision.transforms.Normalize((0.1306,), (0.3081,)), 84 | torchvision.transforms.Lambda((lambda y: lambda x: torch.max(x, noise_patches[y]))(m)), 85 | ]) 86 | 87 | trainset = torchvision.datasets.MNIST(root=datapath, train=True, 88 | download=False, transform=transformer) 89 | 90 | testset = torchvision.datasets.MNIST(root=datapath, train=False, 91 | download=False, transform=transformer) 92 | 93 | if m != M-1: 94 | idx_trainsetlist_loc = get_actual_data_idx(trainset_original.targets, subset_prop=subset_prop, interY=False) 95 | idx_testsetlist_loc = get_actual_data_idx(testset_original.targets, subset_prop=subset_prop, interY=False) 96 | else: 97 | idx_trainsetlist_loc = get_actual_data_idx(trainset_original.targets, subset_prop=subset_prop, interY=interY) 98 | idx_testsetlist_loc = get_actual_data_idx(testset_original.targets, subset_prop=subset_prop, interY=interY) 99 | print('actual env=%d trainsize %s, testsize %s' %(m, idx_trainsetlist_loc.shape, idx_testsetlist_loc.shape)) 100 | 101 | trainloaders[m] = torch.utils.data.DataLoader(trainset, batch_size=train_batch_size, num_workers=0, 102 | sampler = torch.utils.data.sampler.SubsetRandomSampler(idx_trainsetlist_loc)) 103 | testloaders[m] = torch.utils.data.DataLoader(testset, batch_size=test_batch_size, num_workers=0, 104 | sampler = torch.utils.data.sampler.SubsetRandomSampler(idx_testsetlist_loc)) 105 | trainloaders[m].dataset.targets_mod = trainloaders[m].dataset.targets[idx_trainsetlist_loc] 106 | testloaders[m].dataset.targets_mod = testloaders[m].dataset.targets[idx_testsetlist_loc] 107 | elif perturb == 'whitepatch2M': 108 | # prepare the noise patces 109 | # noise_patches = [torch.zeros(1, 28, 28)] 110 | noise_patches = [] 111 | offset = 10 112 | initpos = 2 113 | # sqsize 12 works to make CIRM better than CIP 114 | # offset = 4, sqsize = 12, initpos = 6, interY = 0.3 works for CIRM better than CIP, DIP 115 | sqsizesmall = 12 116 | for m in range(M): 117 | a = torch.zeros(1, 28, 28) 118 | # this will make the pixel white, 3.25 is because of normalization 119 | a[0, (initpos-m*5+offset):initpos+sqsizesmall-m+offset, (initpos-m+offset):initpos+sqsizesmall-m*5+offset] = 3.25 120 | noise_patches.append(a) 121 | 122 | # now transform the data 123 | for m in range(M): 124 | # load MNIST data 125 | transformer = torchvision.transforms.Compose( 126 | [ 127 | torchvision.transforms.ToTensor(), 128 | torchvision.transforms.Normalize((0.1306,), (0.3081,)), 129 | torchvision.transforms.Lambda((lambda y: lambda x: torch.max(x, noise_patches[y]))(m)), 130 | ]) 131 | 132 | trainset = torchvision.datasets.MNIST(root=datapath, train=True, 133 | download=True, transform=transformer) 134 | 135 | testset = torchvision.datasets.MNIST(root=datapath, train=False, 136 | download=True, transform=transformer) 137 | 138 | if m != M-1: 139 | idx_trainsetlist_loc = get_actual_data_idx(trainset_original.targets, subset_prop=subset_prop, interY=False) 140 | idx_testsetlist_loc = get_actual_data_idx(testset_original.targets, subset_prop=subset_prop, interY=False) 141 | else: 142 | idx_trainsetlist_loc = get_actual_data_idx(trainset_original.targets, subset_prop=subset_prop, interY=interY) 143 | idx_testsetlist_loc = get_actual_data_idx(testset_original.targets, subset_prop=subset_prop, interY=interY) 144 | print('actual env=%d trainsize %s, testsize %s' %(m, idx_trainsetlist_loc.shape, idx_testsetlist_loc.shape)) 145 | 146 | trainloaders[m] = torch.utils.data.DataLoader(trainset, batch_size=train_batch_size, num_workers=0, 147 | sampler = torch.utils.data.sampler.SubsetRandomSampler(idx_trainsetlist_loc)) 148 | testloaders[m] = torch.utils.data.DataLoader(testset, batch_size=test_batch_size, num_workers=0, 149 | sampler = torch.utils.data.sampler.SubsetRandomSampler(idx_testsetlist_loc)) 150 | trainloaders[m].dataset.targets_mod = trainloaders[m].dataset.targets[idx_trainsetlist_loc] 151 | testloaders[m].dataset.targets_mod = testloaders[m].dataset.targets[idx_testsetlist_loc] 152 | elif perturb == 'noisepatch': 153 | # prepare the noise patces 154 | noise_patches = [] 155 | offset = 10 156 | initpos = 2 157 | # sqsize 12 works to make CIRM better than CIP 158 | # offset = 4, sqsize = 12, initpos = 6, interY = 0.3 works for CIRM better than CIP, DIP 159 | sqsizesmall = 12 160 | sqsizelarge = 16 161 | for m in range(M-2): 162 | a = torch.zeros(1, 28, 28) 163 | a[0, (initpos-m+offset):initpos+sqsizelarge-m+offset, (initpos-m+offset):initpos+sqsizelarge-m+offset] = 3.25 * (torch.rand(1, sqsizelarge, sqsizelarge) > 0.5) 164 | noise_patches.append(a) 165 | 166 | for m in [M-2, M-1]: 167 | a = torch.zeros(1, 28, 28) 168 | a[0, (initpos-m+offset):initpos+sqsizesmall-m+offset, (initpos-m+offset):initpos+sqsizesmall-m+offset] = 3.25 * (torch.rand(1, sqsizesmall, sqsizesmall) > 0.5) 169 | noise_patches.append(a) 170 | 171 | # now transform the data 172 | for m in range(M): 173 | # load MNIST data 174 | transformer = torchvision.transforms.Compose( 175 | [ 176 | torchvision.transforms.ToTensor(), 177 | torchvision.transforms.Normalize((0.1306,), (0.3081,)), 178 | torchvision.transforms.Lambda((lambda y: lambda x: torch.max(x, noise_patches[y]))(m)), 179 | ]) 180 | 181 | trainset = torchvision.datasets.MNIST(root=datapath, train=True, 182 | download=True, transform=transformer) 183 | 184 | testset = torchvision.datasets.MNIST(root=datapath, train=False, 185 | download=True, transform=transformer) 186 | 187 | if m != M-1: 188 | idx_trainsetlist_loc = get_actual_data_idx(trainset_original.targets, subset_prop=subset_prop, interY=False) 189 | idx_testsetlist_loc = get_actual_data_idx(testset_original.targets, subset_prop=subset_prop, interY=False) 190 | else: 191 | idx_trainsetlist_loc = get_actual_data_idx(trainset_original.targets, subset_prop=subset_prop, interY=interY) 192 | idx_testsetlist_loc = get_actual_data_idx(testset_original.targets, subset_prop=subset_prop, interY=interY) 193 | print('actual env=%d trainsize %s, testsize %s' %(m, idx_trainsetlist_loc.shape, idx_testsetlist_loc.shape)) 194 | 195 | trainloaders[m] = torch.utils.data.DataLoader(trainset, batch_size=train_batch_size, num_workers=0, 196 | sampler = torch.utils.data.sampler.SubsetRandomSampler(idx_trainsetlist_loc)) 197 | testloaders[m] = torch.utils.data.DataLoader(testset, batch_size=test_batch_size, num_workers=0, 198 | sampler = torch.utils.data.sampler.SubsetRandomSampler(idx_testsetlist_loc)) 199 | trainloaders[m].dataset.targets_mod = trainloaders[m].dataset.targets[idx_trainsetlist_loc] 200 | testloaders[m].dataset.targets_mod = testloaders[m].dataset.targets[idx_testsetlist_loc] 201 | 202 | elif perturb == 'rotation': 203 | angles = np.zeros(M) 204 | if M == 12: 205 | angles = np.arange(M) * 10 - 45 206 | elif M == 10: 207 | angles = np.arange(M) * 10 - 35 208 | angles[M-1] = 50 209 | elif M == 5: 210 | angles = np.arange(M) * 15 - 30 211 | # now transform the data 212 | for m in range(M): 213 | # load MNIST data 214 | transformer = torchvision.transforms.Compose( 215 | [torchvision.transforms.RandomRotation((angles[m], angles[m])), 216 | torchvision.transforms.ToTensor(), 217 | torchvision.transforms.Normalize((0.1306,), (0.3081,)) 218 | ]) 219 | trainset = torchvision.datasets.MNIST(root=datapath, train=True, 220 | download=True, transform=transformer) 221 | 222 | testset = torchvision.datasets.MNIST(root=datapath, train=False, 223 | download=True, transform=transformer) 224 | 225 | if m != M-1: 226 | idx_trainsetlist_loc = get_actual_data_idx(trainset_original.targets, subset_prop=subset_prop, interY=False) 227 | idx_testsetlist_loc = get_actual_data_idx(testset_original.targets, subset_prop=subset_prop, interY=False) 228 | else: 229 | idx_trainsetlist_loc = get_actual_data_idx(trainset_original.targets, subset_prop=subset_prop, interY=interY) 230 | idx_testsetlist_loc = get_actual_data_idx(testset_original.targets, subset_prop=subset_prop, interY=interY) 231 | print('actual env=%d trainsize %s, testsize %s' %(m, idx_trainsetlist_loc.shape, idx_testsetlist_loc.shape)) 232 | 233 | trainloaders[m] = torch.utils.data.DataLoader(trainset, batch_size=train_batch_size, num_workers=0, 234 | sampler = torch.utils.data.sampler.SubsetRandomSampler(idx_trainsetlist_loc)) 235 | testloaders[m] = torch.utils.data.DataLoader(testset, batch_size=test_batch_size, num_workers=0, 236 | sampler = torch.utils.data.sampler.SubsetRandomSampler(idx_testsetlist_loc)) 237 | trainloaders[m].dataset.targets_mod = trainloaders[m].dataset.targets[idx_trainsetlist_loc] 238 | testloaders[m].dataset.targets_mod = testloaders[m].dataset.targets[idx_testsetlist_loc] 239 | elif perturb == 'rotation2M' or perturb == 'rotation2Ma': 240 | if perturb == 'rotation2M': 241 | angles = [30, 45] 242 | else: 243 | angles = [10, 45] 244 | # now transform the data 245 | for m in range(M): 246 | # load MNIST data 247 | transformer = torchvision.transforms.Compose( 248 | [torchvision.transforms.RandomRotation((angles[m], angles[m])), 249 | torchvision.transforms.ToTensor(), 250 | torchvision.transforms.Normalize((0.1306,), (0.3081,)) 251 | ]) 252 | trainset = torchvision.datasets.MNIST(root=datapath, train=True, 253 | download=True, transform=transformer) 254 | 255 | testset = torchvision.datasets.MNIST(root=datapath, train=False, 256 | download=True, transform=transformer) 257 | 258 | if m != M-1: 259 | idx_trainsetlist_loc = get_actual_data_idx(trainset_original.targets, subset_prop=subset_prop, interY=False) 260 | idx_testsetlist_loc = get_actual_data_idx(testset_original.targets, subset_prop=subset_prop, interY=False) 261 | else: 262 | idx_trainsetlist_loc = get_actual_data_idx(trainset_original.targets, subset_prop=subset_prop, interY=interY) 263 | idx_testsetlist_loc = get_actual_data_idx(testset_original.targets, subset_prop=subset_prop, interY=interY) 264 | print('actual env=%d trainsize %s, testsize %s' %(m, idx_trainsetlist_loc.shape, idx_testsetlist_loc.shape)) 265 | 266 | trainloaders[m] = torch.utils.data.DataLoader(trainset, batch_size=train_batch_size, num_workers=0, 267 | sampler = torch.utils.data.sampler.SubsetRandomSampler(idx_trainsetlist_loc)) 268 | testloaders[m] = torch.utils.data.DataLoader(testset, batch_size=test_batch_size, num_workers=0, 269 | sampler = torch.utils.data.sampler.SubsetRandomSampler(idx_testsetlist_loc)) 270 | trainloaders[m].dataset.targets_mod = trainloaders[m].dataset.targets[idx_trainsetlist_loc] 271 | testloaders[m].dataset.targets_mod = testloaders[m].dataset.targets[idx_testsetlist_loc] 272 | 273 | elif perturb == 'translation2M': 274 | translates = [(0.2, 0), (0, 0.2)] 275 | # now transform the data 276 | for m in range(M): 277 | # load MNIST data 278 | transformer = torchvision.transforms.Compose( 279 | [torchvision.transforms.RandomAffine(0, translates[m]), 280 | torchvision.transforms.ToTensor(), 281 | torchvision.transforms.Normalize((0.1306,), (0.3081,)) 282 | ]) 283 | trainset = torchvision.datasets.MNIST(root=datapath, train=True, 284 | download=True, transform=transformer) 285 | 286 | testset = torchvision.datasets.MNIST(root=datapath, train=False, 287 | download=True, transform=transformer) 288 | 289 | if m != M-1: 290 | idx_trainsetlist_loc = get_actual_data_idx(trainset_original.targets, subset_prop=subset_prop, interY=False) 291 | idx_testsetlist_loc = get_actual_data_idx(testset_original.targets, subset_prop=subset_prop, interY=False) 292 | else: 293 | idx_trainsetlist_loc = get_actual_data_idx(trainset_original.targets, subset_prop=subset_prop, interY=interY) 294 | idx_testsetlist_loc = get_actual_data_idx(testset_original.targets, subset_prop=subset_prop, interY=interY) 295 | print('actual env=%d trainsize %s, testsize %s' %(m, idx_trainsetlist_loc.shape, idx_testsetlist_loc.shape)) 296 | 297 | trainloaders[m] = torch.utils.data.DataLoader(trainset, batch_size=train_batch_size, num_workers=0, 298 | sampler = torch.utils.data.sampler.SubsetRandomSampler(idx_trainsetlist_loc)) 299 | testloaders[m] = torch.utils.data.DataLoader(testset, batch_size=test_batch_size, num_workers=0, 300 | sampler = torch.utils.data.sampler.SubsetRandomSampler(idx_testsetlist_loc)) 301 | trainloaders[m].dataset.targets_mod = trainloaders[m].dataset.targets[idx_trainsetlist_loc] 302 | testloaders[m].dataset.targets_mod = testloaders[m].dataset.targets[idx_testsetlist_loc] 303 | 304 | return trainloaders, testloaders 305 | 306 | -------------------------------------------------------------------------------- /MNIST/submit_simu_MNIST_patches.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import numpy as np 5 | import subprocess 6 | 7 | 8 | # perturb = 'whitepatch' 9 | perturb = 'rotation' 10 | epochs=100 11 | M = 5 12 | lamMatches = [10.**(k) for k in (np.arange(10)-5)] 13 | lamCIPs = [10.**(k) for k in (np.arange(10)-5)] 14 | 15 | tag_DA = 'baseline' 16 | for seed in range(10): 17 | for k in [2]: 18 | subprocess.call(['bsub', '-W 3:50', '-n 4', '-R', "rusage[ngpus_excl_p=1,mem=2048]", 19 | "./simu_MNIST_patches.py --perturb=%s --M=%d --subset_prop=%.1f --seed=%d --epochs=%d --tag_DA=%s" %(perturb, M, k/10, seed, epochs, tag_DA)]) 20 | 21 | for tag_DA in ['DACIPmean']: 22 | for lam in lamCIPs: 23 | for seed in range(10): 24 | for k in [2]: 25 | subprocess.call(['bsub', '-W 3:50', '-n 8', '-R', "rusage[ngpus_excl_p=1,mem=2048]", 26 | "./simu_MNIST_patches.py --perturb=%s --M=%d --subset_prop=%.1f --seed=%d --lamMatch=%f --lamCIP=%f --lamMatchMMD=%f --lamCIPMMD=%f --epochs=%d --tag_DA=%s" %(perturb, M, k/10, seed, 1., lam, 1., lam, epochs, tag_DA)]) 27 | 28 | for tag_DA in ['DACIPMMD']: 29 | for lam in lamCIPs: 30 | for seed in range(10): 31 | for k in [2]: 32 | subprocess.call(['bsub', '-W 23:50', '-n 8', '-R', "rusage[ngpus_excl_p=1,mem=2048]", 33 | "./simu_MNIST_patches.py --perturb=%s --M=%d --subset_prop=%.1f --seed=%d --lamMatch=%f --lamCIP=%f --lamMatchMMD=%f --lamCIPMMD=%f --epochs=%d --tag_DA=%s" %(perturb, M, k/10, seed, 1., lam, 1., lam, epochs, tag_DA)]) 34 | 35 | # pick lamCIP after looking at CIP source results 36 | lamCIP = 1. 37 | for tag_DA in ['DAmean']: 38 | for lam in lamMatches: 39 | for seed in range(10): 40 | for k in [2]: 41 | subprocess.call(['bsub', '-W 23:50', '-n 4', '-R', "rusage[ngpus_excl_p=1,mem=2048]", 42 | "./simu_MNIST_patches.py --perturb=%s --M=%d --subset_prop=%.1f --seed=%d --lamMatch=%f --lamCIP=%f --lamMatchMMD=%f --lamCIPMMD=%f --epochs=%d --tag_DA=%s" %(perturb, M, k/10, seed, lam, lamCIP, lam, lamCIP, epochs, tag_DA)]) 43 | 44 | 45 | lamCIP = 1. 46 | for tag_DA in ['DAMMD']: 47 | for lam in lamMatches: 48 | for seed in range(10): 49 | for k in [2]: 50 | subprocess.call(['bsub', '-W 23:50', '-n 8', '-R', "rusage[ngpus_excl_p=1,mem=2048]", 51 | "./simu_MNIST_patches.py --perturb=%s --M=%d --subset_prop=%.1f --seed=%d --lamMatch=%f --lamCIP=%f --lamMatchMMD=%f --lamCIPMMD=%f --epochs=%d --tag_DA=%s" %(perturb, M, k/10, seed, lam, lamCIP, lam, lamCIP, epochs, tag_DA)]) 52 | -------------------------------------------------------------------------------- /MNIST/submit_simu_MNIST_patches_2M.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import numpy as np 5 | import subprocess 6 | 7 | 8 | #perturb = 'whitepatch2M' 9 | #perturb = 'rotation2M' 10 | perturb = 'rotation2Ma' 11 | #perturb = 'translation2M' 12 | tag_DA = 'baseline' 13 | epochs = 100 14 | for seed in range(10): 15 | for k in [2]: 16 | subprocess.call(['bsub', '-W 3:50', '-n 4', '-R', "rusage[ngpus_excl_p=1,mem=2048]", 17 | "./simu_MNIST_patches_2M.py --perturb=%s --subset_prop=%.1f --seed=%d --epochs=%d --tag_DA=%s" %(perturb, k/10, seed, epochs, tag_DA)]) 18 | 19 | tag_DA = 'DIP' 20 | lamMatches = [10.**(k) for k in (np.arange(10)-5)] 21 | for lam in lamMatches: 22 | for seed in range(10): 23 | for k in [2]: 24 | subprocess.call(['bsub', '-W 3:50', '-n 4', '-R', "rusage[ngpus_excl_p=1,mem=2048]", 25 | "./simu_MNIST_patches_2M.py --perturb=%s --subset_prop=%.1f --seed=%d --lamMatch=%f --lamMatchMMD=%f --epochs=%d --tag_DA=%s" %(perturb, k/10, seed, lam, lam, epochs, tag_DA)]) 26 | 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CausalDA 2 | Code to reproduce the numerical experiments in the paper "Domain adaptation under structural causal models" (https://arxiv.org/abs/2010.15764) 3 | by Yuansi Chen and Peter Bühlmann. 4 | 5 | Code written with Python 3.7.1 and PyTorch version as follows 6 | 7 | torch==1.2.0 8 | torchvision==0.4.0 9 | 10 | 11 | 12 | ## User Guide 13 | 14 | - semiclass.py implements the main DA methods 15 | - semitorchclass, semitorchstocclass implement the same functions with PyTorch 16 | - semitorchMNISTclass is tailored to convolutional neural nets 17 | - Linear SCM simulations: first seven experiments 18 | - sim_linearSCM_mean_shift_exp1-7.ipynb can run on a single core and plot 19 | - Linear SCM simulations: last two experiments 20 | - Run the following simulations in a computer cluster 21 | - sim_linearSCM_var_shift_exp8_box_submit.py 22 | - sim_linearSCM_var_shift_exp8_scat_submit.py 23 | - sim_linearSCM_var_shift_exp9_scat_submit.py 24 | - Read the results and plot with sim_linearSCM_variance_shift_exp8-9.ipynb 25 | - MNIST experiments: 26 | - Need to set the MNIST data folder! 27 | - Run mnist_get_pretrained.ipynb to get a pretrained CNN on original MNIST 28 | - Single source exp: run submit_simu_MNIST_patches_2M.py 29 | - Mutiple source exp: run submit_simu_MNIST_patches.py 30 | - Read the results and plot with MNIST_read_and_plot_whitepatch2M.ipynb and MNIST_read_and_plot_rotation5M.ipynb 31 | - Amazon review dataset experiments 32 | - Need to set the Amazon review data folder! 33 | - Preprocess the data with read_and_preprocess_amazon_review_data_2018_subset.ipynb 34 | - Run the simulations with submit_amazon_review_data_2018_subset_regression.py 35 | - Plot with amazon_read_and_plot.ipynb 36 | 37 | 38 | 39 | ## License and Citation 40 | Code is released under MIT License. 41 | Please cite our paper if the code helps your research. 42 | 43 | ```bibtex 44 | @article{chen2020domain, 45 | title={Domain adaptation under structural causal models}, 46 | author={Chen, Yuansi and Peter B{\"u}hlmann}, 47 | journal={arXiv preprint arXiv:2010.15764}, 48 | year={2018} 49 | } 50 | ``` -------------------------------------------------------------------------------- /amazon/amazon_review_data_2018_subset_regression.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import numpy as np 5 | 6 | import pandas as pd 7 | import os 8 | import h5py 9 | import sys 10 | import argparse 11 | 12 | import torch 13 | 14 | import torch.nn as nn 15 | import torch.nn.functional as F 16 | import torch.optim as optim 17 | 18 | np.set_printoptions(precision=3) 19 | 20 | from sklearn.linear_model import Ridge 21 | from sklearn.model_selection import train_test_split 22 | from sklearn.feature_extraction.text import TfidfVectorizer 23 | 24 | # local packages 25 | import sys 26 | sys.path.append('../') 27 | import semiclass 28 | import semitorchclass 29 | import semitorchstocclass 30 | import util 31 | 32 | # check gpu avail 33 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 34 | 35 | # Assuming that we are on a CUDA machine, this should print a CUDA device: 36 | 37 | print(device) 38 | 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument("--lamMatch", type=float, default=1., help="DIP matching penalty") 41 | parser.add_argument("--lamCIP", type=float, default=0.1, help="CIP matching penalty") 42 | parser.add_argument("--lamL2", type=float, default=1., help="L2 penalty") 43 | parser.add_argument("--tag_DA", type=str, default="baseline", help="choose whether to run baseline methods or DA methods") 44 | parser.add_argument("--seed", type=int, default=0, help="seed of experiment") 45 | parser.add_argument("--target", type=int, default=0, help="target category") 46 | parser.add_argument("--minDf", type=float, default=0.008, help="minimum term frequency") 47 | parser.add_argument("--epochs", type=int, default=2000, help="number of epochs") 48 | parser.add_argument("--lr", type=float, default=1e-3, help="learning rate") 49 | myargs = parser.parse_args() 50 | print(myargs) 51 | 52 | data_folder = 'data/amazon_review_data_2018_subset' 53 | 54 | # 'All_Beauty_5', 'AMAZON_FASHION_5', 'Appliances_5', 'Gift_Cards_5', 'Magazine_Subscriptions_5' 55 | # are removed for now for not enough number of data points 56 | categories = [ 57 | 'Arts_Crafts_and_Sewing_5', 'Automotive_5', 'CDs_and_Vinyl_5', 58 | 'Cell_Phones_and_Accessories_5', 'Digital_Music_5', 59 | 'Grocery_and_Gourmet_Food_5', 'Industrial_and_Scientific_5', 'Luxury_Beauty_5', 60 | 'Musical_Instruments_5', 'Office_Products_5', 61 | 'Patio_Lawn_and_Garden_5', 'Pet_Supplies_5', 'Prime_Pantry_5', 62 | 'Software_5', 'Tools_and_Home_Improvement_5', 'Toys_and_Games_5'] 63 | 64 | nb_reviews = 10000 65 | dfs = {} 66 | for i, cate in enumerate(categories): 67 | df = pd.read_csv('%s/%s_%d.csv' %(data_folder, cate, nb_reviews)) 68 | dfs[i] = df 69 | print(cate, dfs[i].shape) 70 | 71 | allReviews = pd.concat([dfs[i]['reviewText'] for i in range(len(categories))]) 72 | ngramMin = 1 73 | ngramMax = 2 74 | stop_words = 'english' 75 | vectTF = TfidfVectorizer(min_df = myargs.minDf, stop_words=stop_words, ngram_range=(ngramMin, ngramMax)).fit(allReviews.values.astype('U')) 76 | print("Number of reviews=%d, feature size=%d" %(allReviews.shape[0], len(vectTF.get_feature_names()))) 77 | print(vectTF.vocabulary_) 78 | 79 | lamL1 = 0. 80 | 81 | if myargs.tag_DA == 'baseline': 82 | methods = [ 83 | semiclass.Tar(lamL2=myargs.lamL2), 84 | semiclass.SrcPool(lamL2=myargs.lamL2), 85 | ] 86 | elif myargs.tag_DA == 'DAmean': 87 | methods = [ 88 | semiclass.DIP(lamMatch=myargs.lamMatch, lamL2=myargs.lamL2, sourceInd=0), 89 | semiclass.DIPOracle(lamMatch=myargs.lamMatch, lamL2=myargs.lamL2, sourceInd=0), 90 | semiclass.DIPweigh(lamMatch=myargs.lamMatch, lamL2=myargs.lamL2), 91 | semiclass.CIP(lamCIP=myargs.lamCIP, lamL2=myargs.lamL2), 92 | semiclass.CIRMweigh(lamCIP=myargs.lamCIP, lamMatch=myargs.lamMatch, lamL2=myargs.lamL2), 93 | ] 94 | elif myargs.tag_DA == 'DAstd': 95 | methods = [ 96 | semitorchclass.DIP(lamMatch=myargs.lamMatch, lamL2=myargs.lamL2, lamL1=lamL1, sourceInd=0, lr=myargs.lr, 97 | epochs=myargs.epochs, wayMatch='mean+std+25p'), 98 | semitorchclass.DIPweigh(lamMatch=myargs.lamMatch, lamL2=myargs.lamL2, lamL1=lamL1, lr=myargs.lr, 99 | epochs=myargs.epochs, wayMatch='mean+std+25p'), 100 | semitorchclass.CIP(lamCIP=myargs.lamCIP, lamL2=myargs.lamL2, lamL1=lamL1, lr=myargs.lr, 101 | epochs=myargs.epochs, wayMatch='mean+std+25p'), 102 | semitorchclass.CIRMweigh(lamMatch=myargs.lamMatch, lamL2=myargs.lamL2, lamL1=lamL1, lr=myargs.lr, 103 | epochs=myargs.epochs, wayMatch='mean+std+25p'), 104 | ] 105 | elif myargs.tag_DA == 'DAMMD': 106 | methods = [ 107 | semitorchstocclass.DIP(lamMatch=myargs.lamMatch, lamL2=myargs.lamL2, lamL1=lamL1, sourceInd = 0, lr=myargs.lr, 108 | epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1.]), 109 | semitorchstocclass.DIPweigh(lamMatch=myargs.lamMatch, lamL2=myargs.lamL2, lamL1=lamL1, lr=myargs.lr, 110 | epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1.]), 111 | semitorchstocclass.CIP(lamCIP=myargs.lamCIP, lamL2=myargs.lamL2, lamL1=lamL1, lr=myargs.lr, 112 | epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1.]), 113 | semitorchstocclass.CIRMweigh(lamMatch=myargs.lamMatch, lamCIP=myargs.lamCIP, lamL2=myargs.lamL2, lamL1=lamL1, lr=myargs.lr, 114 | epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1.]) 115 | ] 116 | 117 | names = [str(m) for m in methods] 118 | print(names) 119 | 120 | # random data split 121 | random_state = 123456 + myargs.seed 122 | datasets = {} 123 | datasets_test = {} 124 | for i, cate in enumerate(categories): 125 | X_train_raw, X_test_raw, y_train, y_test = train_test_split(dfs[i]['reviewText'].astype('U'), dfs[i]['overall'].astype('U'), test_size=0.10, random_state = random_state) 126 | 127 | 128 | X_train = vectTF.transform(X_train_raw) 129 | X_test = vectTF.transform(X_test_raw) 130 | 131 | 132 | datasets[i] = np.array(X_train.todense()), np.array(y_train, dtype=np.float32) 133 | datasets_test[i] = np.array(X_test.todense()), np.array(y_test, dtype=np.float32) 134 | 135 | print(cate, X_train.shape, X_test.shape) 136 | 137 | # normalize 138 | Xmean = 0 139 | N = 0 140 | for i, cate in enumerate(categories): 141 | Xmean += np.sum(datasets[i][0], axis=0) 142 | N += datasets[i][0].shape[0] 143 | Xmean /= N 144 | 145 | 146 | Xvar = 0 147 | for i, cate in enumerate(categories): 148 | Xvar += np.sum((datasets[i][0] - Xmean.reshape(1, -1))**2, axis=0) 149 | N += datasets[i][0].shape[0] 150 | 151 | Xvar /= N 152 | Xstd = np.sqrt(Xvar) 153 | 154 | for i, cate in enumerate(categories): 155 | x, y = datasets[i] 156 | x_test, y_test = datasets_test[i] 157 | datasets[i] = (x - Xmean)/Xstd, y 158 | datasets_test[i] = (x_test - Xmean)/Xstd, y_test 159 | 160 | # create torch format data 161 | dataTorch = {} 162 | dataTorchTest = {} 163 | 164 | for i in range(len(categories)): 165 | dataTorch[i] = [torch.from_numpy(datasets[i][0].astype(np.float32)).to(device), 166 | torch.from_numpy(datasets[i][1].astype(np.float32)).to(device)] 167 | dataTorchTest[i] = [torch.from_numpy(datasets_test[i][0].astype(np.float32)).to(device), 168 | torch.from_numpy(datasets_test[i][1].astype(np.float32)).to(device)] 169 | 170 | train_batch_size = 500 171 | test_batch_size = 500 172 | 173 | trainloaders = {} 174 | testloaders = {} 175 | 176 | for i in range(len(categories)): 177 | train_dataset = torch.utils.data.TensorDataset(torch.Tensor(datasets[i][0]), 178 | torch.Tensor(datasets[i][1])) 179 | test_dataset = torch.utils.data.TensorDataset(torch.Tensor(datasets_test[i][0]), 180 | torch.Tensor(datasets_test[i][1])) 181 | trainloaders[i] = torch.utils.data.DataLoader(train_dataset, batch_size=train_batch_size) 182 | testloaders[i] = torch.utils.data.DataLoader(test_dataset, batch_size=test_batch_size) 183 | 184 | 185 | M = len(categories) 186 | source = [i for i in range(M)] 187 | source.remove(myargs.target) 188 | 189 | print("source =", source, "target =", myargs.target, flush=True) 190 | results_src_all = np.zeros((M-1, len(methods), 2)) 191 | results_tar_all = np.zeros((len(methods), 2)) 192 | results_minDiffIndx = {} 193 | labeledsize_list = np.arange(1, 11) * 20 194 | results_tar_sub_all = np.zeros((len(methods), len(labeledsize_list))) 195 | for i, m in enumerate(methods): 196 | if m.__module__ == 'semiclass': 197 | me = m.fit(datasets, source=source, target=myargs.target) 198 | if hasattr(me, 'minDiffIndx'): 199 | print("best index="+str(me.minDiffIndx)) 200 | results_minDiffIndx[(myargs.tag_DA, i)] = me.minDiffIndx 201 | xtar, ytar= datasets[myargs.target] 202 | xtar_test, ytar_test= datasets_test[myargs.target] 203 | targetE = util.MSE(me.ypred, ytar) 204 | targetNE = util.MSE(me.predict(xtar_test), ytar_test) 205 | for j, sourcej in enumerate(source): 206 | results_src_all[j, i, 0] = util.MSE(me.predict(datasets[sourcej][0]), datasets[sourcej][1]) 207 | results_src_all[j, i, 1] = util.MSE(me.predict(datasets_test[sourcej][0]), datasets_test[sourcej][1]) 208 | # obtain target error for each labeledsize 209 | for k, labeledsize in enumerate(labeledsize_list): 210 | xtar_sub = xtar[:labeledsize, :] 211 | ytar_sub = ytar[:labeledsize] 212 | results_tar_sub_all[i, k] = util.MSE(me.predict(xtar_sub), ytar_sub) 213 | elif m.__module__ == 'semitorchclass': 214 | me = m.fit(dataTorch, source=source, target=myargs.target) 215 | if hasattr(me, 'minDiffIndx'): 216 | print("best index="+str(me.minDiffIndx)) 217 | results_minDiffIndx[(myargs.tag_DA, i)] = me.minDiffIndx 218 | xtar, ytar= dataTorch[myargs.target] 219 | xtar_test, ytar_test= dataTorchTest[myargs.target] 220 | targetE = util.torchMSE(me.ypred, ytar) 221 | targetNE = util.torchMSE(me.predict(xtar_test), ytar_test) 222 | for j, sourcej in enumerate(source): 223 | results_src_all[j, i, 0] = util.torchMSE(me.predict(dataTorch[sourcej][0]), dataTorch[sourcej][1]) 224 | results_src_all[j, i, 1] = util.torchMSE(me.predict(dataTorchTest[sourcej][0]), dataTorchTest[sourcej][1]) 225 | for k, labeledsize in enumerate(labeledsize_list): 226 | xtar_sub = xtar[:labeledsize, :] 227 | ytar_sub = ytar[:labeledsize] 228 | results_tar_sub_all[i, k] = util.torchMSE(me.predict(xtar_sub), ytar_sub) 229 | elif m.__module__ == 'semitorchstocclass': 230 | me = m.fit(trainloaders, source=source, target=myargs.target) 231 | targetE = util.torchloaderMSE(me, trainloaders[myargs.target], device) 232 | targetNE = util.torchloaderMSE(me, testloaders[myargs.target], device) 233 | for j, sourcej in enumerate(source): 234 | results_src_all[j, i, 0] = util.torchloaderMSE(me, trainloaders[sourcej], device) 235 | results_src_all[j, i, 1] = util.torchloaderMSE(me, testloaders[sourcej], device) 236 | else: 237 | raise ValueError('error') 238 | results_tar_all[i, 0] = targetE 239 | results_tar_all[i, 1] = targetNE 240 | 241 | res_all = {} 242 | res_all['src'] = results_src_all 243 | res_all['tar'] = results_tar_all 244 | res_all['minDiffIndx'] = results_minDiffIndx 245 | res_all['tar_sub'] = results_tar_sub_all 246 | res_all['labeledsize_list'] = labeledsize_list 247 | 248 | np.save('results_amazon/amazon_review_data_2018_N%d_%s_minDf%s_lamL2%s_lamMatch%s_lamCIP%s_target%d_seed%d.npy' %( 249 | nb_reviews, myargs.tag_DA, myargs.minDf, myargs.lamL2, myargs.lamMatch, myargs.lamCIP, myargs.target, myargs.seed), res_all) 250 | 251 | 252 | -------------------------------------------------------------------------------- /amazon/read_and_preprocess_amazon_review_data_2018_subset.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np\n", 10 | "import matplotlib.pyplot as plt\n", 11 | "import pandas as pd\n", 12 | "import gzip\n", 13 | "import json\n", 14 | "import os\n", 15 | "import h5py\n", 16 | "\n", 17 | "\n", 18 | "plt.rcParams['axes.facecolor'] = 'lightgray'\n", 19 | "\n", 20 | "np.set_printoptions(precision=3)" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 2, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "def parse(path):\n", 30 | " g = gzip.open(path, 'rb')\n", 31 | " for l in g:\n", 32 | " yield json.loads(l)\n", 33 | "\n", 34 | "def getDF(path):\n", 35 | " i = 0\n", 36 | " df = {}\n", 37 | " for d in parse(path):\n", 38 | " df[i] = d\n", 39 | " i += 1\n", 40 | " return pd.DataFrame.from_dict(df, orient='index')" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 3, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "categories = [\n", 50 | " 'Arts_Crafts_and_Sewing_5', 'Automotive_5', 'CDs_and_Vinyl_5',\n", 51 | " 'Cell_Phones_and_Accessories_5', 'Digital_Music_5',\n", 52 | " 'Grocery_and_Gourmet_Food_5', 'Industrial_and_Scientific_5', 'Luxury_Beauty_5',\n", 53 | " 'Musical_Instruments_5', 'Office_Products_5',\n", 54 | " 'Patio_Lawn_and_Garden_5', 'Pet_Supplies_5', 'Prime_Pantry_5',\n", 55 | " 'Software_5', 'Tools_and_Home_Improvement_5', 'Toys_and_Games_5']" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 4, 61 | "metadata": {}, 62 | "outputs": [ 63 | { 64 | "name": "stdout", 65 | "output_type": "stream", 66 | "text": [ 67 | "Clothing_Shoes_and_Jewelry_5 (10000, 12)\n", 68 | "Electronics_5 (10000, 12)\n", 69 | "Home_and_Kitchen_5 (10000, 12)\n", 70 | "Kindle_Store_5 (10000, 12)\n", 71 | "Movies_and_TV_5 (10000, 12)\n", 72 | "Sports_and_Outdoors_5 (10000, 12)\n", 73 | "Video_Games_5 (10000, 12)\n" 74 | ] 75 | } 76 | ], 77 | "source": [ 78 | "N = 10000\n", 79 | "np.random.seed(123456)\n", 80 | "dfs = {}\n", 81 | "for i, cate in enumerate(categories):\n", 82 | " df = getDF('data/amazon_review_data_2018_subset/%s.json.gz' %cate)\n", 83 | " df = df[~df.reviewText.isna()]\n", 84 | " if df.shape[0] > N:\n", 85 | " df = df.sample(n=N)\n", 86 | " dfs[i] = df\n", 87 | " \n", 88 | " dfs[i].to_csv(\"data/amazon_review_data_2018_subset/%s_%d.csv\" %(cate, N), index=False)\n", 89 | " print(cate, dfs[i].shape)\n", 90 | " \n", 91 | " " 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": {}, 112 | "outputs": [], 113 | "source": [] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": null, 118 | "metadata": {}, 119 | "outputs": [], 120 | "source": [] 121 | } 122 | ], 123 | "metadata": { 124 | "kernelspec": { 125 | "display_name": "Python 3", 126 | "language": "python", 127 | "name": "python3" 128 | }, 129 | "language_info": { 130 | "codemirror_mode": { 131 | "name": "ipython", 132 | "version": 3 133 | }, 134 | "file_extension": ".py", 135 | "mimetype": "text/x-python", 136 | "name": "python", 137 | "nbconvert_exporter": "python", 138 | "pygments_lexer": "ipython3", 139 | "version": "3.6.5" 140 | } 141 | }, 142 | "nbformat": 4, 143 | "nbformat_minor": 2 144 | } 145 | -------------------------------------------------------------------------------- /amazon/results_amazon/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /amazon/submit_amazon_review_data_2018_subset_regression.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | 5 | import subprocess 6 | import numpy as np 7 | 8 | tag_DA = 'baseline' 9 | lamL2s = [10.**(k) for k in (np.arange(10)-5)] 10 | for target in range(16): 11 | for lamL2 in lamL2s: 12 | for myseed in range(10): 13 | subprocess.call(['bsub', '-W 03:50', '-n 8', '-R', "rusage[mem=4096]", "./amazon_review_data_2018_subset_regression.py --target=%d --tag_DA=%s --lamL2=%s --seed=%d" %(target, tag_DA, lamL2, myseed)]) 14 | 15 | tag_DA = 'DAmean' 16 | lamMatches = [10.**(k) for k in (np.arange(10)-5)] 17 | for target in range(16): 18 | for lam in lamMatches: 19 | for myseed in range(10): 20 | subprocess.call(['bsub', '-W 03:50', '-n 8', '-R', "rusage[mem=4096]", "./amazon_review_data_2018_subset_regression.py --target=%d --tag_DA=%s --lamL2=%s --lamMatch=%s --epochs=%d --seed=%d" %(target, tag_DA, 1.0, lam, 20000, myseed)]) 21 | 22 | 23 | # tag_DA = 'DAstd' 24 | # for target in range(16): 25 | # for lam in lamMatches: 26 | # for myseed in range(10): 27 | # subprocess.call(['bsub', '-W 23:50', '-n 8', '-R', "rusage[mem=4096]", "./amazon_review_data_2018_subset_regression.py --target=%d --tag_DA=%s --lamMatch=%s --epochs=%d --seed=%d" %(target, tag_DA, lam, 20000, myseed)]) 28 | -------------------------------------------------------------------------------- /mmd.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def _mix_rbf_kernel(X, Y, sigma_list): 4 | m = X.size(0) 5 | 6 | Z = torch.cat((X, Y), 0) 7 | ZZT = torch.mm(Z, Z.t()) 8 | diag_ZZT = torch.diag(ZZT).unsqueeze(1) 9 | Z_norm_sqr = diag_ZZT.expand_as(ZZT) 10 | exponent = Z_norm_sqr - 2 * ZZT + Z_norm_sqr.t() 11 | 12 | K = 0.0 13 | for sigma in sigma_list: 14 | gamma = 1.0 / (2 * sigma**2) 15 | K += torch.exp(-gamma * exponent) 16 | 17 | return K[:m, :m], K[:m, m:], K[m:, m:], len(sigma_list) 18 | 19 | def mix_rbf_mmd2(X, Y, sigma_list): 20 | K_XX, K_XY, K_YY, d = _mix_rbf_kernel(X, Y, sigma_list) 21 | return _mmd2(K_XX, K_XY, K_YY) 22 | 23 | 24 | ################################################################################ 25 | # Helper functions to compute variances based on kernel matrices 26 | ################################################################################ 27 | 28 | 29 | def _mmd2(K_XX, K_XY, K_YY): 30 | m = K_XX.size(0) 31 | l = K_YY.size(0) 32 | 33 | K_XX_sum = K_XX.sum() 34 | K_YY_sum = K_YY.sum() 35 | K_XY_sum = K_XY.sum() 36 | 37 | mmd2 = (K_XX_sum / (m * m) 38 | + K_YY_sum / (l * l) 39 | - 2.0 * K_XY_sum / (m * l)) 40 | 41 | return mmd2 42 | -------------------------------------------------------------------------------- /myrandom.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from abc import abstractmethod 3 | 4 | class RandomGen(): 5 | """Base class for random number generation""" 6 | @abstractmethod 7 | def generate(self, m, n, d): 8 | pass 9 | 10 | class Gaussian(RandomGen): 11 | def __init__(self, M, meanAs, varAs): 12 | self.M = M 13 | self.meanAs = meanAs 14 | self.varAs = varAs 15 | 16 | def generate(self, m, n, d): 17 | vec = np.random.randn(n, d).dot(np.diag(np.sqrt(self.varAs[m, ]))) 18 | vec += self.meanAs[m, :].reshape(1, -1) 19 | return vec 20 | 21 | class Mix2Gaussian(RandomGen): 22 | def __init__(self, M, meanAsList, varAs): 23 | self.M = M 24 | self.meanAsList = meanAsList 25 | self.varAs = varAs 26 | 27 | def generate(self, m, n, d): 28 | vec = np.random.randn(n, d).dot(np.diag(np.sqrt(self.varAs[m, ]))) 29 | mixture_idx = np.random.choice(2, size=n, replace=True, p=[0.5, 0.5]) 30 | for i in range(n): 31 | vec[i, :] += self.meanAsList[mixture_idx[i]][m, :] 32 | return vec 33 | 34 | class MixkGaussian(RandomGen): 35 | def __init__(self, M, meanAsList, varAs): 36 | self.M = M 37 | self.meanAsList = meanAsList 38 | self.varAs = varAs 39 | self.k = len(self.meanAsList) 40 | 41 | def generate(self, m, n, d): 42 | vec = np.random.randn(n, d).dot(np.diag(np.sqrt(self.varAs[m, ]))) 43 | mixture_idx = np.random.choice(self.k, size=n, replace=True, p=np.ones(self.k)/self.k) 44 | for i in range(n): 45 | vec[i, :] += self.meanAsList[mixture_idx[i]][m, :] 46 | return vec 47 | -------------------------------------------------------------------------------- /sem.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import matplotlib.pyplot as plt 4 | 5 | from networkx import nx 6 | 7 | # basic structural equation 8 | class SEM(): 9 | def __init__(self, B, noisef, interAf, invariantList=[], message=None): 10 | self.B = B 11 | self.dp1 = B.shape[0] 12 | self.M = interAf.M 13 | self.interAf = interAf 14 | self.noisef = noisef 15 | 16 | # just for plot purpose 17 | self.invariantList = invariantList 18 | self.message = message 19 | 20 | # raise an error if it is not invertible 21 | self.IBinv = np.linalg.inv(np.eye(self.dp1) - self.B) 22 | 23 | def generateSamples(self, n, m=0): 24 | # generate n samples from mth environment 25 | noise = self.noisef.generate(0, n, self.dp1) 26 | interA = self.interAf.generate(m, n, self.dp1) 27 | 28 | data = (noise + interA).dot(self.IBinv.T) 29 | 30 | # return x and y separately 31 | return data[:, :-1], data[:, -1] 32 | 33 | def generateAllSamples(self, n): 34 | res = {} 35 | for m in range(self.M): 36 | res[m] = self.generateSamples(n, m) 37 | 38 | return res 39 | 40 | def draw(self, layout='circular', figsize=(12, 8)): 41 | plt.figure(figsize=figsize) 42 | G = nx.DiGraph() 43 | G.add_nodes_from(np.arange(self.dp1)) 44 | for i in range(self.dp1): 45 | for j in range(self.dp1): 46 | if not np.isclose(self.B[i, j], 0): 47 | G.add_edge(j, i, weight=self.B[i, j]) 48 | 49 | # labels 50 | labels={} 51 | for i in range(self.dp1-1): 52 | labels[i] = "X"+str(i) 53 | labels[self.dp1-1] = "Y" 54 | 55 | # position 56 | if layout=="spring": 57 | pos = nx.spring_layout(G) 58 | elif layout=="kamada_kawai": 59 | pos = nx.kamada_kawai_layout(G) 60 | else: 61 | pos = nx.circular_layout(G) 62 | 63 | nx.draw_networkx_nodes(G,pos,nodelist = list(np.arange(self.dp1-1)), node_color='b', node_size=1000, alpha=0.5) 64 | nx.draw_networkx_nodes(G,pos,nodelist = [self.dp1-1], node_color='r', 65 | node_size=1000, alpha=0.8) 66 | nx.draw_networkx_nodes(G,pos,nodelist = list(self.invariantList), node_color='y', node_size=1000, alpha=0.8) 67 | nx.draw_networkx_edges(G,pos,width=3.0,alpha=0.5, arrowsize=40) 68 | 69 | arc_weight=nx.get_edge_attributes(G,'weight') 70 | arc_weight_format = {i:'{:.2f}'.format(arc_weight[i]) for i in arc_weight} 71 | 72 | 73 | nx.draw_networkx_edge_labels(G, pos,edge_color= 'k', label_pos=0.7, edge_labels=arc_weight_format) 74 | 75 | nx.draw_networkx_labels(G,pos,labels,font_size=16) 76 | plt.draw() 77 | plt.show() 78 | 79 | 80 | -------------------------------------------------------------------------------- /semitorchclass.py: -------------------------------------------------------------------------------- 1 | """ 2 | ``semitorchclass`` provides classes implementing various domain adaptation methods using torch and gradient method. 3 | All domain adaptation methods have to be subclass of BaseEstimator. 4 | This implementation takes advantage of gradient method to optimize covariance match or MMD match in addition to mean match. 5 | """ 6 | 7 | from abc import abstractmethod 8 | import numpy as np 9 | 10 | import torch 11 | import torch.nn as nn 12 | import torch.nn.functional as F 13 | import torch.optim as optim 14 | 15 | import mmd 16 | 17 | # check gpu avail 18 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 19 | 20 | 21 | # simple linear model in torch 22 | class LinearModel(nn.Module): 23 | def __init__(self, d): 24 | super(LinearModel, self).__init__() 25 | self.lin1 = nn.Linear(d, 1, bias=True) 26 | 27 | def forward(self, x): 28 | x = self.lin1(x) 29 | return x 30 | 31 | class BaseEstimator(): 32 | """Base class for domain adaptation""" 33 | @abstractmethod 34 | def fit(self, data, source, target): 35 | """Fit model. 36 | Arguments: 37 | data (dict of (X, y) pairs): maps env index to the (X, y) pair in that env 38 | source (list of indexes): indexes of source envs 39 | target (int): single index of the target env 40 | """ 41 | self.source = source 42 | self.target = target 43 | 44 | return self 45 | 46 | @abstractmethod 47 | def predict(self, X): 48 | """Use the learned estimator to predict labels on fresh target data X 49 | """ 50 | 51 | def __str__(self): 52 | """For easy name printing 53 | """ 54 | return self.__class__.__name__ 55 | 56 | class ZeroBeta(BaseEstimator): 57 | """Estimator that sets beta to zero""" 58 | 59 | def fit(self, data, source, target): 60 | super().fit(data, source, target) 61 | 62 | d = data[target][0].shape[1] 63 | model = LinearModel(d).to(device) 64 | with torch.no_grad(): 65 | model.lin1.weight.data = torch.zeros_like(model.lin1.weight) 66 | model.lin1.bias.data = torch.zeros_like(model.lin1.bias) 67 | 68 | self.model = model 69 | 70 | xtar, _ = data[target] 71 | self.ypred = self.model(xtar) 72 | 73 | return self 74 | 75 | def predict(self, X): 76 | ypredX = self.model(X) 77 | return ypredX 78 | 79 | class Tar(BaseEstimator): 80 | """Oracle Linear regression (with l1 or l2 penalty) trained on the target domain""" 81 | def __init__(self, lamL2=0.0, lamL1=0.0, lr=1e-4, epochs=10): 82 | self.lamL2 = lamL2 83 | self.lamL1 = lamL1 84 | self.lr = lr 85 | self.epochs = epochs 86 | 87 | def fit(self, data, source, target): 88 | super().fit(data, source, target) 89 | 90 | d = data[target][0].shape[1] 91 | model = LinearModel(d).to(device) 92 | with torch.no_grad(): 93 | model.lin1.bias.data = torch.zeros_like(model.lin1.bias) 94 | # torch.nn.init.kaiming_normal_(model.lin1.weight, mode='fan_in') 95 | torch.nn.init.xavier_normal_(model.lin1.weight, gain=0.01) 96 | # Define loss function 97 | loss_fn = F.mse_loss 98 | opt = optim.Adam(model.parameters(), lr=self.lr) 99 | # opt = optim.SGD(model.parameters(), lr=self.lr, momentum=0.9) 100 | 101 | self.losses = np.zeros(self.epochs) 102 | xtar, ytar = data[target] 103 | # oracle estimator uses target labels 104 | for epoch in range(self.epochs): 105 | opt.zero_grad() 106 | loss = loss_fn(model(xtar), ytar.view(-1, 1)) + \ 107 | self.lamL2 * torch.sum(model.lin1.weight ** 2) + \ 108 | self.lamL1 * torch.sum(torch.abs(model.lin1.weight)) 109 | # Perform gradient descent 110 | loss.backward() 111 | opt.step() 112 | self.losses[epoch] = loss.item() 113 | 114 | self.model = model 115 | 116 | self.ypred = self.model(xtar) 117 | 118 | return self 119 | 120 | def predict(self, X): 121 | ypredX = self.model(X) 122 | return ypredX 123 | 124 | def __str__(self): 125 | return self.__class__.__name__ + "_L2={:.1f}".format(self.lamL2) + "_L1={:.1f}".format(self.lamL1) 126 | 127 | 128 | class SrcPool(BaseEstimator): 129 | """Pool all source data together and then run linear regression 130 | with l1 or l2 penalty """ 131 | def __init__(self, lamL2=0.0, lamL1=0.0, lr=1e-4, epochs=10): 132 | self.lamL2 = lamL2 133 | self.lamL1 = lamL1 134 | self.lr = lr 135 | self.epochs = epochs 136 | 137 | def fit(self, data, source, target): 138 | super().fit(data, source, target) 139 | 140 | d = data[target][0].shape[1] 141 | model = LinearModel(d).to(device) 142 | # custom initialization 143 | with torch.no_grad(): 144 | model.lin1.bias.data = torch.zeros_like(model.lin1.bias) 145 | torch.nn.init.xavier_normal_(model.lin1.weight, gain=0.01) 146 | # Define loss function 147 | loss_fn = F.mse_loss 148 | opt = optim.Adam(model.parameters(), lr=self.lr) 149 | # opt = optim.SGD(model.parameters(), lr=self.lr, momentum=0.9) 150 | 151 | self.losses = np.zeros(self.epochs) 152 | 153 | for epoch in range(self.epochs): 154 | loss = 0 155 | opt.zero_grad() 156 | for m in source: 157 | x, y = data[m] 158 | loss += loss_fn(model(x), y.view(-1, 1))/len(source) 159 | 160 | loss += self.lamL2 * torch.sum(model.lin1.weight ** 2) 161 | loss += self.lamL1 * torch.sum(torch.abs(model.lin1.weight)) 162 | # Perform gradient descent 163 | loss.backward() 164 | opt.step() 165 | self.losses[epoch] = loss.item() 166 | self.model = model 167 | 168 | xtar, _ = data[target] 169 | self.ypred = self.model(xtar) 170 | 171 | return self 172 | 173 | def predict(self, X): 174 | ypredX = self.model(X) 175 | return ypredX 176 | 177 | def __str__(self): 178 | return self.__class__.__name__ + "_L2={:.1f}".format(self.lamL2) + "_L1={:.1f}".format(self.lamL1) 179 | 180 | def wayMatchSelector(wayMatch='mean'): 181 | if wayMatch == 'mean': 182 | return [lambda x: torch.mean(x, dim=0)] 183 | elif wayMatch == 'std': 184 | return [lambda x: torch.std(x, dim=0)] 185 | elif wayMatch == '25p': 186 | return [lambda x: torch.kthvalue(x, (1 + round(.25 * (x.shape[0] - 1))), dim=0)] 187 | elif wayMatch == '75p': 188 | return [lambda x: torch.kthvalue(x, (1 + round(.75 * (x.shape[0] - 1))), dim=0)] 189 | elif wayMatch == 'mean+std': 190 | return [lambda x: torch.mean(x, dim=0), lambda x: torch.std(x, dim=0)] 191 | elif wayMatch == 'mean+std+25p': 192 | return [lambda x: torch.mean(x, dim=0), lambda x: torch.std(x, dim=0), lambda x: torch.kthvalue(x, (1 + round(.25 * (x.shape[0] - 1))), dim=0)[0]] 193 | elif wayMatch == 'mean+std+25p+75p': 194 | return [lambda x: torch.mean(x, dim=0), lambda x: torch.std(x, dim=0), lambda x: torch.kthvalue(x, (1 + round(.25 * (x.shape[0] - 1))), dim=0)[0], 195 | lambda x: torch.kthvalue(x, (1 + round(.75 * (x.shape[0] - 1))), dim=0)[0]] 196 | else: 197 | print("Error: wayMatch not specified correctly, using mean") 198 | return [lambda x: torch.mean(x, 0)] 199 | 200 | 201 | 202 | class DIP(BaseEstimator): 203 | """Pick one source, match mean of X * beta between source and target""" 204 | def __init__(self, lamMatch=10., lamL2=0., lamL1=0., sourceInd = 0, lr=1e-4, epochs=10, wayMatch='mean'): 205 | self.lamMatch = lamMatch 206 | self.lamL2 = lamL2 207 | self.lamL1 = lamL1 208 | self.sourceInd = sourceInd 209 | self.lr = lr 210 | self.epochs = epochs 211 | self.wayMatch = wayMatchSelector(wayMatch) 212 | 213 | def fit(self, data, source, target): 214 | super().fit(data, source, target) 215 | 216 | d = data[target][0].shape[1] 217 | model = LinearModel(d).to(device) 218 | # custom initialization 219 | with torch.no_grad(): 220 | model.lin1.bias.data = torch.zeros_like(model.lin1.bias) 221 | torch.nn.init.xavier_normal_(model.lin1.weight, gain=0.01) 222 | # Define loss function 223 | loss_fn = F.mse_loss 224 | opt = optim.Adam(model.parameters(), lr=self.lr) 225 | 226 | self.losses = np.zeros(self.epochs) 227 | 228 | for epoch in range(self.epochs): 229 | x, y = data[source[self.sourceInd]] 230 | xtar, ytar = data[target] 231 | opt.zero_grad() 232 | loss = loss_fn(model(x), y.view(-1, 1)) + \ 233 | self.lamL2 * torch.sum(model.lin1.weight ** 2) + \ 234 | self.lamL1 * torch.sum(torch.abs(model.lin1.weight)) 235 | for wayMatchLocal in self.wayMatch: 236 | loss += self.lamMatch * loss_fn(wayMatchLocal(model(x)), wayMatchLocal(model(xtar))) 237 | 238 | # Perform gradient descent 239 | loss.backward() 240 | opt.step() 241 | 242 | self.losses[epoch] = loss.item() 243 | self.model = model 244 | 245 | xtar, _ = data[target] 246 | self.ypred = self.model(xtar) 247 | 248 | return self 249 | 250 | def predict(self, X): 251 | ypredX = self.model(X) 252 | return ypredX 253 | 254 | def __str__(self): 255 | return self.__class__.__name__ + "_Match{:.1f}".format(self.lamMatch) + "_L2={:.1f}".format(self.lamL2) + "_L1={:.1f}".format(self.lamL1) 256 | 257 | 258 | class DIPOracle(BaseEstimator): 259 | """Pick one source, match mean of X * beta between source and target, use target labels to fit (oracle)""" 260 | def __init__(self, lamMatch=10., lamL2=0., lamL1=0., sourceInd = 0, lr=1e-4, epochs=10, wayMatch='mean'): 261 | self.lamMatch = lamMatch 262 | self.lamL2 = lamL2 263 | self.lamL1 = lamL1 264 | self.sourceInd = sourceInd 265 | self.lr = lr 266 | self.epochs = epochs 267 | self.wayMatch = wayMatchSelector(wayMatch) 268 | 269 | def fit(self, data, source, target): 270 | super().fit(data, source, target) 271 | 272 | d = data[target][0].shape[1] 273 | model = LinearModel(d).to(device) 274 | # custom initialization 275 | with torch.no_grad(): 276 | model.lin1.bias.data = torch.zeros_like(model.lin1.bias) 277 | torch.nn.init.xavier_normal_(model.lin1.weight, gain=0.01) 278 | # Define loss function 279 | loss_fn = F.mse_loss 280 | opt = optim.Adam(model.parameters(), lr=self.lr) 281 | 282 | self.losses = np.zeros(self.epochs) 283 | 284 | for epoch in range(self.epochs): 285 | x, y = data[source[self.sourceInd]] 286 | xtar, ytar = data[target] 287 | opt.zero_grad() 288 | loss = loss_fn(model(xtar), ytar.view(-1, 1)) + \ 289 | self.lamL2 * torch.sum(model.lin1.weight ** 2) + \ 290 | self.lamL1 * torch.sum(torch.abs(model.lin1.weight)) 291 | 292 | for wayMatchLocal in self.wayMatch: 293 | loss += self.lamMatch * loss_fn(wayMatchLocal(model(x)), wayMatchLocal(model(xtar))) 294 | 295 | # Perform gradient descent 296 | loss.backward() 297 | opt.step() 298 | 299 | self.losses[epoch] = loss.item() 300 | self.model = model 301 | 302 | xtar, _ = data[target] 303 | self.ypred = self.model(xtar) 304 | 305 | return self 306 | 307 | def predict(self, X): 308 | ypredX = self.model(X) 309 | return ypredX 310 | 311 | def __str__(self): 312 | return self.__class__.__name__ + "_Match{:.1f}".format(self.lamMatch) + "_L2={:.1f}".format(self.lamL2) + "_L1={:.1f}".format(self.lamL1) 313 | 314 | 315 | class DIPweigh(BaseEstimator): 316 | '''loop throught all source envs, match the mean of X * beta between source env i and target, weigh the final prediction based loss of env i''' 317 | def __init__(self, lamMatch=10., lamL2=0., lamL1=0., lr=1e-4, 318 | epochs=10, wayMatch='mean'): 319 | self.lamMatch = lamMatch 320 | self.lamL2 = lamL2 321 | self.lamL1 = lamL1 322 | self.lr = lr 323 | self.epochs = epochs 324 | self.wayMatch = wayMatchSelector(wayMatch) 325 | 326 | def fit(self, data, source, target): 327 | super().fit(data, source, target) 328 | 329 | d = data[target][0].shape[1] 330 | models = {} 331 | diffs = {} 332 | ypreds = {} 333 | losses_all = {} 334 | for m in source: 335 | model = LinearModel(d).to(device) 336 | models[m] = model 337 | # custom initialization 338 | with torch.no_grad(): 339 | model.lin1.bias.data = torch.zeros_like(model.lin1.bias) 340 | torch.nn.init.xavier_normal_(model.lin1.weight, gain=0.01) 341 | # Define loss function 342 | loss_fn = F.mse_loss 343 | opt = optim.Adam(model.parameters(), lr=self.lr) 344 | 345 | losses_all[m] = np.zeros(self.epochs) 346 | 347 | for epoch in range(self.epochs): 348 | x, y = data[m] 349 | xtar, ytar = data[target] 350 | opt.zero_grad() 351 | 352 | loss = loss_fn(model(x), y.view(-1, 1)) + \ 353 | self.lamL2* torch.sum(model.lin1.weight ** 2) + \ 354 | self.lamL1 * torch.sum(torch.abs(model.lin1.weight)) 355 | 356 | for wayMatchLocal in self.wayMatch: 357 | loss += self.lamMatch * loss_fn(wayMatchLocal(model(x)), wayMatchLocal(model(xtar))) 358 | # Perform gradient descent 359 | loss.backward() 360 | opt.step() 361 | 362 | losses_all[m][epoch] = loss.item() 363 | 364 | diffs[m] =0. 365 | for wayMatchLocal in self.wayMatch: 366 | diffs[m] += loss_fn(wayMatchLocal(model(x)), wayMatchLocal(model(xtar))) 367 | ypreds[m] = models[m](xtar) 368 | 369 | # take the min diff loss to be current best losses and model 370 | minDiff = diffs[source[0]] 371 | minDiffIndx = source[0] 372 | self.losses = losses_all[source[0]] 373 | for m in source: 374 | if diffs[m] < minDiff: 375 | minDiff = diffs[m] 376 | minDiffIndx = m 377 | self.losses = losses_all[m] 378 | self.model = models[m] 379 | 380 | self.minDiffIndx = minDiffIndx 381 | self.total_weight = 0 382 | self.ypred = 0 383 | for m in self.source: 384 | self.ypred += torch.exp(-100.*diffs[m]) * ypreds[m] 385 | self.total_weight += torch.exp(-100.*diffs[m]) 386 | self.ypred /= self.total_weight 387 | self.models = models 388 | self.diffs = diffs 389 | 390 | return self 391 | 392 | def predict(self, X): 393 | ypredX1 = 0 394 | for m in self.source: 395 | ypredX1 += torch.exp(-100.*self.diffs[m]) * self.models[m](X) 396 | ypredX1 /= self.total_weight 397 | return ypredX1 398 | 399 | def __str__(self): 400 | return self.__class__.__name__ + "_Match{:.1f}".format(self.lamMatch) + "_L2={:.1f}".format(self.lamL2) + "_L1={:.1f}".format(self.lamL1) 401 | 402 | 403 | class CIP(BaseEstimator): 404 | """Match the conditional (on Y) mean of X * beta across source envs, no target env is needed""" 405 | def __init__(self, lamCIP=10., lamL2=0., lamL1=0., lr=1e-4, epochs=10, wayMatch='mean'): 406 | self.lamCIP = lamCIP 407 | self.lamL2 = lamL2 408 | self.lamL1 = lamL1 409 | self.lr = lr 410 | self.epochs = epochs 411 | self.wayMatch = wayMatchSelector(wayMatch) 412 | 413 | def fit(self, data, source, target): 414 | super().fit(data, source, target) 415 | 416 | d = data[target][0].shape[1] 417 | model = LinearModel(d).to(device) 418 | # custom initialization 419 | with torch.no_grad(): 420 | model.lin1.bias.data = torch.zeros_like(model.lin1.bias) 421 | torch.nn.init.xavier_normal_(model.lin1.weight, gain=0.01) 422 | 423 | # Define loss function 424 | loss_fn = F.mse_loss 425 | opt = optim.Adam(model.parameters(), lr=self.lr) 426 | 427 | self.losses = np.zeros(self.epochs) 428 | for epoch in range(self.epochs): 429 | loss = 0 430 | avgmodelxList = [0.] * len(self.wayMatch) 431 | opt.zero_grad() 432 | for m in source: 433 | x, y = data[m] 434 | # do the conditional on y 435 | xmod = x - torch.mm(y.view(-1, 1), torch.mm(y.view(1, -1), x))/torch.sum(y**2) 436 | for i, wayMatchLocal in enumerate(self.wayMatch): 437 | avgmodelxList[i] += wayMatchLocal(model(xmod))/len(source) 438 | for m in source: 439 | x, y = data[m] 440 | xmod = x - torch.mm(y.view(-1, 1), torch.mm(y.view(1, -1), x))/torch.sum(y**2) 441 | loss += loss_fn(model(x), y.view(-1, 1))/len(source) 442 | for i, wayMatchLocal in enumerate(self.wayMatch): 443 | loss += self.lamCIP * loss_fn(avgmodelxList[i], wayMatchLocal(model(xmod)))/len(source) 444 | loss += self.lamL2 * torch.sum(model.lin1.weight ** 2) 445 | loss += self.lamL1 * torch.sum(torch.abs(model.lin1.weight)) 446 | # Perform gradient descent 447 | loss.backward() 448 | opt.step() 449 | 450 | self.losses[epoch] = loss.item() 451 | self.model = model 452 | 453 | xtar, _ = data[target] 454 | self.ypred = self.model(xtar) 455 | 456 | return self 457 | 458 | def predict(self, X): 459 | ypredX = self.model(X) 460 | return ypredX 461 | 462 | def __str__(self): 463 | return self.__class__.__name__ + "_CIP{:.1f}".format(self.lamCIP) + "_L2={:.1f}".format(self.lamL2) + "_L1={:.1f}".format(self.lamL1) 464 | 465 | 466 | class CIRMweigh(BaseEstimator): 467 | """Match the conditional (on Y) mean of X * beta across source envs, use Yhat as proxy of Y to remove the Y parts in X. 468 | Match on the residual between one source env and target env""" 469 | def __init__(self, lamMatch=10., lamL2=0., lamL1=0., lr=1e-4, epochs=10, wayMatch='mean'): 470 | self.lamMatch = lamMatch 471 | self.lamL2 = lamL2 472 | self.lamL1 = lamL1 473 | self.lr = lr 474 | self.epochs = epochs 475 | self.wayMatch = wayMatchSelector(wayMatch) 476 | 477 | def fit(self, data, source, target): 478 | super().fit(data, source, target) 479 | 480 | d = data[target][0].shape[1] 481 | # Step 1: use source envs to match the conditional mean 482 | # find beta_invariant 483 | models1 = LinearModel(d).to(device) 484 | # custom initialization 485 | with torch.no_grad(): 486 | models1.lin1.bias.data = torch.zeros_like(models1.lin1.bias) 487 | torch.nn.init.xavier_normal_(models1.lin1.weight, gain=0.01) 488 | 489 | # Define loss function 490 | loss_fn = F.mse_loss 491 | opt = optim.Adam(models1.parameters(), lr=self.lr) 492 | 493 | losses1 = np.zeros(self.epochs) 494 | for epoch in range(self.epochs): 495 | loss = 0 496 | avgmodelxList = [0.] * len(self.wayMatch) 497 | opt.zero_grad() 498 | for m in source: 499 | x, y = data[m] 500 | # do the conditional on y 501 | xmod = x - torch.mm(y.view(-1, 1), torch.mm(y.view(1, -1), x))/torch.sum(y**2) 502 | for i, wayMatchLocal in enumerate(self.wayMatch): 503 | avgmodelxList[i] += wayMatchLocal(models1(xmod))/len(source) 504 | for m in source: 505 | x, y = data[m] 506 | xmod = x - torch.mm(y.view(-1, 1), torch.mm(y.view(1, -1), x))/torch.sum(y**2) 507 | loss += loss_fn(models1(x), y.view(-1, 1))/len(source) 508 | for i, wayMatchLocal in enumerate(self.wayMatch): 509 | loss += self.lamMatch * loss_fn(avgmodelxList[i], wayMatchLocal(models1(xmod)))/len(source) 510 | loss += self.lamL2 * torch.sum(models1.lin1.weight ** 2) 511 | loss += self.lamL1 * torch.sum(torch.abs(models1.lin1.weight)) 512 | # Perform gradient descent 513 | loss.backward() 514 | opt.step() 515 | losses1[epoch] = loss.item() 516 | 517 | self.models1 = models1 518 | 519 | # fix grads now 520 | for param in models1.lin1.parameters(): 521 | param.requires_grad = False 522 | 523 | # Step 2: remove the invariant part on all source envs, so that everything is independent of Y 524 | # get that coefficient b 525 | YsrcMean = 0 526 | ntotal = 0 527 | for m in source: 528 | YsrcMean += torch.sum(data[m][1]) 529 | ntotal += data[m][1].shape[0] 530 | YsrcMean /= ntotal 531 | 532 | YTX = 0 533 | YTY = 0 534 | for m in source: 535 | x, y = data[m] 536 | yguess = self.models1(x) 537 | yCentered = y - YsrcMean 538 | YTY += torch.sum(yguess.t() * yCentered) 539 | YTX += torch.mm(yCentered.view(1, -1), x) 540 | 541 | b = YTX / YTY 542 | self.b = b 543 | 544 | 545 | # Step 3: mean match between source and target on the residual, after transforming the covariates X - (X * beta_invariant) * b_invariant 546 | models = {} 547 | diffs = {} 548 | ypreds = {} 549 | losses_all = {} 550 | for m in source: 551 | models[m] = LinearModel(d).to(device) 552 | # custom initialization 553 | with torch.no_grad(): 554 | models[m].lin1.bias.data = torch.zeros_like(models[m].lin1.bias) 555 | torch.nn.init.xavier_normal_(models[m].lin1.weight, gain=0.01) 556 | # Define loss function 557 | loss_fn = F.mse_loss 558 | opt = optim.Adam(models[m].parameters(), lr=self.lr) 559 | 560 | losses_all[m] = np.zeros(self.epochs) 561 | x, y = data[m] 562 | xmod = x - torch.mm(self.models1(x), b) 563 | xtar, ytar = data[target] 564 | xtarmod = xtar - torch.mm(self.models1(xtar), b) 565 | 566 | for epoch in range(self.epochs): 567 | loss = loss_fn(models[m](x), y.view(-1, 1)) + \ 568 | self.lamL2 * torch.sum(models[m].lin1.weight ** 2) + \ 569 | self.lamL1 * torch.sum(torch.abs(models[m].lin1.weight)) 570 | 571 | for wayMatchLocal in self.wayMatch: 572 | loss += self.lamMatch * loss_fn(wayMatchLocal(models[m](xmod)), wayMatchLocal(models[m](xtarmod))) 573 | # Perform gradient descent 574 | loss.backward() 575 | opt.step() 576 | opt.zero_grad() 577 | losses_all[m][epoch] = loss.item() 578 | 579 | diffs[m] = 0. 580 | for wayMatchLocal in self.wayMatch: 581 | diffs[m] += loss_fn(wayMatchLocal(models[m](xmod)), wayMatchLocal(models[m](xtarmod))) 582 | ypreds[m] = models[m](xtar) 583 | 584 | # take the min diff loss to be current best losses and model 585 | minDiff = diffs[source[0]] 586 | minDiffIndx = source[0] 587 | self.losses = losses_all[source[0]] 588 | for m in source: 589 | if diffs[m] < minDiff: 590 | minDiff = diffs[m] 591 | minDiffIndx = m 592 | self.losses = losses_all[m] 593 | self.model = models[m] 594 | 595 | self.minDiffIndx = minDiffIndx 596 | self.total_weight = 0 597 | self.ypred = 0 598 | for m in self.source: 599 | self.ypred += torch.exp(-100.*diffs[m]) * ypreds[m] 600 | self.total_weight += torch.exp(-100.*diffs[m]) 601 | self.ypred /= self.total_weight 602 | self.models = models 603 | self.diffs = diffs 604 | 605 | return self 606 | 607 | def predict(self, X): 608 | ypredX1 = 0 609 | for m in self.source: 610 | ypredX1 += torch.exp(-100.*self.diffs[m]) * self.models[m](X) 611 | ypredX1 /= self.total_weight 612 | return ypredX1 613 | 614 | def __str__(self): 615 | return self.__class__.__name__ + "_Match{:.1f}".format(self.lamMatch) + "_L2={:.1f}".format(self.lamL2) + "_L1={:.1f}".format(self.lamL1) 616 | 617 | 618 | 619 | -------------------------------------------------------------------------------- /semitorchstocclass.py: -------------------------------------------------------------------------------- 1 | """ 2 | ``semitorchstocclass`` provides classes implementing various domain adaptation methods using torch and stochastic gradient method. 3 | All domain adaptation methods have to be subclass of BaseEstimator. 4 | This implementation takes advantage of gradient method to optimize covariance match or MMD match in addition to mean match. 5 | """ 6 | 7 | from abc import abstractmethod 8 | import numpy as np 9 | 10 | import torch 11 | import torch.nn as nn 12 | import torch.nn.functional as F 13 | import torch.optim as optim 14 | 15 | import mmd 16 | 17 | # check gpu avail 18 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 19 | 20 | 21 | # simple linear model in torch 22 | class LinearModel(nn.Module): 23 | def __init__(self, d): 24 | super(LinearModel, self).__init__() 25 | self.lin1 = nn.Linear(d, 1, bias=True) 26 | 27 | def forward(self, x): 28 | x = self.lin1(x) 29 | return x 30 | 31 | class BaseEstimator(): 32 | """Base class for domain adaptation""" 33 | @abstractmethod 34 | def fit(self, data, source, target): 35 | """Fit model. 36 | Arguments: 37 | data (dict of (X, y) pairs): maps env index to the (X, y) pair in that env 38 | source (list of indexes): indexes of source envs 39 | target (int): single index of the target env 40 | """ 41 | self.source = source 42 | self.target = target 43 | 44 | return self 45 | 46 | @abstractmethod 47 | def predict(self, X): 48 | """Use the learned estimator to predict labels on fresh target data X 49 | """ 50 | 51 | def __str__(self): 52 | """For easy name printing 53 | """ 54 | return self.__class__.__name__ 55 | 56 | 57 | class Tar(BaseEstimator): 58 | """Oracle Linear regression (with l1 or l2 penalty) trained on the target domain""" 59 | def __init__(self, lamL2=0.0, lamL1=0.0, lr=1e-4, epochs=10): 60 | self.lamL2 = lamL2 61 | self.lamL1 = lamL1 62 | self.lr = lr 63 | self.epochs = epochs 64 | 65 | def fit(self, dataloaders, source, target): 66 | super().fit(dataloaders, source, target) 67 | 68 | # get the input dimension 69 | # assume it is a TensorDataset 70 | d = dataloaders[target].dataset[0][0].shape[0] 71 | model = LinearModel(d).to(device) 72 | with torch.no_grad(): 73 | model.lin1.bias.data = torch.zeros_like(model.lin1.bias) 74 | torch.nn.init.xavier_normal_(model.lin1.weight, gain=0.01) 75 | # Define loss function 76 | loss_fn = F.mse_loss 77 | opt = optim.Adam(model.parameters(), lr=self.lr) 78 | 79 | self.losses = np.zeros(self.epochs) 80 | for epoch in range(self.epochs): 81 | running_loss = 0.0 82 | for i, data in enumerate(dataloaders[target]): 83 | xtar, ytar = data[0].to(device), data[1].to(device) 84 | opt.zero_grad() 85 | loss = loss_fn(model(xtar).view(-1), ytar) + \ 86 | self.lamL2 * torch.sum(model.lin1.weight ** 2) + \ 87 | self.lamL1 * torch.sum(torch.abs(model.lin1.weight)) 88 | # Perform gradient descent 89 | loss.backward() 90 | opt.step() 91 | running_loss += loss.item() 92 | 93 | self.losses[epoch] = running_loss 94 | 95 | self.model = model 96 | 97 | return self 98 | 99 | def predict(self, X): 100 | ypredX = self.model(X) 101 | return ypredX 102 | 103 | def __str__(self): 104 | return self.__class__.__name__ + "_L2={:.1f}".format(self.lamL2) + "_L1={:.1f}".format(self.lamL1) 105 | 106 | 107 | 108 | class Src(BaseEstimator): 109 | """Src Linear regression (with l1 or l2 penalty) trained on the source domain""" 110 | def __init__(self, lamL2=0.0, lamL1=0.0, sourceInd = 0, lr=1e-4, epochs=10): 111 | self.lamL2 = lamL2 112 | self.lamL1 = lamL1 113 | self.sourceInd = sourceInd 114 | self.lr = lr 115 | self.epochs = epochs 116 | 117 | def fit(self, dataloaders, source, target): 118 | super().fit(dataloaders, source, target) 119 | 120 | # get the input dimension 121 | # assume it is a TensorDataset 122 | d = dataloaders[target].dataset[0][0].shape[0] 123 | model = LinearModel(d).to(device) 124 | with torch.no_grad(): 125 | model.lin1.bias.data = torch.zeros_like(model.lin1.bias) 126 | torch.nn.init.xavier_normal_(model.lin1.weight, gain=0.01) 127 | # Define loss function 128 | loss_fn = F.mse_loss 129 | opt = optim.Adam(model.parameters(), lr=self.lr) 130 | 131 | self.losses = np.zeros(self.epochs) 132 | for epoch in range(self.epochs): 133 | running_loss = 0.0 134 | for i, data in enumerate(dataloaders[source[self.sourceInd]]): 135 | x, y = data[0].to(device), data[1].to(device) 136 | opt.zero_grad() 137 | loss = loss_fn(model(x).view(-1), y) + \ 138 | self.lamL2 * torch.sum(model.lin1.weight ** 2) + \ 139 | self.lamL1 * torch.sum(torch.abs(model.lin1.weight)) 140 | # Perform gradient descent 141 | loss.backward() 142 | opt.step() 143 | running_loss += loss.item() 144 | 145 | self.losses[epoch] = running_loss 146 | 147 | self.model = model 148 | 149 | return self 150 | 151 | def predict(self, X): 152 | ypredX = self.model(X) 153 | return ypredX 154 | 155 | def __str__(self): 156 | return self.__class__.__name__ + "_L2={:.1f}".format(self.lamL2) + "_L1={:.1f}".format(self.lamL1) 157 | 158 | 159 | 160 | class SrcPool(BaseEstimator): 161 | """Pool all source data together and then run linear regression 162 | with l1 or l2 penalty """ 163 | def __init__(self, lamL2=0.0, lamL1=0.0, lr=1e-4, epochs=10): 164 | self.lamL2 = lamL2 165 | self.lamL1 = lamL1 166 | self.lr = lr 167 | self.epochs = epochs 168 | 169 | def fit(self, dataloaders, source, target): 170 | super().fit(dataloaders, source, target) 171 | 172 | # get the input dimension 173 | # assume it is a TensorDataset 174 | d = dataloaders[target].dataset[0][0].shape[0] 175 | model = LinearModel(d).to(device) 176 | # custom initialization 177 | with torch.no_grad(): 178 | model.lin1.bias.data = torch.zeros_like(model.lin1.bias) 179 | torch.nn.init.xavier_normal_(model.lin1.weight, gain=0.01) 180 | # Define loss function 181 | loss_fn = F.mse_loss 182 | opt = optim.Adam(model.parameters(), lr=self.lr) 183 | 184 | self.losses = np.zeros(self.epochs) 185 | 186 | for epoch in range(self.epochs): 187 | running_loss = 0.0 188 | for i, data in enumerate(zip(*[dataloaders[m] for m in source])): 189 | opt.zero_grad() 190 | loss = 0 191 | for mindex, m in enumerate(source): 192 | x, y = data[mindex][0].to(device), data[mindex][1].to(device) 193 | loss += loss_fn(model(x).view(-1), y) / len(source) 194 | 195 | loss += self.lamL2 * torch.sum(model.lin1.weight ** 2) 196 | loss += self.lamL1 * torch.sum(torch.abs(model.lin1.weight)) 197 | 198 | loss.backward() 199 | opt.step() 200 | 201 | running_loss += loss.item() 202 | 203 | self.losses[epoch] = running_loss 204 | self.model = model 205 | 206 | return self 207 | 208 | def predict(self, X): 209 | ypredX = self.model(X) 210 | return ypredX 211 | 212 | def __str__(self): 213 | return self.__class__.__name__ + "_L2={:.1f}".format(self.lamL2) + "_L1={:.1f}".format(self.lamL1) 214 | 215 | 216 | class DIP(BaseEstimator): 217 | """Pick one source, match mean of X * beta between source and target""" 218 | def __init__(self, lamMatch=10., lamL2=0., lamL1=0., sourceInd = 0, lr=1e-4, epochs=10, 219 | wayMatch='mean', sigma_list=[0.1, 1, 10, 100]): 220 | self.lamMatch = lamMatch 221 | self.lamL2 = lamL2 222 | self.lamL1 = lamL1 223 | self.sourceInd = sourceInd 224 | self.lr = lr 225 | self.epochs = epochs 226 | self.wayMatch = wayMatch 227 | self.sigma_list = sigma_list 228 | 229 | def fit(self, dataloaders, source, target): 230 | super().fit(dataloaders, source, target) 231 | 232 | d = dataloaders[target].dataset[0][0].shape[0] 233 | model = LinearModel(d).to(device) 234 | # custom initialization 235 | with torch.no_grad(): 236 | model.lin1.bias.data = torch.zeros_like(model.lin1.bias) 237 | torch.nn.init.xavier_normal_(model.lin1.weight, gain=0.01) 238 | # Define loss function 239 | loss_fn = F.mse_loss 240 | opt = optim.Adam(model.parameters(), lr=self.lr) 241 | self.losses = np.zeros(self.epochs) 242 | 243 | for epoch in range(self.epochs): 244 | running_loss = 0.0 245 | 246 | for i, data in enumerate(zip(dataloaders[source[self.sourceInd]], dataloaders[target])): 247 | opt.zero_grad() 248 | loss = 0 249 | x, y = data[0][0].to(device), data[0][1].to(device) 250 | xtar = data[1][0].to(device) 251 | loss += loss_fn(model(x).view(-1), y) 252 | if self.wayMatch == 'mean': 253 | discrepancy = torch.nn.MSELoss() 254 | loss += self.lamMatch * discrepancy(model(x), model(xtar)) 255 | elif self.wayMatch == 'mmd': 256 | loss += self.lamMatch * mmd.mix_rbf_mmd2(model(x), model(xtar), self.sigma_list) 257 | else: 258 | print('error discrepancy') 259 | loss += self.lamL2 * torch.sum(model.lin1.weight ** 2) + \ 260 | self.lamL1 * torch.sum(torch.abs(model.lin1.weight)) 261 | 262 | 263 | loss.backward() 264 | opt.step() 265 | 266 | running_loss += loss.item() 267 | 268 | self.losses[epoch] = running_loss 269 | self.model = model 270 | 271 | return self 272 | 273 | def predict(self, X): 274 | ypredX = self.model(X) 275 | return ypredX 276 | 277 | def __str__(self): 278 | return self.__class__.__name__ + self.wayMatch + "_Match{:.1f}".format(self.lamMatch) + "_L2={:.1f}".format(self.lamL2) + "_L1={:.1f}".format(self.lamL1) 279 | 280 | 281 | class DIPweigh(BaseEstimator): 282 | '''loop throught all source envs, match the mean of X * beta between source env i and target, weigh the final prediction based loss of env i''' 283 | def __init__(self, lamMatch=10., lamL2=0., lamL1=0., lr=1e-4, 284 | epochs=10, wayMatch='mean', sigma_list=[0.1, 1, 10, 100]): 285 | self.lamMatch = lamMatch 286 | self.lamL2 = lamL2 287 | self.lamL1 = lamL1 288 | self.lr = lr 289 | self.epochs = epochs 290 | self.wayMatch = wayMatch 291 | self.sigma_list = sigma_list 292 | 293 | def fit(self, dataloaders, source, target): 294 | super().fit(dataloaders, source, target) 295 | 296 | d = dataloaders[target].dataset[0][0].shape[0] 297 | models = {} 298 | diffs = {} 299 | ypreds = {} 300 | losses_all = {} 301 | self.total_weight = 0 302 | for m in source: 303 | model = LinearModel(d).to(device) 304 | models[m] = model 305 | # custom initialization 306 | with torch.no_grad(): 307 | model.lin1.bias.data = torch.zeros_like(model.lin1.bias) 308 | torch.nn.init.xavier_normal_(model.lin1.weight, gain=0.01) 309 | # Define loss function 310 | loss_fn = F.mse_loss 311 | opt = optim.Adam(model.parameters(), lr=self.lr) 312 | 313 | losses_all[m] = np.zeros(self.epochs) 314 | 315 | for epoch in range(self.epochs): 316 | running_loss = 0.0 317 | 318 | for i, data in enumerate(zip(dataloaders[m], dataloaders[target])): 319 | opt.zero_grad() 320 | loss = 0 321 | x, y = data[0][0].to(device), data[0][1].to(device) 322 | xtar = data[1][0].to(device) 323 | loss += loss_fn(model(x).view(-1), y) 324 | if self.wayMatch == 'mean': 325 | discrepancy = torch.nn.MSELoss() 326 | loss += self.lamMatch * discrepancy(model(x), model(xtar)) 327 | elif self.wayMatch == 'mmd': 328 | loss += self.lamMatch * mmd.mix_rbf_mmd2(model(x), model(xtar), self.sigma_list) 329 | else: 330 | raise('error discrepancy') 331 | loss += self.lamL2 * torch.sum(model.lin1.weight ** 2) + \ 332 | self.lamL1 * torch.sum(torch.abs(model.lin1.weight)) 333 | 334 | 335 | loss.backward() 336 | opt.step() 337 | 338 | running_loss += loss.item() 339 | 340 | 341 | losses_all[m][epoch] = running_loss 342 | 343 | # need to calculate the diffs 344 | diffs[m] = 0 345 | with torch.no_grad(): 346 | for i, data in enumerate(zip(dataloaders[m], dataloaders[target])): 347 | x, y = data[0][0].to(device), data[0][1].to(device) 348 | xtar = data[1][0].to(device) 349 | if self.wayMatch == 'mean': 350 | discrepancy = torch.nn.MSELoss() 351 | local_match_res = discrepancy(model(x), model(xtar)) 352 | elif self.wayMatch == 'mmd': 353 | local_match_res = mmd.mix_rbf_mmd2(model(x), model(xtar), self.sigma_list) 354 | else: 355 | raise('error discrepancy') 356 | diffs[m] += local_match_res / self.epochs / (len(dataloaders[m].dataset)/dataloaders[m].batch_size) 357 | self.total_weight += torch.exp(-100.*diffs[m]) 358 | 359 | self.models = models 360 | self.diffs = diffs 361 | 362 | minDiff = diffs[source[0]] 363 | minDiffIndx = source[0] 364 | for m in source: 365 | if diffs[m] < minDiff: 366 | minDiff = diffs[m] 367 | minDiffIndx = m 368 | self.minDiffIndx = minDiffIndx 369 | print(minDiffIndx) 370 | self.losses = losses_all[minDiffIndx] 371 | 372 | return self 373 | 374 | def predict(self, X): 375 | ypredX1 = 0 376 | for m in self.source: 377 | ypredX1 += torch.exp(-100.*self.diffs[m]) * self.models[m](X) 378 | ypredX1 /= self.total_weight 379 | return ypredX1 380 | 381 | def __str__(self): 382 | return self.__class__.__name__ + self.wayMatch + "_Match{:.1f}".format(self.lamMatch) + "_L2={:.1f}".format(self.lamL2) + "_L1={:.1f}".format(self.lamL1) 383 | 384 | 385 | 386 | 387 | class CIP(BaseEstimator): 388 | """Match the conditional (on Y) mean of X * beta across source envs, no target env is needed""" 389 | def __init__(self, lamCIP=10., lamL2=0., lamL1=0., lr=1e-4, epochs=10, 390 | wayMatch='mean', sigma_list = [0.1, 1, 10, 100]): 391 | self.lamCIP = lamCIP 392 | self.lamL2 = lamL2 393 | self.lamL1 = lamL1 394 | self.lr = lr 395 | self.epochs = epochs 396 | self.wayMatch = wayMatch 397 | self.sigma_list = sigma_list 398 | 399 | def fit(self, dataloaders, source, target): 400 | super().fit(dataloaders, source, target) 401 | 402 | d = dataloaders[target].dataset[0][0].shape[0] 403 | model = LinearModel(d).to(device) 404 | # custom initialization 405 | with torch.no_grad(): 406 | model.lin1.bias.data = torch.zeros_like(model.lin1.bias) 407 | torch.nn.init.xavier_normal_(model.lin1.weight, gain=0.01) 408 | 409 | # Define loss function 410 | loss_fn = F.mse_loss 411 | opt = optim.Adam(model.parameters(), lr=self.lr) 412 | 413 | self.losses = np.zeros(self.epochs) 414 | for epoch in range(self.epochs): 415 | running_loss = 0.0 416 | for i, data in enumerate(zip(*[dataloaders[m] for m in source])): 417 | opt.zero_grad() 418 | loss = 0 419 | for mindex, m in enumerate(source): 420 | x, y = data[mindex][0].to(device), data[mindex][1].to(device) 421 | loss += loss_fn(model(x).view(-1), y)/float(len(source)) 422 | xmod = x - torch.mm(y.view(-1, 1), torch.mm(y.view(1, -1), x))/torch.sum(y**2) 423 | 424 | # conditional invariance penalty 425 | for jindex, j in enumerate(source): 426 | if j > m: 427 | xj, yj = data[jindex][0].to(device), data[jindex][1].to(device) 428 | xmodj = xj - torch.mm(yj.view(-1, 1), torch.mm(yj.view(1, -1), xj))/torch.sum(yj**2) 429 | if self.wayMatch == 'mean': 430 | discrepancy = torch.nn.MSELoss() 431 | loss += self.lamCIP/float(len(source)**2) * discrepancy(model(xmod), model(xmodj)) 432 | elif self.wayMatch == 'mmd': 433 | loss += self.lamCIP/float(len(source)**2) * \ 434 | mmd.mix_rbf_mmd2(model(xmod), model(xmodj), self.sigma_list) 435 | else: 436 | raise('error discrepancy') 437 | 438 | loss += self.lamL2 * torch.sum(model.lin1.weight ** 2) + \ 439 | self.lamL1 * torch.sum(torch.abs(model.lin1.weight)) 440 | # Perform gradient descent 441 | loss.backward() 442 | opt.step() 443 | running_loss += loss.item() 444 | 445 | self.losses[epoch] = running_loss 446 | self.model = model 447 | 448 | return self 449 | 450 | def predict(self, X): 451 | ypredX = self.model(X) 452 | return ypredX 453 | 454 | def __str__(self): 455 | return self.__class__.__name__ + self.wayMatch + "_CIP{:.1f}".format(self.lamCIP) + "_L2={:.1f}".format(self.lamL2) + "_L1={:.1f}".format(self.lamL1) 456 | 457 | 458 | 459 | class CIRMweigh(BaseEstimator): 460 | """Match the conditional (on Y) mean of X * beta across source envs, use Yhat as proxy of Y to remove the Y parts in X. 461 | Match on the residual between one source env and target env""" 462 | def __init__(self, lamMatch=10., lamCIP=10., lamL2=0., lamL1=0., lr=1e-4, epochs=10, 463 | wayMatch='mean', sigma_list=[0.1, 1, 10, 100]): 464 | self.lamMatch = lamMatch 465 | self.lamCIP = lamCIP 466 | self.lamL2 = lamL2 467 | self.lamL1 = lamL1 468 | self.lr = lr 469 | self.epochs = epochs 470 | self.wayMatch = wayMatch 471 | self.sigma_list = sigma_list 472 | 473 | def fit(self, dataloaders, source, target): 474 | super().fit(dataloaders, source, target) 475 | 476 | d = dataloaders[target].dataset[0][0].shape[0] 477 | # Step 1: use source envs to match the conditional mean 478 | # find beta_invariant 479 | models1 = LinearModel(d).to(device) 480 | # custom initialization 481 | with torch.no_grad(): 482 | models1.lin1.bias.data = torch.zeros_like(models1.lin1.bias) 483 | torch.nn.init.xavier_normal_(models1.lin1.weight, gain=0.01) 484 | 485 | # Define loss function 486 | loss_fn = F.mse_loss 487 | opt = optim.Adam(models1.parameters(), lr=self.lr) 488 | 489 | losses1 = np.zeros(self.epochs) 490 | for epoch in range(self.epochs): 491 | running_loss = 0.0 492 | for i, data in enumerate(zip(*[dataloaders[m] for m in source])): 493 | loss = 0 494 | for mindex, m in enumerate(source): 495 | x, y = data[mindex][0].to(device), data[mindex][1].to(device) 496 | loss += loss_fn(models1(x).view(-1), y)/float(len(source)) 497 | xmod = x - torch.mm(y.view(-1, 1), torch.mm(y.view(1, -1), x))/torch.sum(y**2) 498 | 499 | # conditional invariance penalty 500 | for jindex, j in enumerate(source): 501 | xj, yj = data[jindex][0].to(device), data[jindex][1].to(device) 502 | if j > m: 503 | xmodj = xj - torch.mm(yj.view(-1, 1), torch.mm(yj.view(1, -1), xj))/torch.sum(yj**2) 504 | if self.wayMatch == 'mean': 505 | discrepancy = torch.nn.MSELoss() 506 | loss += self.lamCIP/float(len(source)**2) * discrepancy(models1(xmod), models1(xmodj)) 507 | elif self.wayMatch == 'mmd': 508 | loss += self.lamCIP/float(len(source)**2) * \ 509 | mmd.mix_rbf_mmd2(models1(xmod), models1(xmodj), self.sigma_list) 510 | else: 511 | raise('error discrepancy') 512 | loss += self.lamL2 * torch.sum(models1.lin1.weight ** 2) + \ 513 | self.lamL1 * torch.sum(torch.abs(models1.lin1.weight)) 514 | # Perform gradient descent 515 | loss.backward() 516 | opt.step() 517 | opt.zero_grad() 518 | running_loss += loss.item() 519 | losses1[epoch] = running_loss 520 | 521 | self.models1 = models1 522 | 523 | # fix grads now 524 | for param in models1.lin1.parameters(): 525 | param.requires_grad = False 526 | 527 | # Step 2: remove the invariant part on all source envs, so that everything is independent of Y 528 | # get that coefficient b 529 | YsrcMean = 0 530 | ntotal = 0 531 | for m in source: 532 | YsrcMean += torch.sum(dataloaders[m].dataset.tensors[1]) 533 | ntotal += dataloaders[m].dataset.tensors[1].shape[0] 534 | YsrcMean /= ntotal 535 | 536 | YTX = 0 537 | YTY = 0 538 | for m in source: 539 | for i, data in enumerate(dataloaders[m]): 540 | x, y = data[0].to(device), data[1].to(device) 541 | yguess = self.models1(x) 542 | yCentered = y - YsrcMean 543 | YTY += torch.sum(yguess.t() * yCentered) 544 | YTX += torch.mm(yCentered.view(1, -1), x) 545 | 546 | b = YTX / YTY 547 | self.b = b 548 | 549 | 550 | # Step 3: mean match between source and target on the residual, after transforming the covariates X - (X * beta_invariant) * b_invariant 551 | models = {} 552 | diffs = {} 553 | losses_all = {} 554 | self.total_weight = 0 555 | 556 | for m in source: 557 | models[m] = LinearModel(d).to(device) 558 | # custom initialization 559 | with torch.no_grad(): 560 | models[m].lin1.bias.data = torch.zeros_like(models[m].lin1.bias) 561 | torch.nn.init.xavier_normal_(models[m].lin1.weight, gain=0.01) 562 | # Define loss function 563 | loss_fn = F.mse_loss 564 | opt = optim.Adam(models[m].parameters(), lr=self.lr) 565 | 566 | losses_all[m] = np.zeros(self.epochs) 567 | 568 | for epoch in range(self.epochs): # loop over the dataset multiple times 569 | running_loss = 0.0 570 | 571 | for i, data in enumerate(zip(dataloaders[m], dataloaders[target])): 572 | opt.zero_grad() 573 | loss = 0 574 | x, y = data[0][0].to(device), data[0][1].to(device) 575 | yguess = self.models1(x) 576 | xmod = x - torch.mm(yguess, b) 577 | 578 | xtar = data[1][0].to(device) 579 | ytarguess = self.models1(xtar) 580 | xtarmod = xtar - torch.mm(ytarguess, b) 581 | 582 | loss += loss_fn(models[m](x).view(-1), y) 583 | if self.wayMatch == 'mean': 584 | discrepancy = torch.nn.MSELoss() 585 | loss += self.lamMatch * discrepancy(models[m](xmod), 586 | models[m](xtarmod)) 587 | elif self.wayMatch == 'mmd': 588 | loss += self.lamMatch * mmd.mix_rbf_mmd2(models[m](xmod), 589 | models[m](xtarmod), 590 | self.sigma_list) 591 | else: 592 | raise('error discrepancy') 593 | loss += self.lamL2 * torch.sum(models[m].lin1.weight ** 2) + \ 594 | self.lamL1 * torch.sum(torch.abs(models[m].lin1.weight)) 595 | 596 | loss.backward() 597 | opt.step() 598 | 599 | running_loss += loss.item() 600 | 601 | losses_all[m][epoch] = running_loss 602 | 603 | # need to compute diff after training 604 | 605 | 606 | diffs[m] = 0. 607 | with torch.no_grad(): 608 | for i, data in enumerate(zip(dataloaders[m], dataloaders[target])): 609 | x, y = data[0][0].to(device), data[0][1].to(device) 610 | yguess = self.models1(x) 611 | xmod = x - torch.mm(yguess, b) 612 | 613 | xtar = data[1][0].to(device) 614 | ytarguess = self.models1(xtar) 615 | xtarmod = xtar - torch.mm(ytarguess, b) 616 | 617 | if self.wayMatch == 'mean': 618 | discrepancy = torch.nn.MSELoss() 619 | diffs[m] += discrepancy(models[m](xmod), models[m](xtarmod)) / \ 620 | self.epochs / (len(dataloaders[m].dataset)/dataloaders[m].batch_size) 621 | elif self.wayMatch == 'mmd': 622 | diffs[m] += mmd.mix_rbf_mmd2(models[m](xmod), models[m](xtarmod), self.sigma_list) / \ 623 | self.epochs / (len(dataloaders[m].dataset)/dataloaders[m].batch_size) 624 | else: 625 | raise('error discrepancy') 626 | 627 | 628 | self.total_weight += torch.exp(-100.*diffs[m]) 629 | 630 | # take the min diff loss to be current best losses and model 631 | minDiff = diffs[source[0]] 632 | minDiffIndx = source[0] 633 | self.losses = losses_all[source[0]] 634 | for m in source: 635 | if diffs[m] < minDiff: 636 | minDiff = diffs[m] 637 | minDiffIndx = m 638 | self.losses = losses_all[m] 639 | self.model = models[m] 640 | self.minDiffIndx = minDiffIndx 641 | 642 | self.models = models 643 | self.diffs = diffs 644 | 645 | return self 646 | 647 | def predict(self, X): 648 | ypredX1 = 0 649 | for m in self.source: 650 | ypredX1 += torch.exp(-100.*self.diffs[m]) * self.models[m](X) 651 | ypredX1 /= self.total_weight 652 | return ypredX1 653 | 654 | def __str__(self): 655 | return self.__class__.__name__ + self.wayMatch + "_Match{:.1f}".format(self.lamMatch) + "_L2={:.1f}".format(self.lamL2) + "_L1={:.1f}".format(self.lamL1) 656 | -------------------------------------------------------------------------------- /sim/sim_linearSCM_var_shift_exp8_box_run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import numpy as np 5 | import pandas as pd 6 | import sys 7 | import argparse 8 | 9 | import torch 10 | 11 | import torch.nn as nn 12 | import torch.nn.functional as F 13 | import torch.optim as optim 14 | 15 | # local packages 16 | import sys 17 | sys.path.append('../') 18 | import semiclass 19 | import semitorchclass 20 | import semitorchstocclass 21 | import util 22 | import simudata 23 | 24 | # check gpu avail 25 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 26 | # Assuming that we are on a CUDA machine, this should print a CUDA device: 27 | print(device) 28 | 29 | # parse args 30 | parser = argparse.ArgumentParser() 31 | parser.add_argument("--interv_type", type=str, default="sv1", help="type of intervention") 32 | parser.add_argument("--lamMatch", type=float, default=1., help="DIP matching penalty") 33 | parser.add_argument("--epochs", type=int, default=4000, help="number of epochs") 34 | parser.add_argument("--lr", type=float, default=1e-3, help="learning rate") 35 | parser.add_argument("--tag_DA", type=str, default="baseline", help="choose whether to run baseline methods or DA methods") 36 | parser.add_argument("--n", type=int, default=5000, help="sample size") 37 | myargs = parser.parse_args() 38 | print(myargs) 39 | 40 | lamL2 = 0. 41 | lamL1 = 0. 42 | 43 | if myargs.tag_DA == "baseline": 44 | methods = [ 45 | semiclass.Tar(lamL2=lamL2), 46 | semiclass.SrcPool(lamL2=lamL2), 47 | semitorchclass.Tar(lamL2=lamL2, lamL1=lamL1, lr=myargs.lr, epochs=myargs.epochs), 48 | semitorchclass.SrcPool(lamL2=lamL2, lamL1=lamL1, lr=myargs.lr, epochs=myargs.epochs), 49 | ] 50 | elif myargs.tag_DA == "DAmean": 51 | methods = [ 52 | semiclass.DIP(lamMatch=myargs.lamMatch, lamL2=lamL2, sourceInd=0), 53 | semiclass.DIPOracle(lamMatch=myargs.lamMatch, lamL2=lamL2, sourceInd=0), 54 | ] 55 | elif myargs.tag_DA == "DAstd": 56 | methods = [ 57 | semitorchclass.DIP(lamMatch=myargs.lamMatch, lamL2=lamL2, lamL1=lamL1, sourceInd=0, lr=myargs.lr, epochs=myargs.epochs, wayMatch='std'), 58 | semitorchclass.DIP(lamMatch=myargs.lamMatch, lamL2=lamL2, lamL1=lamL1, sourceInd=0, lr=myargs.lr, epochs=myargs.epochs, wayMatch='mean+std+25p'), 59 | ] 60 | else: # "DAMMD" 61 | methods = [ 62 | semitorchstocclass.DIP(lamMatch=myargs.lamMatch, lamL2=lamL2, lamL1=lamL1, sourceInd = 0, lr=myargs.lr, 63 | epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1.]), 64 | ] 65 | 66 | 67 | names = [str(m) for m in methods] 68 | print(names) 69 | names_short = [str(m).split('_')[0] for m in methods] 70 | print(names_short) 71 | 72 | seed1 = int(123456 + np.exp(1) * 1000) 73 | 74 | def simple_run_sem(sem_nums, ds, methods, i2n_ratios=[1.], n=2000, repeats=10): 75 | res_all = {} 76 | for i, sem_num in enumerate(sem_nums): 77 | for j, inter2noise_ratio_local in enumerate(i2n_ratios): 78 | print("Number of envs M=%d, inter2noise_ratio=%.1f" %(2, inter2noise_ratio_local), flush=True) 79 | params = {'M': 2, 'inter2noise_ratio': inter2noise_ratio_local, 'd': ds[i]} 80 | 81 | sem1 = simudata.pick_sem(sem_num, 82 | params=params, 83 | seed=seed1) 84 | 85 | 86 | # run methods on data generated from sem 87 | results_src_all, results_tar_all = util.run_all_methods(sem1, methods, n=n, repeats=repeats) 88 | res_all[(i, j)] = results_src_all, results_tar_all 89 | return res_all 90 | 91 | repeats = 10 92 | res_all = simple_run_sem(sem_nums=['r0%sd3x1' %myargs.interv_type, 93 | 'r0%sd?x1' %myargs.interv_type, 94 | 'r0%sd?x1' %myargs.interv_type], 95 | ds=[3, 10, 20], 96 | i2n_ratios=[1.], 97 | methods=methods, 98 | n=myargs.n, 99 | repeats=repeats) 100 | 101 | np.save("simu_results/sim_exp8_box_r0%sd31020_%s_lamMatch%s_n%d_epochs%d_repeats%d.npy" %(myargs.interv_type, 102 | myargs.tag_DA, myargs.lamMatch, myargs.n, myargs.epochs, repeats), res_all) -------------------------------------------------------------------------------- /sim/sim_linearSCM_var_shift_exp8_box_submit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import numpy as np 5 | import subprocess 6 | 7 | interv_type = 'sv1' 8 | epochs = 20000 9 | epochs_MMD = 2000 10 | n = 5000 11 | 12 | for tag_DA in ['baseline']: 13 | subprocess.call(['bsub', '-W 03:50', '-n 4', "./sim_linearSCM_var_shift_exp8_box_run.py --interv_type=%s --tag_DA=%s --n=%d --epochs=%d" %(interv_type, tag_DA, n, epochs)]) 14 | 15 | lamMatches = [10.**(k) for k in (np.arange(10)-5)] 16 | for tag_DA in ['DAmean', 'DAstd']: 17 | for lam in lamMatches: 18 | subprocess.call(['bsub', '-W 03:50', '-n 4', "./sim_linearSCM_var_shift_exp8_box_run.py --interv_type=%s --lamMatch=%f --tag_DA=%s --n=%d --epochs=%d" %(interv_type, lam, tag_DA, n, epochs)]) 19 | 20 | for tag_DA in ['DAMMD']: 21 | for lam in lamMatches: 22 | subprocess.call(['bsub', '-W 23:50', '-n 4', "./sim_linearSCM_var_shift_exp8_box_run.py --interv_type=%s --lamMatch=%f --tag_DA=%s --n=%d --epochs=%d" %(interv_type, lam, tag_DA, n, epochs_MMD)]) -------------------------------------------------------------------------------- /sim/sim_linearSCM_var_shift_exp8_scat_run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import numpy as np 5 | import pandas as pd 6 | import sys 7 | import argparse 8 | 9 | import torch 10 | 11 | import torch.nn as nn 12 | import torch.nn.functional as F 13 | import torch.optim as optim 14 | 15 | # local packages 16 | import sys 17 | sys.path.append('../') 18 | import semiclass 19 | import semitorchclass 20 | import semitorchstocclass 21 | import util 22 | import simudata 23 | 24 | # check gpu avail 25 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 26 | # Assuming that we are on a CUDA machine, this should print a CUDA device: 27 | print(device) 28 | 29 | # parse args 30 | parser = argparse.ArgumentParser() 31 | parser.add_argument("--interv_type", type=str, default="sv1", help="type of intervention") 32 | parser.add_argument("--lamMatch", type=float, default=1., help="DIP matching penalty") 33 | parser.add_argument("--epochs", type=int, default=4000, help="number of epochs") 34 | parser.add_argument("--lr", type=float, default=1e-3, help="learning rate") 35 | parser.add_argument("--tag_DA", type=str, default="baseline", help="choose whether to run baseline methods or DA methods") 36 | parser.add_argument("--n", type=int, default=5000, help="sample size") 37 | parser.add_argument("--seed", type=int, default=0, help="seed of experiment") 38 | myargs = parser.parse_args() 39 | print(myargs) 40 | 41 | lamL2 = 0. 42 | lamL1 = 0. 43 | 44 | if myargs.tag_DA == "baseline": 45 | methods = [ 46 | semiclass.Tar(lamL2=lamL2), 47 | semiclass.SrcPool(lamL2=lamL2), 48 | semitorchclass.Tar(lamL2=lamL2, lamL1=lamL1, lr=myargs.lr, epochs=myargs.epochs), 49 | semitorchclass.SrcPool(lamL2=lamL2, lamL1=lamL1, lr=myargs.lr, epochs=myargs.epochs), 50 | ] 51 | elif myargs.tag_DA == "DAmean": 52 | methods = [ 53 | semiclass.DIP(lamMatch=myargs.lamMatch, lamL2=lamL2, sourceInd=0), 54 | semiclass.DIPOracle(lamMatch=myargs.lamMatch, lamL2=lamL2, sourceInd=0), 55 | ] 56 | elif myargs.tag_DA == "DAstd": 57 | methods = [ 58 | semitorchclass.DIP(lamMatch=myargs.lamMatch, lamL2=lamL2, lamL1=lamL1, sourceInd=0, lr=myargs.lr, epochs=myargs.epochs, wayMatch='std'), 59 | semitorchclass.DIP(lamMatch=myargs.lamMatch, lamL2=lamL2, lamL1=lamL1, sourceInd=0, lr=myargs.lr, epochs=myargs.epochs, wayMatch='mean+std+25p'), 60 | ] 61 | else: # "DAMMD" 62 | methods = [ 63 | semitorchstocclass.DIP(lamMatch=myargs.lamMatch, lamL2=lamL2, lamL1=lamL1, sourceInd = 0, lr=myargs.lr, 64 | epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1.]), 65 | ] 66 | 67 | names = [str(m) for m in methods] 68 | print(names) 69 | names_short = [str(m).split('_')[0] for m in methods] 70 | print(names_short) 71 | 72 | seed1 = int(123456 + np.exp(1) * 1000) 73 | 74 | params = {'M': 2, 'inter2noise_ratio': 1.0, 'd': 10} 75 | 76 | sem1 = simudata.pick_sem('r0%sd?x1' %myargs.interv_type, 77 | params=params, 78 | seed=seed1+myargs.seed) 79 | # run methods on data generated from sem 80 | results_src_all, results_tar_all = util.run_all_methods(sem1, methods, n=myargs.n, repeats=1) 81 | res_all = {} 82 | res_all['src'] = results_src_all 83 | res_all['tar'] = results_tar_all 84 | 85 | np.save("simu_results/sim_exp8_scat_r0%sd10_%s_lamMatch%s_n%d_epochs%d_seed%d.npy" %(myargs.interv_type, 86 | myargs.tag_DA, myargs.lamMatch, myargs.n, myargs.epochs, myargs.seed), res_all) 87 | 88 | 89 | -------------------------------------------------------------------------------- /sim/sim_linearSCM_var_shift_exp8_scat_submit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import numpy as np 5 | import subprocess 6 | 7 | interv_type = 'sv1' 8 | epochs = 20000 9 | epochs_MMD = 2000 10 | n = 5000 11 | 12 | for tag_DA in ['baseline']: 13 | for myseed in range(100): 14 | subprocess.call(['bsub', '-W 03:50', '-n 4', "./sim_linearSCM_var_shift_exp8_scat_run.py --interv_type=%s --tag_DA=%s --n=%d --epochs=%d --seed=%d" %(interv_type, tag_DA, n, epochs, myseed)]) 15 | 16 | 17 | lamMatches = [10.**(k) for k in (np.arange(10)-5)] 18 | for tag_DA in ['DAmean', 'DAstd']: 19 | for myseed in range(100): 20 | for lam in lamMatches: 21 | subprocess.call(['bsub', '-W 03:50', '-n 4', "./sim_linearSCM_var_shift_exp8_scat_run.py --interv_type=%s --lamMatch=%f --tag_DA=%s --n=%d --epochs=%d --seed=%d" %(interv_type, lam, tag_DA, n, epochs, myseed)]) 22 | 23 | for tag_DA in ['DAMMD']: 24 | for myseed in range(100): 25 | for lam in lamMatches: 26 | subprocess.call(['bsub', '-W 23:50', '-n 4', "./sim_linearSCM_var_shift_exp8_scat_run.py --interv_type=%s --lamMatch=%f --tag_DA=%s --n=%d --epochs=%d --seed=%d" %(interv_type, lam, tag_DA, n, epochs_MMD, myseed)]) -------------------------------------------------------------------------------- /sim/sim_linearSCM_var_shift_exp9_scat_run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import numpy as np 5 | import pandas as pd 6 | import sys 7 | import argparse 8 | 9 | import torch 10 | 11 | import torch.nn as nn 12 | import torch.nn.functional as F 13 | import torch.optim as optim 14 | 15 | # local packages 16 | import sys 17 | sys.path.append('../') 18 | import semiclass 19 | import semitorchclass 20 | import semitorchstocclass 21 | import util 22 | import simudata 23 | 24 | # check gpu avail 25 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 26 | 27 | # Assuming that we are on a CUDA machine, this should print a CUDA device: 28 | 29 | print(device) 30 | 31 | parser = argparse.ArgumentParser() 32 | parser.add_argument("--interv_type", type=str, default="smv1", help="type of intervention") 33 | parser.add_argument("--lamMatch", type=float, default=1., help="DIP matching penalty") 34 | parser.add_argument("--lamCIP", type=float, default=0.1, help="CIP penalty") 35 | parser.add_argument("--epochs", type=int, default=4000, help="number of epochs") 36 | parser.add_argument("--lr", type=float, default=1e-3, help="learning rate") 37 | parser.add_argument("--tag_DA", type=str, default="baseline", help="choose whether to run baseline methods or DA methods") 38 | parser.add_argument("--n", type=int, default=5000, help="sample size") 39 | parser.add_argument("--seed", type=int, default=0, help="seed of experiment") 40 | myargs = parser.parse_args() 41 | print(myargs) 42 | 43 | 44 | lamL2 = 0. 45 | lamL1 = 0. 46 | 47 | if myargs.tag_DA == 'baseline': 48 | methods = [ 49 | semiclass.Tar(lamL2=lamL2), 50 | semiclass.SrcPool(lamL2=lamL2), 51 | semitorchclass.Tar(lamL2=lamL2, lamL1=lamL1, lr=myargs.lr, epochs=myargs.epochs), 52 | semitorchclass.SrcPool(lamL2=lamL2, lamL1=lamL1, lr=myargs.lr, epochs=myargs.epochs), 53 | ] 54 | elif myargs.tag_DA == 'DAmean': 55 | methods = [ 56 | semiclass.DIP(lamMatch=myargs.lamMatch, lamL2=lamL2, sourceInd=0), 57 | semiclass.DIPOracle(lamMatch=myargs.lamMatch, lamL2=lamL2, sourceInd=0), 58 | semiclass.DIPweigh(lamMatch=myargs.lamMatch, lamL2=lamL2), 59 | semiclass.CIP(lamCIP=myargs.lamCIP, lamL2=lamL2), 60 | semiclass.CIRMweigh(lamCIP=myargs.lamCIP, lamMatch=myargs.lamMatch, lamL2=lamL2), 61 | ] 62 | elif myargs.tag_DA == 'DAstd': 63 | methods = [ 64 | semitorchclass.DIP(lamMatch=myargs.lamMatch, lamL2=lamL2, lamL1=lamL1, sourceInd=0, lr=myargs.lr, 65 | epochs=myargs.epochs, wayMatch='mean+std+25p'), 66 | semitorchclass.DIPweigh(lamMatch=myargs.lamMatch, lamL2=lamL2, lamL1=lamL1, lr=myargs.lr, 67 | epochs=myargs.epochs, wayMatch='mean+std+25p'), 68 | semitorchclass.CIP(lamCIP=myargs.lamCIP, lamL2=lamL2, lamL1=lamL1, lr=myargs.lr, 69 | epochs=myargs.epochs, wayMatch='mean+std+25p'), 70 | semitorchclass.CIRMweigh(lamMatch=myargs.lamMatch, lamL2=lamL2, lamL1=lamL1, lr=myargs.lr, 71 | epochs=myargs.epochs, wayMatch='mean+std+25p'), 72 | ] 73 | elif myargs.tag_DA == 'DAMMD': 74 | methods = [ 75 | semitorchstocclass.DIP(lamMatch=myargs.lamMatch, lamL2=lamL2, lamL1=lamL1, sourceInd = 0, lr=myargs.lr, 76 | epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1.]), 77 | semitorchstocclass.DIPweigh(lamMatch=myargs.lamMatch, lamL2=lamL2, lamL1=lamL1, lr=myargs.lr, 78 | epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1.]), 79 | semitorchstocclass.CIP(lamCIP=myargs.lamCIP, lamL2=lamL2, lamL1=lamL1, lr=myargs.lr, 80 | epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1.]), 81 | semitorchstocclass.CIRMweigh(lamMatch=myargs.lamMatch, lamCIP=myargs.lamCIP, lamL2=lamL2, lamL1=lamL1, lr=myargs.lr, 82 | epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1.]) 83 | ] 84 | elif myargs.tag_DA == 'DACIP': 85 | methods = [ 86 | semiclass.CIP(lamCIP=myargs.lamCIP, lamL2=lamL2), 87 | semitorchclass.CIP(lamCIP=myargs.lamCIP, lamL2=lamL2, lamL1=lamL1, lr=myargs.lr, 88 | epochs=myargs.epochs, wayMatch='mean+std+25p'), 89 | ] 90 | elif myargs.tag_DA == 'DACIPMMD': 91 | methods = [ 92 | semitorchstocclass.CIP(lamCIP=myargs.lamCIP, lamL2=lamL2, lamL1=lamL1, lr=myargs.lr, 93 | epochs=myargs.epochs, wayMatch='mmd', sigma_list=[1.]), 94 | ] 95 | 96 | 97 | names = [str(m) for m in methods] 98 | print(names) 99 | names_short = [str(m).split('_')[0] for m in methods] 100 | print(names_short) 101 | 102 | seed1 = int(123456 + np.exp(2) * 1000) 103 | 104 | params = {'M': 15, 'inter2noise_ratio': 1., 'd': 20, 'cicnum': 10,'interY': 1.} 105 | 106 | sem1 = simudata.pick_sem('r0%sd?x4' %myargs.interv_type, 107 | params=params, 108 | seed=seed1+myargs.seed) 109 | 110 | # run methods on data generated from sem 111 | results_src_all, results_tar_all, results_minDiffIndx = util.run_all_methods(sem1, 112 | methods, 113 | n=myargs.n, 114 | repeats=1, 115 | returnMinDiffIndx=True, 116 | tag_DA=myargs.tag_DA) 117 | res_all = {} 118 | res_all['src'] = results_src_all 119 | res_all['tar'] = results_tar_all 120 | res_all['minDiffIndx'] = results_minDiffIndx 121 | 122 | 123 | np.save("simu_results/sim_exp9_scat_r0%sd20x4_%s_lamMatch%s_lamCIP%s_n%d_epochs%d_seed%d.npy" %(myargs.interv_type, 124 | myargs.tag_DA, myargs.lamMatch, myargs.lamCIP, myargs.n, myargs.epochs, myargs.seed), res_all) 125 | 126 | 127 | -------------------------------------------------------------------------------- /sim/sim_linearSCM_var_shift_exp9_scat_submit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | 5 | import numpy as np 6 | import subprocess 7 | import os 8 | 9 | interv_type = 'smv1' 10 | epochs = 20000 11 | epochs_MMD = 2000 12 | n = 5000 13 | 14 | for tag_DA in ['baseline']: 15 | for myseed in range(100): 16 | subprocess.call(['bsub', '-W 03:50', '-n 4', "./sim_linearSCM_var_shift_exp9_scat_run.py --interv_type=%s --tag_DA=%s --epochs=%d --seed=%d" %(interv_type, tag_DA, 20000, myseed)]) 17 | 18 | 19 | # You don't have to run all the lamMatch choices!!! 20 | lamMatches = [10.**(k) for k in (np.arange(10)-5)] 21 | for tag_DA in ['DAmean', 'DAstd']: 22 | for myseed in range(100): 23 | for lam in lamMatches: 24 | subprocess.call(['bsub', '-W 03:50', '-n 4', "./sim_linearSCM_var_shift_exp9_scat_run.py --interv_type=%s --lamMatch=%f --tag_DA=%s --epochs=%d --seed=%d" %(interv_type, lam, tag_DA, epochs, myseed)]) 25 | 26 | for tag_DA in ['DAMMD']: 27 | for myseed in range(100): 28 | for lam in lamMatches: 29 | subprocess.call(['bsub', '-W 119:50', '-n 4', "./sim_linearSCM_var_shift_exp9_scat_run.py --interv_type=%s --lamMatch=%f --tag_DA=%s --epochs=%d --seed=%d" %(interv_type, lam, tag_DA, epochs_MMD, myseed)]) 30 | 31 | 32 | # You don't have to run all the lamCIP choices!!! 33 | lamCIPs = [10.**(k) for k in (np.arange(10)-5)] 34 | for tag_DA in ['DACIP']: 35 | for myseed in range(100): 36 | for lam in lamCIPs: 37 | subprocess.call(['bsub', '-W 03:50', '-n 4', "./sim_linearSCM_var_shift_exp9_scat_run.py --interv_type=%s --lamCIP=%f --tag_DA=%s --epochs=%d --seed=%d" %(interv_type, lam, tag_DA, epochs, myseed)]) 38 | 39 | for tag_DA in ['DACIPMMD']: 40 | for myseed in range(100): 41 | for lam in lamCIPs: 42 | subprocess.call(['bsub', '-W 23:50', '-n 4', "./sim_linearSCM_var_shift_exp9_scat_run.py --interv_type=%s --lamCIP=%f --tag_DA=%s --epochs=%d --seed=%d" %(interv_type, lam, tag_DA, epochs_MMD, myseed)]) 43 | -------------------------------------------------------------------------------- /sim/simu_results/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /sim/simudata.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sys 3 | sys.path.append('../') 4 | import sem 5 | import myrandom 6 | 7 | def pick_intervention_and_noise(M, dp1, inter2noise_ratio, interY=0., cic=[], typeshift='sm1', varAs=None, varnoiseY=1.): 8 | if typeshift == 'sm1': 9 | meanAs = inter2noise_ratio * np.random.randn(M, dp1) 10 | 11 | # meanAs[:, -1] = interY * np.random.randn(M) # 0 means no intervention on Y 12 | meanAs[:, -1] = 0. 13 | meanAs[-1, -1] = interY * np.random.randn(1) # 0 means no intervention on Y 14 | 15 | meanAs[:, cic] = 0 # set conditional invariant components 16 | 17 | if not varAs: 18 | varAs = np.zeros((M, dp1)) 19 | interAf = myrandom.Gaussian(M, meanAs, varAs) 20 | 21 | noisevar = np.ones((1, dp1)) 22 | noisevar[-1] = varnoiseY 23 | noisef = myrandom.Gaussian(1, np.zeros((1, dp1)), noisevar) 24 | elif typeshift == 'sm2': 25 | # mixture 26 | meanAs1 = inter2noise_ratio * np.random.randn(M, dp1) 27 | meanAs2 = inter2noise_ratio * np.random.randn(M, dp1) 28 | meanAs1[:, -1] = 0. # 0 intervY only for the target env 29 | meanAs2[:, -1] = 0. # 0 intervY only for the target env 30 | meanAs1[-1, -1] = interY * np.random.randn(1) # 0 means no intervention on Y 31 | meanAs2[-1, -1] = interY * np.random.randn(1) # 0 means no intervention on Y 32 | meanAs1[:, cic] = 0 # set conditional invariant components 33 | meanAs2[:, cic] = 0 # set conditional invariant components 34 | meanAsList = [meanAs1, meanAs2] 35 | 36 | if not varAs: 37 | varAs = np.zeros((M, dp1)) 38 | interAf = myrandom.Mix2Gaussian(M, meanAsList, varAs) 39 | 40 | noisevar = np.ones((1, dp1)) 41 | noisevar[-1] = varnoiseY 42 | noisef = myrandom.Gaussian(1, np.zeros((1, dp1)), noisevar) 43 | elif typeshift == 'sm3': 44 | # mixture 45 | meanAs1 = inter2noise_ratio * np.random.randn(M, dp1) 46 | meanAs2 = inter2noise_ratio * np.random.randn(M, dp1) 47 | meanAs3 = inter2noise_ratio * np.random.randn(M, dp1) 48 | meanAs1[:, -1] = 0. # 0 intervY only for the target env 49 | meanAs2[:, -1] = 0. # 0 intervY only for the target env 50 | meanAs3[:, -1] = 0. # 0 intervY only for the target env 51 | meanAs1[-1, -1] = interY * np.random.randn(1) # 0 means no intervention on Y 52 | meanAs2[-1, -1] = interY * np.random.randn(1) # 0 means no intervention on Y 53 | meanAs3[-1, -1] = interY * np.random.randn(1) # 0 means no intervention on Y 54 | meanAs1[:, cic] = 0 # set conditional invariant components 55 | meanAs2[:, cic] = 0 # set conditional invariant components 56 | meanAs3[:, cic] = 0 # set conditional invariant components 57 | meanAsList = [meanAs1, meanAs2, meanAs3] 58 | 59 | if not varAs: 60 | varAs = np.zeros((M, dp1)) 61 | interAf = myrandom.MixkGaussian(M, meanAsList, varAs) 62 | 63 | noisevar = np.ones((1, dp1)) 64 | noisevar[-1] = varnoiseY 65 | noisef = myrandom.Gaussian(1, np.zeros((1, dp1)), noisevar) 66 | 67 | elif typeshift == 'sv1': 68 | meanAs = np.zeros((M, dp1)) 69 | varAs_inter = inter2noise_ratio * (np.abs(np.random.randn(M, dp1))) 70 | varAs_inter[:, -1] = 1. 71 | varAs = varAs_inter 72 | 73 | interAf = myrandom.Gaussian(M, meanAs, varAs) 74 | 75 | # the noise is always zero, is taken care of by interAf 76 | noisef = myrandom.Gaussian(1, np.zeros((1, dp1)), np.zeros((1, dp1))) 77 | 78 | elif typeshift == 'smv1': 79 | meanAs = inter2noise_ratio * np.random.randn(M, dp1) 80 | # meanAs[:, -1] = interY * np.random.randn(M) # 0 means no intervention on Y 81 | meanAs[:, -1] = 0. 82 | meanAs[-1, -1] = interY * np.random.randn(1) # 0 means no intervention on Y 83 | meanAs[:, cic] = 0 # set conditional invariant components 84 | varAs_inter = inter2noise_ratio * np.abs(np.random.randn(M, dp1)) 85 | varAs_inter[:, -1] = 1. 86 | varAs_inter[:, cic] = 1. # set conditional invariant components 87 | varAs = varAs_inter 88 | 89 | interAf = myrandom.Gaussian(M, meanAs, varAs) 90 | 91 | # the noise is always zero, is taken care of by interAf 92 | noisef = myrandom.Gaussian(1, np.zeros((1, dp1)), np.zeros((1, dp1))) 93 | 94 | return interAf, noisef 95 | 96 | def pick_random_B(pred_dir = 'anticausal', dp1=1): 97 | B = np.zeros((dp1, dp1)) 98 | if pred_dir == 'anticausal': 99 | # triangular B 100 | for i in range(0, dp1-1): 101 | for j in range(i+1, dp1-1): 102 | B[j, i] = 0.5 * np.random.randn(1) 103 | B[:, -1] = 1.0 * np.random.randn(dp1) 104 | B[-1, -1] = 0 105 | elif pred_dir == 'causal': 106 | for i in range(0, dp1-1): 107 | for j in range(i+1, dp1-1): 108 | B[j, i] = 0.5 * np.random.randn(1) 109 | # so y should not change X 110 | B[:, -1] = 0 111 | # causal prediction 112 | B[-1, :-1] = 1.0 * np.random.randn(dp1-1) 113 | B[-1, -1] = 0 114 | elif pred_dir == 'halfhalf': 115 | # make them mixed causal and anti-causal 116 | B = np.zeros((dp1, dp1)) 117 | for i in range(0, dp1-1): 118 | for j in range(i+1, dp1-1): 119 | B[j, i] = 0.5 * np.random.randn(1) 120 | for i in range((dp1-1)//2): 121 | # half anti-causal 122 | B[2*i, -1] = 1.0 * np.random.randn(1) 123 | # half causal 124 | if 2*i+1 <= dp1-2: 125 | B[-1, 2*i+1] = 1.0 * np.random.randn(1) 126 | for j in range((dp1-1)//2): 127 | # anti-causal node should not point to causal node, to ensure acyclic graph 128 | B[2*i+1, 2*j] = 0 129 | B[-1, -1] = 0 130 | else: 131 | raise ValueError('case not recognized.') 132 | 133 | 134 | return B 135 | 136 | 137 | def pick_sem(data_num, params = None, seed=123456): 138 | np.random.seed(seed) 139 | # name rules 140 | # r0: r0 regression Y is cause 141 | # r1 regression Y is effect 142 | # r2 regression Y is in the middle 143 | # sm1: 1 dimensional mean shift 144 | # case 1 1 dimensional mean shift 145 | # case 2 2 mixture of mean shift 146 | # sv1 change of variance 147 | # d3: dimension of the problem is 3 148 | # x1: no intervention on Y, only intervention on X, 149 | # case 1 (no conditional invariant components, no inter Y) 150 | # case 2 (with conditional invariant components, no inter Y) 151 | # case 3 (no conditional invariant components, inter Y) 152 | # case 4 (with conditional invariant components, inter Y) 153 | if 'd3' in data_num: 154 | # Y cause, d = 3 155 | M = params['M'] 156 | # d plus 1 157 | dp1 = 4 158 | 159 | 160 | inter2noise_ratio = params['inter2noise_ratio'] 161 | if 'x1' in data_num: 162 | # conditional invariant components 163 | cic = [] 164 | # intervention on Y 165 | interY = 0 166 | elif 'x3' in data_num: 167 | cic = [] 168 | if 'interY' in params.keys(): 169 | interY = params['interY'] 170 | else: 171 | interY = 1. 172 | elif 'x2' in data_num: 173 | # conditional invariant components 174 | cic = [0] 175 | # intervention on Y 176 | interY = 0 177 | elif 'x4' in data_num: 178 | cic = [0] 179 | if 'interY' in params.keys(): 180 | interY = params['interY'] 181 | else: 182 | interY = 1. 183 | else: 184 | raise ValueError('case not recognized.') 185 | 186 | if 'sm1' in data_num: 187 | typeshift = 'sm1' 188 | elif 'sm2' in data_num: 189 | typeshift = 'sm2' 190 | elif 'sm3' in data_num: 191 | typeshift = 'sm3' 192 | elif 'sv1' in data_num: 193 | typeshift = 'sv1' 194 | elif 'smv1' in data_num: 195 | typeshift = 'smv1' 196 | elif 'smm2' in data_num: 197 | typeshift = 'smm2' 198 | else: 199 | typeshift = 'sm1' 200 | 201 | 202 | interAf, noisef = pick_intervention_and_noise(M, dp1, inter2noise_ratio, interY=interY, 203 | cic=cic, typeshift=typeshift, varAs=None, varnoiseY=1.) 204 | 205 | if 'r0' in data_num: 206 | pred_dir = 'anticausal' 207 | B = np.array([[0, 0, 0, 1], [0, 0, 0, -1], [0, 0, 0, 3], [0, 0, 0, 0]]) 208 | elif 'r1' in data_num: 209 | pred_dir = 'causal' 210 | B = np.array([[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [1, -1, 3, 0]]) 211 | # elif 'r2' in data_num: 212 | # pred_dir = 'halfhalf' 213 | else: 214 | raise ValueError('case not recognized.') 215 | 216 | 217 | 218 | invariantList = [] 219 | message = "%sM%di%d, fixed simple B" %(data_num, M, inter2noise_ratio) 220 | elif 'd?' in data_num: 221 | # Y cause, d = ? 222 | M = params['M'] 223 | # d plus 1 224 | if 'd' in params.keys(): 225 | dp1 = params['d'] + 1 226 | else: 227 | # set default dimension to 10 228 | dp1 = 11 229 | if 'interY' in params.keys(): 230 | interY = params['interY'] 231 | else: 232 | interY = 1. 233 | 234 | inter2noise_ratio = params['inter2noise_ratio'] 235 | 236 | if 'r0' in data_num: 237 | pred_dir = 'anticausal' 238 | varnoiseY = 1. 239 | elif 'r1' in data_num: 240 | pred_dir = 'causal' 241 | varnoiseY = 1. 242 | elif 'r2' in data_num: 243 | pred_dir = 'halfhalf' 244 | varnoiseY = 0.01 245 | else: 246 | raise ValueError('case not recognized.') 247 | 248 | B = pick_random_B(pred_dir, dp1) 249 | 250 | if 'x1' in data_num: 251 | # conditional invariant components 252 | cic = [] 253 | # intervention on Y 254 | interY = 0 255 | elif 'x3' in data_num: 256 | cic = [] 257 | if 'interY' in params.keys(): 258 | interY = params['interY'] 259 | else: 260 | interY = 1. 261 | elif 'x2' in data_num: 262 | if 'cicnum' in params.keys(): 263 | cicnum = params['cicnum'] 264 | else: 265 | cicnum = int(dp1/2) 266 | # conditional invariant components 267 | cic = np.random.choice(dp1-1, cicnum, replace=False) 268 | # intervention on Y 269 | interY = 0 270 | elif 'x4' in data_num: 271 | if 'cicnum' in params.keys(): 272 | cicnum = params['cicnum'] 273 | else: 274 | cicnum = int(dp1/2) 275 | # cic = np.arange(0, cicnum) 276 | cic = np.random.choice(dp1-1, cicnum, replace=False) 277 | if 'interY' in params.keys(): 278 | interY = params['interY'] 279 | else: 280 | interY = 1. 281 | else: 282 | raise ValueError('case not recognized.') 283 | 284 | if 'sm1' in data_num: 285 | typeshift = 'sm1' 286 | elif 'sm2' in data_num: 287 | typeshift = 'sm2' 288 | elif 'sv1' in data_num: 289 | typeshift = 'sv1' 290 | elif 'smv1' in data_num: 291 | typeshift = 'smv1' 292 | elif 'smm2' in data_num: 293 | typeshift = 'smm2' 294 | else: 295 | typeshift = 'sm1' 296 | 297 | interAf, noisef = pick_intervention_and_noise(M, dp1, inter2noise_ratio, interY=interY, cic=cic, typeshift=typeshift, varAs=None, varnoiseY=varnoiseY) 298 | 299 | invariantList = cic 300 | message = "%sM%dd%di%d, fixed simple B" %(data_num, M, dp1-1, inter2noise_ratio) 301 | # generate sem 302 | sem1 = sem.SEM(B, noisef, interAf, invariantList=invariantList, message=message) 303 | 304 | return sem1 305 | -------------------------------------------------------------------------------- /util.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sem 3 | 4 | import torch 5 | 6 | # check gpu avail 7 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 8 | 9 | 10 | def MSE(yhat, y): 11 | return np.mean((yhat-y)**2) 12 | 13 | def torchMSE(a, b): 14 | return torch.mean((a.squeeze() - b)**2) 15 | 16 | def torchloaderMSE(me, dataloader, device): 17 | # get MSE from a torch model with dataloader 18 | error = 0 19 | n = 0 20 | with torch.no_grad(): 21 | for data in dataloader: 22 | x, y = data[0].to(device), data[1].to(device) 23 | ypred = me.predict(x) 24 | n += x.shape[0] 25 | error += torch.sum((ypred.squeeze() - y)**2) 26 | return error.item()/n 27 | 28 | 29 | # given a SEM, run all methods and return target risks and target test risks 30 | def run_all_methods(sem1, methods, n=1000, repeats=10, returnMinDiffIndx=False, tag_DA='DAMMD'): 31 | M = sem1.M 32 | results_src_all = np.zeros((M-1, len(methods), 2, repeats)) 33 | results_tar_all = np.zeros((len(methods), 2, repeats)) 34 | results_minDiffIndx = {} 35 | 36 | # generate data 37 | for repeat in range(repeats): 38 | data = sem1.generateAllSamples(n) 39 | dataTest = sem1.generateAllSamples(n) 40 | # may use other target as well 41 | source = np.arange(M-1) 42 | target = M-1 43 | 44 | # prepare torch format data 45 | dataTorch = {} 46 | dataTorchTest = {} 47 | 48 | for i in range(M): 49 | dataTorch[i] = [torch.from_numpy(data[i][0].astype(np.float32)).to(device), 50 | torch.from_numpy(data[i][1].astype(np.float32)).to(device)] 51 | dataTorchTest[i] = [torch.from_numpy(dataTest[i][0].astype(np.float32)).to(device), 52 | torch.from_numpy(dataTest[i][1].astype(np.float32)).to(device)] 53 | 54 | # prepare torch format data for batch stochastic gradient descent 55 | train_batch_size = 500 56 | test_batch_size = 500 57 | 58 | trainloaders = {} 59 | testloaders = {} 60 | 61 | for i in range(M): 62 | train_dataset = torch.utils.data.TensorDataset(torch.Tensor(data[i][0]), 63 | torch.Tensor(data[i][1])) 64 | test_dataset = torch.utils.data.TensorDataset(torch.Tensor(dataTest[i][0]), 65 | torch.Tensor(dataTest[i][1])) 66 | trainloaders[i] = torch.utils.data.DataLoader(train_dataset, batch_size=train_batch_size) 67 | testloaders[i] = torch.utils.data.DataLoader(test_dataset, batch_size=test_batch_size) 68 | 69 | for i, m in enumerate(methods): 70 | if m.__module__ == 'semiclass': 71 | me = m.fit(data, source=source, target=target) 72 | if hasattr(me, 'minDiffIndx'): 73 | print("best index="+str(me.minDiffIndx)) 74 | results_minDiffIndx[(tag_DA, i, repeat)] = me.minDiffIndx 75 | xtar, ytar = data[target] 76 | xtar_test, ytar_test = dataTest[target] 77 | targetE = MSE(me.ypred, ytar) 78 | targetNE = MSE(me.predict(xtar_test), ytar_test) 79 | for j, sourcej in enumerate(source): 80 | results_src_all[j, i, 0, repeat] = MSE(me.predict(data[sourcej][0]), data[sourcej][1]) 81 | results_src_all[j, i, 1, repeat] = MSE(me.predict(dataTest[sourcej][0]), dataTest[sourcej][1]) 82 | elif m.__module__ == 'semitorchclass': 83 | me = m.fit(dataTorch, source=source, target=target) 84 | if hasattr(me, 'minDiffIndx'): 85 | print("best index="+str(me.minDiffIndx)) 86 | results_minDiffIndx[(tag_DA, i, repeat)] = me.minDiffIndx 87 | xtar, ytar= dataTorch[target] 88 | xtar_test, ytar_test= dataTorchTest[target] 89 | targetE = torchMSE(me.ypred, ytar) 90 | targetNE = torchMSE(me.predict(xtar_test), ytar_test) 91 | for j, sourcej in enumerate(source): 92 | results_src_all[j, i, 0, repeat] = torchMSE(me.predict(dataTorch[sourcej][0]), dataTorch[sourcej][1]) 93 | results_src_all[j, i, 1, repeat] = torchMSE(me.predict(dataTorchTest[sourcej][0]), dataTorchTest[sourcej][1]) 94 | elif m.__module__ == 'semitorchstocclass': 95 | me = m.fit(trainloaders, source=source, target=target) 96 | targetE = torchloaderMSE(me, trainloaders[target], device) 97 | targetNE = torchloaderMSE(me, testloaders[target], device) 98 | for j, sourcej in enumerate(source): 99 | results_src_all[j, i, 0, repeat] = torchloaderMSE(me, trainloaders[sourcej], device) 100 | results_src_all[j, i, 1, repeat] = torchloaderMSE(me, testloaders[sourcej], device) 101 | else: 102 | raise ValueError("Unexpected method class") 103 | results_tar_all[i, 0, repeat] = targetE 104 | results_tar_all[i, 1, repeat] = targetNE 105 | print("Repeat %d Target %-30s error=%.3f errorTest=%.3f" %(repeat, str(m), targetE, targetNE), flush=True) 106 | if returnMinDiffIndx: 107 | return results_src_all, results_tar_all, results_minDiffIndx 108 | else: 109 | return results_src_all, results_tar_all 110 | --------------------------------------------------------------------------------