├── README.md └── synthetic_experiments.ipynb /README.md: -------------------------------------------------------------------------------- 1 | In this repository, we open-source the code accompanying the simulation analyses of the paper: 2 | 3 | - Ye, Zikun and Zhang, Zhiqi and Zhang, Dennis and Zhang, Heng and Zhang, Renyu. 2025. Deep-Learning-Based Causal Inference for Large-Scale Combinatorial Experiments: Theory and Empirical Evidence. *Management Science*, forthcoming. Available at SSRN: https://ssrn.com/abstract=4375327. 4 | 5 | ## File 6 | * synthetic_experiments.ipynb: This is the Jupyter notebook code for the synthetic experiments in the paper (Appendix D). 7 | -------------------------------------------------------------------------------- /synthetic_experiments.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "80fe70ab", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2023-02-10T16:08:33.845190Z", 10 | "start_time": "2023-02-10T16:08:30.779708Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "import numpy as np\n", 16 | "from numpy.linalg import inv\n", 17 | "from numpy import linalg as LA\n", 18 | "import seaborn as sns\n", 19 | "import matplotlib.pyplot as plt\n", 20 | "from scipy import stats\n", 21 | "from scipy.optimize import curve_fit\n", 22 | "import pandas as pd\n", 23 | "import scipy.stats as st\n", 24 | "\n", 25 | "import torch\n", 26 | "import torchvision\n", 27 | "import torchvision.transforms as transforms\n", 28 | "import torch.nn as nn\n", 29 | "import torch.utils.data\n", 30 | "import torch.optim as optim\n", 31 | "import torch.backends.cudnn as cudnn\n", 32 | "import os\n", 33 | "import os.path\n", 34 | "import argparse\n", 35 | "from torch.autograd import Variable\n", 36 | "import statsmodels.api as sm" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "id": "d89963d9", 43 | "metadata": { 44 | "ExecuteTime": { 45 | "end_time": "2023-02-11T04:27:27.694674Z", 46 | "start_time": "2023-02-11T04:27:27.669666Z" 47 | } 48 | }, 49 | "outputs": [], 50 | "source": [ 51 | "def round_p(value, round_digits):\n", 52 | " return format(round(value,round_digits), \".\"+str(round_digits)+\"f\")\n", 53 | "def mse_loss(beta_, c, d, y, t):\n", 54 | " return(c/(1+np.exp(-beta_.dot(t)))+d-y)**2\n", 55 | "\n", 56 | "def avg_y(beta_, c, d, t):\n", 57 | " return c/(1+np.exp(-beta_.dot(t)))+d\n", 58 | "\n", 59 | "def exp(x):\n", 60 | " return np.exp(x)\n", 61 | "\n", 62 | "def generate_y(coef, c, d, x, t, n):\n", 63 | " y = np.zeros(n)\n", 64 | " y_error = 0.05*np.random.uniform(-1, 1, n)\n", 65 | " for i in range(n):\n", 66 | " y[i] = c/(1+np.exp(-(x[i].dot(coef)).dot(t[i]))) + d + y_error[i]\n", 67 | " return y, y_error\n", 68 | "\n", 69 | "def generate_x(d_c, n):\n", 70 | " return np.random.uniform(0, 1, [n, d_c])\n", 71 | "\n", 72 | "def generate_t(t_combo, t_dist, n):\n", 73 | " return t_combo[np.random.choice(np.shape(t_dist)[0], n, p=t_dist),]\n", 74 | "\n", 75 | "\n", 76 | "def powerset(s):\n", 77 | " x = len(s)\n", 78 | " masks = [1 << i for i in range(x)]\n", 79 | " for i in range(1 << x):\n", 80 | " yield [ss for mask, ss in zip(masks, s) if i & mask]\n", 81 | "\n", 82 | "def calculate_mse(loader, is_gpu, net):\n", 83 | " \"\"\"Calculate accuracy.\n", 84 | "\n", 85 | " Args:\n", 86 | " loader (torch.utils.data.DataLoader): training / test set loader\n", 87 | " is_gpu (bool): whether to run on GPU\n", 88 | " Returns:\n", 89 | " tuple: (overall accuracy, class level accuracy)\n", 90 | " \"\"\"\n", 91 | " cnt = 0\n", 92 | " total_loss = 0\n", 93 | "\n", 94 | " for data in loader:\n", 95 | " inputs, labels = data\n", 96 | " if is_gpu:\n", 97 | " inputs = inputs.cuda()\n", 98 | " labels = labels.cuda()\n", 99 | " inputs, labels = Variable(inputs), Variable(labels)\n", 100 | " outputs = net(inputs)\n", 101 | " \n", 102 | " \n", 103 | " cnt += labels.size(0)\n", 104 | " total_loss += sum((outputs-labels)**2)\n", 105 | "\n", 106 | " return total_loss/float(cnt)\n", 107 | "\n", 108 | "def plot_dev(loader, is_gpu, net):\n", 109 | " cnt = 0\n", 110 | " total_loss = 0\n", 111 | " \n", 112 | " real_y = []\n", 113 | " pred_y = []\n", 114 | "\n", 115 | " for data in loader:\n", 116 | " inputs, labels = data\n", 117 | " if is_gpu:\n", 118 | " inputs = inputs.cuda()\n", 119 | " labels = labels.cuda()\n", 120 | " inputs, labels = Variable(inputs), Variable(labels)\n", 121 | " outputs = net(inputs)\n", 122 | " \n", 123 | " real_y.append(labels.tolist())\n", 124 | " pred_y.append(outputs.tolist())\n", 125 | " \n", 126 | " cnt += labels.size(0)\n", 127 | " total_loss += sum((outputs-labels)**2)\n", 128 | " real_y = [x for sublist in real_y for x in sublist]\n", 129 | " pred_y = [x for sublist in pred_y for x in sublist]\n", 130 | " print('MSELoss= ', total_loss/float(cnt))\n", 131 | " plt.scatter(real_y, pred_y)\n", 132 | " plt.xlabel('real y')\n", 133 | " plt.ylabel('pred y')\n", 134 | " plt.show()\n", 135 | " \n", 136 | "\n", 137 | "parser = argparse.ArgumentParser()\n", 138 | "# hyperparameters settings\n", 139 | "parser.add_argument('--lr', type=float, default=0.001, help='learning rate')\n", 140 | "parser.add_argument('--wd', type=float, default=5e-4, help='weight decay')#lr/(c+wd)\n", 141 | "parser.add_argument('--epochs', type=int, default=50,\n", 142 | " help='number of epochs to train')\n", 143 | "parser.add_argument('--batch_size_train', type=int,\n", 144 | " default=1000, help='training set input batch size')\n", 145 | "parser.add_argument('--batch_size_test', type=int,\n", 146 | " default=1000, help='test set input batch size')\n", 147 | "parser.add_argument('--is_gpu', type=bool, default=False,\n", 148 | " help='whether training using GPU')\n", 149 | "import sys\n", 150 | "sys.argv=['']\n", 151 | "del sys" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": null, 157 | "id": "c46f61e9", 158 | "metadata": { 159 | "ExecuteTime": { 160 | "end_time": "2023-02-11T04:27:31.019119Z", 161 | "start_time": "2023-02-11T04:27:31.017447Z" 162 | } 163 | }, 164 | "outputs": [], 165 | "source": [ 166 | "n_rep = 5#we replicate 200 times in the paper" 167 | ] 168 | }, 169 | { 170 | "cell_type": "markdown", 171 | "id": "9bfed773", 172 | "metadata": {}, 173 | "source": [ 174 | "# Validation of DeDL" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "id": "03a16db9", 180 | "metadata": {}, 181 | "source": [ 182 | "## Data generating process definition" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": null, 188 | "id": "2946056b", 189 | "metadata": { 190 | "ExecuteTime": { 191 | "end_time": "2023-02-11T04:27:34.418396Z", 192 | "start_time": "2023-02-11T04:27:34.411017Z" 193 | } 194 | }, 195 | "outputs": [], 196 | "source": [ 197 | "m = 4 #number of experiments\n", 198 | "d_c = 10 #number of features\n", 199 | "lr = 0.05\n", 200 | "\n", 201 | "all_combo = list(powerset(list(np.arange(1, m+1))))\n", 202 | "t_combo = []\n", 203 | "for i in all_combo:\n", 204 | " t = np.zeros(m+1)\n", 205 | " t[0] = 1\n", 206 | " t[i] = 1\n", 207 | " t_combo.append(t)\n", 208 | "t_combo = np.int16(t_combo)\n", 209 | "t_dist = (1/2**m)*np.ones(2**m)\n", 210 | "t_combo_obs = []\n", 211 | "for i in range(m+1):\n", 212 | " t = np.zeros(m+1)\n", 213 | " t[0] = 1\n", 214 | " t[i] = 1\n", 215 | " t_combo_obs.append(t)\n", 216 | "t = np.zeros(m+1)\n", 217 | "t[[0,1,2]] = 1\n", 218 | "t_combo_obs.append(t)\n", 219 | "t_combo_obs = np.int16(t_combo_obs)\n", 220 | "t_dist_obs = (1/(m+2))*np.ones(m+2)\n", 221 | "\n", 222 | "t_star_all = t_combo.copy()\n", 223 | "idx = (t_combo[:,None]!=t_combo_obs).any(-1).all(1)\n", 224 | "t_star_unobs = t_combo[idx]\n", 225 | "\n", 226 | "n_est = int(m*500)\n", 227 | "n_train = int(m*500)\n", 228 | "reg_term = 0.0005\n", 229 | "#reg_loss = 0.001\n", 230 | "reg_loss = 0\n", 231 | "train_epochs = 2000\n", 232 | "test_thres = 0.3\n", 233 | "n_cnver_test = 10000\n", 234 | "\n", 235 | "feature_list = []\n", 236 | "for i in range(d_c):\n", 237 | " feature_list.append(str('x')+str(i+1))\n", 238 | "t_list = []\n", 239 | "for i in range(m+1):\n", 240 | " t_list.append(str('t')+str(i))\n", 241 | "\n", 242 | "true_0 = []#real ATE in base\n", 243 | "true_1 = []#real ATE in treatment\n", 244 | "\n", 245 | "estimator_LR_0 = []#ATE by LR in base\n", 246 | "estimator_LR_1 = []#ATE by LR in treatment\n", 247 | "\n", 248 | "estimator_0 = []#ATE by SDL in base\n", 249 | "estimator_1 = []#ATE by SDL in treatment\n", 250 | "\n", 251 | "estimator_debias_0 = []#ATE by DeDL in base\n", 252 | "estimator_debias_1 = []#ATE by DeDL in treatment\n", 253 | "\n", 254 | "estimator_additive_1 = []#ATE by LA in treatment\n", 255 | "mse_c = []\n", 256 | "mse_theta = []\n", 257 | "mae_theta = []\n", 258 | "\n", 259 | "p_ = []\n", 260 | "p_LA = []\n", 261 | "p_LR = []\n", 262 | "p_DL = []#p value by SDL\n", 263 | "p_DDL = []#p value by DeDL\n", 264 | "test_err = []" 265 | ] 266 | }, 267 | { 268 | "cell_type": "markdown", 269 | "id": "139293f3", 270 | "metadata": {}, 271 | "source": [ 272 | "## Comparison of LA, LR, SDL, DeDL" 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": null, 278 | "id": "3c2c19db", 279 | "metadata": { 280 | "ExecuteTime": { 281 | "end_time": "2023-02-11T04:27:37.723447Z", 282 | "start_time": "2023-02-11T04:27:37.719124Z" 283 | } 284 | }, 285 | "outputs": [], 286 | "source": [ 287 | "# FNN+poly with k=4 and 4 exps\n", 288 | "class FNN_asig(nn.Module):\n", 289 | " \"\"\"FNN.\"\"\"\n", 290 | "\n", 291 | " def __init__(self):\n", 292 | " \"\"\"FNN Builder.\"\"\"\n", 293 | " super(FNN_asig, self).__init__()\n", 294 | " \n", 295 | "\n", 296 | " self.layer1 = nn.Sequential(\n", 297 | " nn.Linear(d_c, 10, bias=False),\n", 298 | " nn.ReLU(inplace=True),\n", 299 | " nn.Linear(10, m+1)\n", 300 | " )\n", 301 | " self.siglayer = nn.Sigmoid()\n", 302 | " self.layer3 = nn.Linear(1, 1, bias=False)\n", 303 | "\n", 304 | "\n", 305 | " def forward(self, x):\n", 306 | " \"\"\"Perform forward.\"\"\"\n", 307 | " b = self.layer1(x[:,0:d_c])\n", 308 | " u = torch.sum(b*x[:, d_c:], 1)\n", 309 | " u = self.siglayer(u)\n", 310 | " u = u.unsqueeze(1)\n", 311 | " u = self.layer3(u)\n", 312 | " return torch.reshape(u, (-1,))\n", 313 | " \n", 314 | "def get_activation(name):\n", 315 | " def hook(model, input, output):\n", 316 | " activation[name] = output.detach()\n", 317 | " return hook" 318 | ] 319 | }, 320 | { 321 | "cell_type": "code", 322 | "execution_count": null, 323 | "id": "6036fb85", 324 | "metadata": { 325 | "ExecuteTime": { 326 | "end_time": "2023-02-11T04:29:40.835514Z", 327 | "start_time": "2023-02-11T04:27:41.025676Z" 328 | }, 329 | "scrolled": true 330 | }, 331 | "outputs": [], 332 | "source": [ 333 | "rep_index = 0\n", 334 | "\n", 335 | "while rep_index < n_rep:\n", 336 | " \n", 337 | " print('# of replication:', rep_index)\n", 338 | " coef = np.random.uniform(-0.5, 0.5, [d_c, m+1])\n", 339 | " c_true = np.random.uniform(10, 20)\n", 340 | " d_true = 0\n", 341 | "\n", 342 | " def generate_y_true(coef, c, d, x, t, n):\n", 343 | " y = np.zeros(n)\n", 344 | " y_error = 0.05*np.random.uniform(-1, 1, n)\n", 345 | " for i in range(n):\n", 346 | " y[i] = c/(1+np.exp(-((x[i].dot(coef))**3).dot(t[i]))) + d + y_error[i]\n", 347 | " return y, y_error\n", 348 | " def generate_y_true_1(coef, x, t):\n", 349 | " return c_true/(1+np.exp(-((x.dot(coef))**3).dot(t)))\n", 350 | " def generate_beta_true_1(coef, x, t):\n", 351 | " return (x.dot(coef))**3\n", 352 | " \n", 353 | " #generate samples for estimation\n", 354 | " samples_x_est = generate_x(d_c, n_train)\n", 355 | " samples_t_est = generate_t(t_combo_obs, t_dist_obs, n_train)\n", 356 | " samples_y_est, samples_y_error_est = generate_y_true(coef, c_true, d_true, samples_x_est, samples_t_est, n_train)\n", 357 | " full_data_est = np.append(samples_x_est, samples_t_est, axis=1)\n", 358 | " full_data_est = np.append(full_data_est, samples_y_est.reshape(n_train,1), axis=1)\n", 359 | " data_obs = pd.DataFrame(full_data_est, columns = feature_list + t_list + ['y'])\n", 360 | " \n", 361 | " model_LR = sm.OLS(samples_y_est.reshape(n_train,1), np.append(samples_x_est, samples_t_est, axis=1))\n", 362 | " results_LR = model_LR.fit()\n", 363 | " coef_LR = results_LR.params\n", 364 | "\n", 365 | " opt = parser.parse_args()\n", 366 | " train_samples = int(0.7*data_obs.shape[0])\n", 367 | " x_train_set = np.float32(data_obs)[0:train_samples, 0:-1]\n", 368 | " y_train_set = np.float32(data_obs)[0:train_samples, -1]\n", 369 | " x_test_set = np.float32(data_obs)[train_samples:, 0:-1]\n", 370 | " y_test_set = np.float32(data_obs)[train_samples:, -1]\n", 371 | "\n", 372 | " trainset = torch.utils.data.TensorDataset(torch.Tensor(x_train_set), torch.Tensor(y_train_set)) # create your datset\n", 373 | " trainloader = torch.utils.data.DataLoader(\n", 374 | " trainset, batch_size=opt.batch_size_train, shuffle=True)\n", 375 | "\n", 376 | " testset = torch.utils.data.TensorDataset(torch.Tensor(x_test_set), torch.Tensor(y_test_set)) # create your datset\n", 377 | " testloader = torch.utils.data.DataLoader(\n", 378 | " testset, batch_size=opt.batch_size_test, shuffle=True)\n", 379 | " \n", 380 | " net = FNN_asig()\n", 381 | " test_accuracy = 100\n", 382 | " print('---------warm-up train---------')\n", 383 | " wp_cnt = 0\n", 384 | " while(test_accuracy >= 1):\n", 385 | " wp_cnt += 1\n", 386 | " net = FNN_asig()\n", 387 | " with torch.no_grad():\n", 388 | " net.layer3.weight[0, 0] = float(np.max(y_train_set))\n", 389 | " criterion = nn.MSELoss()\n", 390 | " #criterion = nn.L1Loss()\n", 391 | " #warm-up train\n", 392 | " test_accuracy = 100\n", 393 | " optimizer = optim.Adam(net.parameters(), lr = 0.1, weight_decay = opt.wd)\n", 394 | " for epoch in range(200):\n", 395 | " for i, data_i in enumerate(trainloader, 0):\n", 396 | " inputs, labels = data_i\n", 397 | " inputs, labels = Variable(inputs), Variable(labels)\n", 398 | " optimizer.zero_grad()\n", 399 | " outputs = net(inputs)\n", 400 | " l1_norm = sum(p.abs().sum() for p in net.parameters())\n", 401 | " loss = criterion(outputs, labels) + reg_loss*l1_norm\n", 402 | " loss.backward()\n", 403 | " optimizer.step()\n", 404 | " #train_accuracy = calculate_mse(trainloader, opt.is_gpu, net)\n", 405 | " test_accuracy = calculate_mse(testloader, opt.is_gpu, net)\n", 406 | " if test_accuracy < 1:\n", 407 | " break\n", 408 | " if wp_cnt >= 5:\n", 409 | " break\n", 410 | " #training \n", 411 | " print('---------train---------')\n", 412 | " optimizer = optim.Adam(net.parameters(), lr = lr, weight_decay = opt.wd)\n", 413 | " criterion = nn.MSELoss()\n", 414 | " for epoch in range(train_epochs):\n", 415 | " for i, data_i in enumerate(trainloader, 0):\n", 416 | " inputs, labels = data_i\n", 417 | " inputs, labels = Variable(inputs), Variable(labels)\n", 418 | " optimizer.zero_grad()\n", 419 | " outputs = net(inputs)\n", 420 | " l1_norm = sum(p.abs().sum() for p in net.parameters())\n", 421 | " loss = criterion(outputs, labels) + reg_loss*l1_norm\n", 422 | " loss.backward()\n", 423 | " optimizer.step()\n", 424 | " train_accuracy = calculate_mse(trainloader, opt.is_gpu, net)\n", 425 | " test_accuracy = calculate_mse(testloader, opt.is_gpu, net)\n", 426 | " if epoch%100 == 0:\n", 427 | " print(\"Iteration: {0} | Training MSE: {1} | Test MSE: {2}\".format(\n", 428 | " epoch, train_accuracy, test_accuracy))\n", 429 | " if test_accuracy < test_thres and epoch > 500:\n", 430 | " break \n", 431 | " if test_accuracy > test_thres:\n", 432 | " continue\n", 433 | " else:\n", 434 | " rep_index += 1\n", 435 | " test_err.append(test_accuracy)\n", 436 | " \n", 437 | " activation = {}\n", 438 | " net.layer1.register_forward_hook(get_activation('layer1'))\n", 439 | " \n", 440 | " \n", 441 | " #generate samples for inference\n", 442 | " samples_x_est = generate_x(d_c, n_est)\n", 443 | " samples_t_est = generate_t(t_combo_obs, t_dist_obs, n_est)\n", 444 | " samples_y_est, samples_y_error_est = generate_y_true(coef, c_true, d_true, samples_x_est, samples_t_est, n_est)\n", 445 | " full_data_est = np.append(samples_x_est, samples_t_est, axis=1)\n", 446 | " full_data_est = np.append(full_data_est, samples_y_est.reshape(n_est,1), axis=1)\n", 447 | "\n", 448 | " for t_star in t_star_all:\n", 449 | " data_est = pd.DataFrame(full_data_est, columns = feature_list + t_list + ['y'])\n", 450 | " #at t=0\n", 451 | " t_star_base = np.zeros(m+1)\n", 452 | " t_star_base[0] = 1\n", 453 | " x_all_set=np.float32(data_est)[:, 0:-1]\n", 454 | " y_all_set=np.float32(data_est)[:, -1]\n", 455 | " \n", 456 | " allset = torch.utils.data.TensorDataset(torch.Tensor(x_all_set), torch.Tensor(y_all_set)) # create your datset\n", 457 | " allloader = torch.utils.data.DataLoader(\n", 458 | " allset, batch_size=1000, shuffle=False)\n", 459 | " \n", 460 | " beta_ = []\n", 461 | " pred_y_loss = []\n", 462 | " for i, data_ in enumerate(allloader, 0):\n", 463 | " inputs, labels = data_\n", 464 | " inputs, labels = Variable(inputs), Variable(labels)\n", 465 | " outputs = net(inputs)\n", 466 | " beta_.append(activation['layer1'].tolist()) \n", 467 | " pred_y_loss.append(outputs.tolist())\n", 468 | " beta_ = np.array(beta_).reshape(n_est, m+1)\n", 469 | " pred_y_loss = np.array(pred_y_loss).reshape(n_est)\n", 470 | " \n", 471 | " real_y = samples_y_est.copy()\n", 472 | " \n", 473 | " for j in range(m):\n", 474 | " x_all_set[:, -m+j] = t_star_base[j+1]\n", 475 | " allset = torch.utils.data.TensorDataset(torch.Tensor(x_all_set), torch.Tensor(y_all_set)) # create your datset\n", 476 | " allloader = torch.utils.data.DataLoader(\n", 477 | " allset, batch_size=1000, shuffle=False)\n", 478 | " pred_y = []\n", 479 | " for i, data_ in enumerate(allloader, 0):\n", 480 | " inputs, labels = data_\n", 481 | " inputs, labels = Variable(inputs), Variable(labels)\n", 482 | " outputs = net(inputs)\n", 483 | " pred_y.append(outputs.tolist())\n", 484 | " pred_y = np.array(pred_y).reshape(n_est)\n", 485 | " \n", 486 | " real_y_star = []\n", 487 | " for i in range(n_est):\n", 488 | " real_y_star.append(0.05*np.random.uniform(-1, 1)+generate_y_true_1(coef,samples_x_est[i],t_star_base))\n", 489 | "\n", 490 | " lambda_inv = []\n", 491 | " G_theta = []\n", 492 | " G_theta_loss = []\n", 493 | " t_obs_ = samples_t_est.copy()\n", 494 | " cnt = 0\n", 495 | "\n", 496 | " for beta_temp in beta_:\n", 497 | "\n", 498 | " lambda_ = np.zeros([m+2, m+2])#asig use m+2\n", 499 | " u_ = beta_temp.dot(t_star_base)\n", 500 | " G_theta.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*t_star_base, [1/(np.exp(-u_)+1)]))\n", 501 | "\n", 502 | " u_ = beta_temp.dot(samples_t_est[cnt])\n", 503 | " G_theta_loss.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*samples_t_est[cnt], [1/(np.exp(-u_)+1)]))\n", 504 | "\n", 505 | " cnt += 1\n", 506 | "\n", 507 | " for i in range(t_combo_obs.shape[0]):\n", 508 | " t = t_combo_obs[i]\n", 509 | " y = avg_y(beta_temp, c_true, d_true, t)\n", 510 | " u = beta_temp.dot(t)\n", 511 | " G_prime = np.append(c_est*np.exp(-u)/(np.exp(-u)+1)**2*t, [1/(np.exp(-u)+1)])\n", 512 | " lambda_ += t_dist_obs[i]*2*np.outer(G_prime, G_prime)\n", 513 | "\n", 514 | " try:\n", 515 | " lambda_inv.append(inv(lambda_ + reg_term*np.eye(m+2)))\n", 516 | " except:\n", 517 | " print('Singular matrix')\n", 518 | "\n", 519 | " lambda_inv_loss_prime = []\n", 520 | " for i in range(n_est):\n", 521 | " lambda_inv_loss_prime.append(2*(pred_y_loss[i]-real_y[i])*lambda_inv[i].dot(G_theta_loss[i]))\n", 522 | " lambda_inv_loss_prime = np.array(lambda_inv_loss_prime)\n", 523 | "\n", 524 | " pred_y_debiased = []\n", 525 | " for i in range(n_est):\n", 526 | " pred_y_debiased.append(pred_y[i]-G_theta[i].dot(lambda_inv_loss_prime[i]))\n", 527 | " \n", 528 | " pred_y_LR = []\n", 529 | " for i in range(n_est): \n", 530 | " pred_y_LR.append(np.append(samples_x_est[i], t_star_base).dot(coef_LR))\n", 531 | " pred_y_LR = np.array(pred_y_LR)\n", 532 | "\n", 533 | " estimator_LR_0.append(np.mean(pred_y_LR))#LR\n", 534 | " estimator_0.append(np.mean(pred_y))#SDL\n", 535 | " estimator_debias_0.append(np.mean(pred_y_debiased))#DeDL\n", 536 | " true_0.append(np.mean(real_y_star))#ground truth\n", 537 | " \n", 538 | " #record for comparison - t test\n", 539 | " real_y_star_0 = real_y_star.copy()\n", 540 | " pred_y_LR_0 = pred_y_LR.copy()\n", 541 | " pred_y_0 = pred_y.copy()\n", 542 | " pred_y_debiased_0 = pred_y_debiased.copy()\n", 543 | " \n", 544 | " #at t=t_star\n", 545 | " t_star_base = t_star.copy()\n", 546 | " \n", 547 | " for i in range(n_est):\n", 548 | " for j in range(m):\n", 549 | " x_all_set[i][-m+j] = t_star_base[j+1]\n", 550 | " allset = torch.utils.data.TensorDataset(torch.Tensor(x_all_set), torch.Tensor(y_all_set)) # create your datset\n", 551 | " allloader = torch.utils.data.DataLoader(\n", 552 | " allset, batch_size=1000, shuffle=False)\n", 553 | " pred_y = []\n", 554 | " for i, data_ in enumerate(allloader, 0):\n", 555 | " inputs, labels = data_\n", 556 | " inputs, labels = Variable(inputs), Variable(labels)\n", 557 | " outputs = net(inputs)\n", 558 | " pred_y.append(outputs.tolist())\n", 559 | " pred_y = np.array(pred_y).reshape(n_est)\n", 560 | " \n", 561 | " real_y_star = []\n", 562 | " for i in range(n_est):\n", 563 | " real_y_star.append(0.05*np.random.uniform(-1, 1)+generate_y_true_1(coef,samples_x_est[i],t_star_base))\n", 564 | "\n", 565 | "\n", 566 | " G_theta = []\n", 567 | " \n", 568 | " G_theta_loss = []\n", 569 | " t_obs_ = samples_t_est.copy()\n", 570 | " cnt = 0\n", 571 | "\n", 572 | " for beta_temp in beta_:\n", 573 | " lambda_ = np.zeros([m+2, m+2])#asig use m+2\n", 574 | " u_ = beta_temp.dot(t_star_base)\n", 575 | " G_theta.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*t_star_base, [1/(np.exp(-u_)+1)]))\n", 576 | " u_ = beta_temp.dot(samples_t_est[cnt])\n", 577 | " G_theta_loss.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*samples_t_est[cnt], [1/(np.exp(-u_)+1)]))\n", 578 | " cnt += 1\n", 579 | "\n", 580 | "\n", 581 | " lambda_inv_loss_prime = []\n", 582 | " for i in range(n_est):\n", 583 | " lambda_inv_loss_prime.append(2*(pred_y_loss[i]-real_y[i])*lambda_inv[i].dot(G_theta_loss[i]))\n", 584 | " lambda_inv_loss_prime = np.array(lambda_inv_loss_prime)\n", 585 | "\n", 586 | " pred_y_debiased = []\n", 587 | " for i in range(n_est):\n", 588 | " pred_y_debiased.append(pred_y[i]-G_theta[i].dot(lambda_inv_loss_prime[i]))\n", 589 | " \n", 590 | " pred_y_LR = []\n", 591 | " for i in range(n_est): \n", 592 | " pred_y_LR.append(np.append(samples_x_est[i], t_star_base).dot(coef_LR))\n", 593 | " pred_y_LR = np.array(pred_y_LR)\n", 594 | "\n", 595 | " true_1.append(np.mean(real_y_star))\n", 596 | " estimator_LR_1.append(np.mean(pred_y_LR))\n", 597 | " estimator_1.append(np.mean(pred_y))\n", 598 | " estimator_debias_1.append(np.mean(pred_y_debiased))\n", 599 | "\n", 600 | " p_.append(stats.ttest_ind(real_y_star, real_y_star_0)[1])\n", 601 | " p_LR.append(stats.ttest_ind(pred_y_LR, pred_y_LR_0)[1]) \n", 602 | " p_DL.append(stats.ttest_ind(pred_y, pred_y_0)[1])\n", 603 | " p_DDL.append(stats.ttest_ind(pred_y_debiased, pred_y_debiased_0)[1])\n", 604 | " \n", 605 | " #calculate additive\n", 606 | " pred_y_additive = 0\n", 607 | " pred_y_additive_var = 0\n", 608 | " n_LA_samples = 0\n", 609 | " for single_exp_index in np.where(t_star == 1)[0]:\n", 610 | " ttt = np.zeros(m+1)\n", 611 | " ttt[0] = 1\n", 612 | " ttt[single_exp_index] = 1\n", 613 | " pred_y_additive += samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].mean()\n", 614 | " pred_y_additive_var += samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].var()\n", 615 | " n_LA_samples += (samples_t_est == ttt).sum()\n", 616 | " ttt = np.zeros(m+1)\n", 617 | " ttt[0] = 1\n", 618 | " pred_y_additive -= (np.sum(t_star))*samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].mean()\n", 619 | " estimator_additive_1.append(pred_y_additive)\n", 620 | " t_value = pred_y_additive/np.sqrt(pred_y_additive_var*(np.sum(t_star))/n_LA_samples)\n", 621 | " p_LA.append(stats.t.sf(abs(t_value), df=1))\n", 622 | " \n", 623 | " " 624 | ] 625 | }, 626 | { 627 | "cell_type": "code", 628 | "execution_count": null, 629 | "id": "b67de56b", 630 | "metadata": { 631 | "ExecuteTime": { 632 | "end_time": "2023-02-11T04:29:52.624292Z", 633 | "start_time": "2023-02-11T04:29:52.579177Z" 634 | } 635 | }, 636 | "outputs": [], 637 | "source": [ 638 | "test_err_np = np.zeros(n_rep)\n", 639 | "for i in range(n_rep):\n", 640 | " test_err_np[i] = test_err[i].tolist()\n", 641 | "test_err = test_err_np.copy()\n", 642 | "del test_err_np\n", 643 | "\n", 644 | "p_ = np.array(p_).reshape((n_rep, 2**m))\n", 645 | "p_LA = np.array(p_LA).reshape((n_rep, 2**m))\n", 646 | "p_LR = np.array(p_LR).reshape((n_rep, 2**m))\n", 647 | "p_DL = np.array(p_DL).reshape((n_rep, 2**m))\n", 648 | "p_DDL = np.array(p_DDL).reshape((n_rep, 2**m))\n", 649 | "true_0 = np.array(true_0).reshape((n_rep, 2**m))\n", 650 | "estimator_LR_0 = np.array(estimator_LR_0).reshape((n_rep, 2**m))\n", 651 | "estimator_0 = np.array(estimator_0).reshape((n_rep, 2**m))\n", 652 | "estimator_debias_0 = np.array(estimator_debias_0).reshape((n_rep, 2**m))\n", 653 | "true_1 = np.array(true_1).reshape((n_rep, 2**m))\n", 654 | "estimator_LR_1 = np.array(estimator_LR_1).reshape((n_rep, 2**m))\n", 655 | "estimator_1 = np.array(estimator_1).reshape((n_rep, 2**m))\n", 656 | "estimator_debias_1 = np.array(estimator_debias_1).reshape((n_rep, 2**m))\n", 657 | "estimator_additive_1 = np.array(estimator_additive_1).reshape((n_rep, 2**m))\n", 658 | "index = 0\n", 659 | "\n", 660 | "#LA LR SDL DeDL\n", 661 | "ape_2 = np.zeros([2**m, 4, n_rep])\n", 662 | "se = np.zeros([2**m, 4, n_rep])\n", 663 | "ae = np.zeros([2**m, 4, n_rep])\n", 664 | "CDR = np.zeros([2**m, 4, n_rep])\n", 665 | "for i in range(n_rep):\n", 666 | " CDR[0,:,i] = 1\n", 667 | " for j in range(1,2**m):\n", 668 | " true_eff = true_1[i,j]-true_0[i,j]\n", 669 | " la = estimator_additive_1[i,j]\n", 670 | " lr = estimator_LR_1[i,j]-estimator_LR_0[i,j]\n", 671 | " dl = estimator_1[i,j]-estimator_0[i,j]\n", 672 | " dedl = estimator_debias_1[i,j]-estimator_debias_0[i,j]\n", 673 | " if (p_[i,j]<=0.05 and p_LA[i,j]<=0.05 and la*true_eff>0) or (p_[i,j]>0.05 and p_LA[i,j]>0.05):\n", 674 | " CDR[j,0,i] = 1\n", 675 | " if (p_[i,j]<=0.05 and p_LR[i,j]<=0.05 and lr*true_eff>0) or (p_[i,j]>0.05 and p_LR[i,j]>0.05):\n", 676 | " CDR[j,1,i] = 1\n", 677 | " if (p_[i,j]<=0.05 and p_DL[i,j]<=0.05 and dl*true_eff>0) or (p_[i,j]>0.05 and p_DL[i,j]>0.05):\n", 678 | " CDR[j,2,i] = 1\n", 679 | " if (p_[i,j]<=0.05 and p_DDL[i,j]<=0.05 and dedl*true_eff>0) or (p_[i,j]>0.05 and p_DDL[i,j]>0.05):\n", 680 | " CDR[j,3,i] = 1\n", 681 | " \n", 682 | "\n", 683 | "print('----------------All combos--------------------')\n", 684 | "for t_star in t_star_all:\n", 685 | " #print('Treatment effect when t = ', t_star[1:])\n", 686 | " true_effect = true_1[:,index]-true_0[:,index]\n", 687 | " add_effect = estimator_additive_1[:,index]\n", 688 | " LR_effect = estimator_LR_1[:,index]-estimator_LR_0[:,index] \n", 689 | " no_bias_effect = estimator_1[:,index]-estimator_0[:,index]\n", 690 | " debias_effect = estimator_debias_1[:,index]-estimator_debias_0[:,index]\n", 691 | " \n", 692 | " ape_2[index, 0, :] = np.abs((add_effect-true_effect))/(np.abs(true_effect))\n", 693 | " ape_2[index, 1, :] = np.abs((LR_effect-true_effect))/(np.abs(true_effect))\n", 694 | " ape_2[index, 2, :] = np.abs((no_bias_effect-true_effect))/(np.abs(true_effect))\n", 695 | " ape_2[index, 3, :] = np.abs((debias_effect-true_effect))/(np.abs(true_effect))\n", 696 | " \n", 697 | " #if p_[:,index]>=0.05:#not significant treatment effect\n", 698 | " ape_2[index, 0, p_[:,index]>=0.05] = 0\n", 699 | " ape_2[index, 1, p_[:,index]>=0.05] = 0\n", 700 | " ape_2[index, 2, p_[:,index]>=0.05] = 0\n", 701 | " ape_2[index, 3, p_[:,index]>=0.05] = 0\n", 702 | " \n", 703 | " se[index, 0, :] = ((add_effect - true_effect)**2)\n", 704 | " se[index, 1, :] = ((LR_effect - true_effect)**2) \n", 705 | " se[index, 2, :] = ((no_bias_effect - true_effect)**2)\n", 706 | " se[index, 3, :] = ((debias_effect - true_effect)**2)\n", 707 | " \n", 708 | " ae[index, 0, :] = np.abs((add_effect - true_effect))\n", 709 | " ae[index, 1, :] = np.abs((LR_effect - true_effect))\n", 710 | " ae[index, 2, :] = np.abs((no_bias_effect - true_effect))\n", 711 | " ae[index, 3, :] = np.abs((debias_effect - true_effect))\n", 712 | " \n", 713 | " index += 1\n", 714 | " \n", 715 | "print('------------------------------------')\n", 716 | "print('MSE of LA ','|', np.mean(se[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,0,:], axis=0)), scale=st.sem(np.mean(se[:,0,:], axis=0))))\n", 717 | "print('MSE of LR ','|', np.mean(se[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,1,:], axis=0)), scale=st.sem(np.mean(se[:,1,:], axis=0))))\n", 718 | "print('MSE of SDL ','|', np.mean(se[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,2,:], axis=0)), scale=st.sem(np.mean(se[:,2,:], axis=0))))\n", 719 | "print('MSE of DeDL ','|', np.mean(se[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,3,:], axis=0)), scale=st.sem(np.mean(se[:,3,:], axis=0))))\n", 720 | "print('------------------------------------')\n", 721 | "print('MAE of LA ','|', np.mean(ae[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,0,:], axis=0)), scale=st.sem(np.mean(ae[:,0,:], axis=0))))\n", 722 | "print('MAE of LR ','|', np.mean(ae[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,1,:], axis=0)), scale=st.sem(np.mean(ae[:,1,:], axis=0))))\n", 723 | "print('MAE of SDL ','|', np.mean(ae[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,2,:], axis=0)), scale=st.sem(np.mean(ae[:,2,:], axis=0))))\n", 724 | "print('MAE of DeDL ','|', np.mean(ae[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,3,:], axis=0)), scale=st.sem(np.mean(ae[:,3,:], axis=0))))\n", 725 | "print('------------------------------------')\n", 726 | "print('MAPE of LA ','|', np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)), \n", 727 | " scale=st.sem(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0))))\n", 728 | "print('MAPE of LR ','|', np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)), \n", 729 | " scale=st.sem(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0))))\n", 730 | "print('MAPE of SDL ','|', np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)), \n", 731 | " scale=st.sem(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0))))\n", 732 | "print('MAPE of DeDL ','|', np.mean(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0)), \n", 733 | " scale=st.sem(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0))))\n", 734 | "\n", 735 | "print('------------------------------------')\n", 736 | "print('CDR of LA ','|', np.mean(CDR[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,0,:], axis=0)), scale=st.sem(np.mean(CDR[:,0,:], axis=0))))\n", 737 | "print('CDR of LR ','|', np.mean(CDR[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,1,:], axis=0)), scale=st.sem(np.mean(CDR[:,1,:], axis=0))))\n", 738 | "print('CDR of SDL ','|', np.mean(CDR[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,2,:], axis=0)), scale=st.sem(np.mean(CDR[:,2,:], axis=0))))\n", 739 | "print('CDR of DeDL ','|', np.mean(CDR[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,3,:], axis=0)), scale=st.sem(np.mean(CDR[:,3,:], axis=0))))\n", 740 | "\n", 741 | "print('------------------------------------')\n", 742 | "\n", 743 | "cnt_bai = np.zeros([n_rep, 4])\n", 744 | "for i in range(n_rep):\n", 745 | " max_effect = -10000*np.ones(5)\n", 746 | " max_effect_index = np.zeros(5)\n", 747 | " for index in range(2**m):\n", 748 | " true_effect = true_1[i,index]-true_0[i,index]\n", 749 | " add_effect = estimator_additive_1[i,index]\n", 750 | " LR_effect = estimator_LR_1[i,index]-estimator_LR_0[i,index]\n", 751 | " no_bias_effect = estimator_1[i,index]-estimator_0[i,index]\n", 752 | " debias_effect = estimator_debias_1[i,index]-estimator_debias_0[i,index]\n", 753 | " \n", 754 | " if true_effect > max_effect[4]:\n", 755 | " max_effect[4] = true_effect\n", 756 | " max_effect_index[4] = index\n", 757 | " if add_effect > max_effect[0]:\n", 758 | " max_effect[0] = add_effect\n", 759 | " max_effect_index[0] = index\n", 760 | " if LR_effect > max_effect[1]:\n", 761 | " max_effect[1] = LR_effect\n", 762 | " max_effect_index[1] = index\n", 763 | " if no_bias_effect > max_effect[2]:\n", 764 | " max_effect[2] = no_bias_effect\n", 765 | " max_effect_index[2] = index\n", 766 | " if debias_effect > max_effect[3]:\n", 767 | " max_effect[3] = debias_effect\n", 768 | " max_effect_index[3] = index\n", 769 | " \n", 770 | " if max_effect_index[0] == max_effect_index[4]:\n", 771 | " cnt_bai[i, 0] = 1\n", 772 | " if max_effect_index[1] == max_effect_index[4]:\n", 773 | " cnt_bai[i, 1] = 1\n", 774 | " if max_effect_index[2] == max_effect_index[4]:\n", 775 | " cnt_bai[i, 2] = 1\n", 776 | " if max_effect_index[3] == max_effect_index[4]:\n", 777 | " cnt_bai[i, 3] = 1\n", 778 | " \n", 779 | "print('BAI of LA ','|', np.mean(cnt_bai[:, 0]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 0]), scale=st.sem(cnt_bai[:, 0])))\n", 780 | "print('BAI of LR ','|', np.mean(cnt_bai[:, 1]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 1]), scale=st.sem(cnt_bai[:, 1])))\n", 781 | "print('BAI of SDL ','|', np.mean(cnt_bai[:, 2]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 2]), scale=st.sem(cnt_bai[:, 2])))\n", 782 | "print('BAI of DeDL ','|', np.mean(cnt_bai[:, 3]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 3]), scale=st.sem(cnt_bai[:, 3])))\n", 783 | "print('------------------------------------')\n", 784 | "print('ABS of ATE ','|', np.mean(np.mean(np.abs(true_1-true_0), axis=1)),'|',st.t.interval(0.95, n_rep-1, \n", 785 | " loc=np.mean(np.mean(np.abs(true_1-true_0), axis=1)), \n", 786 | " scale=st.sem(np.mean(np.abs(true_1-true_0), axis=1))))" 787 | ] 788 | }, 789 | { 790 | "cell_type": "markdown", 791 | "id": "5df2ab8e", 792 | "metadata": {}, 793 | "source": [ 794 | "## PDL Performance under varying NN size and partially/all observed data" 795 | ] 796 | }, 797 | { 798 | "cell_type": "code", 799 | "execution_count": null, 800 | "id": "97cbcb6c", 801 | "metadata": { 802 | "ExecuteTime": { 803 | "end_time": "2023-02-11T04:29:58.501841Z", 804 | "start_time": "2023-02-11T04:29:58.494969Z" 805 | } 806 | }, 807 | "outputs": [], 808 | "source": [ 809 | "is_small_dnn = 1#by default for fair comparison\n", 810 | "is_all_obs = 0#by default for fair comparison\n", 811 | "\n", 812 | "m = 4 #number of experiments\n", 813 | "d_c = 10 #number of features\n", 814 | "\n", 815 | "all_combo = list(powerset(list(np.arange(1, m+1))))\n", 816 | "t_combo = []\n", 817 | "for i in all_combo:\n", 818 | " t = np.zeros(m+1)\n", 819 | " t[0] = 1\n", 820 | " t[i] = 1\n", 821 | " t_combo.append(t)\n", 822 | "t_combo = np.int16(t_combo)\n", 823 | "t_dist = (1/2**m)*np.ones(2**m)\n", 824 | "t_combo_obs = []\n", 825 | "for i in range(m+1):\n", 826 | " t = np.zeros(m+1)\n", 827 | " t[0] = 1\n", 828 | " t[i] = 1\n", 829 | " t_combo_obs.append(t)\n", 830 | "t = np.zeros(m+1)\n", 831 | "t[[0,1,2]] = 1\n", 832 | "t_combo_obs.append(t)\n", 833 | "t_combo_obs = np.int16(t_combo_obs)\n", 834 | "t_dist_obs = (1/(m+2))*np.ones(m+2)\n", 835 | "\n", 836 | "t_star_all = t_combo.copy()\n", 837 | "idx = (t_combo[:,None]!=t_combo_obs).any(-1).all(1)\n", 838 | "t_star_unobs = t_combo[idx]\n", 839 | "\n", 840 | "is_all_obs = 1\n", 841 | "is_small_dnn = 0\n", 842 | "n_est = int(m*500)\n", 843 | "\n", 844 | "n_train = int(m*500)\n", 845 | "reg_term = 0.0005\n", 846 | "#reg_loss = 0.05\n", 847 | "reg_loss = 0\n", 848 | "train_epochs = 1000\n", 849 | "test_thres = 1.0\n", 850 | "\n", 851 | "lr = 0.05\n", 852 | "\n", 853 | "feature_list = []\n", 854 | "for i in range(d_c):\n", 855 | " feature_list.append(str('x')+str(i+1))\n", 856 | "t_list = []\n", 857 | "for i in range(m+1):\n", 858 | " t_list.append(str('t')+str(i))\n", 859 | "\n", 860 | "\n", 861 | "\n", 862 | "true_0 = []#real ATE in base\n", 863 | "true_1 = []#real ATE in treatment\n", 864 | "\n", 865 | "estimator_0 = []#ATE by PDL in base\n", 866 | "estimator_1 = []#ATE by PDL in treatment\n", 867 | "\n", 868 | "estimator_LR_0 = []#ATE by LR in base\n", 869 | "estimator_LR_1 = []#ATE by LR in treatment\n", 870 | "estimator_PDL_0 = []#ATE by PDL in base\n", 871 | "estimator_PDL_1 = []#ATE by PDL in treatment\n", 872 | "\n", 873 | "estimator_additive_1 = []#ATE by additive in treatment\n", 874 | "p_ = []\n", 875 | "p_LA = []\n", 876 | "p_LR = []\n", 877 | "p_PDL = []\n", 878 | "\n", 879 | "test_err = []" 880 | ] 881 | }, 882 | { 883 | "cell_type": "code", 884 | "execution_count": null, 885 | "id": "af18e6c9", 886 | "metadata": { 887 | "ExecuteTime": { 888 | "end_time": "2023-02-11T04:30:04.300330Z", 889 | "start_time": "2023-02-11T04:30:04.294633Z" 890 | } 891 | }, 892 | "outputs": [], 893 | "source": [ 894 | "if is_small_dnn == 0:\n", 895 | " \n", 896 | " class FNN_PDL(nn.Module):\n", 897 | " \"\"\"FNN.\"\"\"\n", 898 | "\n", 899 | " def __init__(self, xavier_init = None):\n", 900 | " \"\"\"FNN Builder.\"\"\"\n", 901 | " super(FNN_PDL, self).__init__()\n", 902 | "\n", 903 | "\n", 904 | " self.layer1 = nn.Sequential(\n", 905 | " nn.Linear(d_c+m+1, 40, bias=False),\n", 906 | " nn.ReLU(inplace=True),\n", 907 | " nn.Linear(40, 40),\n", 908 | " nn.ReLU(inplace=True),\n", 909 | " nn.Linear(40, 1)\n", 910 | " )\n", 911 | "\n", 912 | " if (xavier_init is not None):\n", 913 | " for m_ in self.modules():\n", 914 | " if isinstance(m_, nn.Linear):\n", 915 | " nn.init.xavier_normal_(m_.weight)\n", 916 | "\n", 917 | "\n", 918 | "\n", 919 | " def forward(self, x):\n", 920 | " \"\"\"Perform forward.\"\"\"\n", 921 | " x = self.layer1(x) \n", 922 | " return torch.reshape(x, (-1,))\n", 923 | "\n", 924 | "else:\n", 925 | " \n", 926 | " class FNN_PDL(nn.Module):\n", 927 | " \"\"\"FNN.\"\"\"\n", 928 | "\n", 929 | " def __init__(self, xavier_init = None):\n", 930 | " \"\"\"FNN Builder.\"\"\"\n", 931 | " super(FNN_PDL, self).__init__()\n", 932 | "\n", 933 | "\n", 934 | " self.layer1 = nn.Sequential(\n", 935 | " nn.Linear(d_c+m+1, 10, bias=False),\n", 936 | " nn.ReLU(inplace=True),\n", 937 | " nn.Linear(10, 10),\n", 938 | " nn.ReLU(inplace=True),\n", 939 | " nn.Linear(10, 1)\n", 940 | " )\n", 941 | "\n", 942 | " if (xavier_init is not None):\n", 943 | " for m_ in self.modules():\n", 944 | " if isinstance(m_, nn.Linear):\n", 945 | " nn.init.xavier_normal_(m_.weight)\n", 946 | "\n", 947 | "\n", 948 | "\n", 949 | " def forward(self, x):\n", 950 | " \"\"\"Perform forward.\"\"\"\n", 951 | " x = self.layer1(x) \n", 952 | " return torch.reshape(x, (-1,))\n", 953 | "\n" 954 | ] 955 | }, 956 | { 957 | "cell_type": "code", 958 | "execution_count": null, 959 | "id": "9252fc88", 960 | "metadata": { 961 | "ExecuteTime": { 962 | "end_time": "2023-02-11T04:31:01.842736Z", 963 | "start_time": "2023-02-11T04:30:10.106485Z" 964 | } 965 | }, 966 | "outputs": [], 967 | "source": [ 968 | "rep_index = 0\n", 969 | "\n", 970 | "while rep_index < n_rep:\n", 971 | " \n", 972 | " print('# of replication:', rep_index)\n", 973 | " coef = np.random.uniform(-0.5, 0.5, [d_c, m+1])\n", 974 | " c_true = np.random.uniform(10, 20)\n", 975 | " d_true = 0\n", 976 | "\n", 977 | " def generate_y_true(coef, c, d, x, t, n):\n", 978 | " y = np.zeros(n)\n", 979 | " y_error = 0.05*np.random.uniform(-1, 1, n)\n", 980 | " for i in range(n):\n", 981 | " y[i] = c/(1+np.exp(-((x[i].dot(coef))**3).dot(t[i]))) + d + y_error[i]\n", 982 | " return y, y_error\n", 983 | " def generate_y_true_1(coef, x, t):\n", 984 | " return c_true/(1+np.exp(-((x.dot(coef))**3).dot(t)))\n", 985 | " def generate_beta_true_1(coef, x, t):\n", 986 | " return (x.dot(coef))**3\n", 987 | " \n", 988 | " #generate samples for estimation\n", 989 | " samples_x_est = generate_x(d_c, n_train)\n", 990 | " if is_all_obs == 0:\n", 991 | " samples_t_est = generate_t(t_combo_obs, t_dist_obs, n_train)\n", 992 | " else:\n", 993 | " samples_t_est = generate_t(t_combo, t_dist, n_train)\n", 994 | " samples_y_est, samples_y_error_est = generate_y_true(coef, c_true, d_true, samples_x_est, samples_t_est, n_train)\n", 995 | " full_data_est = np.append(samples_x_est, samples_t_est, axis=1)\n", 996 | " full_data_est = np.append(full_data_est, samples_y_est.reshape(n_train,1), axis=1)\n", 997 | " data_obs = pd.DataFrame(full_data_est, columns = feature_list + t_list + ['y'])\n", 998 | " \n", 999 | " model_LR = sm.OLS(samples_y_est.reshape(n_train,1), np.append(samples_x_est, samples_t_est, axis=1))\n", 1000 | " results_LR = model_LR.fit()\n", 1001 | " coef_LR = results_LR.params\n", 1002 | " #results.bse\n", 1003 | "\n", 1004 | " opt = parser.parse_args()\n", 1005 | " train_samples = int(0.7*data_obs.shape[0])\n", 1006 | " x_train_set = np.float32(data_obs)[0:train_samples, 0:-1]\n", 1007 | " y_train_set = np.float32(data_obs)[0:train_samples, -1]\n", 1008 | " x_test_set = np.float32(data_obs)[train_samples:, 0:-1]\n", 1009 | " y_test_set = np.float32(data_obs)[train_samples:, -1]\n", 1010 | "\n", 1011 | " trainset = torch.utils.data.TensorDataset(torch.Tensor(x_train_set), torch.Tensor(y_train_set)) # create your datset\n", 1012 | " trainloader = torch.utils.data.DataLoader(\n", 1013 | " trainset, batch_size=opt.batch_size_train, shuffle=True)\n", 1014 | "\n", 1015 | " testset = torch.utils.data.TensorDataset(torch.Tensor(x_test_set), torch.Tensor(y_test_set)) # create your datset\n", 1016 | " testloader = torch.utils.data.DataLoader(\n", 1017 | " testset, batch_size=opt.batch_size_test, shuffle=True)\n", 1018 | " \n", 1019 | " net = FNN_PDL(xavier_init=1)\n", 1020 | " test_accuracy = 100\n", 1021 | " print('---------warm-up train---------')\n", 1022 | " wp_cnt = 0\n", 1023 | " while(test_accuracy >= 1):\n", 1024 | " wp_cnt += 1\n", 1025 | " net = FNN_PDL()\n", 1026 | " criterion = nn.MSELoss()\n", 1027 | " test_accuracy = 100\n", 1028 | " optimizer = optim.Adam(net.parameters(), lr = 0.1, weight_decay = opt.wd)\n", 1029 | " for epoch in range(200):\n", 1030 | " for i, data_i in enumerate(trainloader, 0):\n", 1031 | " inputs, labels = data_i\n", 1032 | " inputs, labels = Variable(inputs), Variable(labels)\n", 1033 | " optimizer.zero_grad()\n", 1034 | " outputs = net(inputs)\n", 1035 | " l1_norm = sum(p.abs().sum() for p in net.parameters())\n", 1036 | " loss = criterion(outputs, labels) + reg_loss*l1_norm\n", 1037 | " loss.backward()\n", 1038 | " optimizer.step()\n", 1039 | " #train_accuracy = calculate_mse(trainloader, opt.is_gpu, net)\n", 1040 | " test_accuracy = calculate_mse(testloader, opt.is_gpu, net)\n", 1041 | " #if epoch%100 == 0:\n", 1042 | " # print(\"Iteration: {0} | Training MSE: {1} | Test MSE: {2}\".format(\n", 1043 | " # epoch, train_accuracy, test_accuracy))\n", 1044 | " if test_accuracy < 5:\n", 1045 | " break\n", 1046 | " if wp_cnt >= 5:\n", 1047 | " break\n", 1048 | " #training \n", 1049 | " print('---------train---------')\n", 1050 | " optimizer = optim.Adam(net.parameters(), lr = lr, weight_decay = opt.wd)\n", 1051 | " criterion = nn.MSELoss()\n", 1052 | " for epoch in range(train_epochs):\n", 1053 | " for i, data_i in enumerate(trainloader, 0):\n", 1054 | " inputs, labels = data_i\n", 1055 | " inputs, labels = Variable(inputs), Variable(labels)\n", 1056 | " optimizer.zero_grad()\n", 1057 | " outputs = net(inputs)\n", 1058 | " l1_norm = sum(p.abs().sum() for p in net.parameters())\n", 1059 | " loss = criterion(outputs, labels) + reg_loss*l1_norm\n", 1060 | " loss.backward()\n", 1061 | " optimizer.step()\n", 1062 | " train_accuracy = calculate_mse(trainloader, opt.is_gpu, net)\n", 1063 | " test_accuracy = calculate_mse(testloader, opt.is_gpu, net)\n", 1064 | " if epoch%100 == 0:\n", 1065 | " print(\"Iteration: {0} | Training MSE: {1} | Test MSE: {2}\".format(\n", 1066 | " epoch, train_accuracy, test_accuracy))\n", 1067 | " if test_accuracy < test_thres and epoch > 200:\n", 1068 | " break \n", 1069 | " if test_accuracy > test_thres:\n", 1070 | " continue\n", 1071 | " else:\n", 1072 | " rep_index += 1\n", 1073 | " test_err.append(test_accuracy)\n", 1074 | " \n", 1075 | " \n", 1076 | " \n", 1077 | " #generate samples for inference\n", 1078 | " samples_x_est = generate_x(d_c, n_est)\n", 1079 | " samples_t_est = generate_t(t_combo_obs, t_dist_obs, n_est)\n", 1080 | " samples_y_est, samples_y_error_est = generate_y_true(coef, c_true, d_true, samples_x_est, samples_t_est, n_est)\n", 1081 | " full_data_est = np.append(samples_x_est, samples_t_est, axis=1)\n", 1082 | " full_data_est = np.append(full_data_est, samples_y_est.reshape(n_est,1), axis=1)\n", 1083 | "\n", 1084 | " for t_star in t_star_all:\n", 1085 | " \n", 1086 | " \n", 1087 | " data_est = pd.DataFrame(full_data_est, columns = feature_list + t_list + ['y'])\n", 1088 | " #at t=0\n", 1089 | " t_star_base = np.zeros(m+1)\n", 1090 | " t_star_base[0] = 1\n", 1091 | " x_all_set=np.float32(data_est)[:, 0:-1]\n", 1092 | " y_all_set=np.float32(data_est)[:, -1]\n", 1093 | " \n", 1094 | " for j in range(m):\n", 1095 | " x_all_set[:, -m+j] = t_star_base[j+1]\n", 1096 | " allset = torch.utils.data.TensorDataset(torch.Tensor(x_all_set), torch.Tensor(y_all_set)) # create your datset\n", 1097 | " allloader = torch.utils.data.DataLoader(\n", 1098 | " allset, batch_size=1000, shuffle=False)\n", 1099 | " pred_y = []\n", 1100 | " for i, data_ in enumerate(allloader, 0):\n", 1101 | " inputs, labels = data_\n", 1102 | " inputs, labels = Variable(inputs), Variable(labels)\n", 1103 | " outputs = net(inputs)\n", 1104 | " pred_y.append(outputs.tolist())\n", 1105 | " pred_y = np.array(pred_y).reshape(n_est)\n", 1106 | " \n", 1107 | " pred_y_LR = []\n", 1108 | " for i in range(n_est): \n", 1109 | " pred_y_LR.append(np.append(samples_x_est[i], t_star_base).dot(coef_LR))\n", 1110 | " pred_y_LR = np.array(pred_y_LR)\n", 1111 | " \n", 1112 | " real_y_star = []\n", 1113 | " for i in range(n_est):\n", 1114 | " real_y_star.append(0.05*np.random.uniform(-1, 1)+generate_y_true_1(coef,samples_x_est[i],t_star_base))\n", 1115 | " \n", 1116 | " \n", 1117 | " true_0.append(np.mean(real_y_star))\n", 1118 | " estimator_LR_0.append(np.mean(pred_y_LR))\n", 1119 | " estimator_PDL_0.append(np.mean(pred_y))\n", 1120 | "\n", 1121 | " \n", 1122 | " #record\n", 1123 | " real_y_star_0 = real_y_star.copy()\n", 1124 | " pred_y_LR_0 = pred_y_LR.copy()\n", 1125 | " pred_y_0 = pred_y.copy()\n", 1126 | " \n", 1127 | " #at t=t_star\n", 1128 | " t_star_base = t_star.copy()\n", 1129 | " \n", 1130 | " for j in range(m):\n", 1131 | " x_all_set[:, -m+j] = t_star_base[j+1]\n", 1132 | " allset = torch.utils.data.TensorDataset(torch.Tensor(x_all_set), torch.Tensor(y_all_set)) # create your datset\n", 1133 | " allloader = torch.utils.data.DataLoader(\n", 1134 | " allset, batch_size=1000, shuffle=False)\n", 1135 | " pred_y = []\n", 1136 | " for i, data_ in enumerate(allloader, 0):\n", 1137 | " inputs, labels = data_\n", 1138 | " inputs, labels = Variable(inputs), Variable(labels)\n", 1139 | " outputs = net(inputs)\n", 1140 | " pred_y.append(outputs.tolist())\n", 1141 | " pred_y = np.array(pred_y).reshape(n_est)\n", 1142 | " \n", 1143 | " pred_y_LR = []\n", 1144 | " for i in range(n_est): \n", 1145 | " pred_y_LR.append(np.append(samples_x_est[i], t_star_base).dot(coef_LR))\n", 1146 | " pred_y_LR = np.array(pred_y_LR)\n", 1147 | " \n", 1148 | " real_y_star = []\n", 1149 | " for i in range(n_est):\n", 1150 | " real_y_star.append(0.05*np.random.uniform(-1, 1)+generate_y_true_1(coef,samples_x_est[i],t_star_base))\n", 1151 | " \n", 1152 | " \n", 1153 | " true_1.append(np.mean(real_y_star))\n", 1154 | " estimator_LR_1.append(np.mean(pred_y_LR))\n", 1155 | " estimator_PDL_1.append(np.mean(pred_y))\n", 1156 | " p_.append(stats.ttest_ind(real_y_star, real_y_star_0)[1])\n", 1157 | " p_LR.append(stats.ttest_ind(pred_y_LR, pred_y_LR_0)[1]) \n", 1158 | " p_PDL.append(stats.ttest_ind(pred_y, pred_y_0)[1]) \n", 1159 | " \n", 1160 | " #calculate additive\n", 1161 | " pred_y_additive = 0\n", 1162 | " pred_y_additive_var = 0\n", 1163 | " n_LA_samples = 0\n", 1164 | " for single_exp_index in np.where(t_star == 1)[0]:\n", 1165 | " ttt = np.zeros(m+1)\n", 1166 | " ttt[0] = 1\n", 1167 | " ttt[single_exp_index] = 1\n", 1168 | " pred_y_additive += samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].mean()\n", 1169 | " pred_y_additive_var += samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].var()\n", 1170 | " n_LA_samples += (samples_t_est == ttt).sum()\n", 1171 | " ttt = np.zeros(m+1)\n", 1172 | " ttt[0] = 1\n", 1173 | " pred_y_additive -= (np.sum(t_star))*samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].mean()\n", 1174 | " estimator_additive_1.append(pred_y_additive)\n", 1175 | " t_value = pred_y_additive/np.sqrt(pred_y_additive_var*(np.sum(t_star))/n_LA_samples)\n", 1176 | " p_LA.append(stats.t.sf(abs(t_value), df=1))\n", 1177 | " " 1178 | ] 1179 | }, 1180 | { 1181 | "cell_type": "code", 1182 | "execution_count": null, 1183 | "id": "b615ff48", 1184 | "metadata": { 1185 | "ExecuteTime": { 1186 | "end_time": "2023-02-11T04:31:07.759253Z", 1187 | "start_time": "2023-02-11T04:31:07.690743Z" 1188 | } 1189 | }, 1190 | "outputs": [], 1191 | "source": [ 1192 | "test_err_np = np.zeros(n_rep)\n", 1193 | "for i in range(n_rep):\n", 1194 | " test_err_np[i] = test_err[i].tolist()\n", 1195 | "test_err = test_err_np.copy()\n", 1196 | "del test_err_np\n", 1197 | "\n", 1198 | "p_ = np.array(p_).reshape((n_rep, 2**m))\n", 1199 | "p_LA = np.array(p_LA).reshape((n_rep, 2**m))\n", 1200 | "p_LR = np.array(p_LR).reshape((n_rep, 2**m))\n", 1201 | "p_PDL = np.array(p_PDL).reshape((n_rep, 2**m))\n", 1202 | "true_0 = np.array(true_0).reshape((n_rep, 2**m))\n", 1203 | "estimator_LR_0 = np.array(estimator_LR_0).reshape((n_rep, 2**m))\n", 1204 | "estimator_PDL_0 = np.array(estimator_PDL_0).reshape((n_rep, 2**m))\n", 1205 | "true_1 = np.array(true_1).reshape((n_rep, 2**m))\n", 1206 | "estimator_LR_1 = np.array(estimator_LR_1).reshape((n_rep, 2**m))\n", 1207 | "estimator_PDL_1 = np.array(estimator_PDL_1).reshape((n_rep, 2**m))\n", 1208 | "estimator_additive_1 = np.array(estimator_additive_1).reshape((n_rep, 2**m))\n", 1209 | "index = 0\n", 1210 | "\n", 1211 | "#LA LR PDL\n", 1212 | "ape_2 = np.zeros([2**m, 3, n_rep])\n", 1213 | "se = np.zeros([2**m, 3, n_rep])\n", 1214 | "ae = np.zeros([2**m, 3, n_rep])\n", 1215 | "CDR = np.zeros([2**m, 3, n_rep])\n", 1216 | "for i in range(n_rep):\n", 1217 | " for j in range(2**m):\n", 1218 | " true_eff = true_1[i,j]-true_0[i,j]\n", 1219 | " la = estimator_additive_1[i,j]\n", 1220 | " lr = estimator_LR_1[i,j]-estimator_LR_0[i,j]\n", 1221 | " pdl = estimator_PDL_1[i,j]-estimator_PDL_0[i,j]\n", 1222 | " if (p_[i,j]<=0.05 and p_LA[i,j]<=0.05 and la*true_eff>0) or (p_[i,j]>0.05 and p_LA[i,j]>0.05):\n", 1223 | " CDR[j,0,i] = 1\n", 1224 | " if (p_[i,j]<=0.05 and p_LR[i,j]<=0.05 and lr*true_eff>0) or (p_[i,j]>0.05 and p_LR[i,j]>0.05):\n", 1225 | " CDR[j,1,i] = 1\n", 1226 | " if (p_[i,j]<=0.05 and p_PDL[i,j]<=0.05 and pdl*true_eff>0) or (p_[i,j]>0.05 and p_PDL[i,j]>0.05):\n", 1227 | " CDR[j,2,i] = 1\n", 1228 | "print('----------------All combos--------------------')\n", 1229 | "for t_star in t_star_all:\n", 1230 | " #print('Treatment effect when t = ', t_star[1:])\n", 1231 | " true_effect = true_1[:,index]-true_0[:,index]\n", 1232 | " \n", 1233 | " add_effect = estimator_additive_1[:,index]\n", 1234 | " LR_effect = estimator_LR_1[:,index]-estimator_LR_0[:,index]\n", 1235 | " PDL_effect = estimator_PDL_1[:,index]-estimator_PDL_0[:,index]\n", 1236 | " \n", 1237 | " ape_2[index, 0, :] = np.abs((add_effect-true_effect))/(np.abs(true_effect))\n", 1238 | " ape_2[index, 1, :] = np.abs((LR_effect-true_effect))/(np.abs(true_effect))\n", 1239 | " ape_2[index, 2, :] = np.abs((PDL_effect-true_effect))/(np.abs(true_effect))\n", 1240 | " \n", 1241 | " #if p_[:,index]>=0.05:#not significant treatment effect\n", 1242 | " ape_2[index, 0, p_[:,index]>=0.05] = 0\n", 1243 | " ape_2[index, 1, p_[:,index]>=0.05] = 0\n", 1244 | " ape_2[index, 2, p_[:,index]>=0.05] = 0\n", 1245 | " \n", 1246 | " se[index, 0, :] = ((add_effect - true_effect)**2)\n", 1247 | " se[index, 1, :] = ((LR_effect - true_effect)**2)\n", 1248 | " se[index, 2, :] = ((PDL_effect - true_effect)**2)\n", 1249 | " \n", 1250 | " ae[index, 0, :] = np.abs((add_effect - true_effect))\n", 1251 | " ae[index, 1, :] = np.abs((LR_effect - true_effect))\n", 1252 | " ae[index, 2, :] = np.abs((PDL_effect - true_effect))\n", 1253 | " \n", 1254 | " \n", 1255 | " \n", 1256 | " index += 1\n", 1257 | " \n", 1258 | " \n", 1259 | "print('------------------------------------')\n", 1260 | "print('MSE of LA ','|', np.mean(se[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,0,:], axis=0)), scale=st.sem(np.mean(se[:,0,:], axis=0))))\n", 1261 | "print('MSE of LR ','|', np.mean(se[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,1,:], axis=0)), scale=st.sem(np.mean(se[:,1,:], axis=0))))\n", 1262 | "print('MSE of PDL ','|', np.mean(se[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,2,:], axis=0)), scale=st.sem(np.mean(se[:,2,:], axis=0))))\n", 1263 | "print('------------------------------------')\n", 1264 | "print('MAE of LA ','|', np.mean(ae[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,0,:], axis=0)), scale=st.sem(np.mean(ae[:,0,:], axis=0))))\n", 1265 | "print('MAE of LR ','|', np.mean(ae[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,1,:], axis=0)), scale=st.sem(np.mean(ae[:,1,:], axis=0))))\n", 1266 | "print('MAE of PDL ','|', np.mean(ae[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,2,:], axis=0)), scale=st.sem(np.mean(ae[:,2,:], axis=0))))\n", 1267 | "print('------------------------------------')\n", 1268 | "print('MAPE2 of LA ','|', np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)), \n", 1269 | " scale=st.sem(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0))))\n", 1270 | "print('MAPE2 of LR ','|', np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)), \n", 1271 | " scale=st.sem(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0))))\n", 1272 | "print('MAPE2 of PDL ','|', np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)), \n", 1273 | " scale=st.sem(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0))))\n", 1274 | "print('------------------------------------')\n", 1275 | "print('CDR of LA ','|', np.mean(CDR[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,0,:], axis=0)), scale=st.sem(np.mean(CDR[:,0,:], axis=0))))\n", 1276 | "print('CDR of LR ','|', np.mean(CDR[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,1,:], axis=0)), scale=st.sem(np.mean(CDR[:,1,:], axis=0))))\n", 1277 | "print('CDR of PDL ','|', np.mean(CDR[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,2,:], axis=0)), scale=st.sem(np.mean(CDR[:,2,:], axis=0))))\n", 1278 | "print('------------------------------------')\n", 1279 | "\n", 1280 | "cnt_bai = np.zeros([n_rep, 3])\n", 1281 | "for i in range(n_rep):\n", 1282 | " max_effect = -10000*np.ones(4)\n", 1283 | " max_effect_index = np.zeros(4)\n", 1284 | " for index in range(2**m):\n", 1285 | " true_effect = true_1[i,index]-true_0[i,index]\n", 1286 | " add_effect = estimator_additive_1[i,index]\n", 1287 | " LR_effect = estimator_LR_1[i,index]-estimator_LR_0[i,index]\n", 1288 | " PDL_effect = estimator_PDL_1[i,index]-estimator_PDL_0[i,index]\n", 1289 | " \n", 1290 | " if true_effect > max_effect[3]:\n", 1291 | " max_effect[3] = true_effect\n", 1292 | " max_effect_index[3] = index\n", 1293 | " if add_effect > max_effect[0]:\n", 1294 | " max_effect[0] = add_effect\n", 1295 | " max_effect_index[0] = index\n", 1296 | " if LR_effect > max_effect[1]:\n", 1297 | " max_effect[1] = LR_effect\n", 1298 | " max_effect_index[1] = index\n", 1299 | " if PDL_effect > max_effect[2]:\n", 1300 | " max_effect[2] = PDL_effect\n", 1301 | " max_effect_index[2] = index\n", 1302 | " if max_effect_index[0] == max_effect_index[3]:\n", 1303 | " cnt_bai[i, 0] = 1\n", 1304 | " if max_effect_index[1] == max_effect_index[3]:\n", 1305 | " cnt_bai[i, 1] = 1\n", 1306 | " if max_effect_index[2] == max_effect_index[3]:\n", 1307 | " cnt_bai[i, 2] = 1\n", 1308 | "\n", 1309 | "\n", 1310 | "print('BAI of LA ','|', np.mean(cnt_bai[:, 0]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 0]), scale=st.sem(cnt_bai[:, 0])))\n", 1311 | "print('BAI of LR ','|', np.mean(cnt_bai[:, 1]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 1]), scale=st.sem(cnt_bai[:, 1])))\n", 1312 | "print('BAI of PDL ','|', np.mean(cnt_bai[:, 2]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 2]), scale=st.sem(cnt_bai[:, 2])))\n", 1313 | "print('------------------------------------')\n", 1314 | "print('ABS of ATE ','|', np.mean(np.mean(np.abs(true_1-true_0), axis=1)),'|',st.t.interval(0.95, n_rep-1, \n", 1315 | " loc=np.mean(np.mean(np.abs(true_1-true_0), axis=1)), \n", 1316 | " scale=st.sem(np.mean(np.abs(true_1-true_0), axis=1))))\n", 1317 | "\n", 1318 | "\n", 1319 | "print('----------------observed combos--------------------')\n", 1320 | "\n", 1321 | "index = 0\n", 1322 | "index_1 = 0\n", 1323 | "ape_2 = np.zeros([2**m-np.shape(t_star_unobs)[0], 3, n_rep])\n", 1324 | "se = np.zeros([2**m-np.shape(t_star_unobs)[0], 3, n_rep])\n", 1325 | "ae = np.zeros([2**m-np.shape(t_star_unobs)[0], 3, n_rep])\n", 1326 | "\n", 1327 | "\n", 1328 | "for t_star in t_star_all:\n", 1329 | " if list(t_star) not in t_star_unobs.tolist():\n", 1330 | " #print('Unobserved treatment effect when t = ', t_star[1:])\n", 1331 | " \n", 1332 | " true_effect = true_1[:,index]-true_0[:,index]\n", 1333 | " \n", 1334 | " add_effect = estimator_additive_1[:,index]\n", 1335 | " LR_effect = estimator_LR_1[:,index]-estimator_LR_0[:,index]\n", 1336 | " PDL_effect = estimator_PDL_1[:,index]-estimator_PDL_0[:,index]\n", 1337 | "\n", 1338 | " ape_2[index_1, 0, :] = np.abs((add_effect-true_effect))/(np.abs(true_effect))\n", 1339 | " ape_2[index_1, 1, :] = np.abs((LR_effect-true_effect))/(np.abs(true_effect))\n", 1340 | " ape_2[index_1, 2, :] = np.abs((PDL_effect-true_effect))/(np.abs(true_effect))\n", 1341 | "\n", 1342 | " #if p_[:,index]>=0.05:#not significant treatment effect\n", 1343 | " ape_2[index_1, 0, p_[:,index]>=0.05] = 0\n", 1344 | " ape_2[index_1, 1, p_[:,index]>=0.05] = 0\n", 1345 | " ape_2[index_1, 2, p_[:,index]>=0.05] = 0\n", 1346 | "\n", 1347 | " se[index_1, 0, :] = ((add_effect - true_effect)**2)\n", 1348 | " se[index_1, 1, :] = ((LR_effect - true_effect)**2)\n", 1349 | " se[index_1, 2, :] = ((PDL_effect - true_effect)**2)\n", 1350 | "\n", 1351 | " ae[index_1, 0, :] = np.abs((add_effect - true_effect))\n", 1352 | " ae[index_1, 1, :] = np.abs((LR_effect - true_effect))\n", 1353 | " ae[index_1, 2, :] = np.abs((PDL_effect - true_effect))\n", 1354 | " \n", 1355 | " \n", 1356 | " index_1 += 1\n", 1357 | "\n", 1358 | " index += 1\n", 1359 | "print('------------------------------------')\n", 1360 | "print('MSE of LA ','|', np.mean(se[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,0,:], axis=0)), scale=st.sem(np.mean(se[:,0,:], axis=0))))\n", 1361 | "print('MSE of LR ','|', np.mean(se[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,1,:], axis=0)), scale=st.sem(np.mean(se[:,1,:], axis=0))))\n", 1362 | "print('MSE of PDL ','|', np.mean(se[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,2,:], axis=0)), scale=st.sem(np.mean(se[:,2,:], axis=0))))\n", 1363 | "print('------------------------------------')\n", 1364 | "print('MAE of LA ','|', np.mean(ae[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,0,:], axis=0)), scale=st.sem(np.mean(ae[:,0,:], axis=0))))\n", 1365 | "print('MAE of LR ','|', np.mean(ae[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,1,:], axis=0)), scale=st.sem(np.mean(ae[:,1,:], axis=0))))\n", 1366 | "print('MAE of PDL ','|', np.mean(ae[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,2,:], axis=0)), scale=st.sem(np.mean(ae[:,2,:], axis=0))))\n", 1367 | "print('------------------------------------')\n", 1368 | "print('MAPE2 of LA ','|', np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)), \n", 1369 | " scale=st.sem(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0))))\n", 1370 | "print('MAPE2 of LR ','|', np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)), \n", 1371 | " scale=st.sem(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0))))\n", 1372 | "print('MAPE2 of PDL ','|', np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)), \n", 1373 | " scale=st.sem(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0))))\n", 1374 | "\n", 1375 | "\n", 1376 | "\n", 1377 | "\n", 1378 | "\n", 1379 | "\n", 1380 | "print('----------------unobserved combos--------------------')\n", 1381 | "\n", 1382 | "index = 0\n", 1383 | "index_1 = 0\n", 1384 | "ape_2 = np.zeros([np.shape(t_star_unobs)[0], 3, n_rep])\n", 1385 | "se = np.zeros([np.shape(t_star_unobs)[0], 3, n_rep])\n", 1386 | "ae = np.zeros([np.shape(t_star_unobs)[0], 3, n_rep])\n", 1387 | "\n", 1388 | "\n", 1389 | "for t_star in t_star_all:\n", 1390 | " if list(t_star) in t_star_unobs.tolist():\n", 1391 | " #print('Unobserved treatment effect when t = ', t_star[1:])\n", 1392 | " \n", 1393 | " true_effect = true_1[:,index]-true_0[:,index]\n", 1394 | " \n", 1395 | " add_effect = estimator_additive_1[:,index]\n", 1396 | " LR_effect = estimator_LR_1[:,index]-estimator_LR_0[:,index]\n", 1397 | " PDL_effect = estimator_PDL_1[:,index]-estimator_PDL_0[:,index]\n", 1398 | "\n", 1399 | " ape_2[index_1, 0, :] = np.abs((add_effect-true_effect))/(np.abs(true_effect))\n", 1400 | " ape_2[index_1, 1, :] = np.abs((LR_effect-true_effect))/(np.abs(true_effect))\n", 1401 | " ape_2[index_1, 2, :] = np.abs((PDL_effect-true_effect))/(np.abs(true_effect))\n", 1402 | "\n", 1403 | " #if p_[:,index]>=0.05:#not significant treatment effect\n", 1404 | " ape_2[index_1, 0, p_[:,index]>=0.05] = 0\n", 1405 | " ape_2[index_1, 1, p_[:,index]>=0.05] = 0\n", 1406 | " ape_2[index_1, 2, p_[:,index]>=0.05] = 0\n", 1407 | "\n", 1408 | " se[index_1, 0, :] = ((add_effect - true_effect)**2)\n", 1409 | " se[index_1, 1, :] = ((LR_effect - true_effect)**2)\n", 1410 | " se[index_1, 2, :] = ((PDL_effect - true_effect)**2)\n", 1411 | "\n", 1412 | " ae[index_1, 0, :] = np.abs((add_effect - true_effect))\n", 1413 | " ae[index_1, 1, :] = np.abs((LR_effect - true_effect))\n", 1414 | " ae[index_1, 2, :] = np.abs((PDL_effect - true_effect))\n", 1415 | " \n", 1416 | " \n", 1417 | " index_1 += 1\n", 1418 | "\n", 1419 | " index += 1\n", 1420 | "print('------------------------------------')\n", 1421 | "print('MSE of LA ','|', np.mean(se[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,0,:], axis=0)), scale=st.sem(np.mean(se[:,0,:], axis=0))))\n", 1422 | "print('MSE of LR ','|', np.mean(se[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,1,:], axis=0)), scale=st.sem(np.mean(se[:,1,:], axis=0))))\n", 1423 | "print('MSE of PDL ','|', np.mean(se[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,2,:], axis=0)), scale=st.sem(np.mean(se[:,2,:], axis=0))))\n", 1424 | "print('------------------------------------')\n", 1425 | "print('MAE of LA ','|', np.mean(ae[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,0,:], axis=0)), scale=st.sem(np.mean(ae[:,0,:], axis=0))))\n", 1426 | "print('MAE of LR ','|', np.mean(ae[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,1,:], axis=0)), scale=st.sem(np.mean(ae[:,1,:], axis=0))))\n", 1427 | "print('MAE of PDL ','|', np.mean(ae[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,2,:], axis=0)), scale=st.sem(np.mean(ae[:,2,:], axis=0))))\n", 1428 | "print('------------------------------------')\n", 1429 | "print('MAPE2 of LA ','|', np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)), \n", 1430 | " scale=st.sem(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0))))\n", 1431 | "print('MAPE2 of LR ','|', np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)), \n", 1432 | " scale=st.sem(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0))))\n", 1433 | "print('MAPE2 of PDL ','|', np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)), \n", 1434 | " scale=st.sem(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0))))\n", 1435 | "\n", 1436 | "\n" 1437 | ] 1438 | }, 1439 | { 1440 | "cell_type": "markdown", 1441 | "id": "d3f4247e", 1442 | "metadata": {}, 1443 | "source": [ 1444 | "# Robustness to DNN Convergence" 1445 | ] 1446 | }, 1447 | { 1448 | "cell_type": "markdown", 1449 | "id": "f624fe40", 1450 | "metadata": {}, 1451 | "source": [ 1452 | "## Data generating process definition\n", 1453 | "define delta = 0.1, 0.2, 0.3" 1454 | ] 1455 | }, 1456 | { 1457 | "cell_type": "code", 1458 | "execution_count": null, 1459 | "id": "e1ea9c58", 1460 | "metadata": { 1461 | "ExecuteTime": { 1462 | "end_time": "2023-02-11T04:31:13.658089Z", 1463 | "start_time": "2023-02-11T04:31:13.651547Z" 1464 | } 1465 | }, 1466 | "outputs": [], 1467 | "source": [ 1468 | "err = 0.1#delta coefficient in the paper\n", 1469 | "m = 4 #number of experiments\n", 1470 | "d_c = 10 #number of features\n", 1471 | "\n", 1472 | "#No need to change\n", 1473 | "all_combo = list(powerset(list(np.arange(1, m+1))))\n", 1474 | "t_combo = []\n", 1475 | "for i in all_combo:\n", 1476 | " t = np.zeros(m+1)\n", 1477 | " t[0] = 1\n", 1478 | " t[i] = 1\n", 1479 | " t_combo.append(t)\n", 1480 | "t_combo = np.int16(t_combo)\n", 1481 | "t_dist = (1/2**m)*np.ones(2**m)\n", 1482 | "t_combo_obs = []\n", 1483 | "for i in range(m+1):\n", 1484 | " t = np.zeros(m+1)\n", 1485 | " t[0] = 1\n", 1486 | " t[i] = 1\n", 1487 | " t_combo_obs.append(t)\n", 1488 | "t = np.zeros(m+1)\n", 1489 | "t[[0,1,2]] = 1\n", 1490 | "t_combo_obs.append(t)\n", 1491 | "t_combo_obs = np.int16(t_combo_obs)\n", 1492 | "t_dist_obs = (1/(m+2))*np.ones(m+2)\n", 1493 | "\n", 1494 | "t_star_all = t_combo.copy()\n", 1495 | "idx = (t_combo[:,None]!=t_combo_obs).any(-1).all(1)\n", 1496 | "t_star_unobs = t_combo[idx]\n", 1497 | "\n", 1498 | "n_est = int(m*500)\n", 1499 | "n_train = int(m*500)\n", 1500 | "reg_term = 0.0005\n", 1501 | "\n", 1502 | "feature_list = []\n", 1503 | "for i in range(d_c):\n", 1504 | " feature_list.append(str('x')+str(i+1))\n", 1505 | "t_list = []\n", 1506 | "for i in range(m+1):\n", 1507 | " t_list.append(str('t')+str(i))\n", 1508 | "\n", 1509 | "true_0 = []#real ATE in base\n", 1510 | "estimator_0 = []#ATE by ML in base\n", 1511 | "estimator_debias_0 = []#ATE by DML in base\n", 1512 | "true_1 = []#real ATE in treatment\n", 1513 | "estimator_1 = []#ATE by ML in treatment\n", 1514 | "estimator_debias_1 = []#ATE by DML in treatment\n", 1515 | "estimator_additive_1 = []#ATE by additive in treatment\n", 1516 | "estimator_LR_0 = []#ATE by LR in base\n", 1517 | "estimator_LR_1 = []#ATE by LR in treatment\n", 1518 | "p_ = []\n", 1519 | "p_LA = []\n", 1520 | "p_LR = []\n", 1521 | "p_DL = []\n", 1522 | "p_DDL = []" 1523 | ] 1524 | }, 1525 | { 1526 | "cell_type": "markdown", 1527 | "id": "727964b0", 1528 | "metadata": {}, 1529 | "source": [ 1530 | "## Comparison of LA, LR, SDL, DeDL" 1531 | ] 1532 | }, 1533 | { 1534 | "cell_type": "code", 1535 | "execution_count": null, 1536 | "id": "957ad252", 1537 | "metadata": { 1538 | "ExecuteTime": { 1539 | "end_time": "2023-02-11T04:31:50.794763Z", 1540 | "start_time": "2023-02-11T04:31:19.425827Z" 1541 | } 1542 | }, 1543 | "outputs": [], 1544 | "source": [ 1545 | "for rep_index in range(n_rep):\n", 1546 | " print('# of replication:', rep_index)\n", 1547 | " #generate coef and biased coef\n", 1548 | " \n", 1549 | " coef = np.random.uniform(-0.5, 0.5, [d_c, m+1])\n", 1550 | " c_true = np.random.uniform(10, 20)\n", 1551 | " d_true = 0\n", 1552 | " \n", 1553 | " coef_ = coef.copy()\n", 1554 | " c_est = c_true*(1+np.random.uniform(-err, err))\n", 1555 | " d_est = 0\n", 1556 | " \n", 1557 | "\n", 1558 | " def generate_y_true(coef, c, d, x, t, n):\n", 1559 | " y = np.zeros(n)\n", 1560 | " y_error = 0.05*np.random.uniform(-1, 1, n)\n", 1561 | " for i in range(n):\n", 1562 | " y[i] = (c/(1+np.exp(-((x[i].dot(coef))).dot(t[i]))) + d + y_error[i])\n", 1563 | " return y, y_error\n", 1564 | " def generate_y_true_1(coef, x, t):\n", 1565 | " return c_true/(1+np.exp(-((x.dot(coef))).dot(t)))\n", 1566 | " \n", 1567 | " \n", 1568 | " #generate samples for training LR\n", 1569 | " samples_x_est = generate_x(d_c, n_est)\n", 1570 | " samples_t_est = generate_t(t_combo_obs, t_dist_obs, n_est)\n", 1571 | " samples_y_est, samples_y_error_est = generate_y(coef, c_true, d_true, samples_x_est, samples_t_est, n_est)\n", 1572 | " full_data_est_xt = np.append(samples_x_est, samples_t_est, axis=1)\n", 1573 | " full_data_est = np.append(full_data_est_xt, samples_y_est.reshape(n_est,1), axis=1)\n", 1574 | " model_LR = sm.OLS(samples_y_est.reshape(n_est,1), full_data_est_xt)\n", 1575 | " results_LR = model_LR.fit()\n", 1576 | " coef_LR = results_LR.params\n", 1577 | " \n", 1578 | " #generate samples for inference\n", 1579 | " samples_x_est = generate_x(d_c, n_est)\n", 1580 | " samples_t_est = generate_t(t_combo_obs, t_dist_obs, n_est)\n", 1581 | " samples_y_est, samples_y_error_est = generate_y(coef, c_true, d_true, samples_x_est, samples_t_est, n_est)\n", 1582 | " full_data_est = np.append(samples_x_est, samples_t_est, axis=1)\n", 1583 | " full_data_est = np.append(full_data_est, samples_y_est.reshape(n_est,1), axis=1)\n", 1584 | " \n", 1585 | "\n", 1586 | " for t_star in t_star_all:\n", 1587 | " \n", 1588 | "\n", 1589 | " t_star_base = np.zeros(m+1)\n", 1590 | " t_star_base[0] = 1\n", 1591 | " \n", 1592 | " pred_y_loss = []\n", 1593 | " beta_ = []\n", 1594 | " for i in range(n_est):\n", 1595 | " beta_biased = samples_x_est[i].dot(coef_)*(1+np.random.uniform(-err, err, m+1))#add bias\n", 1596 | " beta_.append(beta_biased) \n", 1597 | " pred_y_loss.append(c_est/(1+np.exp(-(beta_biased).dot(samples_t_est[i]))))\n", 1598 | " beta_ = np.array(beta_)\n", 1599 | " pred_y_loss = np.array(pred_y_loss)\n", 1600 | " real_y = samples_y_est.copy()\n", 1601 | " pred_y = []\n", 1602 | " for i in range(n_est): \n", 1603 | " pred_y.append(c_est/(1+np.exp(-(beta_[i]).dot(t_star_base))))\n", 1604 | " pred_y = np.array(pred_y)\n", 1605 | " \n", 1606 | " real_y_star = []\n", 1607 | " for i in range(n_est):\n", 1608 | " real_y_star.append(0.05*np.random.uniform(-1, 1)+generate_y_true_1(coef,samples_x_est[i],t_star_base))\n", 1609 | "\n", 1610 | " \n", 1611 | " \n", 1612 | " lambda_inv = []\n", 1613 | " G_theta = []\n", 1614 | " G_theta_loss = []\n", 1615 | " t_obs_ = samples_t_est.copy()\n", 1616 | " cnt = 0\n", 1617 | "\n", 1618 | " for beta_temp in beta_:\n", 1619 | "\n", 1620 | " lambda_ = np.zeros([m+2, m+2])#asig use m+2\n", 1621 | " u_ = beta_temp.dot(t_star_base)\n", 1622 | " #G_theta.append(c_true*np.exp(-u_)/(np.exp(-u_)+1)**2*t_star_base)\n", 1623 | " G_theta.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*t_star_base, [1/(np.exp(-u_)+1)]))\n", 1624 | "\n", 1625 | " u_ = beta_temp.dot(samples_t_est[cnt])\n", 1626 | " #G_theta_loss.append(c_true*np.exp(-u_)/(np.exp(-u_)+1)**2*samples_t_est[cnt])\n", 1627 | " G_theta_loss.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*samples_t_est[cnt], [1/(np.exp(-u_)+1)]))\n", 1628 | "\n", 1629 | " cnt += 1\n", 1630 | "\n", 1631 | " for i in range(t_combo_obs.shape[0]):\n", 1632 | " t = t_combo_obs[i]\n", 1633 | " y = avg_y(beta_temp, c_true, d_true, t)\n", 1634 | " u = beta_temp.dot(t)\n", 1635 | " #G_prime = c_true*np.exp(-u)/(np.exp(-u)+1)**2*t\n", 1636 | " G_prime = np.append(c_est*np.exp(-u)/(np.exp(-u)+1)**2*t, [1/(np.exp(-u)+1)])\n", 1637 | " lambda_ += t_dist_obs[i]*2*np.outer(G_prime, G_prime)\n", 1638 | "\n", 1639 | " try:\n", 1640 | " lambda_inv.append(inv(lambda_ + reg_term*np.eye(m+2)))\n", 1641 | " #lambda_inv.append(inv(lambda_))\n", 1642 | " except:\n", 1643 | " print('Singular matrix')\n", 1644 | "\n", 1645 | " lambda_inv_loss_prime = []\n", 1646 | " for i in range(n_est):\n", 1647 | " lambda_inv_loss_prime.append(2*(pred_y_loss[i]-real_y[i])*lambda_inv[i].dot(G_theta_loss[i]))\n", 1648 | " lambda_inv_loss_prime = np.array(lambda_inv_loss_prime)\n", 1649 | "\n", 1650 | " pred_y_debiased = []\n", 1651 | " for i in range(n_est):\n", 1652 | " pred_y_debiased.append(pred_y[i]-G_theta[i].dot(lambda_inv_loss_prime[i]))\n", 1653 | " \n", 1654 | " pred_y_LR = []\n", 1655 | " for i in range(n_est): \n", 1656 | " pred_y_LR.append(np.append(samples_x_est[i], t_star_base).dot(coef_LR))\n", 1657 | " pred_y_LR = np.array(pred_y_LR)\n", 1658 | "\n", 1659 | " true_0.append(np.mean(real_y_star))\n", 1660 | " estimator_LR_0.append(np.mean(pred_y_LR))\n", 1661 | " estimator_0.append(np.mean(pred_y))\n", 1662 | " estimator_debias_0.append(np.mean(pred_y_debiased))\n", 1663 | " \n", 1664 | " #record\n", 1665 | " real_y_star_0 = real_y_star.copy()\n", 1666 | " pred_y_LR_0 = pred_y_LR.copy()\n", 1667 | " pred_y_0 = pred_y.copy()\n", 1668 | " pred_y_debiased_0 = pred_y_debiased.copy()\n", 1669 | " \n", 1670 | " \n", 1671 | " \n", 1672 | " #at t=t_star\n", 1673 | " t_star_base = t_star.copy()\n", 1674 | " \n", 1675 | " pred_y = []\n", 1676 | " for i in range(n_est): \n", 1677 | " pred_y.append(c_est/(1+np.exp(-(beta_[i]).dot(t_star_base))))\n", 1678 | " pred_y = np.array(pred_y)\n", 1679 | " \n", 1680 | "\n", 1681 | " real_y_star = []\n", 1682 | " for i in range(n_est):\n", 1683 | " real_y_star.append(0.05*np.random.uniform(-1, 1)+generate_y_true_1(coef,samples_x_est[i],t_star_base))\n", 1684 | "\n", 1685 | "\n", 1686 | " #lambda_inv = []\n", 1687 | " G_theta = []\n", 1688 | " G_theta_loss = []\n", 1689 | " t_obs_ = samples_t_est.copy()\n", 1690 | " cnt = 0\n", 1691 | "\n", 1692 | " for beta_temp in beta_:\n", 1693 | "\n", 1694 | " lambda_ = np.zeros([m+2, m+2])#asig use m+2\n", 1695 | " u_ = beta_temp.dot(t_star_base)\n", 1696 | " G_theta.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*t_star_base, [1/(np.exp(-u_)+1)]))\n", 1697 | "\n", 1698 | " u_ = beta_temp.dot(samples_t_est[cnt])\n", 1699 | " G_theta_loss.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*samples_t_est[cnt], [1/(np.exp(-u_)+1)]))\n", 1700 | "\n", 1701 | " cnt += 1\n", 1702 | "\n", 1703 | " #for i in range(t_combo_obs.shape[0]):\n", 1704 | " # t = t_combo_obs[i]\n", 1705 | " # y = avg_y(beta_temp, c_true, d_true, t)\n", 1706 | " # u = beta_temp.dot(t)\n", 1707 | " # G_prime = np.append(c_est*np.exp(-u)/(np.exp(-u)+1)**2*t, [1/(np.exp(-u)+1)])\n", 1708 | " # lambda_ += t_dist_obs[i]*2*np.outer(G_prime, G_prime)\n", 1709 | "\n", 1710 | " #try:\n", 1711 | " # lambda_inv.append(inv(lambda_ + reg_term*np.eye(m+2)))\n", 1712 | " #except:\n", 1713 | " # print('Singular matrix')\n", 1714 | "\n", 1715 | " lambda_inv_loss_prime = []\n", 1716 | " for i in range(n_est):\n", 1717 | " lambda_inv_loss_prime.append(2*(pred_y_loss[i]-real_y[i])*lambda_inv[i].dot(G_theta_loss[i]))\n", 1718 | " lambda_inv_loss_prime = np.array(lambda_inv_loss_prime)\n", 1719 | "\n", 1720 | " pred_y_debiased = []\n", 1721 | " for i in range(n_est):\n", 1722 | " pred_y_debiased.append(pred_y[i]-G_theta[i].dot(lambda_inv_loss_prime[i]))\n", 1723 | " \n", 1724 | " pred_y_LR = []\n", 1725 | " for i in range(n_est): \n", 1726 | " pred_y_LR.append(np.append(samples_x_est[i], t_star_base).dot(coef_LR))\n", 1727 | " pred_y_LR = np.array(pred_y_LR)\n", 1728 | "\n", 1729 | " true_1.append(np.mean(real_y_star)) \n", 1730 | " estimator_LR_1.append(np.mean(pred_y_LR))\n", 1731 | " estimator_1.append(np.mean(pred_y))\n", 1732 | " estimator_debias_1.append(np.mean(pred_y_debiased))\n", 1733 | " \n", 1734 | " p_.append(stats.ttest_rel(real_y_star, real_y_star_0)[1])\n", 1735 | " p_LR.append(stats.ttest_ind(pred_y_LR, pred_y_LR_0)[1])\n", 1736 | " p_DL.append(stats.ttest_rel(pred_y, pred_y_0)[1])\n", 1737 | " p_DDL.append(stats.ttest_rel(pred_y_debiased, pred_y_debiased_0)[1])\n", 1738 | " \n", 1739 | " \n", 1740 | " #calculate additive\n", 1741 | " pred_y_additive = 0\n", 1742 | " pred_y_additive_var = 0\n", 1743 | " n_LA_samples = 0\n", 1744 | " for single_exp_index in np.where(t_star == 1)[0]:\n", 1745 | " ttt = np.zeros(m+1)\n", 1746 | " ttt[0] = 1\n", 1747 | " ttt[single_exp_index] = 1\n", 1748 | " pred_y_additive += samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].mean()\n", 1749 | " pred_y_additive_var += samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].var()\n", 1750 | " n_LA_samples += (samples_t_est == ttt).sum()\n", 1751 | " ttt = np.zeros(m+1)\n", 1752 | " ttt[0] = 1\n", 1753 | " pred_y_additive -= (np.sum(t_star))*samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].mean()\n", 1754 | " estimator_additive_1.append(pred_y_additive)\n", 1755 | " t_value = pred_y_additive/np.sqrt(pred_y_additive_var*(np.sum(t_star))/n_LA_samples)\n", 1756 | " p_LA.append(stats.t.sf(abs(t_value), df=1))" 1757 | ] 1758 | }, 1759 | { 1760 | "cell_type": "code", 1761 | "execution_count": null, 1762 | "id": "65e04614", 1763 | "metadata": { 1764 | "ExecuteTime": { 1765 | "end_time": "2023-02-11T04:31:56.674454Z", 1766 | "start_time": "2023-02-11T04:31:56.630226Z" 1767 | } 1768 | }, 1769 | "outputs": [], 1770 | "source": [ 1771 | "p_ = np.array(p_).reshape((n_rep, 2**m))\n", 1772 | "p_LA = np.array(p_LA).reshape((n_rep, 2**m))\n", 1773 | "p_LR = np.array(p_LR).reshape((n_rep, 2**m))\n", 1774 | "p_DL = np.array(p_DL).reshape((n_rep, 2**m))\n", 1775 | "p_DDL = np.array(p_DDL).reshape((n_rep, 2**m))\n", 1776 | "true_0 = np.array(true_0).reshape((n_rep, 2**m))\n", 1777 | "estimator_LR_0 = np.array(estimator_LR_0).reshape((n_rep, 2**m))\n", 1778 | "estimator_0 = np.array(estimator_0).reshape((n_rep, 2**m))\n", 1779 | "estimator_debias_0 = np.array(estimator_debias_0).reshape((n_rep, 2**m))\n", 1780 | "true_1 = np.array(true_1).reshape((n_rep, 2**m))\n", 1781 | "estimator_LR_1 = np.array(estimator_LR_1).reshape((n_rep, 2**m))\n", 1782 | "estimator_1 = np.array(estimator_1).reshape((n_rep, 2**m))\n", 1783 | "estimator_debias_1 = np.array(estimator_debias_1).reshape((n_rep, 2**m))\n", 1784 | "estimator_additive_1 = np.array(estimator_additive_1).reshape((n_rep, 2**m))\n", 1785 | "index = 0\n", 1786 | "\n", 1787 | "#LA LR SDL DeDL\n", 1788 | "ape_2 = np.zeros([2**m, 4, n_rep])\n", 1789 | "se = np.zeros([2**m, 4, n_rep])\n", 1790 | "ae = np.zeros([2**m, 4, n_rep])\n", 1791 | "CDR = np.zeros([2**m, 4, n_rep])\n", 1792 | "for i in range(n_rep):\n", 1793 | " CDR[0,:,i] = 1\n", 1794 | " for j in range(1,2**m):\n", 1795 | " true_eff = true_1[i,j]-true_0[i,j]\n", 1796 | " la = estimator_additive_1[i,j]\n", 1797 | " lr = estimator_LR_1[i,j]-estimator_LR_0[i,j]\n", 1798 | " dl = estimator_1[i,j]-estimator_0[i,j]\n", 1799 | " dedl = estimator_debias_1[i,j]-estimator_debias_0[i,j]\n", 1800 | " if (p_[i,j]<=0.05 and p_LA[i,j]<=0.05 and la*true_eff>0) or (p_[i,j]>0.05 and p_LA[i,j]>0.05):\n", 1801 | " CDR[j,0,i] = 1\n", 1802 | " if (p_[i,j]<=0.05 and p_LR[i,j]<=0.05 and lr*true_eff>0) or (p_[i,j]>0.05 and p_LR[i,j]>0.05):\n", 1803 | " CDR[j,1,i] = 1\n", 1804 | " if (p_[i,j]<=0.05 and p_DL[i,j]<=0.05 and dl*true_eff>0) or (p_[i,j]>0.05 and p_DL[i,j]>0.05):\n", 1805 | " CDR[j,2,i] = 1\n", 1806 | " if (p_[i,j]<=0.05 and p_DDL[i,j]<=0.05 and dedl*true_eff>0) or (p_[i,j]>0.05 and p_DDL[i,j]>0.05):\n", 1807 | " CDR[j,3,i] = 1\n", 1808 | " \n", 1809 | "\n", 1810 | "print('----------------All combos--------------------')\n", 1811 | "for t_star in t_star_all:\n", 1812 | " #print('Treatment effect when t = ', t_star[1:])\n", 1813 | " true_effect = true_1[:,index]-true_0[:,index]\n", 1814 | " add_effect = estimator_additive_1[:,index]\n", 1815 | " LR_effect = estimator_LR_1[:,index]-estimator_LR_0[:,index] \n", 1816 | " no_bias_effect = estimator_1[:,index]-estimator_0[:,index]\n", 1817 | " debias_effect = estimator_debias_1[:,index]-estimator_debias_0[:,index]\n", 1818 | " \n", 1819 | " ape_2[index, 0, :] = np.abs((add_effect-true_effect))/(np.abs(true_effect))\n", 1820 | " ape_2[index, 1, :] = np.abs((LR_effect-true_effect))/(np.abs(true_effect))\n", 1821 | " ape_2[index, 2, :] = np.abs((no_bias_effect-true_effect))/(np.abs(true_effect))\n", 1822 | " ape_2[index, 3, :] = np.abs((debias_effect-true_effect))/(np.abs(true_effect))\n", 1823 | " \n", 1824 | " #if p_[:,index]>=0.05:#not significant treatment effect\n", 1825 | " ape_2[index, 0, p_[:,index]>=0.05] = 0\n", 1826 | " ape_2[index, 1, p_[:,index]>=0.05] = 0\n", 1827 | " ape_2[index, 2, p_[:,index]>=0.05] = 0\n", 1828 | " ape_2[index, 3, p_[:,index]>=0.05] = 0\n", 1829 | " \n", 1830 | " se[index, 0, :] = ((add_effect - true_effect)**2)\n", 1831 | " se[index, 1, :] = ((LR_effect - true_effect)**2) \n", 1832 | " se[index, 2, :] = ((no_bias_effect - true_effect)**2)\n", 1833 | " se[index, 3, :] = ((debias_effect - true_effect)**2)\n", 1834 | " \n", 1835 | " ae[index, 0, :] = np.abs((add_effect - true_effect))\n", 1836 | " ae[index, 1, :] = np.abs((LR_effect - true_effect))\n", 1837 | " ae[index, 2, :] = np.abs((no_bias_effect - true_effect))\n", 1838 | " ae[index, 3, :] = np.abs((debias_effect - true_effect))\n", 1839 | " \n", 1840 | " index += 1\n", 1841 | " \n", 1842 | "print('------------------------------------')\n", 1843 | "print('MSE of LA ','|', np.mean(se[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,0,:], axis=0)), scale=st.sem(np.mean(se[:,0,:], axis=0))))\n", 1844 | "print('MSE of LR ','|', np.mean(se[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,1,:], axis=0)), scale=st.sem(np.mean(se[:,1,:], axis=0))))\n", 1845 | "print('MSE of SDL ','|', np.mean(se[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,2,:], axis=0)), scale=st.sem(np.mean(se[:,2,:], axis=0))))\n", 1846 | "print('MSE of DeDL ','|', np.mean(se[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,3,:], axis=0)), scale=st.sem(np.mean(se[:,3,:], axis=0))))\n", 1847 | "print('------------------------------------')\n", 1848 | "print('MAE of LA ','|', np.mean(ae[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,0,:], axis=0)), scale=st.sem(np.mean(ae[:,0,:], axis=0))))\n", 1849 | "print('MAE of LR ','|', np.mean(ae[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,1,:], axis=0)), scale=st.sem(np.mean(ae[:,1,:], axis=0))))\n", 1850 | "print('MAE of SDL ','|', np.mean(ae[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,2,:], axis=0)), scale=st.sem(np.mean(ae[:,2,:], axis=0))))\n", 1851 | "print('MAE of DeDL ','|', np.mean(ae[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,3,:], axis=0)), scale=st.sem(np.mean(ae[:,3,:], axis=0))))\n", 1852 | "print('------------------------------------')\n", 1853 | "print('MAPE of LA ','|', np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)), \n", 1854 | " scale=st.sem(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0))))\n", 1855 | "print('MAPE of LR ','|', np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)), \n", 1856 | " scale=st.sem(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0))))\n", 1857 | "print('MAPE of SDL ','|', np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)), \n", 1858 | " scale=st.sem(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0))))\n", 1859 | "print('MAPE of DeDL ','|', np.mean(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0)), \n", 1860 | " scale=st.sem(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0))))\n", 1861 | "\n", 1862 | "print('------------------------------------')\n", 1863 | "print('CDR of LA ','|', np.mean(CDR[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,0,:], axis=0)), scale=st.sem(np.mean(CDR[:,0,:], axis=0))))\n", 1864 | "print('CDR of LR ','|', np.mean(CDR[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,1,:], axis=0)), scale=st.sem(np.mean(CDR[:,1,:], axis=0))))\n", 1865 | "print('CDR of SDL ','|', np.mean(CDR[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,2,:], axis=0)), scale=st.sem(np.mean(CDR[:,2,:], axis=0))))\n", 1866 | "print('CDR of DeDL ','|', np.mean(CDR[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,3,:], axis=0)), scale=st.sem(np.mean(CDR[:,3,:], axis=0))))\n", 1867 | "\n", 1868 | "print('------------------------------------')\n", 1869 | "\n", 1870 | "cnt_bai = np.zeros([n_rep, 4])\n", 1871 | "for i in range(n_rep):\n", 1872 | " max_effect = -10000*np.ones(5)\n", 1873 | " max_effect_index = np.zeros(5)\n", 1874 | " for index in range(2**m):\n", 1875 | " true_effect = true_1[i,index]-true_0[i,index]\n", 1876 | " add_effect = estimator_additive_1[i,index]\n", 1877 | " LR_effect = estimator_LR_1[i,index]-estimator_LR_0[i,index]\n", 1878 | " no_bias_effect = estimator_1[i,index]-estimator_0[i,index]\n", 1879 | " debias_effect = estimator_debias_1[i,index]-estimator_debias_0[i,index]\n", 1880 | " \n", 1881 | " if true_effect > max_effect[4]:\n", 1882 | " max_effect[4] = true_effect\n", 1883 | " max_effect_index[4] = index\n", 1884 | " if add_effect > max_effect[0]:\n", 1885 | " max_effect[0] = add_effect\n", 1886 | " max_effect_index[0] = index\n", 1887 | " if LR_effect > max_effect[1]:\n", 1888 | " max_effect[1] = LR_effect\n", 1889 | " max_effect_index[1] = index\n", 1890 | " if no_bias_effect > max_effect[2]:\n", 1891 | " max_effect[2] = no_bias_effect\n", 1892 | " max_effect_index[2] = index\n", 1893 | " if debias_effect > max_effect[3]:\n", 1894 | " max_effect[3] = debias_effect\n", 1895 | " max_effect_index[3] = index\n", 1896 | " \n", 1897 | " if max_effect_index[0] == max_effect_index[4]:\n", 1898 | " cnt_bai[i, 0] = 1\n", 1899 | " if max_effect_index[1] == max_effect_index[4]:\n", 1900 | " cnt_bai[i, 1] = 1\n", 1901 | " if max_effect_index[2] == max_effect_index[4]:\n", 1902 | " cnt_bai[i, 2] = 1\n", 1903 | " if max_effect_index[3] == max_effect_index[4]:\n", 1904 | " cnt_bai[i, 3] = 1\n", 1905 | " \n", 1906 | "print('BAI of LA ','|', np.mean(cnt_bai[:, 0]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 0]), scale=st.sem(cnt_bai[:, 0])))\n", 1907 | "print('BAI of LR ','|', np.mean(cnt_bai[:, 1]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 1]), scale=st.sem(cnt_bai[:, 1])))\n", 1908 | "print('BAI of SDL ','|', np.mean(cnt_bai[:, 2]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 2]), scale=st.sem(cnt_bai[:, 2])))\n", 1909 | "print('BAI of DeDL ','|', np.mean(cnt_bai[:, 3]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 3]), scale=st.sem(cnt_bai[:, 3])))\n", 1910 | "print('------------------------------------')\n", 1911 | "print('ABS of ATE ','|', np.mean(np.mean(np.abs(true_1-true_0), axis=1)),'|',st.t.interval(0.95, n_rep-1, \n", 1912 | " loc=np.mean(np.mean(np.abs(true_1-true_0), axis=1)), \n", 1913 | " scale=st.sem(np.mean(np.abs(true_1-true_0), axis=1))))" 1914 | ] 1915 | }, 1916 | { 1917 | "cell_type": "markdown", 1918 | "id": "6f3e853a", 1919 | "metadata": {}, 1920 | "source": [ 1921 | "# Link Function Misspecification" 1922 | ] 1923 | }, 1924 | { 1925 | "cell_type": "code", 1926 | "execution_count": null, 1927 | "id": "0f807dab", 1928 | "metadata": { 1929 | "ExecuteTime": { 1930 | "end_time": "2023-02-10T17:53:40.076064Z", 1931 | "start_time": "2023-02-10T17:53:26.359568Z" 1932 | } 1933 | }, 1934 | "outputs": [], 1935 | "source": [ 1936 | "y_linear = []\n", 1937 | "y_sigmoid = []\n", 1938 | "n_cnver_test = 10000\n", 1939 | "gamma_ = 1\n", 1940 | "\n", 1941 | "rep_index = 0\n", 1942 | "while rep_index < 100:\n", 1943 | " coef = np.random.uniform(-0.5, 0.5, [d_c, m+1])\n", 1944 | " coef_linear = np.random.uniform(-0.5, 0.5, [d_c, m+1])\n", 1945 | " c_true = np.random.uniform(10, 20)\n", 1946 | " d_true = 0\n", 1947 | " \n", 1948 | " def generate_y_true_linear(coef, c, d, x, t, n):\n", 1949 | " y = np.zeros(n)\n", 1950 | " for i in range(n):\n", 1951 | " y[i] = gamma_*(x[i].dot(coef_linear)).dot(t[i]) \n", 1952 | " return y\n", 1953 | " \n", 1954 | " def generate_y_true_sigmoid(coef, c, d, x, t, n):\n", 1955 | " y = np.zeros(n)\n", 1956 | " for i in range(n):\n", 1957 | " y[i] = c/(1+np.exp(-((x[i].dot(coef))).dot(t[i])))\n", 1958 | " return y\n", 1959 | " \n", 1960 | " samples_x_cnver = generate_x(d_c, n_cnver_test)\n", 1961 | " samples_t_cnver = generate_t(t_combo, t_dist, n_cnver_test)\n", 1962 | " y_linear.append(generate_y_true_linear(coef, c_true, d_true, samples_x_cnver, samples_t_cnver, n_cnver_test))\n", 1963 | " y_sigmoid.append(generate_y_true_sigmoid(coef, c_true, d_true, samples_x_cnver, samples_t_cnver, n_cnver_test))\n", 1964 | " rep_index+=1\n", 1965 | " #print(rep_index)\n", 1966 | "y_linear = np.array(y_linear).reshape(-1)\n", 1967 | "y_sigmoid = np.array(y_sigmoid).reshape(-1) \n", 1968 | "#plt.hist(y_linear, label='linear', density=True, alpha=0.6,bins=20)\n", 1969 | "#plt.hist(y_sigmoid, label='sigmoid', density=True, alpha=0.6,bins=20)\n", 1970 | "#plt.xlabel('HTE')\n", 1971 | "#plt.xlim(-20, 20)\n", 1972 | "#plt.ylabel('density')\n", 1973 | "#plt.legend()\n", 1974 | "#plt.show()\n", 1975 | "\n", 1976 | "#print('HTE Linear ','|', np.mean(y_linear),'|',st.t.interval(0.95, len(y_linear)-1, loc=np.mean(y_linear), scale=st.sem(y_linear)))\n", 1977 | "#print('HTE Sigmoid ','|', np.mean(y_sigmoid),'|',st.t.interval(0.95, len(y_sigmoid)-1, loc=np.mean(y_sigmoid), scale=st.sem(y_sigmoid)))\n", 1978 | "from IPython.core.pylabtools import figsize\n", 1979 | "from matplotlib.ticker import FixedFormatter\n", 1980 | "figsize(6, 4)\n", 1981 | "fig, ax = plt.subplots(1, 1)\n", 1982 | "plt.rcParams['savefig.dpi'] = 1200\n", 1983 | "plt.rcParams['figure.dpi'] = 1200\n", 1984 | "ax.yaxis.set_ticks_position('both')\n", 1985 | "ax.xaxis.set_ticks_position('both')\n", 1986 | "plt.rcParams['xtick.direction'] = 'in'\n", 1987 | "plt.rcParams['ytick.direction'] = 'in'\n", 1988 | "plt.rc('font',family='Times New Roman')\n", 1989 | "y_linear_1 = y_linear + y_sigmoid\n", 1990 | "y_linear_2 = 3*y_linear+y_sigmoid\n", 1991 | "y_linear_5 = 5*y_linear+y_sigmoid\n", 1992 | "#y_linear_1 = y_linear \n", 1993 | "#y_linear_2 = 2*y_linear\n", 1994 | "#y_linear_5 = 5*y_linear\n", 1995 | "sns.set_theme(style=\"white\",font=\"Times New Roman\")\n", 1996 | "\n", 1997 | "sns.kdeplot(y_sigmoid, label='\\u03B3 = 0',fill=True,bw_adjust=5)\n", 1998 | "sns.kdeplot(y_linear_1[np.where((y_linear_1 > -10) & (y_linear_1 < 30))],color=\"r\", label='\\u03B3 = 1',fill=True,bw_adjust=5)\n", 1999 | "sns.kdeplot(y_linear_2[np.where((y_linear_2 > -10) & (y_linear_2 < 30))],color=\"g\", label='\\u03B3 = 3',fill=True,bw_adjust=5)\n", 2000 | "sns.kdeplot(y_linear_5[np.where((y_linear_5 > -10) & (y_linear_5 < 30))],color=\"orange\", label='\\u03B3 = 5',fill=True,bw_adjust=5)\n", 2001 | "plt.xlabel('Outcome Y')\n", 2002 | "plt.xlim(-10, 30)\n", 2003 | "plt.ylabel('density')\n", 2004 | "plt.legend(loc=\"upper left\")\n", 2005 | "#plt.savefig('misspecific_HTE.png', dpi=600, bbox_inches='tight',pad_inches=0.0) \n" 2006 | ] 2007 | }, 2008 | { 2009 | "cell_type": "markdown", 2010 | "id": "f5e9d28a", 2011 | "metadata": {}, 2012 | "source": [ 2013 | "## Data generating process definition\n", 2014 | "define the level of gamma = 0, 1, 3, 5" 2015 | ] 2016 | }, 2017 | { 2018 | "cell_type": "code", 2019 | "execution_count": null, 2020 | "id": "0364669a", 2021 | "metadata": { 2022 | "ExecuteTime": { 2023 | "end_time": "2023-02-11T04:32:02.637548Z", 2024 | "start_time": "2023-02-11T04:32:02.630371Z" 2025 | } 2026 | }, 2027 | "outputs": [], 2028 | "source": [ 2029 | "gamma_ = 1\n", 2030 | "m = 4 #number of experiments\n", 2031 | "d_c = 10 #number of features\n", 2032 | "\n", 2033 | "all_combo = list(powerset(list(np.arange(1, m+1))))\n", 2034 | "t_combo = []\n", 2035 | "for i in all_combo:\n", 2036 | " t = np.zeros(m+1)\n", 2037 | " t[0] = 1\n", 2038 | " t[i] = 1\n", 2039 | " t_combo.append(t)\n", 2040 | "t_combo = np.int16(t_combo)\n", 2041 | "t_dist = (1/2**m)*np.ones(2**m)\n", 2042 | "t_combo_obs = []\n", 2043 | "for i in range(m+1):\n", 2044 | " t = np.zeros(m+1)\n", 2045 | " t[0] = 1\n", 2046 | " t[i] = 1\n", 2047 | " t_combo_obs.append(t)\n", 2048 | "t = np.zeros(m+1)\n", 2049 | "t[[0,1,2]] = 1\n", 2050 | "t_combo_obs.append(t)\n", 2051 | "t_combo_obs = np.int16(t_combo_obs)\n", 2052 | "t_dist_obs = (1/(m+2))*np.ones(m+2)\n", 2053 | "\n", 2054 | "t_star_all = t_combo.copy()\n", 2055 | "idx = (t_combo[:,None]!=t_combo_obs).any(-1).all(1)\n", 2056 | "t_star_unobs = t_combo[idx]\n", 2057 | "\n", 2058 | "\n", 2059 | "n_est = int(m*500)\n", 2060 | "n_train = int(m*500)\n", 2061 | "reg_term = 0.0005\n", 2062 | "#reg_loss = 0.001\n", 2063 | "reg_loss = 0\n", 2064 | "train_epochs = 2000\n", 2065 | "test_thres = 0.2*gamma_+0.1\n", 2066 | "if gamma_ == 0:\n", 2067 | " test_thres = 0.03\n", 2068 | " \n", 2069 | "n_cnver_test = 10000\n", 2070 | "lr = 0.05\n", 2071 | "\n", 2072 | "feature_list = []\n", 2073 | "for i in range(d_c):\n", 2074 | " feature_list.append(str('x')+str(i+1))\n", 2075 | "t_list = []\n", 2076 | "for i in range(m+1):\n", 2077 | " t_list.append(str('t')+str(i))\n", 2078 | "\n", 2079 | "true_0 = []#real ATE in base\n", 2080 | "estimator_LR_0 = []#ATE by LR in base\n", 2081 | "estimator_0 = []#ATE by SDL in base\n", 2082 | "estimator_debias_0 = []#ATE by DEDL in base\n", 2083 | "\n", 2084 | "true_1 = []#real ATE in treatment\n", 2085 | "estimator_LR_1 = []#ATE by LR in treatment\n", 2086 | "estimator_1 = []#ATE by SDL in treatment\n", 2087 | "estimator_debias_1 = []#ATE by DEDL in treatment\n", 2088 | "\n", 2089 | "estimator_additive_1 = []#ATE by additive in treatment\n", 2090 | "p_ = []\n", 2091 | "p_LA = []\n", 2092 | "p_LR = []\n", 2093 | "p_DL = []\n", 2094 | "p_DDL = []\n", 2095 | "test_err = []" 2096 | ] 2097 | }, 2098 | { 2099 | "cell_type": "markdown", 2100 | "id": "6dec4937", 2101 | "metadata": {}, 2102 | "source": [ 2103 | "## Comparison of LA, LR, SDL, DeDL" 2104 | ] 2105 | }, 2106 | { 2107 | "cell_type": "code", 2108 | "execution_count": null, 2109 | "id": "1da448c8", 2110 | "metadata": { 2111 | "ExecuteTime": { 2112 | "end_time": "2023-02-11T04:34:08.984786Z", 2113 | "start_time": "2023-02-11T04:32:08.583560Z" 2114 | } 2115 | }, 2116 | "outputs": [], 2117 | "source": [ 2118 | "rep_index = 0\n", 2119 | "\n", 2120 | "while rep_index < n_rep:\n", 2121 | " print('# of replication:', rep_index)\n", 2122 | " coef = np.random.uniform(-0.5, 0.5, [d_c, m+1])\n", 2123 | " coef_linear = np.random.uniform(-0.5, 0.5, [d_c, m+1])\n", 2124 | " c_true = np.random.uniform(10, 20)\n", 2125 | " d_true = 0\n", 2126 | " \n", 2127 | "\n", 2128 | " def generate_y_true(coef, c, d, x, t, n):\n", 2129 | " y = np.zeros(n)\n", 2130 | " y_error = 0.05*np.random.uniform(-1, 1, n)\n", 2131 | " for i in range(n):\n", 2132 | " y[i] = gamma_*(x[i].dot(coef_linear)).dot(t[i]) + (c/(1+np.exp(-((x[i].dot(coef))).dot(t[i]))) + d + y_error[i])\n", 2133 | " return y, y_error\n", 2134 | " def generate_y_true_1(coef, x, t):\n", 2135 | " return gamma_*(x.dot(coef_linear)).dot(t) + c_true/(1+np.exp(-((x.dot(coef))).dot(t)))\n", 2136 | "\n", 2137 | " \n", 2138 | " #generate samples for estimation\n", 2139 | " samples_x_est = generate_x(d_c, n_train)\n", 2140 | " samples_t_est = generate_t(t_combo_obs, t_dist_obs, n_train)\n", 2141 | " samples_y_est, samples_y_error_est = generate_y_true(coef, c_true, d_true, samples_x_est, samples_t_est, n_train)\n", 2142 | " full_data_est = np.append(samples_x_est, samples_t_est, axis=1)\n", 2143 | " full_data_est = np.append(full_data_est, samples_y_est.reshape(n_train,1), axis=1)\n", 2144 | " data_obs = pd.DataFrame(full_data_est, columns = feature_list + t_list + ['y'])\n", 2145 | "\n", 2146 | " model_LR = sm.OLS(samples_y_est.reshape(n_train,1), np.append(samples_x_est, samples_t_est, axis=1))\n", 2147 | " results_LR = model_LR.fit()\n", 2148 | " coef_LR = results_LR.params\n", 2149 | " \n", 2150 | " opt = parser.parse_args()\n", 2151 | " train_samples = int(0.7*data_obs.shape[0])\n", 2152 | " x_train_set = np.float32(data_obs)[0:train_samples, 0:-1]\n", 2153 | " y_train_set = np.float32(data_obs)[0:train_samples, -1]\n", 2154 | " x_test_set = np.float32(data_obs)[train_samples:, 0:-1]\n", 2155 | " y_test_set = np.float32(data_obs)[train_samples:, -1]\n", 2156 | "\n", 2157 | " trainset = torch.utils.data.TensorDataset(torch.Tensor(x_train_set), torch.Tensor(y_train_set)) # create your datset\n", 2158 | " trainloader = torch.utils.data.DataLoader(\n", 2159 | " trainset, batch_size=opt.batch_size_train, shuffle=True)\n", 2160 | "\n", 2161 | " testset = torch.utils.data.TensorDataset(torch.Tensor(x_test_set), torch.Tensor(y_test_set)) # create your datset\n", 2162 | " testloader = torch.utils.data.DataLoader(\n", 2163 | " testset, batch_size=opt.batch_size_test, shuffle=True)\n", 2164 | " net = FNN_asig()\n", 2165 | " test_accuracy = 100\n", 2166 | " print('---------warm-up train---------')\n", 2167 | " wp_cnt = 0\n", 2168 | " while(test_accuracy >= 1):\n", 2169 | " wp_cnt += 1\n", 2170 | " net = FNN_asig()\n", 2171 | " with torch.no_grad():\n", 2172 | " net.layer3.weight[0, 0] = float(np.max(y_train_set))\n", 2173 | " criterion = nn.MSELoss()\n", 2174 | " #criterion = nn.L1Loss()\n", 2175 | " #warm-up train\n", 2176 | " test_accuracy = 100\n", 2177 | " optimizer = optim.Adam(net.parameters(), lr = 0.1, weight_decay = opt.wd)\n", 2178 | " for epoch in range(200):\n", 2179 | " for i, data_i in enumerate(trainloader, 0):\n", 2180 | " inputs, labels = data_i\n", 2181 | " inputs, labels = Variable(inputs), Variable(labels)\n", 2182 | " optimizer.zero_grad()\n", 2183 | " outputs = net(inputs)\n", 2184 | " l1_norm = sum(p.abs().sum() for p in net.parameters())\n", 2185 | " loss = criterion(outputs, labels) + reg_loss*l1_norm\n", 2186 | " loss.backward()\n", 2187 | " optimizer.step()\n", 2188 | " #train_accuracy = calculate_mse(trainloader, opt.is_gpu, net)\n", 2189 | " test_accuracy = calculate_mse(testloader, opt.is_gpu, net)\n", 2190 | " #if epoch%100 == 0:\n", 2191 | " # print(\"Iteration: {0} | Training MSE: {1} | Test MSE: {2}\".format(\n", 2192 | " # epoch, train_accuracy, test_accuracy))\n", 2193 | " if test_accuracy < 1:\n", 2194 | " break\n", 2195 | " if wp_cnt >= 5:\n", 2196 | " break\n", 2197 | " #training \n", 2198 | " print('---------train---------')\n", 2199 | " optimizer = optim.Adam(net.parameters(), lr = lr, weight_decay = opt.wd)\n", 2200 | " criterion = nn.MSELoss()\n", 2201 | " for epoch in range(train_epochs):\n", 2202 | " for i, data_i in enumerate(trainloader, 0):\n", 2203 | " inputs, labels = data_i\n", 2204 | " inputs, labels = Variable(inputs), Variable(labels)\n", 2205 | " optimizer.zero_grad()\n", 2206 | " outputs = net(inputs)\n", 2207 | " l1_norm = sum(p.abs().sum() for p in net.parameters())\n", 2208 | " loss = criterion(outputs, labels) + reg_loss*l1_norm\n", 2209 | " loss.backward()\n", 2210 | " optimizer.step()\n", 2211 | " train_accuracy = calculate_mse(trainloader, opt.is_gpu, net)\n", 2212 | " test_accuracy = calculate_mse(testloader, opt.is_gpu, net)\n", 2213 | " if epoch%100 == 0:\n", 2214 | " print(\"Iteration: {0} | Training MSE: {1} | Test MSE: {2}\".format(\n", 2215 | " epoch, train_accuracy, test_accuracy))\n", 2216 | " if test_accuracy < test_thres and epoch > 500:\n", 2217 | " break \n", 2218 | " if test_accuracy > test_thres:\n", 2219 | " continue\n", 2220 | " else:\n", 2221 | " rep_index += 1\n", 2222 | " test_err.append(test_accuracy)\n", 2223 | " #rep_index += 1\n", 2224 | " \n", 2225 | " activation = {}\n", 2226 | " net.layer1.register_forward_hook(get_activation('layer1'))\n", 2227 | "\n", 2228 | " \n", 2229 | " #generate samples for inference\n", 2230 | " samples_x_est = generate_x(d_c, n_est)\n", 2231 | " samples_t_est = generate_t(t_combo_obs, t_dist_obs, n_est)\n", 2232 | " samples_y_est, samples_y_error_est = generate_y_true(coef, c_true, d_true, samples_x_est, samples_t_est, n_est)\n", 2233 | " full_data_est = np.append(samples_x_est, samples_t_est, axis=1)\n", 2234 | " full_data_est = np.append(full_data_est, samples_y_est.reshape(n_est,1), axis=1)\n", 2235 | "\n", 2236 | " for t_star in t_star_all:\n", 2237 | " data_est = pd.DataFrame(full_data_est, columns = feature_list + t_list + ['y'])\n", 2238 | " #at t=0\n", 2239 | " t_star_base = np.zeros(m+1)\n", 2240 | " t_star_base[0] = 1\n", 2241 | " x_all_set=np.float32(data_est)[:, 0:-1]\n", 2242 | " y_all_set=np.float32(data_est)[:, -1]\n", 2243 | " allset = torch.utils.data.TensorDataset(torch.Tensor(x_all_set), torch.Tensor(y_all_set)) # create your datset\n", 2244 | " allloader = torch.utils.data.DataLoader(\n", 2245 | " allset, batch_size=1000, shuffle=False)\n", 2246 | " \n", 2247 | " beta_ = []\n", 2248 | " pred_y_loss = []\n", 2249 | " for i, data_ in enumerate(allloader, 0):\n", 2250 | " inputs, labels = data_\n", 2251 | " inputs, labels = Variable(inputs), Variable(labels)\n", 2252 | " outputs = net(inputs)\n", 2253 | " beta_.append(activation['layer1'].tolist()) \n", 2254 | " pred_y_loss.append(outputs.tolist())\n", 2255 | " beta_ = np.array(beta_).reshape(n_est, m+1)\n", 2256 | " pred_y_loss = np.array(pred_y_loss).reshape(n_est)\n", 2257 | " \n", 2258 | " real_y = samples_y_est.copy()\n", 2259 | " \n", 2260 | " for j in range(m):\n", 2261 | " x_all_set[:, -m+j] = t_star_base[j+1]\n", 2262 | " allset = torch.utils.data.TensorDataset(torch.Tensor(x_all_set), torch.Tensor(y_all_set)) # create your datset\n", 2263 | " allloader = torch.utils.data.DataLoader(\n", 2264 | " allset, batch_size=1000, shuffle=False)\n", 2265 | " pred_y = []\n", 2266 | " for i, data_ in enumerate(allloader, 0):\n", 2267 | " inputs, labels = data_\n", 2268 | " inputs, labels = Variable(inputs), Variable(labels)\n", 2269 | " outputs = net(inputs)\n", 2270 | " pred_y.append(outputs.tolist())\n", 2271 | " pred_y = np.array(pred_y).reshape(n_est)\n", 2272 | " \n", 2273 | " real_y_star = []\n", 2274 | " for i in range(n_est):\n", 2275 | " real_y_star.append(0.05*np.random.uniform(-1, 1)+generate_y_true_1(coef,samples_x_est[i],t_star_base))\n", 2276 | "\n", 2277 | " lambda_inv = []\n", 2278 | " G_theta = []\n", 2279 | " G_theta_loss = []\n", 2280 | " t_obs_ = samples_t_est.copy()\n", 2281 | " cnt = 0\n", 2282 | "\n", 2283 | " for beta_temp in beta_:\n", 2284 | "\n", 2285 | " lambda_ = np.zeros([m+2, m+2])#asig use m+2\n", 2286 | " u_ = beta_temp.dot(t_star_base)\n", 2287 | " G_theta.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*t_star_base, [1/(np.exp(-u_)+1)]))\n", 2288 | "\n", 2289 | " u_ = beta_temp.dot(samples_t_est[cnt])\n", 2290 | " G_theta_loss.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*samples_t_est[cnt], [1/(np.exp(-u_)+1)]))\n", 2291 | "\n", 2292 | " cnt += 1\n", 2293 | "\n", 2294 | " for i in range(t_combo_obs.shape[0]):\n", 2295 | " t = t_combo_obs[i]\n", 2296 | " y = avg_y(beta_temp, c_true, d_true, t)\n", 2297 | " u = beta_temp.dot(t)\n", 2298 | " G_prime = np.append(c_est*np.exp(-u)/(np.exp(-u)+1)**2*t, [1/(np.exp(-u)+1)])\n", 2299 | " lambda_ += t_dist_obs[i]*2*np.outer(G_prime, G_prime)\n", 2300 | "\n", 2301 | " try:\n", 2302 | " lambda_inv.append(inv(lambda_ + reg_term*np.eye(m+2)))\n", 2303 | " except:\n", 2304 | " print('Singular matrix')\n", 2305 | "\n", 2306 | " lambda_inv_loss_prime = []\n", 2307 | " for i in range(n_est):\n", 2308 | " lambda_inv_loss_prime.append(2*(pred_y_loss[i]-real_y[i])*lambda_inv[i].dot(G_theta_loss[i]))\n", 2309 | " lambda_inv_loss_prime = np.array(lambda_inv_loss_prime)\n", 2310 | "\n", 2311 | " pred_y_debiased = []\n", 2312 | " for i in range(n_est):\n", 2313 | " pred_y_debiased.append(pred_y[i]-G_theta[i].dot(lambda_inv_loss_prime[i]))\n", 2314 | " \n", 2315 | " pred_y_LR = []\n", 2316 | " for i in range(n_est): \n", 2317 | " pred_y_LR.append(np.append(samples_x_est[i], t_star_base).dot(coef_LR))\n", 2318 | " pred_y_LR = np.array(pred_y_LR)\n", 2319 | "\n", 2320 | " true_0.append(np.mean(real_y_star))\n", 2321 | " estimator_LR_0.append(np.mean(pred_y_LR))\n", 2322 | " estimator_0.append(np.mean(pred_y))\n", 2323 | " estimator_debias_0.append(np.mean(pred_y_debiased))\n", 2324 | " \n", 2325 | " #record\n", 2326 | " real_y_star_0 = real_y_star.copy()\n", 2327 | " pred_y_LR_0 = pred_y_LR.copy()\n", 2328 | " pred_y_0 = pred_y.copy()\n", 2329 | " pred_y_debiased_0 = pred_y_debiased.copy()\n", 2330 | " \n", 2331 | " #at t=t_star\n", 2332 | " t_star_base = t_star.copy()\n", 2333 | " \n", 2334 | " for i in range(n_est):\n", 2335 | " for j in range(m):\n", 2336 | " x_all_set[i][-m+j] = t_star_base[j+1]\n", 2337 | " allset = torch.utils.data.TensorDataset(torch.Tensor(x_all_set), torch.Tensor(y_all_set)) # create your datset\n", 2338 | " allloader = torch.utils.data.DataLoader(\n", 2339 | " allset, batch_size=1000, shuffle=False)\n", 2340 | " pred_y = []\n", 2341 | " for i, data_ in enumerate(allloader, 0):\n", 2342 | " inputs, labels = data_\n", 2343 | " inputs, labels = Variable(inputs), Variable(labels)\n", 2344 | " outputs = net(inputs)\n", 2345 | " pred_y.append(outputs.tolist())\n", 2346 | " pred_y = np.array(pred_y).reshape(n_est)\n", 2347 | " \n", 2348 | "\n", 2349 | " real_y_star = []\n", 2350 | " for i in range(n_est):\n", 2351 | " real_y_star.append(0.05*np.random.uniform(-1, 1)+generate_y_true_1(coef,samples_x_est[i],t_star_base))\n", 2352 | "\n", 2353 | "\n", 2354 | " G_theta = []\n", 2355 | " G_theta_loss = []\n", 2356 | " t_obs_ = samples_t_est.copy()\n", 2357 | " cnt = 0\n", 2358 | "\n", 2359 | " for beta_temp in beta_:\n", 2360 | "\n", 2361 | " lambda_ = np.zeros([m+2, m+2])#asig use m+2\n", 2362 | " u_ = beta_temp.dot(t_star_base)\n", 2363 | " G_theta.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*t_star_base, [1/(np.exp(-u_)+1)]))\n", 2364 | "\n", 2365 | " u_ = beta_temp.dot(samples_t_est[cnt])\n", 2366 | " G_theta_loss.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*samples_t_est[cnt], [1/(np.exp(-u_)+1)]))\n", 2367 | "\n", 2368 | " cnt += 1\n", 2369 | "\n", 2370 | "\n", 2371 | " lambda_inv_loss_prime = []\n", 2372 | " for i in range(n_est):\n", 2373 | " lambda_inv_loss_prime.append(2*(pred_y_loss[i]-real_y[i])*lambda_inv[i].dot(G_theta_loss[i]))\n", 2374 | " lambda_inv_loss_prime = np.array(lambda_inv_loss_prime)\n", 2375 | "\n", 2376 | " pred_y_debiased = []\n", 2377 | " for i in range(n_est):\n", 2378 | " pred_y_debiased.append(pred_y[i]-G_theta[i].dot(lambda_inv_loss_prime[i]))\n", 2379 | " \n", 2380 | " pred_y_LR = []\n", 2381 | " for i in range(n_est): \n", 2382 | " pred_y_LR.append(np.append(samples_x_est[i], t_star_base).dot(coef_LR))\n", 2383 | " pred_y_LR = np.array(pred_y_LR)\n", 2384 | "\n", 2385 | " true_1.append(np.mean(real_y_star))\n", 2386 | " estimator_LR_1.append(np.mean(pred_y_LR))\n", 2387 | " estimator_1.append(np.mean(pred_y))\n", 2388 | " estimator_debias_1.append(np.mean(pred_y_debiased))\n", 2389 | " \n", 2390 | " p_.append(stats.ttest_ind(real_y_star, real_y_star_0)[1])\n", 2391 | " p_LR.append(stats.ttest_ind(pred_y_LR, pred_y_LR_0)[1]) \n", 2392 | " p_DL.append(stats.ttest_ind(pred_y, pred_y_0)[1])\n", 2393 | " p_DDL.append(stats.ttest_ind(pred_y_debiased, pred_y_debiased_0)[1])\n", 2394 | " \n", 2395 | " \n", 2396 | " #calculate additive\n", 2397 | " pred_y_additive = 0\n", 2398 | " pred_y_additive_var = 0\n", 2399 | " n_LA_samples = 0\n", 2400 | " for single_exp_index in np.where(t_star == 1)[0]:\n", 2401 | " ttt = np.zeros(m+1)\n", 2402 | " ttt[0] = 1\n", 2403 | " ttt[single_exp_index] = 1\n", 2404 | " pred_y_additive += samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].mean()\n", 2405 | " pred_y_additive_var += samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].var()\n", 2406 | " n_LA_samples += (samples_t_est == ttt).sum()\n", 2407 | " ttt = np.zeros(m+1)\n", 2408 | " ttt[0] = 1\n", 2409 | " pred_y_additive -= (np.sum(t_star))*samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].mean()\n", 2410 | " estimator_additive_1.append(pred_y_additive)\n", 2411 | " t_value = pred_y_additive/np.sqrt(pred_y_additive_var*(np.sum(t_star))/n_LA_samples)\n", 2412 | " p_LA.append(stats.t.sf(abs(t_value), df=1))\n", 2413 | "\n", 2414 | " " 2415 | ] 2416 | }, 2417 | { 2418 | "cell_type": "code", 2419 | "execution_count": null, 2420 | "id": "ab8e34ef", 2421 | "metadata": { 2422 | "ExecuteTime": { 2423 | "end_time": "2023-02-11T04:34:14.870076Z", 2424 | "start_time": "2023-02-11T04:34:14.824707Z" 2425 | } 2426 | }, 2427 | "outputs": [], 2428 | "source": [ 2429 | "test_err_np = np.zeros(n_rep)\n", 2430 | "for i in range(n_rep):\n", 2431 | " test_err_np[i] = test_err[i].tolist()\n", 2432 | "test_err = test_err_np.copy()\n", 2433 | "del test_err_np\n", 2434 | "\n", 2435 | "p_ = np.array(p_).reshape((n_rep, 2**m))\n", 2436 | "p_LA = np.array(p_LA).reshape((n_rep, 2**m))\n", 2437 | "p_LR = np.array(p_LR).reshape((n_rep, 2**m))\n", 2438 | "p_DL = np.array(p_DL).reshape((n_rep, 2**m))\n", 2439 | "p_DDL = np.array(p_DDL).reshape((n_rep, 2**m))\n", 2440 | "true_0 = np.array(true_0).reshape((n_rep, 2**m))\n", 2441 | "estimator_LR_0 = np.array(estimator_LR_0).reshape((n_rep, 2**m))\n", 2442 | "estimator_0 = np.array(estimator_0).reshape((n_rep, 2**m))\n", 2443 | "estimator_debias_0 = np.array(estimator_debias_0).reshape((n_rep, 2**m))\n", 2444 | "true_1 = np.array(true_1).reshape((n_rep, 2**m))\n", 2445 | "estimator_LR_1 = np.array(estimator_LR_1).reshape((n_rep, 2**m))\n", 2446 | "estimator_1 = np.array(estimator_1).reshape((n_rep, 2**m))\n", 2447 | "estimator_debias_1 = np.array(estimator_debias_1).reshape((n_rep, 2**m))\n", 2448 | "estimator_additive_1 = np.array(estimator_additive_1).reshape((n_rep, 2**m))\n", 2449 | "index = 0\n", 2450 | "\n", 2451 | "#LA LR SDL DeDL\n", 2452 | "ape_2 = np.zeros([2**m, 4, n_rep])\n", 2453 | "se = np.zeros([2**m, 4, n_rep])\n", 2454 | "ae = np.zeros([2**m, 4, n_rep])\n", 2455 | "CDR = np.zeros([2**m, 4, n_rep])\n", 2456 | "for i in range(n_rep):\n", 2457 | " CDR[0,:,i] = 1\n", 2458 | " for j in range(1,2**m):\n", 2459 | " true_eff = true_1[i,j]-true_0[i,j]\n", 2460 | " la = estimator_additive_1[i,j]\n", 2461 | " lr = estimator_LR_1[i,j]-estimator_LR_0[i,j]\n", 2462 | " dl = estimator_1[i,j]-estimator_0[i,j]\n", 2463 | " dedl = estimator_debias_1[i,j]-estimator_debias_0[i,j]\n", 2464 | " if (p_[i,j]<=0.05 and p_LA[i,j]<=0.05 and la*true_eff>0) or (p_[i,j]>0.05 and p_LA[i,j]>0.05):\n", 2465 | " CDR[j,0,i] = 1\n", 2466 | " if (p_[i,j]<=0.05 and p_LR[i,j]<=0.05 and lr*true_eff>0) or (p_[i,j]>0.05 and p_LR[i,j]>0.05):\n", 2467 | " CDR[j,1,i] = 1\n", 2468 | " if (p_[i,j]<=0.05 and p_DL[i,j]<=0.05 and dl*true_eff>0) or (p_[i,j]>0.05 and p_DL[i,j]>0.05):\n", 2469 | " CDR[j,2,i] = 1\n", 2470 | " if (p_[i,j]<=0.05 and p_DDL[i,j]<=0.05 and dedl*true_eff>0) or (p_[i,j]>0.05 and p_DDL[i,j]>0.05):\n", 2471 | " CDR[j,3,i] = 1\n", 2472 | " \n", 2473 | "\n", 2474 | "print('----------------All combos--------------------')\n", 2475 | "for t_star in t_star_all:\n", 2476 | " #print('Treatment effect when t = ', t_star[1:])\n", 2477 | " true_effect = true_1[:,index]-true_0[:,index]\n", 2478 | " add_effect = estimator_additive_1[:,index]\n", 2479 | " LR_effect = estimator_LR_1[:,index]-estimator_LR_0[:,index] \n", 2480 | " no_bias_effect = estimator_1[:,index]-estimator_0[:,index]\n", 2481 | " debias_effect = estimator_debias_1[:,index]-estimator_debias_0[:,index]\n", 2482 | " \n", 2483 | " ape_2[index, 0, :] = np.abs((add_effect-true_effect))/(np.abs(true_effect))\n", 2484 | " ape_2[index, 1, :] = np.abs((LR_effect-true_effect))/(np.abs(true_effect))\n", 2485 | " ape_2[index, 2, :] = np.abs((no_bias_effect-true_effect))/(np.abs(true_effect))\n", 2486 | " ape_2[index, 3, :] = np.abs((debias_effect-true_effect))/(np.abs(true_effect))\n", 2487 | " \n", 2488 | " #if p_[:,index]>=0.05:#not significant treatment effect\n", 2489 | " ape_2[index, 0, p_[:,index]>=0.05] = 0\n", 2490 | " ape_2[index, 1, p_[:,index]>=0.05] = 0\n", 2491 | " ape_2[index, 2, p_[:,index]>=0.05] = 0\n", 2492 | " ape_2[index, 3, p_[:,index]>=0.05] = 0\n", 2493 | " \n", 2494 | " se[index, 0, :] = ((add_effect - true_effect)**2)\n", 2495 | " se[index, 1, :] = ((LR_effect - true_effect)**2) \n", 2496 | " se[index, 2, :] = ((no_bias_effect - true_effect)**2)\n", 2497 | " se[index, 3, :] = ((debias_effect - true_effect)**2)\n", 2498 | " \n", 2499 | " ae[index, 0, :] = np.abs((add_effect - true_effect))\n", 2500 | " ae[index, 1, :] = np.abs((LR_effect - true_effect))\n", 2501 | " ae[index, 2, :] = np.abs((no_bias_effect - true_effect))\n", 2502 | " ae[index, 3, :] = np.abs((debias_effect - true_effect))\n", 2503 | " \n", 2504 | " index += 1\n", 2505 | " \n", 2506 | "print('------------------------------------')\n", 2507 | "print('MSE of LA ','|', np.mean(se[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,0,:], axis=0)), scale=st.sem(np.mean(se[:,0,:], axis=0))))\n", 2508 | "print('MSE of LR ','|', np.mean(se[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,1,:], axis=0)), scale=st.sem(np.mean(se[:,1,:], axis=0))))\n", 2509 | "print('MSE of SDL ','|', np.mean(se[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,2,:], axis=0)), scale=st.sem(np.mean(se[:,2,:], axis=0))))\n", 2510 | "print('MSE of DeDL ','|', np.mean(se[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,3,:], axis=0)), scale=st.sem(np.mean(se[:,3,:], axis=0))))\n", 2511 | "print('------------------------------------')\n", 2512 | "print('MAE of LA ','|', np.mean(ae[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,0,:], axis=0)), scale=st.sem(np.mean(ae[:,0,:], axis=0))))\n", 2513 | "print('MAE of LR ','|', np.mean(ae[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,1,:], axis=0)), scale=st.sem(np.mean(ae[:,1,:], axis=0))))\n", 2514 | "print('MAE of SDL ','|', np.mean(ae[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,2,:], axis=0)), scale=st.sem(np.mean(ae[:,2,:], axis=0))))\n", 2515 | "print('MAE of DeDL ','|', np.mean(ae[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,3,:], axis=0)), scale=st.sem(np.mean(ae[:,3,:], axis=0))))\n", 2516 | "print('------------------------------------')\n", 2517 | "print('MAPE of LA ','|', np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)), \n", 2518 | " scale=st.sem(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0))))\n", 2519 | "print('MAPE of LR ','|', np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)), \n", 2520 | " scale=st.sem(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0))))\n", 2521 | "print('MAPE of SDL ','|', np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)), \n", 2522 | " scale=st.sem(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0))))\n", 2523 | "print('MAPE of DeDL ','|', np.mean(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0)), \n", 2524 | " scale=st.sem(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0))))\n", 2525 | "\n", 2526 | "print('------------------------------------')\n", 2527 | "print('CDR of LA ','|', np.mean(CDR[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,0,:], axis=0)), scale=st.sem(np.mean(CDR[:,0,:], axis=0))))\n", 2528 | "print('CDR of LR ','|', np.mean(CDR[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,1,:], axis=0)), scale=st.sem(np.mean(CDR[:,1,:], axis=0))))\n", 2529 | "print('CDR of SDL ','|', np.mean(CDR[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,2,:], axis=0)), scale=st.sem(np.mean(CDR[:,2,:], axis=0))))\n", 2530 | "print('CDR of DeDL ','|', np.mean(CDR[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,3,:], axis=0)), scale=st.sem(np.mean(CDR[:,3,:], axis=0))))\n", 2531 | "\n", 2532 | "print('------------------------------------')\n", 2533 | "\n", 2534 | "cnt_bai = np.zeros([n_rep, 4])\n", 2535 | "for i in range(n_rep):\n", 2536 | " max_effect = -10000*np.ones(5)\n", 2537 | " max_effect_index = np.zeros(5)\n", 2538 | " for index in range(2**m):\n", 2539 | " true_effect = true_1[i,index]-true_0[i,index]\n", 2540 | " add_effect = estimator_additive_1[i,index]\n", 2541 | " LR_effect = estimator_LR_1[i,index]-estimator_LR_0[i,index]\n", 2542 | " no_bias_effect = estimator_1[i,index]-estimator_0[i,index]\n", 2543 | " debias_effect = estimator_debias_1[i,index]-estimator_debias_0[i,index]\n", 2544 | " \n", 2545 | " if true_effect > max_effect[4]:\n", 2546 | " max_effect[4] = true_effect\n", 2547 | " max_effect_index[4] = index\n", 2548 | " if add_effect > max_effect[0]:\n", 2549 | " max_effect[0] = add_effect\n", 2550 | " max_effect_index[0] = index\n", 2551 | " if LR_effect > max_effect[1]:\n", 2552 | " max_effect[1] = LR_effect\n", 2553 | " max_effect_index[1] = index\n", 2554 | " if no_bias_effect > max_effect[2]:\n", 2555 | " max_effect[2] = no_bias_effect\n", 2556 | " max_effect_index[2] = index\n", 2557 | " if debias_effect > max_effect[3]:\n", 2558 | " max_effect[3] = debias_effect\n", 2559 | " max_effect_index[3] = index\n", 2560 | " \n", 2561 | " if max_effect_index[0] == max_effect_index[4]:\n", 2562 | " cnt_bai[i, 0] = 1\n", 2563 | " if max_effect_index[1] == max_effect_index[4]:\n", 2564 | " cnt_bai[i, 1] = 1\n", 2565 | " if max_effect_index[2] == max_effect_index[4]:\n", 2566 | " cnt_bai[i, 2] = 1\n", 2567 | " if max_effect_index[3] == max_effect_index[4]:\n", 2568 | " cnt_bai[i, 3] = 1\n", 2569 | " \n", 2570 | "print('BAI of LA ','|', np.mean(cnt_bai[:, 0]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 0]), scale=st.sem(cnt_bai[:, 0])))\n", 2571 | "print('BAI of LR ','|', np.mean(cnt_bai[:, 1]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 1]), scale=st.sem(cnt_bai[:, 1])))\n", 2572 | "print('BAI of SDL ','|', np.mean(cnt_bai[:, 2]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 2]), scale=st.sem(cnt_bai[:, 2])))\n", 2573 | "print('BAI of DeDL ','|', np.mean(cnt_bai[:, 3]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 3]), scale=st.sem(cnt_bai[:, 3])))\n", 2574 | "print('------------------------------------')\n", 2575 | "print('ABS of ATE ','|', np.mean(np.mean(np.abs(true_1-true_0), axis=1)),'|',st.t.interval(0.95, n_rep-1, \n", 2576 | " loc=np.mean(np.mean(np.abs(true_1-true_0), axis=1)), \n", 2577 | " scale=st.sem(np.mean(np.abs(true_1-true_0), axis=1))))" 2578 | ] 2579 | }, 2580 | { 2581 | "cell_type": "markdown", 2582 | "id": "08b78b53", 2583 | "metadata": {}, 2584 | "source": [ 2585 | "## Model Misspecified - High Order Interactions" 2586 | ] 2587 | }, 2588 | { 2589 | "cell_type": "code", 2590 | "execution_count": null, 2591 | "id": "92d3a502", 2592 | "metadata": {}, 2593 | "outputs": [], 2594 | "source": [ 2595 | "gamma_ = 1\n", 2596 | "m = 4 #number of experiments\n", 2597 | "d_c = 10 #number of features\n", 2598 | "\n", 2599 | "all_combo = list(powerset(list(np.arange(1, m+1))))\n", 2600 | "t_combo = []\n", 2601 | "for i in all_combo:\n", 2602 | " t = np.zeros(m+1)\n", 2603 | " t[0] = 1\n", 2604 | " t[i] = 1\n", 2605 | " t_combo.append(t)\n", 2606 | "t_combo = np.int16(t_combo)\n", 2607 | "t_dist = (1/2**m)*np.ones(2**m)\n", 2608 | "t_combo_obs = []\n", 2609 | "for i in range(m+1):\n", 2610 | " t = np.zeros(m+1)\n", 2611 | " t[0] = 1\n", 2612 | " t[i] = 1\n", 2613 | " t_combo_obs.append(t)\n", 2614 | "t = np.zeros(m+1)\n", 2615 | "t[[0,1,2]] = 1\n", 2616 | "t_combo_obs.append(t)\n", 2617 | "t_combo_obs = np.int16(t_combo_obs)\n", 2618 | "t_dist_obs = (1/(m+2))*np.ones(m+2)\n", 2619 | "\n", 2620 | "t_star_all = t_combo.copy()\n", 2621 | "idx = (t_combo[:,None]!=t_combo_obs).any(-1).all(1)\n", 2622 | "t_star_unobs = t_combo[idx]\n", 2623 | "\n", 2624 | "\n", 2625 | "n_est = int(m*500)\n", 2626 | "n_train = int(m*500)\n", 2627 | "reg_term = 0.0005\n", 2628 | "#reg_loss = 0.001\n", 2629 | "reg_loss = 0\n", 2630 | "train_epochs = 2000\n", 2631 | "test_thres = 0.2*gamma_+0.1\n", 2632 | "if gamma_ == 0:\n", 2633 | " test_thres = 0.03\n", 2634 | " \n", 2635 | "n_cnver_test = 10000\n", 2636 | "lr = 0.05\n", 2637 | "\n", 2638 | "feature_list = []\n", 2639 | "for i in range(d_c):\n", 2640 | " feature_list.append(str('x')+str(i+1))\n", 2641 | "t_list = []\n", 2642 | "for i in range(m+1):\n", 2643 | " t_list.append(str('t')+str(i))\n", 2644 | "\n", 2645 | "true_0 = []#real ATE in base\n", 2646 | "estimator_LR_0 = []#ATE by LR in base\n", 2647 | "estimator_0 = []#ATE by SDL in base\n", 2648 | "estimator_debias_0 = []#ATE by DEDL in base\n", 2649 | "\n", 2650 | "true_1 = []#real ATE in treatment\n", 2651 | "estimator_LR_1 = []#ATE by LR in treatment\n", 2652 | "estimator_1 = []#ATE by SDL in treatment\n", 2653 | "estimator_debias_1 = []#ATE by DEDL in treatment\n", 2654 | "\n", 2655 | "estimator_additive_1 = []#ATE by additive in treatment\n", 2656 | "p_ = []\n", 2657 | "p_LA = []\n", 2658 | "p_LR = []\n", 2659 | "p_DL = []\n", 2660 | "p_DDL = []\n", 2661 | "test_err = []" 2662 | ] 2663 | }, 2664 | { 2665 | "cell_type": "code", 2666 | "execution_count": null, 2667 | "id": "53e57c64", 2668 | "metadata": {}, 2669 | "outputs": [], 2670 | "source": [ 2671 | "rep_index = 0\n", 2672 | "\n", 2673 | "while rep_index < n_rep:\n", 2674 | " print('# of replication:', rep_index)\n", 2675 | " coef_linear = np.random.uniform(0, 0.5, [d_c, 2**m-11])\n", 2676 | " c_true = np.random.uniform(10, 20)\n", 2677 | " d_true = 0\n", 2678 | " \n", 2679 | "\n", 2680 | " def generate_y_true(coef_linear, c, d, x, t, n):\n", 2681 | " y = np.zeros(n)\n", 2682 | " y_error = 0.05*np.random.uniform(-1, 1, n)\n", 2683 | " for i in range(n):\n", 2684 | " t_ = t[i]\n", 2685 | " t_ext = np.append(t_, np.array([t_[1]*t_[2], t_[1]*t_[3], t_[1]*t_[4], t_[2]*t_[3], t_[2]*t_[4], t_[3]*t_[4],\n", 2686 | " t_[1]*t_[2]*t_[3], t_[2]*t_[3]*t_[4], t_[3]*t_[4]*t_[1], t_[4]*t_[1]*t_[2], \n", 2687 | " t_[1]*t_[2]*t_[3]*t_[4]]))\n", 2688 | " y[i] = gamma_*(x[i].dot(coef_linear)).dot(t_ext[0:-11]) + y_error[i]\n", 2689 | " return y, y_error\n", 2690 | " def generate_y_true_1(coef_linear, x, t_):\n", 2691 | " t_ext = np.append(t_, np.array([t_[1]*t_[2], t_[1]*t_[3], t_[1]*t_[4], t_[2]*t_[3], t_[2]*t_[4], t_[3]*t_[4],\n", 2692 | " t_[1]*t_[2]*t_[3], t_[2]*t_[3]*t_[4], t_[3]*t_[4]*t_[1], t_[4]*t_[1]*t_[2], \n", 2693 | " t_[1]*t_[2]*t_[3]*t_[4]]))\n", 2694 | " return gamma_*(x.dot(coef_linear)).dot(t_ext[0:-11])\n", 2695 | "\n", 2696 | " \n", 2697 | " #generate samples for estimation\n", 2698 | " samples_x_est = generate_x(d_c, n_train)\n", 2699 | " samples_t_est = generate_t(t_combo_obs, t_dist_obs, n_train)\n", 2700 | " samples_y_est, samples_y_error_est = generate_y_true(coef_linear, c_true, d_true, samples_x_est, samples_t_est, n_train)\n", 2701 | " full_data_est = np.append(samples_x_est, samples_t_est, axis=1)\n", 2702 | " full_data_est = np.append(full_data_est, samples_y_est.reshape(n_train,1), axis=1)\n", 2703 | " data_obs = pd.DataFrame(full_data_est, columns = feature_list + t_list + ['y'])\n", 2704 | "\n", 2705 | " model_LR = sm.OLS(samples_y_est.reshape(n_train,1), np.append(samples_x_est, samples_t_est, axis=1))\n", 2706 | " results_LR = model_LR.fit()\n", 2707 | " coef_LR = results_LR.params\n", 2708 | " \n", 2709 | " opt = parser.parse_args()\n", 2710 | " train_samples = int(0.7*data_obs.shape[0])\n", 2711 | " x_train_set = np.float32(data_obs)[0:train_samples, 0:-1]\n", 2712 | " y_train_set = np.float32(data_obs)[0:train_samples, -1]\n", 2713 | " x_test_set = np.float32(data_obs)[train_samples:, 0:-1]\n", 2714 | " y_test_set = np.float32(data_obs)[train_samples:, -1]\n", 2715 | "\n", 2716 | " trainset = torch.utils.data.TensorDataset(torch.Tensor(x_train_set), torch.Tensor(y_train_set)) # create your datset\n", 2717 | " trainloader = torch.utils.data.DataLoader(\n", 2718 | " trainset, batch_size=opt.batch_size_train, shuffle=True)\n", 2719 | "\n", 2720 | " testset = torch.utils.data.TensorDataset(torch.Tensor(x_test_set), torch.Tensor(y_test_set)) # create your datset\n", 2721 | " testloader = torch.utils.data.DataLoader(\n", 2722 | " testset, batch_size=opt.batch_size_test, shuffle=True)\n", 2723 | " net = FNN_asig()\n", 2724 | " test_accuracy = 100\n", 2725 | " print('---------warm-up train---------')\n", 2726 | " wp_cnt = 0\n", 2727 | " while(test_accuracy >= 1):\n", 2728 | " wp_cnt += 1\n", 2729 | " net = FNN_asig()\n", 2730 | " with torch.no_grad():\n", 2731 | " net.layer3.weight[0, 0] = float(np.max(y_train_set))\n", 2732 | " criterion = nn.MSELoss()\n", 2733 | " #criterion = nn.L1Loss()\n", 2734 | " #warm-up train\n", 2735 | " test_accuracy = 100\n", 2736 | " optimizer = optim.Adam(net.parameters(), lr = 0.1, weight_decay = opt.wd)\n", 2737 | " for epoch in range(200):\n", 2738 | " for i, data_i in enumerate(trainloader, 0):\n", 2739 | " inputs, labels = data_i\n", 2740 | " inputs, labels = Variable(inputs), Variable(labels)\n", 2741 | " optimizer.zero_grad()\n", 2742 | " outputs = net(inputs)\n", 2743 | " l1_norm = sum(p.abs().sum() for p in net.parameters())\n", 2744 | " loss = criterion(outputs, labels) + reg_loss*l1_norm\n", 2745 | " loss.backward()\n", 2746 | " optimizer.step()\n", 2747 | " #train_accuracy = calculate_mse(trainloader, opt.is_gpu, net)\n", 2748 | " test_accuracy = calculate_mse(testloader, opt.is_gpu, net)\n", 2749 | " #if epoch%100 == 0:\n", 2750 | " # print(\"Iteration: {0} | Training MSE: {1} | Test MSE: {2}\".format(\n", 2751 | " # epoch, train_accuracy, test_accuracy))\n", 2752 | " if test_accuracy < 1:\n", 2753 | " break\n", 2754 | " if wp_cnt >= 5:\n", 2755 | " break\n", 2756 | " #training \n", 2757 | " print('---------train---------')\n", 2758 | " optimizer = optim.Adam(net.parameters(), lr = lr, weight_decay = opt.wd)\n", 2759 | " criterion = nn.MSELoss()\n", 2760 | " for epoch in range(train_epochs):\n", 2761 | " for i, data_i in enumerate(trainloader, 0):\n", 2762 | " inputs, labels = data_i\n", 2763 | " inputs, labels = Variable(inputs), Variable(labels)\n", 2764 | " optimizer.zero_grad()\n", 2765 | " outputs = net(inputs)\n", 2766 | " l1_norm = sum(p.abs().sum() for p in net.parameters())\n", 2767 | " loss = criterion(outputs, labels) + reg_loss*l1_norm\n", 2768 | " loss.backward()\n", 2769 | " optimizer.step()\n", 2770 | " train_accuracy = calculate_mse(trainloader, opt.is_gpu, net)\n", 2771 | " test_accuracy = calculate_mse(testloader, opt.is_gpu, net)\n", 2772 | " if epoch%100 == 0:\n", 2773 | " print(\"Iteration: {0} | Training MSE: {1} | Test MSE: {2}\".format(\n", 2774 | " epoch, train_accuracy, test_accuracy))\n", 2775 | " if test_accuracy < test_thres and epoch > 500:\n", 2776 | " break \n", 2777 | " if test_accuracy > test_thres:\n", 2778 | " continue\n", 2779 | " else:\n", 2780 | " rep_index += 1\n", 2781 | " test_err.append(test_accuracy)\n", 2782 | " #rep_index += 1\n", 2783 | " \n", 2784 | " activation = {}\n", 2785 | " net.layer1.register_forward_hook(get_activation('layer1'))\n", 2786 | " true_value=[]\n", 2787 | " for name, param in net.named_parameters():\n", 2788 | " if param.requires_grad:\n", 2789 | " true_value.append(param.data)\n", 2790 | " c_est = np.array(true_value[-1].tolist())[0][0]\n", 2791 | " d_est = 0\n", 2792 | "\n", 2793 | " \n", 2794 | " #generate samples for inference\n", 2795 | " samples_x_est = generate_x(d_c, n_est)\n", 2796 | " samples_t_est = generate_t(t_combo_obs, t_dist_obs, n_est)\n", 2797 | " samples_y_est, samples_y_error_est = generate_y_true(coef_linear, c_true, d_true, samples_x_est, samples_t_est, n_est)\n", 2798 | " full_data_est = np.append(samples_x_est, samples_t_est, axis=1)\n", 2799 | " full_data_est = np.append(full_data_est, samples_y_est.reshape(n_est,1), axis=1)\n", 2800 | "\n", 2801 | " for t_star in t_star_all:\n", 2802 | " data_est = pd.DataFrame(full_data_est, columns = feature_list + t_list + ['y'])\n", 2803 | " #at t=0\n", 2804 | " t_star_base = np.zeros(m+1)\n", 2805 | " t_star_base[0] = 1\n", 2806 | " x_all_set=np.float32(data_est)[:, 0:-1]\n", 2807 | " y_all_set=np.float32(data_est)[:, -1]\n", 2808 | " allset = torch.utils.data.TensorDataset(torch.Tensor(x_all_set), torch.Tensor(y_all_set)) # create your datset\n", 2809 | " allloader = torch.utils.data.DataLoader(\n", 2810 | " allset, batch_size=1000, shuffle=False)\n", 2811 | " \n", 2812 | " beta_ = []\n", 2813 | " pred_y_loss = []\n", 2814 | " for i, data_ in enumerate(allloader, 0):\n", 2815 | " inputs, labels = data_\n", 2816 | " inputs, labels = Variable(inputs), Variable(labels)\n", 2817 | " outputs = net(inputs)\n", 2818 | " beta_.append(activation['layer1'].tolist()) \n", 2819 | " pred_y_loss.append(outputs.tolist())\n", 2820 | " beta_ = np.array(beta_).reshape(n_est, m+1)\n", 2821 | " pred_y_loss = np.array(pred_y_loss).reshape(n_est)\n", 2822 | " \n", 2823 | " real_y = samples_y_est.copy()\n", 2824 | " \n", 2825 | " for j in range(m):\n", 2826 | " x_all_set[:, -m+j] = t_star_base[j+1]\n", 2827 | " allset = torch.utils.data.TensorDataset(torch.Tensor(x_all_set), torch.Tensor(y_all_set)) # create your datset\n", 2828 | " allloader = torch.utils.data.DataLoader(\n", 2829 | " allset, batch_size=1000, shuffle=False)\n", 2830 | " pred_y = []\n", 2831 | " for i, data_ in enumerate(allloader, 0):\n", 2832 | " inputs, labels = data_\n", 2833 | " inputs, labels = Variable(inputs), Variable(labels)\n", 2834 | " outputs = net(inputs)\n", 2835 | " pred_y.append(outputs.tolist())\n", 2836 | " pred_y = np.array(pred_y).reshape(n_est)\n", 2837 | " \n", 2838 | " real_y_star = []\n", 2839 | " for i in range(n_est):\n", 2840 | " real_y_star.append(0.05*np.random.uniform(-1, 1)+generate_y_true_1(coef_linear,samples_x_est[i],t_star_base))\n", 2841 | "\n", 2842 | " lambda_inv = []\n", 2843 | " G_theta = []\n", 2844 | " G_theta_loss = []\n", 2845 | " t_obs_ = samples_t_est.copy()\n", 2846 | " cnt = 0\n", 2847 | "\n", 2848 | " for beta_temp in beta_:\n", 2849 | "\n", 2850 | " lambda_ = np.zeros([m+2, m+2])#asig use m+2\n", 2851 | " u_ = beta_temp.dot(t_star_base)\n", 2852 | " G_theta.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*t_star_base, [1/(np.exp(-u_)+1)]))\n", 2853 | "\n", 2854 | " u_ = beta_temp.dot(samples_t_est[cnt])\n", 2855 | " G_theta_loss.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*samples_t_est[cnt], [1/(np.exp(-u_)+1)]))\n", 2856 | "\n", 2857 | " cnt += 1\n", 2858 | "\n", 2859 | " for i in range(t_combo_obs.shape[0]):\n", 2860 | " t = t_combo_obs[i]\n", 2861 | " y = avg_y(beta_temp, c_true, d_true, t)\n", 2862 | " u = beta_temp.dot(t)\n", 2863 | " G_prime = np.append(c_est*np.exp(-u)/(np.exp(-u)+1)**2*t, [1/(np.exp(-u)+1)])\n", 2864 | " lambda_ += t_dist_obs[i]*2*np.outer(G_prime, G_prime)\n", 2865 | "\n", 2866 | " try:\n", 2867 | " lambda_inv.append(inv(lambda_ + reg_term*np.eye(m+2)))\n", 2868 | " except:\n", 2869 | " print('Singular matrix')\n", 2870 | "\n", 2871 | " lambda_inv_loss_prime = []\n", 2872 | " for i in range(n_est):\n", 2873 | " lambda_inv_loss_prime.append(2*(pred_y_loss[i]-real_y[i])*lambda_inv[i].dot(G_theta_loss[i]))\n", 2874 | " lambda_inv_loss_prime = np.array(lambda_inv_loss_prime)\n", 2875 | "\n", 2876 | " pred_y_debiased = []\n", 2877 | " for i in range(n_est):\n", 2878 | " pred_y_debiased.append(pred_y[i]-G_theta[i].dot(lambda_inv_loss_prime[i]))\n", 2879 | " \n", 2880 | " pred_y_LR = []\n", 2881 | " for i in range(n_est): \n", 2882 | " pred_y_LR.append(np.append(samples_x_est[i], t_star_base).dot(coef_LR))\n", 2883 | " pred_y_LR = np.array(pred_y_LR)\n", 2884 | "\n", 2885 | " true_0.append(np.mean(real_y_star))\n", 2886 | " estimator_LR_0.append(np.mean(pred_y_LR))\n", 2887 | " estimator_0.append(np.mean(pred_y))\n", 2888 | " estimator_debias_0.append(np.mean(pred_y_debiased))\n", 2889 | " \n", 2890 | " #record\n", 2891 | " real_y_star_0 = real_y_star.copy()\n", 2892 | " pred_y_LR_0 = pred_y_LR.copy()\n", 2893 | " pred_y_0 = pred_y.copy()\n", 2894 | " pred_y_debiased_0 = pred_y_debiased.copy()\n", 2895 | " \n", 2896 | " #at t=t_star\n", 2897 | " t_star_base = t_star.copy()\n", 2898 | " \n", 2899 | " for i in range(n_est):\n", 2900 | " for j in range(m):\n", 2901 | " x_all_set[i][-m+j] = t_star_base[j+1]\n", 2902 | " allset = torch.utils.data.TensorDataset(torch.Tensor(x_all_set), torch.Tensor(y_all_set)) # create your datset\n", 2903 | " allloader = torch.utils.data.DataLoader(\n", 2904 | " allset, batch_size=1000, shuffle=False)\n", 2905 | " pred_y = []\n", 2906 | " for i, data_ in enumerate(allloader, 0):\n", 2907 | " inputs, labels = data_\n", 2908 | " inputs, labels = Variable(inputs), Variable(labels)\n", 2909 | " outputs = net(inputs)\n", 2910 | " pred_y.append(outputs.tolist())\n", 2911 | " pred_y = np.array(pred_y).reshape(n_est)\n", 2912 | " \n", 2913 | "\n", 2914 | " real_y_star = []\n", 2915 | " for i in range(n_est):\n", 2916 | " real_y_star.append(0.05*np.random.uniform(-1, 1)+generate_y_true_1(coef_linear,samples_x_est[i],t_star_base))\n", 2917 | "\n", 2918 | "\n", 2919 | " G_theta = []\n", 2920 | " G_theta_loss = []\n", 2921 | " t_obs_ = samples_t_est.copy()\n", 2922 | " cnt = 0\n", 2923 | "\n", 2924 | " for beta_temp in beta_:\n", 2925 | "\n", 2926 | " lambda_ = np.zeros([m+2, m+2])#asig use m+2\n", 2927 | " u_ = beta_temp.dot(t_star_base)\n", 2928 | " G_theta.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*t_star_base, [1/(np.exp(-u_)+1)]))\n", 2929 | "\n", 2930 | " u_ = beta_temp.dot(samples_t_est[cnt])\n", 2931 | " G_theta_loss.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*samples_t_est[cnt], [1/(np.exp(-u_)+1)]))\n", 2932 | "\n", 2933 | " cnt += 1\n", 2934 | "\n", 2935 | "\n", 2936 | " lambda_inv_loss_prime = []\n", 2937 | " for i in range(n_est):\n", 2938 | " lambda_inv_loss_prime.append(2*(pred_y_loss[i]-real_y[i])*lambda_inv[i].dot(G_theta_loss[i]))\n", 2939 | " lambda_inv_loss_prime = np.array(lambda_inv_loss_prime)\n", 2940 | "\n", 2941 | " pred_y_debiased = []\n", 2942 | " for i in range(n_est):\n", 2943 | " pred_y_debiased.append(pred_y[i]-G_theta[i].dot(lambda_inv_loss_prime[i]))\n", 2944 | " \n", 2945 | " pred_y_LR = []\n", 2946 | " for i in range(n_est): \n", 2947 | " pred_y_LR.append(np.append(samples_x_est[i], t_star_base).dot(coef_LR))\n", 2948 | " pred_y_LR = np.array(pred_y_LR)\n", 2949 | "\n", 2950 | " true_1.append(np.mean(real_y_star))\n", 2951 | " estimator_LR_1.append(np.mean(pred_y_LR))\n", 2952 | " estimator_1.append(np.mean(pred_y))\n", 2953 | " estimator_debias_1.append(np.mean(pred_y_debiased))\n", 2954 | " \n", 2955 | " p_.append(stats.ttest_ind(real_y_star, real_y_star_0)[1])\n", 2956 | " p_LR.append(stats.ttest_ind(pred_y_LR, pred_y_LR_0)[1]) \n", 2957 | " p_DL.append(stats.ttest_ind(pred_y, pred_y_0)[1])\n", 2958 | " p_DDL.append(stats.ttest_ind(pred_y_debiased, pred_y_debiased_0)[1])\n", 2959 | " \n", 2960 | " \n", 2961 | " #calculate additive\n", 2962 | " pred_y_additive = 0\n", 2963 | " pred_y_additive_var = 0\n", 2964 | " n_LA_samples = 0\n", 2965 | " for single_exp_index in np.where(t_star == 1)[0]:\n", 2966 | " ttt = np.zeros(m+1)\n", 2967 | " ttt[0] = 1\n", 2968 | " ttt[single_exp_index] = 1\n", 2969 | " pred_y_additive += samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].mean()\n", 2970 | " pred_y_additive_var += samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].var()\n", 2971 | " n_LA_samples += (samples_t_est == ttt).sum()\n", 2972 | " ttt = np.zeros(m+1)\n", 2973 | " ttt[0] = 1\n", 2974 | " pred_y_additive -= (np.sum(t_star))*samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].mean()\n", 2975 | " estimator_additive_1.append(pred_y_additive)\n", 2976 | " t_value = pred_y_additive/np.sqrt(pred_y_additive_var*(np.sum(t_star))/n_LA_samples)\n", 2977 | " p_LA.append(stats.t.sf(abs(t_value), df=1))\n", 2978 | "\n", 2979 | " " 2980 | ] 2981 | }, 2982 | { 2983 | "cell_type": "code", 2984 | "execution_count": null, 2985 | "id": "098ae200", 2986 | "metadata": {}, 2987 | "outputs": [], 2988 | "source": [ 2989 | "test_err_np = np.zeros(n_rep)\n", 2990 | "for i in range(n_rep):\n", 2991 | " test_err_np[i] = test_err[i].tolist()\n", 2992 | "test_err = test_err_np.copy()\n", 2993 | "del test_err_np\n", 2994 | "\n", 2995 | "p_ = np.array(p_).reshape((n_rep, 2**m))\n", 2996 | "p_LA = np.array(p_LA).reshape((n_rep, 2**m))\n", 2997 | "p_LR = np.array(p_LR).reshape((n_rep, 2**m))\n", 2998 | "p_DL = np.array(p_DL).reshape((n_rep, 2**m))\n", 2999 | "p_DDL = np.array(p_DDL).reshape((n_rep, 2**m))\n", 3000 | "true_0 = np.array(true_0).reshape((n_rep, 2**m))\n", 3001 | "estimator_LR_0 = np.array(estimator_LR_0).reshape((n_rep, 2**m))\n", 3002 | "estimator_0 = np.array(estimator_0).reshape((n_rep, 2**m))\n", 3003 | "estimator_debias_0 = np.array(estimator_debias_0).reshape((n_rep, 2**m))\n", 3004 | "true_1 = np.array(true_1).reshape((n_rep, 2**m))\n", 3005 | "estimator_LR_1 = np.array(estimator_LR_1).reshape((n_rep, 2**m))\n", 3006 | "estimator_1 = np.array(estimator_1).reshape((n_rep, 2**m))\n", 3007 | "estimator_debias_1 = np.array(estimator_debias_1).reshape((n_rep, 2**m))\n", 3008 | "estimator_additive_1 = np.array(estimator_additive_1).reshape((n_rep, 2**m))\n", 3009 | "index = 0\n", 3010 | "\n", 3011 | "#LA LR SDL DeDL\n", 3012 | "ape_2 = np.zeros([2**m, 4, n_rep])\n", 3013 | "se = np.zeros([2**m, 4, n_rep])\n", 3014 | "ae = np.zeros([2**m, 4, n_rep])\n", 3015 | "CDR = np.zeros([2**m, 4, n_rep])\n", 3016 | "for i in range(n_rep):\n", 3017 | " CDR[0,:,i] = 1\n", 3018 | " for j in range(1,2**m):\n", 3019 | " true_eff = true_1[i,j]-true_0[i,j]\n", 3020 | " la = estimator_additive_1[i,j]\n", 3021 | " lr = estimator_LR_1[i,j]-estimator_LR_0[i,j]\n", 3022 | " dl = estimator_1[i,j]-estimator_0[i,j]\n", 3023 | " dedl = estimator_debias_1[i,j]-estimator_debias_0[i,j]\n", 3024 | " if (p_[i,j]<=0.05 and p_LA[i,j]<=0.05 and la*true_eff>0) or (p_[i,j]>0.05 and p_LA[i,j]>0.05):\n", 3025 | " CDR[j,0,i] = 1\n", 3026 | " if (p_[i,j]<=0.05 and p_LR[i,j]<=0.05 and lr*true_eff>0) or (p_[i,j]>0.05 and p_LR[i,j]>0.05):\n", 3027 | " CDR[j,1,i] = 1\n", 3028 | " if (p_[i,j]<=0.05 and p_DL[i,j]<=0.05 and dl*true_eff>0) or (p_[i,j]>0.05 and p_DL[i,j]>0.05):\n", 3029 | " CDR[j,2,i] = 1\n", 3030 | " if (p_[i,j]<=0.05 and p_DDL[i,j]<=0.05 and dedl*true_eff>0) or (p_[i,j]>0.05 and p_DDL[i,j]>0.05):\n", 3031 | " CDR[j,3,i] = 1\n", 3032 | " \n", 3033 | "\n", 3034 | "print('----------------All combos--------------------')\n", 3035 | "for t_star in t_star_all:\n", 3036 | " #print('Treatment effect when t = ', t_star[1:])\n", 3037 | " true_effect = true_1[:,index]-true_0[:,index]\n", 3038 | " add_effect = estimator_additive_1[:,index]\n", 3039 | " LR_effect = estimator_LR_1[:,index]-estimator_LR_0[:,index] \n", 3040 | " no_bias_effect = estimator_1[:,index]-estimator_0[:,index]\n", 3041 | " debias_effect = estimator_debias_1[:,index]-estimator_debias_0[:,index]\n", 3042 | " \n", 3043 | " ape_2[index, 0, :] = np.abs((add_effect-true_effect))/(np.abs(true_effect))\n", 3044 | " ape_2[index, 1, :] = np.abs((LR_effect-true_effect))/(np.abs(true_effect))\n", 3045 | " ape_2[index, 2, :] = np.abs((no_bias_effect-true_effect))/(np.abs(true_effect))\n", 3046 | " ape_2[index, 3, :] = np.abs((debias_effect-true_effect))/(np.abs(true_effect))\n", 3047 | " \n", 3048 | " #if p_[:,index]>=0.05:#not significant treatment effect\n", 3049 | " ape_2[index, 0, p_[:,index]>=0.05] = 0\n", 3050 | " ape_2[index, 1, p_[:,index]>=0.05] = 0\n", 3051 | " ape_2[index, 2, p_[:,index]>=0.05] = 0\n", 3052 | " ape_2[index, 3, p_[:,index]>=0.05] = 0\n", 3053 | " \n", 3054 | " se[index, 0, :] = ((add_effect - true_effect)**2)\n", 3055 | " se[index, 1, :] = ((LR_effect - true_effect)**2) \n", 3056 | " se[index, 2, :] = ((no_bias_effect - true_effect)**2)\n", 3057 | " se[index, 3, :] = ((debias_effect - true_effect)**2)\n", 3058 | " \n", 3059 | " ae[index, 0, :] = np.abs((add_effect - true_effect))\n", 3060 | " ae[index, 1, :] = np.abs((LR_effect - true_effect))\n", 3061 | " ae[index, 2, :] = np.abs((no_bias_effect - true_effect))\n", 3062 | " ae[index, 3, :] = np.abs((debias_effect - true_effect))\n", 3063 | " \n", 3064 | " index += 1\n", 3065 | " \n", 3066 | "print('------------------------------------')\n", 3067 | "print('MSE of LA ','|', np.mean(se[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,0,:], axis=0)), scale=st.sem(np.mean(se[:,0,:], axis=0))))\n", 3068 | "print('MSE of LR ','|', np.mean(se[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,1,:], axis=0)), scale=st.sem(np.mean(se[:,1,:], axis=0))))\n", 3069 | "print('MSE of SDL ','|', np.mean(se[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,2,:], axis=0)), scale=st.sem(np.mean(se[:,2,:], axis=0))))\n", 3070 | "print('MSE of DeDL ','|', np.mean(se[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,3,:], axis=0)), scale=st.sem(np.mean(se[:,3,:], axis=0))))\n", 3071 | "print('------------------------------------')\n", 3072 | "print('MAE of LA ','|', np.mean(ae[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,0,:], axis=0)), scale=st.sem(np.mean(ae[:,0,:], axis=0))))\n", 3073 | "print('MAE of LR ','|', np.mean(ae[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,1,:], axis=0)), scale=st.sem(np.mean(ae[:,1,:], axis=0))))\n", 3074 | "print('MAE of SDL ','|', np.mean(ae[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,2,:], axis=0)), scale=st.sem(np.mean(ae[:,2,:], axis=0))))\n", 3075 | "print('MAE of DeDL ','|', np.mean(ae[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,3,:], axis=0)), scale=st.sem(np.mean(ae[:,3,:], axis=0))))\n", 3076 | "print('------------------------------------')\n", 3077 | "print('MAPE of LA ','|', np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)), \n", 3078 | " scale=st.sem(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0))))\n", 3079 | "print('MAPE of LR ','|', np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)), \n", 3080 | " scale=st.sem(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0))))\n", 3081 | "print('MAPE of SDL ','|', np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)), \n", 3082 | " scale=st.sem(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0))))\n", 3083 | "print('MAPE of DeDL ','|', np.mean(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0)), \n", 3084 | " scale=st.sem(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0))))\n", 3085 | "\n", 3086 | "print('------------------------------------')\n", 3087 | "print('CDR of LA ','|', np.mean(CDR[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,0,:], axis=0)), scale=st.sem(np.mean(CDR[:,0,:], axis=0))))\n", 3088 | "print('CDR of LR ','|', np.mean(CDR[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,1,:], axis=0)), scale=st.sem(np.mean(CDR[:,1,:], axis=0))))\n", 3089 | "print('CDR of SDL ','|', np.mean(CDR[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,2,:], axis=0)), scale=st.sem(np.mean(CDR[:,2,:], axis=0))))\n", 3090 | "print('CDR of DeDL ','|', np.mean(CDR[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,3,:], axis=0)), scale=st.sem(np.mean(CDR[:,3,:], axis=0))))\n", 3091 | "\n", 3092 | "print('------------------------------------')\n", 3093 | "\n", 3094 | "cnt_bai = np.zeros([n_rep, 4])\n", 3095 | "for i in range(n_rep):\n", 3096 | " max_effect = -10000*np.ones(5)\n", 3097 | " max_effect_index = np.zeros(5)\n", 3098 | " for index in range(2**m):\n", 3099 | " true_effect = true_1[i,index]-true_0[i,index]\n", 3100 | " add_effect = estimator_additive_1[i,index]\n", 3101 | " LR_effect = estimator_LR_1[i,index]-estimator_LR_0[i,index]\n", 3102 | " no_bias_effect = estimator_1[i,index]-estimator_0[i,index]\n", 3103 | " debias_effect = estimator_debias_1[i,index]-estimator_debias_0[i,index]\n", 3104 | " \n", 3105 | " if true_effect > max_effect[4]:\n", 3106 | " max_effect[4] = true_effect\n", 3107 | " max_effect_index[4] = index\n", 3108 | " if add_effect > max_effect[0]:\n", 3109 | " max_effect[0] = add_effect\n", 3110 | " max_effect_index[0] = index\n", 3111 | " if LR_effect > max_effect[1]:\n", 3112 | " max_effect[1] = LR_effect\n", 3113 | " max_effect_index[1] = index\n", 3114 | " if no_bias_effect > max_effect[2]:\n", 3115 | " max_effect[2] = no_bias_effect\n", 3116 | " max_effect_index[2] = index\n", 3117 | " if debias_effect > max_effect[3]:\n", 3118 | " max_effect[3] = debias_effect\n", 3119 | " max_effect_index[3] = index\n", 3120 | " \n", 3121 | " if max_effect_index[0] == max_effect_index[4]:\n", 3122 | " cnt_bai[i, 0] = 1\n", 3123 | " if max_effect_index[1] == max_effect_index[4]:\n", 3124 | " cnt_bai[i, 1] = 1\n", 3125 | " if max_effect_index[2] == max_effect_index[4]:\n", 3126 | " cnt_bai[i, 2] = 1\n", 3127 | " if max_effect_index[3] == max_effect_index[4]:\n", 3128 | " cnt_bai[i, 3] = 1\n", 3129 | " \n", 3130 | "print('BAI of LA ','|', np.mean(cnt_bai[:, 0]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 0]), scale=st.sem(cnt_bai[:, 0])))\n", 3131 | "print('BAI of LR ','|', np.mean(cnt_bai[:, 1]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 1]), scale=st.sem(cnt_bai[:, 1])))\n", 3132 | "print('BAI of SDL ','|', np.mean(cnt_bai[:, 2]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 2]), scale=st.sem(cnt_bai[:, 2])))\n", 3133 | "print('BAI of DeDL ','|', np.mean(cnt_bai[:, 3]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 3]), scale=st.sem(cnt_bai[:, 3])))\n", 3134 | "print('------------------------------------')\n", 3135 | "print('ABS of ATE ','|', np.mean(np.mean(np.abs(true_1-true_0), axis=1)),'|',st.t.interval(0.95, n_rep-1, \n", 3136 | " loc=np.mean(np.mean(np.abs(true_1-true_0), axis=1)), \n", 3137 | " scale=st.sem(np.mean(np.abs(true_1-true_0), axis=1))))" 3138 | ] 3139 | }, 3140 | { 3141 | "cell_type": "markdown", 3142 | "id": "80f7520b", 3143 | "metadata": {}, 3144 | "source": [ 3145 | "# Imbalanced Covariates" 3146 | ] 3147 | }, 3148 | { 3149 | "cell_type": "markdown", 3150 | "id": "e600ee51", 3151 | "metadata": { 3152 | "ExecuteTime": { 3153 | "end_time": "2023-02-10T18:15:46.310749Z", 3154 | "start_time": "2023-02-10T18:15:46.293567Z" 3155 | } 3156 | }, 3157 | "source": [ 3158 | "## Data generating process definition\n", 3159 | "define the level of lambda = 2, 1, 0.5" 3160 | ] 3161 | }, 3162 | { 3163 | "cell_type": "code", 3164 | "execution_count": null, 3165 | "id": "641582fd", 3166 | "metadata": { 3167 | "ExecuteTime": { 3168 | "end_time": "2023-02-11T04:34:20.840449Z", 3169 | "start_time": "2023-02-11T04:34:20.830487Z" 3170 | } 3171 | }, 3172 | "outputs": [], 3173 | "source": [ 3174 | "lambda_level = 2\n", 3175 | "precision_reb = 1/2 #precision of rebalance, precision_reb=1 means no rebalancing\n", 3176 | "\n", 3177 | "\n", 3178 | "m = 4 #number of experiments\n", 3179 | "d_c = 10 #number of features\n", 3180 | "\n", 3181 | "def generate_x_imb(d_c, n):\n", 3182 | " x = np.random.uniform(0, 1, [n, d_c])\n", 3183 | " x[:,-1] = np.random.exponential(1.0/lambda_level, n)\n", 3184 | " return x\n", 3185 | "\n", 3186 | "def re_x(x_1, prec=1):\n", 3187 | " x_1_b = x_1.copy()\n", 3188 | " if prec == 1:\n", 3189 | " return x_1_b\n", 3190 | " else:\n", 3191 | " n_bucket = int(1/prec)\n", 3192 | " n_samples = x_1_b.shape[0]\n", 3193 | " min_n = 10000\n", 3194 | " for lb in np.arange(0,1,prec):\n", 3195 | " rb = lb + prec\n", 3196 | " min_n = min(min_n, (np.greater(x_1_b[:,-1],lb)*np.less(x_1_b[:,-1],rb)).sum())\n", 3197 | " x_emp = []\n", 3198 | " for lb in np.arange(0,1,prec):\n", 3199 | " rb = lb + prec\n", 3200 | " x_temp = x_1_b[np.greater(x_1_b[:,-1],lb)*np.less(x_1_b[:,-1],rb),:]\n", 3201 | " x_index = np.random.choice(x_temp.shape[0], min_n, replace=False)\n", 3202 | " x_emp.append(x_temp[x_index])\n", 3203 | " return np.array(x_emp).reshape(min_n*n_bucket, x_1_b.shape[1])\n", 3204 | "\n", 3205 | "\n", 3206 | "all_combo = list(powerset(list(np.arange(1, m+1))))\n", 3207 | "t_combo = []\n", 3208 | "for i in all_combo:\n", 3209 | " t = np.zeros(m+1)\n", 3210 | " t[0] = 1\n", 3211 | " t[i] = 1\n", 3212 | " t_combo.append(t)\n", 3213 | "t_combo = np.int16(t_combo)\n", 3214 | "t_dist = (1/2**m)*np.ones(2**m)\n", 3215 | "t_combo_obs = []\n", 3216 | "for i in range(m+1):\n", 3217 | " t = np.zeros(m+1)\n", 3218 | " t[0] = 1\n", 3219 | " t[i] = 1\n", 3220 | " t_combo_obs.append(t)\n", 3221 | "t = np.zeros(m+1)\n", 3222 | "t[[0,1,2]] = 1\n", 3223 | "t_combo_obs.append(t)\n", 3224 | "t_combo_obs = np.int16(t_combo_obs)\n", 3225 | "t_dist_obs = (1/(m+2))*np.ones(m+2)\n", 3226 | "\n", 3227 | "t_star_all = t_combo.copy()\n", 3228 | "idx = (t_combo[:,None]!=t_combo_obs).any(-1).all(1)\n", 3229 | "t_star_unobs = t_combo[idx]\n", 3230 | "\n", 3231 | "\n", 3232 | "n_est = int(m*500)\n", 3233 | "n_train = int(m*500)\n", 3234 | "reg_term = 0.0005\n", 3235 | "reg_loss = 0\n", 3236 | "train_epochs = 2000\n", 3237 | "test_thres = 0.5\n", 3238 | "lr = 0.05\n", 3239 | "\n", 3240 | "feature_list = []\n", 3241 | "for i in range(d_c):\n", 3242 | " feature_list.append(str('x')+str(i+1))\n", 3243 | "t_list = []\n", 3244 | "for i in range(m+1):\n", 3245 | " t_list.append(str('t')+str(i))\n", 3246 | "\n", 3247 | "true_0 = []#real ATE in base\n", 3248 | "estimator_0 = []#ATE by ML in base\n", 3249 | "estimator_debias_0 = []#ATE by DML in base\n", 3250 | "estimator_LR_0 = []#ATE by LR in base\n", 3251 | "\n", 3252 | "true_1 = []#real ATE in treatment\n", 3253 | "estimator_1 = []#ATE by ML in treatment\n", 3254 | "estimator_debias_1 = []#ATE by DML in treatment\n", 3255 | "estimator_LR_1 = []#ATE by LR in treatment\n", 3256 | "\n", 3257 | "estimator_additive_1 = []#ATE by additive in treatment\n", 3258 | "\n", 3259 | "p_ = []\n", 3260 | "p_LA = []\n", 3261 | "p_LR = []\n", 3262 | "p_DL = []\n", 3263 | "p_DDL = []\n", 3264 | "test_err = []" 3265 | ] 3266 | }, 3267 | { 3268 | "cell_type": "markdown", 3269 | "id": "2d25e7e5", 3270 | "metadata": {}, 3271 | "source": [ 3272 | "## Comparison of LA, LR, SDL, DeDL" 3273 | ] 3274 | }, 3275 | { 3276 | "cell_type": "code", 3277 | "execution_count": null, 3278 | "id": "55516121", 3279 | "metadata": { 3280 | "ExecuteTime": { 3281 | "end_time": "2023-02-11T04:35:29.323324Z", 3282 | "start_time": "2023-02-11T04:34:26.773694Z" 3283 | } 3284 | }, 3285 | "outputs": [], 3286 | "source": [ 3287 | "rep_index = 0\n", 3288 | "\n", 3289 | "while rep_index < n_rep:\n", 3290 | " \n", 3291 | " print('# of replication:', rep_index)\n", 3292 | " coef = np.random.uniform(-0.5, 0.5, [d_c, m+1])\n", 3293 | " c_true = np.random.uniform(10, 20)\n", 3294 | " d_true = 0\n", 3295 | " \n", 3296 | " def generate_y_true(coef, c, d, x, t, n):\n", 3297 | " y = np.zeros(n)\n", 3298 | " y_error = 0.05*np.random.uniform(-1, 1, n)\n", 3299 | " for i in range(n):\n", 3300 | " y[i] = c/(1+np.exp(-((x[i].dot(coef))).dot(t[i]))) + d + y_error[i]\n", 3301 | " return y, y_error\n", 3302 | " def generate_y_true_1(coef, x, t):\n", 3303 | " return c_true/(1+np.exp(-((x.dot(coef))).dot(t)))\n", 3304 | " def generate_beta_true_1(coef, x, t):\n", 3305 | " return (x.dot(coef))\n", 3306 | " \n", 3307 | " #generate samples for estimation\n", 3308 | " samples_x_est = generate_x_imb(d_c, n_train)\n", 3309 | " samples_x_est = re_x(samples_x_est, precision_reb)\n", 3310 | " n_train_ = samples_x_est.shape[0]\n", 3311 | " samples_t_est = generate_t(t_combo_obs, t_dist_obs, n_train_)\n", 3312 | " samples_y_est, samples_y_error_est = generate_y_true(coef, c_true, d_true, samples_x_est, samples_t_est, n_train_)\n", 3313 | " full_data_est = np.append(samples_x_est, samples_t_est, axis=1)\n", 3314 | " full_data_est = np.append(full_data_est, samples_y_est.reshape(n_train_,1), axis=1)\n", 3315 | " data_obs = pd.DataFrame(full_data_est, columns = feature_list + t_list + ['y'])\n", 3316 | " \n", 3317 | " model_LR = sm.OLS(samples_y_est.reshape(n_train_,1), np.append(samples_x_est, samples_t_est, axis=1))\n", 3318 | " results_LR = model_LR.fit()\n", 3319 | " coef_LR = results_LR.params\n", 3320 | "\n", 3321 | " opt = parser.parse_args()\n", 3322 | " train_samples = int(0.7*data_obs.shape[0])\n", 3323 | " x_train_set = np.float32(data_obs)[0:train_samples, 0:-1]\n", 3324 | " y_train_set = np.float32(data_obs)[0:train_samples, -1]\n", 3325 | " x_test_set = np.float32(data_obs)[train_samples:, 0:-1]\n", 3326 | " y_test_set = np.float32(data_obs)[train_samples:, -1]\n", 3327 | "\n", 3328 | " trainset = torch.utils.data.TensorDataset(torch.Tensor(x_train_set), torch.Tensor(y_train_set)) # create your datset\n", 3329 | " trainloader = torch.utils.data.DataLoader(\n", 3330 | " trainset, batch_size=opt.batch_size_train, shuffle=True)\n", 3331 | "\n", 3332 | " testset = torch.utils.data.TensorDataset(torch.Tensor(x_test_set), torch.Tensor(y_test_set)) # create your datset\n", 3333 | " testloader = torch.utils.data.DataLoader(\n", 3334 | " testset, batch_size=opt.batch_size_test, shuffle=True)\n", 3335 | " net = FNN_asig()\n", 3336 | " test_accuracy = 100\n", 3337 | " print('---------warm-up train---------')\n", 3338 | " wp_cnt = 0\n", 3339 | " while(test_accuracy >= 1):\n", 3340 | " wp_cnt += 1\n", 3341 | " net = FNN_asig()\n", 3342 | " with torch.no_grad():\n", 3343 | " net.layer3.weight[0, 0] = float(np.max(y_train_set))\n", 3344 | " criterion = nn.MSELoss()\n", 3345 | " #criterion = nn.L1Loss()\n", 3346 | " #warm-up train\n", 3347 | " test_accuracy = 100\n", 3348 | " optimizer = optim.Adam(net.parameters(), lr = 0.1, weight_decay = opt.wd)\n", 3349 | " for epoch in range(200):\n", 3350 | " for i, data_i in enumerate(trainloader, 0):\n", 3351 | " inputs, labels = data_i\n", 3352 | " inputs, labels = Variable(inputs), Variable(labels)\n", 3353 | " optimizer.zero_grad()\n", 3354 | " outputs = net(inputs)\n", 3355 | " l1_norm = sum(p.abs().sum() for p in net.parameters())\n", 3356 | " loss = criterion(outputs, labels) + reg_loss*l1_norm\n", 3357 | " loss.backward()\n", 3358 | " optimizer.step()\n", 3359 | " #train_accuracy = calculate_mse(trainloader, opt.is_gpu, net)\n", 3360 | " test_accuracy = calculate_mse(testloader, opt.is_gpu, net)\n", 3361 | " #if epoch%100 == 0:\n", 3362 | " # print(\"Iteration: {0} | Training MSE: {1} | Test MSE: {2}\".format(\n", 3363 | " # epoch, train_accuracy, test_accuracy))\n", 3364 | " if test_accuracy < 1:\n", 3365 | " break\n", 3366 | " if wp_cnt >= 5:\n", 3367 | " break\n", 3368 | " #training \n", 3369 | " print('---------train---------')\n", 3370 | " optimizer = optim.Adam(net.parameters(), lr = lr, weight_decay = opt.wd)\n", 3371 | " criterion = nn.MSELoss()\n", 3372 | " for epoch in range(train_epochs):\n", 3373 | " for i, data_i in enumerate(trainloader, 0):\n", 3374 | " inputs, labels = data_i\n", 3375 | " inputs, labels = Variable(inputs), Variable(labels)\n", 3376 | " optimizer.zero_grad()\n", 3377 | " outputs = net(inputs)\n", 3378 | " l1_norm = sum(p.abs().sum() for p in net.parameters())\n", 3379 | " loss = criterion(outputs, labels) + reg_loss*l1_norm\n", 3380 | " loss.backward()\n", 3381 | " optimizer.step()\n", 3382 | " train_accuracy = calculate_mse(trainloader, opt.is_gpu, net)\n", 3383 | " test_accuracy = calculate_mse(testloader, opt.is_gpu, net)\n", 3384 | " if epoch%100 == 0:\n", 3385 | " print(\"Iteration: {0} | Training MSE: {1} | Test MSE: {2}\".format(\n", 3386 | " epoch, train_accuracy, test_accuracy))\n", 3387 | " if test_accuracy < test_thres and epoch > 500:\n", 3388 | " break \n", 3389 | " if test_accuracy > test_thres:\n", 3390 | " continue\n", 3391 | " else:\n", 3392 | " rep_index += 1\n", 3393 | " test_err.append(test_accuracy)\n", 3394 | " #rep_index += 1\n", 3395 | " \n", 3396 | " activation = {}\n", 3397 | " net.layer1.register_forward_hook(get_activation('layer1'))\n", 3398 | " \n", 3399 | " #generate samples for inference\n", 3400 | " samples_x_est = generate_x_imb(d_c, n_est)\n", 3401 | " samples_x_est = re_x(samples_x_est, precision_reb)\n", 3402 | " n_est_ = samples_x_est.shape[0]\n", 3403 | " samples_t_est = generate_t(t_combo_obs, t_dist_obs, n_est_)\n", 3404 | " samples_y_est, samples_y_error_est = generate_y_true(coef, c_true, d_true, samples_x_est, samples_t_est, n_est_)\n", 3405 | " full_data_est = np.append(samples_x_est, samples_t_est, axis=1)\n", 3406 | " full_data_est = np.append(full_data_est, samples_y_est.reshape(n_est_,1), axis=1)\n", 3407 | "\n", 3408 | " for t_star in t_star_all:\n", 3409 | " data_est = pd.DataFrame(full_data_est, columns = feature_list + t_list + ['y'])\n", 3410 | " #at t=0\n", 3411 | " t_star_base = np.zeros(m+1)\n", 3412 | " t_star_base[0] = 1\n", 3413 | " x_all_set=np.float32(data_est)[:, 0:-1]\n", 3414 | " y_all_set=np.float32(data_est)[:, -1]\n", 3415 | " allset = torch.utils.data.TensorDataset(torch.Tensor(x_all_set), torch.Tensor(y_all_set)) # create your datset\n", 3416 | " allloader = torch.utils.data.DataLoader(\n", 3417 | " allset, batch_size=n_est_, shuffle=False)\n", 3418 | " \n", 3419 | " beta_ = []\n", 3420 | " pred_y_loss = []\n", 3421 | " for i, data_ in enumerate(allloader, 0):\n", 3422 | " inputs, labels = data_\n", 3423 | " inputs, labels = Variable(inputs), Variable(labels)\n", 3424 | " outputs = net(inputs)\n", 3425 | " beta_.append(activation['layer1'].tolist()) \n", 3426 | " pred_y_loss.append(outputs.tolist())\n", 3427 | " beta_ = np.array(beta_).reshape(n_est_, m+1)\n", 3428 | " pred_y_loss = np.array(pred_y_loss).reshape(n_est_)\n", 3429 | " \n", 3430 | " real_y = samples_y_est.copy()\n", 3431 | " \n", 3432 | " for j in range(m):\n", 3433 | " x_all_set[:, -m+j] = t_star_base[j+1]\n", 3434 | " allset = torch.utils.data.TensorDataset(torch.Tensor(x_all_set), torch.Tensor(y_all_set)) # create your datset\n", 3435 | " allloader = torch.utils.data.DataLoader(\n", 3436 | " allset, batch_size=n_est_, shuffle=False)\n", 3437 | " pred_y = []\n", 3438 | " for i, data_ in enumerate(allloader, 0):\n", 3439 | " inputs, labels = data_\n", 3440 | " inputs, labels = Variable(inputs), Variable(labels)\n", 3441 | " outputs = net(inputs)\n", 3442 | " pred_y.append(outputs.tolist())\n", 3443 | " pred_y = np.array(pred_y).reshape(n_est_)\n", 3444 | " \n", 3445 | " #real_y_star = []\n", 3446 | " #for i in range(n_est):\n", 3447 | " # real_y_star.append(0.05*np.random.uniform(-1, 1)+generate_y_true_1(coef,samples_x_est[i],t_star_base))\n", 3448 | "\n", 3449 | " lambda_inv = []\n", 3450 | " G_theta = []\n", 3451 | " G_theta_loss = []\n", 3452 | " t_obs_ = samples_t_est.copy()\n", 3453 | " cnt = 0\n", 3454 | "\n", 3455 | " for beta_temp in beta_:\n", 3456 | "\n", 3457 | " lambda_ = np.zeros([m+2, m+2])#asig use m+2\n", 3458 | " u_ = beta_temp.dot(t_star_base)\n", 3459 | " G_theta.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*t_star_base, [1/(np.exp(-u_)+1)]))\n", 3460 | "\n", 3461 | " u_ = beta_temp.dot(samples_t_est[cnt])\n", 3462 | " G_theta_loss.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*samples_t_est[cnt], [1/(np.exp(-u_)+1)]))\n", 3463 | "\n", 3464 | " cnt += 1\n", 3465 | "\n", 3466 | " for i in range(t_combo_obs.shape[0]):\n", 3467 | " t = t_combo_obs[i]\n", 3468 | " y = avg_y(beta_temp, c_true, d_true, t)\n", 3469 | " u = beta_temp.dot(t)\n", 3470 | " G_prime = np.append(c_est*np.exp(-u)/(np.exp(-u)+1)**2*t, [1/(np.exp(-u)+1)])\n", 3471 | " lambda_ += t_dist_obs[i]*2*np.outer(G_prime, G_prime)\n", 3472 | "\n", 3473 | " try:\n", 3474 | " lambda_inv.append(inv(lambda_ + reg_term*np.eye(m+2)))\n", 3475 | " except:\n", 3476 | " print('Singular matrix')\n", 3477 | "\n", 3478 | " lambda_inv_loss_prime = []\n", 3479 | " for i in range(n_est_):\n", 3480 | " lambda_inv_loss_prime.append(2*(pred_y_loss[i]-real_y[i])*lambda_inv[i].dot(G_theta_loss[i]))\n", 3481 | " lambda_inv_loss_prime = np.array(lambda_inv_loss_prime)\n", 3482 | "\n", 3483 | " pred_y_debiased = []\n", 3484 | " for i in range(n_est_):\n", 3485 | " pred_y_debiased.append(pred_y[i]-G_theta[i].dot(lambda_inv_loss_prime[i]))\n", 3486 | " \n", 3487 | " pred_y_LR = []\n", 3488 | " for i in range(n_est_): \n", 3489 | " pred_y_LR.append(np.append(samples_x_est[i], t_star_base).dot(coef_LR))\n", 3490 | " pred_y_LR = np.array(pred_y_LR)\n", 3491 | " \n", 3492 | " \n", 3493 | " samples_x_est_true = generate_x(d_c, 10000)\n", 3494 | " real_y_star = []\n", 3495 | " for i in range(10000):\n", 3496 | " real_y_star.append(0.05*np.random.uniform(-1, 1)+generate_y_true_1(coef,samples_x_est_true[i],t_star_base))\n", 3497 | " \n", 3498 | " \n", 3499 | "\n", 3500 | " estimator_0.append(np.mean(pred_y))\n", 3501 | " estimator_debias_0.append(np.mean(pred_y_debiased))\n", 3502 | " true_0.append(np.mean(real_y_star))\n", 3503 | " estimator_LR_0.append(np.mean(pred_y_LR))\n", 3504 | "\n", 3505 | " \n", 3506 | " #record\n", 3507 | " real_y_star_0 = real_y_star.copy()\n", 3508 | " pred_y_LR_0 = pred_y_LR.copy()\n", 3509 | " pred_y_0 = pred_y.copy()\n", 3510 | " \n", 3511 | " #record\n", 3512 | " real_y_star_0 = real_y_star.copy()\n", 3513 | " pred_y_0 = pred_y.copy()\n", 3514 | " pred_y_debiased_0 = pred_y_debiased.copy()\n", 3515 | " \n", 3516 | " #at t=t_star\n", 3517 | " t_star_base = t_star.copy()\n", 3518 | " \n", 3519 | " for i in range(n_est_):\n", 3520 | " for j in range(m):\n", 3521 | " x_all_set[i][-m+j] = t_star_base[j+1]\n", 3522 | " allset = torch.utils.data.TensorDataset(torch.Tensor(x_all_set), torch.Tensor(y_all_set)) # create your datset\n", 3523 | " allloader = torch.utils.data.DataLoader(\n", 3524 | " allset, batch_size=n_est_, shuffle=False)\n", 3525 | " pred_y = []\n", 3526 | " for i, data_ in enumerate(allloader, 0):\n", 3527 | " inputs, labels = data_\n", 3528 | " inputs, labels = Variable(inputs), Variable(labels)\n", 3529 | " outputs = net(inputs)\n", 3530 | " pred_y.append(outputs.tolist())\n", 3531 | " pred_y = np.array(pred_y).reshape(n_est_)\n", 3532 | " \n", 3533 | "\n", 3534 | " G_theta = []\n", 3535 | " G_theta_loss = []\n", 3536 | " t_obs_ = samples_t_est.copy()\n", 3537 | " cnt = 0\n", 3538 | "\n", 3539 | " for beta_temp in beta_:\n", 3540 | "\n", 3541 | " lambda_ = np.zeros([m+2, m+2])#asig use m+2\n", 3542 | " u_ = beta_temp.dot(t_star_base)\n", 3543 | " G_theta.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*t_star_base, [1/(np.exp(-u_)+1)]))\n", 3544 | "\n", 3545 | " u_ = beta_temp.dot(samples_t_est[cnt])\n", 3546 | " G_theta_loss.append(np.append(c_est*np.exp(-u_)/(np.exp(-u_)+1)**2*samples_t_est[cnt], [1/(np.exp(-u_)+1)]))\n", 3547 | "\n", 3548 | " cnt += 1\n", 3549 | "\n", 3550 | "\n", 3551 | " lambda_inv_loss_prime = []\n", 3552 | " for i in range(n_est_):\n", 3553 | " lambda_inv_loss_prime.append(2*(pred_y_loss[i]-real_y[i])*lambda_inv[i].dot(G_theta_loss[i]))\n", 3554 | " lambda_inv_loss_prime = np.array(lambda_inv_loss_prime)\n", 3555 | "\n", 3556 | " pred_y_debiased = []\n", 3557 | " for i in range(n_est_):\n", 3558 | " pred_y_debiased.append(pred_y[i]-G_theta[i].dot(lambda_inv_loss_prime[i]))\n", 3559 | " \n", 3560 | " pred_y_LR = []\n", 3561 | " for i in range(n_est_): \n", 3562 | " pred_y_LR.append(np.append(samples_x_est[i], t_star_base).dot(coef_LR))\n", 3563 | " pred_y_LR = np.array(pred_y_LR)\n", 3564 | " \n", 3565 | " \n", 3566 | " samples_x_est_true = generate_x(d_c, 10000)\n", 3567 | " real_y_star = []\n", 3568 | " for i in range(10000):\n", 3569 | " real_y_star.append(0.05*np.random.uniform(-1, 1)+generate_y_true_1(coef,samples_x_est_true[i],t_star_base))\n", 3570 | " \n", 3571 | " \n", 3572 | " estimator_1.append(np.mean(pred_y))\n", 3573 | " estimator_debias_1.append(np.mean(pred_y_debiased))\n", 3574 | " estimator_LR_1.append(np.mean(pred_y_LR))\n", 3575 | " \n", 3576 | " true_1.append(np.mean(real_y_star))\n", 3577 | " p_.append(stats.ttest_ind(real_y_star, real_y_star_0)[1])\n", 3578 | " p_DL.append(stats.ttest_ind(pred_y, pred_y_0)[1])\n", 3579 | " p_DDL.append(stats.ttest_ind(pred_y_debiased, pred_y_debiased_0)[1])\n", 3580 | " p_LR.append(stats.ttest_ind(pred_y_LR, pred_y_LR_0)[1]) \n", 3581 | " \n", 3582 | " #calculate additive\n", 3583 | " pred_y_additive = 0\n", 3584 | " pred_y_additive_var = 0\n", 3585 | " n_LA_samples = 0\n", 3586 | " for single_exp_index in np.where(t_star == 1)[0]:\n", 3587 | " ttt = np.zeros(m+1)\n", 3588 | " ttt[0] = 1\n", 3589 | " ttt[single_exp_index] = 1\n", 3590 | " pred_y_additive += samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].mean()\n", 3591 | " pred_y_additive_var += samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].var()\n", 3592 | " n_LA_samples += (samples_t_est == ttt).sum()\n", 3593 | " ttt = np.zeros(m+1)\n", 3594 | " ttt[0] = 1\n", 3595 | " pred_y_additive -= (np.sum(t_star))*samples_y_est[np.where((samples_t_est == ttt).all(axis=1))].mean()\n", 3596 | " estimator_additive_1.append(pred_y_additive)\n", 3597 | " t_value = pred_y_additive/np.sqrt(pred_y_additive_var*(np.sum(t_star))/n_LA_samples)\n", 3598 | " p_LA.append(stats.t.sf(abs(t_value), df=1))\n", 3599 | " \n", 3600 | "\n", 3601 | " " 3602 | ] 3603 | }, 3604 | { 3605 | "cell_type": "code", 3606 | "execution_count": null, 3607 | "id": "377b1417", 3608 | "metadata": { 3609 | "ExecuteTime": { 3610 | "end_time": "2023-02-11T04:35:32.919749Z", 3611 | "start_time": "2023-02-11T04:35:32.868686Z" 3612 | } 3613 | }, 3614 | "outputs": [], 3615 | "source": [ 3616 | "test_err_np = np.zeros(n_rep)\n", 3617 | "for i in range(n_rep):\n", 3618 | " test_err_np[i] = test_err[i].tolist()\n", 3619 | "test_err = test_err_np.copy()\n", 3620 | "del test_err_np\n", 3621 | "\n", 3622 | "p_ = np.array(p_).reshape((n_rep, 2**m))\n", 3623 | "p_LA = np.array(p_LA).reshape((n_rep, 2**m))\n", 3624 | "p_LR = np.array(p_LR).reshape((n_rep, 2**m))\n", 3625 | "p_DL = np.array(p_DL).reshape((n_rep, 2**m))\n", 3626 | "p_DDL = np.array(p_DDL).reshape((n_rep, 2**m))\n", 3627 | "true_0 = np.array(true_0).reshape((n_rep, 2**m))\n", 3628 | "estimator_LR_0 = np.array(estimator_LR_0).reshape((n_rep, 2**m))\n", 3629 | "estimator_0 = np.array(estimator_0).reshape((n_rep, 2**m))\n", 3630 | "estimator_debias_0 = np.array(estimator_debias_0).reshape((n_rep, 2**m))\n", 3631 | "true_1 = np.array(true_1).reshape((n_rep, 2**m))\n", 3632 | "estimator_LR_1 = np.array(estimator_LR_1).reshape((n_rep, 2**m))\n", 3633 | "estimator_1 = np.array(estimator_1).reshape((n_rep, 2**m))\n", 3634 | "estimator_debias_1 = np.array(estimator_debias_1).reshape((n_rep, 2**m))\n", 3635 | "estimator_additive_1 = np.array(estimator_additive_1).reshape((n_rep, 2**m))\n", 3636 | "index = 0\n", 3637 | "\n", 3638 | "#LA LR SDL DeDL\n", 3639 | "ape_2 = np.zeros([2**m, 4, n_rep])\n", 3640 | "se = np.zeros([2**m, 4, n_rep])\n", 3641 | "ae = np.zeros([2**m, 4, n_rep])\n", 3642 | "CDR = np.zeros([2**m, 4, n_rep])\n", 3643 | "for i in range(n_rep):\n", 3644 | " CDR[0,:,i] = 1\n", 3645 | " for j in range(1,2**m):\n", 3646 | " true_eff = true_1[i,j]-true_0[i,j]\n", 3647 | " la = estimator_additive_1[i,j]\n", 3648 | " lr = estimator_LR_1[i,j]-estimator_LR_0[i,j]\n", 3649 | " dl = estimator_1[i,j]-estimator_0[i,j]\n", 3650 | " dedl = estimator_debias_1[i,j]-estimator_debias_0[i,j]\n", 3651 | " if (p_[i,j]<=0.05 and p_LA[i,j]<=0.05 and la*true_eff>0) or (p_[i,j]>0.05 and p_LA[i,j]>0.05):\n", 3652 | " CDR[j,0,i] = 1\n", 3653 | " if (p_[i,j]<=0.05 and p_LR[i,j]<=0.05 and lr*true_eff>0) or (p_[i,j]>0.05 and p_LR[i,j]>0.05):\n", 3654 | " CDR[j,1,i] = 1\n", 3655 | " if (p_[i,j]<=0.05 and p_DL[i,j]<=0.05 and dl*true_eff>0) or (p_[i,j]>0.05 and p_DL[i,j]>0.05):\n", 3656 | " CDR[j,2,i] = 1\n", 3657 | " if (p_[i,j]<=0.05 and p_DDL[i,j]<=0.05 and dedl*true_eff>0) or (p_[i,j]>0.05 and p_DDL[i,j]>0.05):\n", 3658 | " CDR[j,3,i] = 1\n", 3659 | " \n", 3660 | "\n", 3661 | "print('----------------All combos--------------------')\n", 3662 | "for t_star in t_star_all:\n", 3663 | " #print('Treatment effect when t = ', t_star[1:])\n", 3664 | " true_effect = true_1[:,index]-true_0[:,index]\n", 3665 | " add_effect = estimator_additive_1[:,index]\n", 3666 | " LR_effect = estimator_LR_1[:,index]-estimator_LR_0[:,index] \n", 3667 | " no_bias_effect = estimator_1[:,index]-estimator_0[:,index]\n", 3668 | " debias_effect = estimator_debias_1[:,index]-estimator_debias_0[:,index]\n", 3669 | " \n", 3670 | " ape_2[index, 0, :] = np.abs((add_effect-true_effect))/(np.abs(true_effect))\n", 3671 | " ape_2[index, 1, :] = np.abs((LR_effect-true_effect))/(np.abs(true_effect))\n", 3672 | " ape_2[index, 2, :] = np.abs((no_bias_effect-true_effect))/(np.abs(true_effect))\n", 3673 | " ape_2[index, 3, :] = np.abs((debias_effect-true_effect))/(np.abs(true_effect))\n", 3674 | " \n", 3675 | " #if p_[:,index]>=0.05:#not significant treatment effect\n", 3676 | " ape_2[index, 0, p_[:,index]>=0.05] = 0\n", 3677 | " ape_2[index, 1, p_[:,index]>=0.05] = 0\n", 3678 | " ape_2[index, 2, p_[:,index]>=0.05] = 0\n", 3679 | " ape_2[index, 3, p_[:,index]>=0.05] = 0\n", 3680 | " \n", 3681 | " se[index, 0, :] = ((add_effect - true_effect)**2)\n", 3682 | " se[index, 1, :] = ((LR_effect - true_effect)**2) \n", 3683 | " se[index, 2, :] = ((no_bias_effect - true_effect)**2)\n", 3684 | " se[index, 3, :] = ((debias_effect - true_effect)**2)\n", 3685 | " \n", 3686 | " ae[index, 0, :] = np.abs((add_effect - true_effect))\n", 3687 | " ae[index, 1, :] = np.abs((LR_effect - true_effect))\n", 3688 | " ae[index, 2, :] = np.abs((no_bias_effect - true_effect))\n", 3689 | " ae[index, 3, :] = np.abs((debias_effect - true_effect))\n", 3690 | " \n", 3691 | " index += 1\n", 3692 | " \n", 3693 | "print('------------------------------------')\n", 3694 | "print('MSE of LA ','|', np.mean(se[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,0,:], axis=0)), scale=st.sem(np.mean(se[:,0,:], axis=0))))\n", 3695 | "print('MSE of LR ','|', np.mean(se[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,1,:], axis=0)), scale=st.sem(np.mean(se[:,1,:], axis=0))))\n", 3696 | "print('MSE of SDL ','|', np.mean(se[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,2,:], axis=0)), scale=st.sem(np.mean(se[:,2,:], axis=0))))\n", 3697 | "print('MSE of DeDL ','|', np.mean(se[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(se[:,3,:], axis=0)), scale=st.sem(np.mean(se[:,3,:], axis=0))))\n", 3698 | "print('------------------------------------')\n", 3699 | "print('MAE of LA ','|', np.mean(ae[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,0,:], axis=0)), scale=st.sem(np.mean(ae[:,0,:], axis=0))))\n", 3700 | "print('MAE of LR ','|', np.mean(ae[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,1,:], axis=0)), scale=st.sem(np.mean(ae[:,1,:], axis=0))))\n", 3701 | "print('MAE of SDL ','|', np.mean(ae[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,2,:], axis=0)), scale=st.sem(np.mean(ae[:,2,:], axis=0))))\n", 3702 | "print('MAE of DeDL ','|', np.mean(ae[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ae[:,3,:], axis=0)), scale=st.sem(np.mean(ae[:,3,:], axis=0))))\n", 3703 | "print('------------------------------------')\n", 3704 | "print('MAPE of LA ','|', np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0)), \n", 3705 | " scale=st.sem(np.mean(ape_2[:,0,:]*2**m/np.sum(ape_2[:,0,:]!=0, axis=0), axis=0))))\n", 3706 | "print('MAPE of LR ','|', np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0)), \n", 3707 | " scale=st.sem(np.mean(ape_2[:,1,:]*2**m/np.sum(ape_2[:,1,:]!=0, axis=0), axis=0))))\n", 3708 | "print('MAPE of SDL ','|', np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0)), \n", 3709 | " scale=st.sem(np.mean(ape_2[:,2,:]*2**m/np.sum(ape_2[:,2,:]!=0, axis=0), axis=0))))\n", 3710 | "print('MAPE of DeDL ','|', np.mean(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0)),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0)), \n", 3711 | " scale=st.sem(np.mean(ape_2[:,3,:]*2**m/np.sum(ape_2[:,3,:]!=0, axis=0), axis=0))))\n", 3712 | "\n", 3713 | "print('------------------------------------')\n", 3714 | "print('CDR of LA ','|', np.mean(CDR[:,0,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,0,:], axis=0)), scale=st.sem(np.mean(CDR[:,0,:], axis=0))))\n", 3715 | "print('CDR of LR ','|', np.mean(CDR[:,1,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,1,:], axis=0)), scale=st.sem(np.mean(CDR[:,1,:], axis=0))))\n", 3716 | "print('CDR of SDL ','|', np.mean(CDR[:,2,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,2,:], axis=0)), scale=st.sem(np.mean(CDR[:,2,:], axis=0))))\n", 3717 | "print('CDR of DeDL ','|', np.mean(CDR[:,3,:]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(np.mean(CDR[:,3,:], axis=0)), scale=st.sem(np.mean(CDR[:,3,:], axis=0))))\n", 3718 | "\n", 3719 | "print('------------------------------------')\n", 3720 | "\n", 3721 | "cnt_bai = np.zeros([n_rep, 4])\n", 3722 | "for i in range(n_rep):\n", 3723 | " max_effect = -10000*np.ones(5)\n", 3724 | " max_effect_index = np.zeros(5)\n", 3725 | " for index in range(2**m):\n", 3726 | " true_effect = true_1[i,index]-true_0[i,index]\n", 3727 | " add_effect = estimator_additive_1[i,index]\n", 3728 | " LR_effect = estimator_LR_1[i,index]-estimator_LR_0[i,index]\n", 3729 | " no_bias_effect = estimator_1[i,index]-estimator_0[i,index]\n", 3730 | " debias_effect = estimator_debias_1[i,index]-estimator_debias_0[i,index]\n", 3731 | " \n", 3732 | " if true_effect > max_effect[4]:\n", 3733 | " max_effect[4] = true_effect\n", 3734 | " max_effect_index[4] = index\n", 3735 | " if add_effect > max_effect[0]:\n", 3736 | " max_effect[0] = add_effect\n", 3737 | " max_effect_index[0] = index\n", 3738 | " if LR_effect > max_effect[1]:\n", 3739 | " max_effect[1] = LR_effect\n", 3740 | " max_effect_index[1] = index\n", 3741 | " if no_bias_effect > max_effect[2]:\n", 3742 | " max_effect[2] = no_bias_effect\n", 3743 | " max_effect_index[2] = index\n", 3744 | " if debias_effect > max_effect[3]:\n", 3745 | " max_effect[3] = debias_effect\n", 3746 | " max_effect_index[3] = index\n", 3747 | " \n", 3748 | " if max_effect_index[0] == max_effect_index[4]:\n", 3749 | " cnt_bai[i, 0] = 1\n", 3750 | " if max_effect_index[1] == max_effect_index[4]:\n", 3751 | " cnt_bai[i, 1] = 1\n", 3752 | " if max_effect_index[2] == max_effect_index[4]:\n", 3753 | " cnt_bai[i, 2] = 1\n", 3754 | " if max_effect_index[3] == max_effect_index[4]:\n", 3755 | " cnt_bai[i, 3] = 1\n", 3756 | " \n", 3757 | "print('BAI of LA ','|', np.mean(cnt_bai[:, 0]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 0]), scale=st.sem(cnt_bai[:, 0])))\n", 3758 | "print('BAI of LR ','|', np.mean(cnt_bai[:, 1]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 1]), scale=st.sem(cnt_bai[:, 1])))\n", 3759 | "print('BAI of SDL ','|', np.mean(cnt_bai[:, 2]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 2]), scale=st.sem(cnt_bai[:, 2])))\n", 3760 | "print('BAI of DeDL ','|', np.mean(cnt_bai[:, 3]),'|',st.t.interval(0.95, n_rep-1, loc=np.mean(cnt_bai[:, 3]), scale=st.sem(cnt_bai[:, 3])))\n", 3761 | "print('------------------------------------')\n", 3762 | "print('ABS of ATE ','|', np.mean(np.mean(np.abs(true_1-true_0), axis=1)),'|',st.t.interval(0.95, n_rep-1, \n", 3763 | " loc=np.mean(np.mean(np.abs(true_1-true_0), axis=1)), \n", 3764 | " scale=st.sem(np.mean(np.abs(true_1-true_0), axis=1))))" 3765 | ] 3766 | }, 3767 | { 3768 | "cell_type": "code", 3769 | "execution_count": null, 3770 | "id": "d28e8428", 3771 | "metadata": {}, 3772 | "outputs": [], 3773 | "source": [] 3774 | } 3775 | ], 3776 | "metadata": { 3777 | "kernelspec": { 3778 | "display_name": "Python 3 (ipykernel)", 3779 | "language": "python", 3780 | "name": "python3" 3781 | }, 3782 | "language_info": { 3783 | "codemirror_mode": { 3784 | "name": "ipython", 3785 | "version": 3 3786 | }, 3787 | "file_extension": ".py", 3788 | "mimetype": "text/x-python", 3789 | "name": "python", 3790 | "nbconvert_exporter": "python", 3791 | "pygments_lexer": "ipython3", 3792 | "version": "3.7.7" 3793 | }, 3794 | "latex_envs": { 3795 | "LaTeX_envs_menu_present": true, 3796 | "autoclose": false, 3797 | "autocomplete": true, 3798 | "bibliofile": "biblio.bib", 3799 | "cite_by": "apalike", 3800 | "current_citInitial": 1, 3801 | "eqLabelWithNumbers": true, 3802 | "eqNumInitial": 1, 3803 | "hotkeys": { 3804 | "equation": "Ctrl-E", 3805 | "itemize": "Ctrl-I" 3806 | }, 3807 | "labels_anchors": false, 3808 | "latex_user_defs": false, 3809 | "report_style_numbering": false, 3810 | "user_envs_cfg": false 3811 | }, 3812 | "toc": { 3813 | "base_numbering": 1, 3814 | "nav_menu": {}, 3815 | "number_sections": true, 3816 | "sideBar": true, 3817 | "skip_h1_title": false, 3818 | "title_cell": "Table of Contents", 3819 | "title_sidebar": "Contents", 3820 | "toc_cell": false, 3821 | "toc_position": {}, 3822 | "toc_section_display": true, 3823 | "toc_window_display": true 3824 | }, 3825 | "varInspector": { 3826 | "cols": { 3827 | "lenName": 16, 3828 | "lenType": 16, 3829 | "lenVar": 40 3830 | }, 3831 | "kernels_config": { 3832 | "python": { 3833 | "delete_cmd_postfix": "", 3834 | "delete_cmd_prefix": "del ", 3835 | "library": "var_list.py", 3836 | "varRefreshCmd": "print(var_dic_list())" 3837 | }, 3838 | "r": { 3839 | "delete_cmd_postfix": ") ", 3840 | "delete_cmd_prefix": "rm(", 3841 | "library": "var_list.r", 3842 | "varRefreshCmd": "cat(var_dic_list()) " 3843 | } 3844 | }, 3845 | "types_to_exclude": [ 3846 | "module", 3847 | "function", 3848 | "builtin_function_or_method", 3849 | "instance", 3850 | "_Feature" 3851 | ], 3852 | "window_display": false 3853 | } 3854 | }, 3855 | "nbformat": 4, 3856 | "nbformat_minor": 5 3857 | } 3858 | --------------------------------------------------------------------------------