├── .gitignore
├── LICENSE
├── PythonCodeBoxes.ipynb
├── RCodeBoxes.R
├── README.md
├── Results.RData
├── StataCodeBoxes.do
├── rhc.Rdata
├── rhc.csv
└── rhc.dta


/.gitignore:
--------------------------------------------------------------------------------
1 | ### Python ###
2 | .ipynb_checkpoints/*
3 | ### R ###
4 | # History files
5 | .Rhistory
6 | .Rapp.history
7 | # User-specific files
8 | .Ruserdata
9 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2020 Miguel Angel Luque Fernandez
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:  
 6 | 
 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.  
 8 | 
 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  
10 | 


--------------------------------------------------------------------------------
/PythonCodeBoxes.ipynb:
--------------------------------------------------------------------------------
   1 | {
   2 |  "cells": [
   3 |   {
   4 |    "cell_type": "markdown",
   5 |    "metadata": {},
   6 |    "source": [
   7 |     "# Tutorial: causal inference methods made easy for applied resarchers/epidemiologists/statisticians \n",
   8 |     "\n",
   9 |     "### ICON-LSHTM, LONDON, 16th October 2020\n",
  10 |     "\n",
  11 |     "Miguel Angel Luque Fernandez PhD, Assistant Professor of Epidemiology and Biostatistics\n",
  12 |     "\n",
  13 |     "Matthew Smith PhD, Research Fellow Inequalities in Cancer Outcomes Network, LSHTM, London, UK\n",
  14 |     "\n",
  15 |     "Paul Zivich, University of North Carolina at Chapel Hill\n",
  16 |     "\n",
  17 |     "Copyright (c) 2020 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n",
  18 |     "\n",
  19 |     "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n",
  20 |     "\n",
  21 |     "Bug reports: miguel-angel.luque at lshtm.ac.uk\n",
  22 |     "\n",
  23 |     "The rhc dataset can be dowloaded at http://biostat.mc.vanderbilt.edu/wiki/Main/DataSets"
  24 |    ]
  25 |   },
  26 |   {
  27 |    "cell_type": "code",
  28 |    "execution_count": 1,
  29 |    "metadata": {},
  30 |    "outputs": [
  31 |     {
  32 |      "name": "stdout",
  33 |      "output_type": "stream",
  34 |      "text": [
  35 |       "patsy       0.5.1\n",
  36 |       "scipy       1.5.2\n",
  37 |       "numpy       1.19.1\n",
  38 |       "pandas      1.1.0\n",
  39 |       "statsmodels 0.11.1\n",
  40 |       "matplotlib  3.3.1\n",
  41 |       "zepid       0.9.0\n"
  42 |      ]
  43 |     }
  44 |    ],
  45 |    "source": [
  46 |     "%matplotlib inline\n",
  47 |     "\n",
  48 |     "# Importing libraries for the tutorial\n",
  49 |     "import patsy\n",
  50 |     "import scipy\n",
  51 |     "import numpy as np\n",
  52 |     "import pandas as pd\n",
  53 |     "import statsmodels.api as sm\n",
  54 |     "import statsmodels.formula.api as smf\n",
  55 |     "import matplotlib\n",
  56 |     "import matplotlib.pyplot as plt\n",
  57 |     "import zepid\n",
  58 |     "\n",
  59 |     "from scipy.stats.kde import gaussian_kde\n",
  60 |     "from scipy.stats import logistic\n",
  61 |     "from zepid.calc import probability_to_odds, odds_to_probability\n",
  62 |     "\n",
  63 |     "print(\"patsy      \", patsy.__version__)\n",
  64 |     "print(\"scipy      \", scipy.__version__)\n",
  65 |     "print(\"numpy      \", np.__version__)\n",
  66 |     "print(\"pandas     \", pd.__version__)\n",
  67 |     "print(\"statsmodels\", sm.__version__)\n",
  68 |     "print(\"matplotlib \", matplotlib.__version__)\n",
  69 |     "print(\"zepid      \", zepid.__version__)"
  70 |    ]
  71 |   },
  72 |   {
  73 |    "cell_type": "markdown",
  74 |    "metadata": {},
  75 |    "source": [
  76 |     "## Setting up the Data"
  77 |    ]
  78 |   },
  79 |   {
  80 |    "cell_type": "code",
  81 |    "execution_count": 2,
  82 |    "metadata": {},
  83 |    "outputs": [],
  84 |    "source": [
  85 |     "# Box 1: Setting up the data\n",
  86 |     "data = pd.read_csv(\"rhc.csv\")\n",
  87 |     "data.rename(columns={\"rhc\": \"A\", \n",
  88 |     "                     \"death_d30\": \"Y\",\n",
  89 |     "                     \"gender\": \"C\",\n",
  90 |     "                     \"age\": \"W1\",\n",
  91 |     "                     \"edu\": \"W2\",\n",
  92 |     "                     \"race\": \"W3\",\n",
  93 |     "                     \"carcinoma\": \"W4\",\n",
  94 |     "            }, inplace=True)\n",
  95 |     "data['A'] = np.where(data['A'] == \"Yes\", 1, 0)\n",
  96 |     "data['C'] = np.where(data['C'] == \"Female\", 0, 1)\n",
  97 |     "\n",
  98 |     "data = data[[\"Y\", \"A\", \"C\", \"W1\", \"W2\", \"W3\", \"W4\"]].copy()"
  99 |    ]
 100 |   },
 101 |   {
 102 |    "cell_type": "markdown",
 103 |    "metadata": {},
 104 |    "source": [
 105 |     "## Naive estimate of the ATE"
 106 |    ]
 107 |   },
 108 |   {
 109 |    "cell_type": "code",
 110 |    "execution_count": 3,
 111 |    "metadata": {},
 112 |    "outputs": [
 113 |     {
 114 |      "name": "stdout",
 115 |      "output_type": "stream",
 116 |      "text": [
 117 |       "0.07352\n"
 118 |      ]
 119 |     },
 120 |     {
 121 |      "data": {
 122 |       "text/html": [
 123 |        "<table class=\"simpletable\">\n",
 124 |        "<caption>OLS Regression Results</caption>\n",
 125 |        "<tr>\n",
 126 |        "  <th>Dep. Variable:</th>            <td>Y</td>        <th>  R-squared:         </th> <td>   0.006</td>\n",
 127 |        "</tr>\n",
 128 |        "<tr>\n",
 129 |        "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.005</td>\n",
 130 |        "</tr>\n",
 131 |        "<tr>\n",
 132 |        "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   16.59</td>\n",
 133 |        "</tr>\n",
 134 |        "<tr>\n",
 135 |        "  <th>Date:</th>             <td>Mon, 21 Dec 2020</td> <th>  Prob (F-statistic):</th> <td>6.58e-08</td>\n",
 136 |        "</tr>\n",
 137 |        "<tr>\n",
 138 |        "  <th>Time:</th>                 <td>06:30:58</td>     <th>  Log-Likelihood:    </th> <td> -3812.9</td>\n",
 139 |        "</tr>\n",
 140 |        "<tr>\n",
 141 |        "  <th>No. Observations:</th>      <td>  5735</td>      <th>  AIC:               </th> <td>   7632.</td>\n",
 142 |        "</tr>\n",
 143 |        "<tr>\n",
 144 |        "  <th>Df Residuals:</th>          <td>  5732</td>      <th>  BIC:               </th> <td>   7652.</td>\n",
 145 |        "</tr>\n",
 146 |        "<tr>\n",
 147 |        "  <th>Df Model:</th>              <td>     2</td>      <th>                     </th>     <td> </td>   \n",
 148 |        "</tr>\n",
 149 |        "<tr>\n",
 150 |        "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>   \n",
 151 |        "</tr>\n",
 152 |        "</table>\n",
 153 |        "<table class=\"simpletable\">\n",
 154 |        "<tr>\n",
 155 |        "      <td></td>         <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
 156 |        "</tr>\n",
 157 |        "<tr>\n",
 158 |        "  <th>Intercept</th> <td>    0.3049</td> <td>    0.010</td> <td>   29.354</td> <td> 0.000</td> <td>    0.285</td> <td>    0.325</td>\n",
 159 |        "</tr>\n",
 160 |        "<tr>\n",
 161 |        "  <th>A</th>         <td>    0.0735</td> <td>    0.013</td> <td>    5.739</td> <td> 0.000</td> <td>    0.048</td> <td>    0.099</td>\n",
 162 |        "</tr>\n",
 163 |        "<tr>\n",
 164 |        "  <th>C</th>         <td>    0.0027</td> <td>    0.013</td> <td>    0.219</td> <td> 0.826</td> <td>   -0.022</td> <td>    0.027</td>\n",
 165 |        "</tr>\n",
 166 |        "</table>\n",
 167 |        "<table class=\"simpletable\">\n",
 168 |        "<tr>\n",
 169 |        "  <th>Omnibus:</th>       <td>36985.427</td> <th>  Durbin-Watson:     </th> <td>   1.981</td> \n",
 170 |        "</tr>\n",
 171 |        "<tr>\n",
 172 |        "  <th>Prob(Omnibus):</th>  <td> 0.000</td>   <th>  Jarque-Bera (JB):  </th> <td> 993.118</td> \n",
 173 |        "</tr>\n",
 174 |        "<tr>\n",
 175 |        "  <th>Skew:</th>           <td> 0.696</td>   <th>  Prob(JB):          </th> <td>2.22e-216</td>\n",
 176 |        "</tr>\n",
 177 |        "<tr>\n",
 178 |        "  <th>Kurtosis:</th>       <td> 1.511</td>   <th>  Cond. No.          </th> <td>    3.07</td> \n",
 179 |        "</tr>\n",
 180 |        "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
 181 |       ],
 182 |       "text/plain": [
 183 |        "<class 'statsmodels.iolib.summary.Summary'>\n",
 184 |        "\"\"\"\n",
 185 |        "                            OLS Regression Results                            \n",
 186 |        "==============================================================================\n",
 187 |        "Dep. Variable:                      Y   R-squared:                       0.006\n",
 188 |        "Model:                            OLS   Adj. R-squared:                  0.005\n",
 189 |        "Method:                 Least Squares   F-statistic:                     16.59\n",
 190 |        "Date:                Mon, 21 Dec 2020   Prob (F-statistic):           6.58e-08\n",
 191 |        "Time:                        06:30:58   Log-Likelihood:                -3812.9\n",
 192 |        "No. Observations:                5735   AIC:                             7632.\n",
 193 |        "Df Residuals:                    5732   BIC:                             7652.\n",
 194 |        "Df Model:                           2                                         \n",
 195 |        "Covariance Type:            nonrobust                                         \n",
 196 |        "==============================================================================\n",
 197 |        "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
 198 |        "------------------------------------------------------------------------------\n",
 199 |        "Intercept      0.3049      0.010     29.354      0.000       0.285       0.325\n",
 200 |        "A              0.0735      0.013      5.739      0.000       0.048       0.099\n",
 201 |        "C              0.0027      0.013      0.219      0.826      -0.022       0.027\n",
 202 |        "==============================================================================\n",
 203 |        "Omnibus:                    36985.427   Durbin-Watson:                   1.981\n",
 204 |        "Prob(Omnibus):                  0.000   Jarque-Bera (JB):              993.118\n",
 205 |        "Skew:                           0.696   Prob(JB):                    2.22e-216\n",
 206 |        "Kurtosis:                       1.511   Cond. No.                         3.07\n",
 207 |        "==============================================================================\n",
 208 |        "\n",
 209 |        "Warnings:\n",
 210 |        "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
 211 |        "\"\"\""
 212 |       ]
 213 |      },
 214 |      "execution_count": 3,
 215 |      "metadata": {},
 216 |      "output_type": "execute_result"
 217 |     }
 218 |    ],
 219 |    "source": [
 220 |     "# Box 2: Regression naive approach\n",
 221 |     "fm = smf.ols(\"Y ~ A + C\", data).fit()\n",
 222 |     "print(np.round(fm.params['A'], 5))  # ATE = 0.07352\n",
 223 |     "fm.summary()  # Full model results"
 224 |    ]
 225 |   },
 226 |   {
 227 |    "cell_type": "code",
 228 |    "execution_count": 4,
 229 |    "metadata": {},
 230 |    "outputs": [
 231 |     {
 232 |      "name": "stdout",
 233 |      "output_type": "stream",
 234 |      "text": [
 235 |       "Prop. Male   0.56\n",
 236 |       "Prop. Female 0.44\n"
 237 |      ]
 238 |     }
 239 |    ],
 240 |    "source": [
 241 |     "# Box 3: Marginal probabilities\n",
 242 |     "pr_c1 = np.mean(data['C'])\n",
 243 |     "pr_c0 = 1 - pr_c1\n",
 244 |     "print(\"Prop. Male  \", np.round(pr_c1, 2))\n",
 245 |     "print(\"Prop. Female\", np.round(pr_c0, 2))"
 246 |    ]
 247 |   },
 248 |   {
 249 |    "cell_type": "markdown",
 250 |    "metadata": {},
 251 |    "source": [
 252 |     "## 3. G-Formula\n",
 253 |     "\n",
 254 |     "### 3.1 Non-parametric g-formula"
 255 |    ]
 256 |   },
 257 |   {
 258 |    "cell_type": "code",
 259 |    "execution_count": 5,
 260 |    "metadata": {},
 261 |    "outputs": [
 262 |     {
 263 |      "name": "stdout",
 264 |      "output_type": "stream",
 265 |      "text": [
 266 |       "ATE 0.073692\n"
 267 |      ]
 268 |     }
 269 |    ],
 270 |    "source": [
 271 |     "# Box 4: Non-parametric g-formula for the ATE\n",
 272 |     "pr_y_a1c1 = np.mean(data.loc[(data['C'] == 1) & (data['A'] == 1), 'Y'])\n",
 273 |     "pr_y_a0c1 = np.mean(data.loc[(data['C'] == 1) & (data['A'] == 0), 'Y'])\n",
 274 |     "pr_y_a1c0 = np.mean(data.loc[(data['C'] == 0) & (data['A'] == 1), 'Y'])\n",
 275 |     "pr_y_a0c0 = np.mean(data.loc[(data['C'] == 0) & (data['A'] == 0), 'Y'])\n",
 276 |     "\n",
 277 |     "ate = (pr_y_a1c1 - pr_y_a0c1)*pr_c1 + (pr_y_a1c0 - pr_y_a0c0)*pr_c0\n",
 278 |     "print(\"ATE\", np.round(ate, 6))"
 279 |    ]
 280 |   },
 281 |   {
 282 |    "cell_type": "code",
 283 |    "execution_count": 6,
 284 |    "metadata": {},
 285 |    "outputs": [
 286 |     {
 287 |      "name": "stdout",
 288 |      "output_type": "stream",
 289 |      "text": [
 290 |       "ATT 0.073248\n"
 291 |      ]
 292 |     }
 293 |    ],
 294 |    "source": [
 295 |     "# Box 5: Non-parametric g-formula for the ATT\n",
 296 |     "pr_c1_a1 = np.mean(data.loc[data['A'] == 1, 'C'])\n",
 297 |     "pr_c0_a1 = 1 - pr_c1_a1\n",
 298 |     "\n",
 299 |     "att = (pr_y_a1c1 - pr_y_a0c1)*pr_c1_a1 + (pr_y_a1c0 - pr_y_a0c0)*pr_c0_a1\n",
 300 |     "print(\"ATT\", np.round(att, 6))"
 301 |    ]
 302 |   },
 303 |   {
 304 |    "cell_type": "code",
 305 |    "execution_count": 7,
 306 |    "metadata": {},
 307 |    "outputs": [
 308 |     {
 309 |      "name": "stdout",
 310 |      "output_type": "stream",
 311 |      "text": [
 312 |       "95% Confidence limits for the ATE\n",
 313 |       "Percentile method:    [0.047741 0.099149]\n",
 314 |       "Normal Approx method: [0.04798  0.099404]\n",
 315 |       "\n",
 316 |       "95% Confidence limits for the ATT\n",
 317 |       "Percentile method:    [0.048054 0.098375]\n",
 318 |       "Normal Approx method: [0.047811 0.098686]\n"
 319 |      ]
 320 |     }
 321 |    ],
 322 |    "source": [
 323 |     "# Box 6: Bootstrap the 95% confidence intervals (CI) for the\n",
 324 |     "#        ATE/ATT estimated using the non-parametric G-Formula\n",
 325 |     "\n",
 326 |     "def ate_nonparm_gformula(d):\n",
 327 |     "    \"\"\"Function to estimate the ATE using the nonparametric\n",
 328 |     "    g-formula\"\"\"\n",
 329 |     "    pr_c1 = np.mean(d['C'])\n",
 330 |     "    pr_c0 = 1 - pr_c1\n",
 331 |     "\n",
 332 |     "    pr_y_11 = np.mean(d.loc[(d['C'] == 1) & (d['A'] == 1), 'Y'])\n",
 333 |     "    pr_y_01 = np.mean(d.loc[(d['C'] == 1) & (d['A'] == 0), 'Y'])\n",
 334 |     "    pr_y_10 = np.mean(d.loc[(d['C'] == 0) & (d['A'] == 1), 'Y'])\n",
 335 |     "    pr_y_00 = np.mean(d.loc[(d['C'] == 0) & (d['A'] == 0), 'Y'])\n",
 336 |     "    \n",
 337 |     "    return (pr_y_11 - pr_y_01)*pr_c1 + (pr_y_10 - pr_y_00)*pr_c0\n",
 338 |     "\n",
 339 |     "## ATE ##\n",
 340 |     "ate_rs = []\n",
 341 |     "for i in range(1000):  # Drawing 1000 bootstrapped samples\n",
 342 |     "    d_star = data.sample(n=data.shape[0], # Same size as input data\n",
 343 |     "                         replace=True)  # Draw with replacement\n",
 344 |     "    ate_rs.append(ate_nonparm_gformula(d=d_star))\n",
 345 |     "\n",
 346 |     "print(\"95% Confidence limits for the ATE\")\n",
 347 |     "ci_perc = np.percentile(ate_rs, q=[2.5, 97.5])\n",
 348 |     "print(\"Percentile method:   \", np.round(ci_perc, 6))\n",
 349 |     "ate_se = np.std(ate_rs, ddof=1)\n",
 350 |     "print(\"Normal Approx method:\", np.round([ate - 1.96*ate_se,\n",
 351 |     "                                          ate + 1.96*ate_se], 6))\n",
 352 |     "\n",
 353 |     "\n",
 354 |     "def att_nonparm_gformula(d):\n",
 355 |     "    \"\"\"Function to estimate the ATT using the nonparametric\n",
 356 |     "    g-formula\"\"\"\n",
 357 |     "    pr_c1_a1 = np.mean(d.loc[data['A'] == 1, 'C'])\n",
 358 |     "    pr_c0_a1 = 1 - pr_c1_a1\n",
 359 |     "\n",
 360 |     "    pr_y_11 = np.mean(d.loc[(d['C'] == 1) & (d['A'] == 1), 'Y'])\n",
 361 |     "    pr_y_01 = np.mean(d.loc[(d['C'] == 1) & (d['A'] == 0), 'Y'])\n",
 362 |     "    pr_y_10 = np.mean(d.loc[(d['C'] == 0) & (d['A'] == 1), 'Y'])\n",
 363 |     "    pr_y_00 = np.mean(d.loc[(d['C'] == 0) & (d['A'] == 0), 'Y'])\n",
 364 |     "    \n",
 365 |     "    return (pr_y_11 - pr_y_01)*pr_c1_a1 + (pr_y_10 - pr_y_00)*pr_c0_a1\n",
 366 |     "\n",
 367 |     "\n",
 368 |     "## ATT ##\n",
 369 |     "att_rs = []\n",
 370 |     "for i in range(1000):  # Drawing 1000 bootstrapped samples\n",
 371 |     "    d_star = data.sample(n=data.shape[0], # Same size as input data\n",
 372 |     "                         replace=True)  # Draw with replacement\n",
 373 |     "    att_rs.append(att_nonparm_gformula(d=d_star))\n",
 374 |     "\n",
 375 |     "print(\"\\n95% Confidence limits for the ATT\")\n",
 376 |     "ci_perc = np.percentile(att_rs, q=[2.5, 97.5])\n",
 377 |     "print(\"Percentile method:   \", np.round(ci_perc, 6))\n",
 378 |     "att_se = np.std(att_rs, ddof=1)\n",
 379 |     "print(\"Normal Approx method:\", np.round([att - 1.96*att_se,\n",
 380 |     "                                          att + 1.96*att_se], 6))    "
 381 |    ]
 382 |   },
 383 |   {
 384 |    "cell_type": "code",
 385 |    "execution_count": 8,
 386 |    "metadata": {},
 387 |    "outputs": [
 388 |     {
 389 |      "name": "stdout",
 390 |      "output_type": "stream",
 391 |      "text": [
 392 |       "ATE 0.073692\n"
 393 |      ]
 394 |     }
 395 |    ],
 396 |    "source": [
 397 |     "# Box 7: Non-parametric g-formula using saturated regression model (A)\n",
 398 |     "data[\"A1\"] = np.where(data['A'] == 1, 1, 0)\n",
 399 |     "data[\"A0\"] = np.where(data['A'] == 0, 1, 0)\n",
 400 |     "fm = smf.ols(\"Y ~ A1 + A0 + A1:C + A0:C - 1\", data).fit()\n",
 401 |     "betas = fm.params\n",
 402 |     "ate = np.mean((betas[\"A1\"] + betas[\"A1:C\"]*data[\"C\"]) -\n",
 403 |     "              (betas[\"A0\"] + betas[\"A0:C\"]*data[\"C\"]))\n",
 404 |     "\n",
 405 |     "print(\"ATE\", np.round(ate, 6))"
 406 |    ]
 407 |   },
 408 |   {
 409 |    "cell_type": "code",
 410 |    "execution_count": 9,
 411 |    "metadata": {},
 412 |    "outputs": [
 413 |     {
 414 |      "name": "stdout",
 415 |      "output_type": "stream",
 416 |      "text": [
 417 |       "ATE 0.073692\n"
 418 |      ]
 419 |     }
 420 |    ],
 421 |    "source": [
 422 |     "# Box 8: G-formula with saturated regression model using zEpid\n",
 423 |     "g_formula = zepid.causal.gformula.TimeFixedGFormula(data, \n",
 424 |     "                                                    exposure=\"A\", \n",
 425 |     "                                                    outcome=\"Y\")\n",
 426 |     "g_formula.outcome_model(\"A + C + A:C\",  # Estimating model\n",
 427 |     "                        print_results=False)\n",
 428 |     "\n",
 429 |     "g_formula.fit(\"all\")  # all sets A=1\n",
 430 |     "y_a1 = g_formula.marginal_outcome\n",
 431 |     "\n",
 432 |     "g_formula.fit(\"none\")  # none sets A=0\n",
 433 |     "y_a0 = g_formula.marginal_outcome\n",
 434 |     "\n",
 435 |     "print(\"ATE\", np.round(y_a1 - y_a0, 6))"
 436 |    ]
 437 |   },
 438 |   {
 439 |    "cell_type": "markdown",
 440 |    "metadata": {},
 441 |    "source": [
 442 |     "### 3.2 Parametric g-formula"
 443 |    ]
 444 |   },
 445 |   {
 446 |    "cell_type": "code",
 447 |    "execution_count": 10,
 448 |    "metadata": {},
 449 |    "outputs": [
 450 |     {
 451 |      "name": "stdout",
 452 |      "output_type": "stream",
 453 |      "text": [
 454 |       "ATE 0.073692\n"
 455 |      ]
 456 |     }
 457 |    ],
 458 |    "source": [
 459 |     "# Box 9: Parametric g-formula by hand\n",
 460 |     "f = sm.families.family.Binomial()  # Using logit model unlike prev\n",
 461 |     "fm_a1 = smf.glm(\"Y ~ C\", data.loc[data[\"A\"] == 1], family=f).fit()\n",
 462 |     "fm_a0 = smf.glm(\"Y ~ C\", data.loc[data[\"A\"] == 0], family=f).fit()\n",
 463 |     "\n",
 464 |     "y_a1 = fm_a1.predict(data['C'])\n",
 465 |     "y_a0 = fm_a0.predict(data['C'])\n",
 466 |     "ate = np.mean(y_a1 - y_a0)\n",
 467 |     "\n",
 468 |     "print(\"ATE\", np.round(ate, 6))"
 469 |    ]
 470 |   },
 471 |   {
 472 |    "cell_type": "code",
 473 |    "execution_count": 11,
 474 |    "metadata": {},
 475 |    "outputs": [
 476 |     {
 477 |      "name": "stdout",
 478 |      "output_type": "stream",
 479 |      "text": [
 480 |       "ATE 0.073514\n"
 481 |      ]
 482 |     }
 483 |    ],
 484 |    "source": [
 485 |     "# Box 10: Parametric Regression Adjustment\n",
 486 |     "f = sm.families.family.Binomial()\n",
 487 |     "fm = smf.glm(\"Y ~ A + C\", data, family=f).fit()\n",
 488 |     "\n",
 489 |     "da1 = data.copy()\n",
 490 |     "da1['A'] = 1\n",
 491 |     "y_a1 = fm.predict(da1)\n",
 492 |     "\n",
 493 |     "da0 = data.copy()\n",
 494 |     "da0['A'] = 0\n",
 495 |     "y_a0 = fm.predict(da0)\n",
 496 |     "ate = np.mean(y_a1 - y_a0)\n",
 497 |     "\n",
 498 |     "print(\"ATE\", np.round(ate, 6))"
 499 |    ]
 500 |   },
 501 |   {
 502 |    "cell_type": "code",
 503 |    "execution_count": 12,
 504 |    "metadata": {},
 505 |    "outputs": [
 506 |     {
 507 |      "name": "stdout",
 508 |      "output_type": "stream",
 509 |      "text": [
 510 |       "95% Confidence limits for the ATE\n",
 511 |       "Percentile method:    [0.048036 0.100379]\n",
 512 |       "Normal approx method: [0.047652 0.099376]\n"
 513 |      ]
 514 |     }
 515 |    ],
 516 |    "source": [
 517 |     "# Box 11: Bootstrap for the parametric regression adjustment\n",
 518 |     "ate_rs = []\n",
 519 |     "for i in range(1000):  # Drawing 1000 bootstrapped samples\n",
 520 |     "    d_star = data.sample(n=data.shape[0], # Same size as input data\n",
 521 |     "                         replace=True)  # Draw with replacement\n",
 522 |     "    fm = smf.glm(\"Y ~ A + C\", d_star, family=f).fit()\n",
 523 |     "    da = d_star.copy()\n",
 524 |     "    da['A'] = 1\n",
 525 |     "    y_a1 = fm.predict(da)\n",
 526 |     "    da['A'] = 0\n",
 527 |     "    y_a0 = fm.predict(da)\n",
 528 |     "    ate_rs.append(np.mean(y_a1 - y_a0))\n",
 529 |     "\n",
 530 |     "print(\"95% Confidence limits for the ATE\")\n",
 531 |     "ci_perc = np.percentile(ate_rs, q=[2.5, 97.5])\n",
 532 |     "print(\"Percentile method:   \", np.round(ci_perc, 6))\n",
 533 |     "ate_se = np.std(ate_rs, ddof=1)\n",
 534 |     "print(\"Normal approx method:\", np.round([ate - 1.96*ate_se,\n",
 535 |     "                                          ate + 1.96*ate_se], 6))"
 536 |    ]
 537 |   },
 538 |   {
 539 |    "cell_type": "code",
 540 |    "execution_count": 13,
 541 |    "metadata": {},
 542 |    "outputs": [
 543 |     {
 544 |      "name": "stdout",
 545 |      "output_type": "stream",
 546 |      "text": [
 547 |       "ATE 0.083929\n"
 548 |      ]
 549 |     }
 550 |    ],
 551 |    "source": [
 552 |     "# Box 12: Parametric multivariate regression adjustment implementation\n",
 553 |     "f = sm.families.family.Binomial()\n",
 554 |     "fm_a1 = smf.glm(\"Y ~ C + W1 + W2 + W3 + W4\", \n",
 555 |     "                data.loc[data[\"A\"] == 1], family=f).fit()\n",
 556 |     "fm_a0 = smf.glm(\"Y ~ C + W1 + W2 + W3 + W4\", \n",
 557 |     "                data.loc[data[\"A\"] == 0], family=f).fit()\n",
 558 |     "\n",
 559 |     "y_a1 = fm_a1.predict(data)\n",
 560 |     "y_a0 = fm_a0.predict(data)\n",
 561 |     "ate = np.mean(y_a1 - y_a0)\n",
 562 |     "\n",
 563 |     "print(\"ATE\", np.round(ate, 6))"
 564 |    ]
 565 |   },
 566 |   {
 567 |    "cell_type": "code",
 568 |    "execution_count": 14,
 569 |    "metadata": {},
 570 |    "outputs": [
 571 |     {
 572 |      "name": "stdout",
 573 |      "output_type": "stream",
 574 |      "text": [
 575 |       "ATE 0.083929\n"
 576 |      ]
 577 |     }
 578 |    ],
 579 |    "source": [
 580 |     "# Box 13: Multivariate regression with zEpid\n",
 581 |     "g_formula = zepid.causal.gformula.TimeFixedGFormula(data, \n",
 582 |     "                                                    exposure=\"A\", \n",
 583 |     "                                                    outcome=\"Y\")\n",
 584 |     "g_formula.outcome_model(\"A + C + W1 + W2 + W3 + W4 + \"\n",
 585 |     "                        \"A:C + A:W1 + A:W2 + A:W3 + A:W4\",\n",
 586 |     "                        print_results=False)\n",
 587 |     "\n",
 588 |     "g_formula.fit(\"all\")  # all sets A=1\n",
 589 |     "y_a1 = g_formula.marginal_outcome\n",
 590 |     "\n",
 591 |     "g_formula.fit(\"none\")  # none sets A=0\n",
 592 |     "y_a0 = g_formula.marginal_outcome\n",
 593 |     "\n",
 594 |     "print(\"ATE\", np.round(y_a1 - y_a0, 6))"
 595 |    ]
 596 |   },
 597 |   {
 598 |    "cell_type": "code",
 599 |    "execution_count": 15,
 600 |    "metadata": {},
 601 |    "outputs": [],
 602 |    "source": [
 603 |     "# Box 14: Not Available for Python\n",
 604 |     "# zEpid does not support two version of the parametric g-formula"
 605 |    ]
 606 |   },
 607 |   {
 608 |    "cell_type": "code",
 609 |    "execution_count": 16,
 610 |    "metadata": {},
 611 |    "outputs": [
 612 |     {
 613 |      "name": "stdout",
 614 |      "output_type": "stream",
 615 |      "text": [
 616 |       "95% Confidence limits for the ATE\n",
 617 |       "Percentile method:    [0.059649 0.106662]\n",
 618 |       "Normal approx method: [0.058851 0.109006]\n"
 619 |      ]
 620 |     }
 621 |    ],
 622 |    "source": [
 623 |     "# Box 15: Bootstrap for multivariate adjustment\n",
 624 |     "ate_rs = []\n",
 625 |     "for i in range(1000):  # Drawing 1000 bootstrapped samples\n",
 626 |     "    d_star = data.sample(n=data.shape[0], # Same size as input data\n",
 627 |     "                         replace=True)  # Draw with replacement\n",
 628 |     "    fm_a1 = smf.glm(\"Y ~ C + W1 + W2 + W3 + W4\", \n",
 629 |     "                    d_star.loc[d_star[\"A\"] == 1], \n",
 630 |     "                    family=f).fit()\n",
 631 |     "    fm_a0 = smf.glm(\"Y ~ C + W1 + W2 + W3 + W4\", \n",
 632 |     "                    d_star.loc[d_star[\"A\"] == 0], \n",
 633 |     "                    family=f).fit()\n",
 634 |     "    ate_rs.append(np.mean(fm_a1.predict(data) - fm_a0.predict(data)))\n",
 635 |     "\n",
 636 |     "print(\"95% Confidence limits for the ATE\")\n",
 637 |     "ci_perc = np.percentile(ate_rs, q=[2.5, 97.5])\n",
 638 |     "print(\"Percentile method:   \", np.round(ci_perc, 6))\n",
 639 |     "ate_se = np.std(ate_rs, ddof=1)\n",
 640 |     "print(\"Normal approx method:\", np.round([ate - 1.96*ate_se,\n",
 641 |     "                                          ate + 1.96*ate_se], 6))"
 642 |    ]
 643 |   },
 644 |   {
 645 |    "cell_type": "code",
 646 |    "execution_count": 17,
 647 |    "metadata": {},
 648 |    "outputs": [
 649 |     {
 650 |      "name": "stdout",
 651 |      "output_type": "stream",
 652 |      "text": [
 653 |       "RR 1.2766\n"
 654 |      ]
 655 |     }
 656 |    ],
 657 |    "source": [
 658 |     "# Box 16: Computing the parametric marginal risk ratio\n",
 659 |     "f = sm.families.family.Binomial()\n",
 660 |     "fm_a1 = smf.glm(\"Y ~ C + W1 + W2 + W3 + W4\", \n",
 661 |     "                data.loc[data[\"A\"] == 1], family=f).fit()\n",
 662 |     "fm_a0 = smf.glm(\"Y ~ C + W1 + W2 + W3 + W4\", \n",
 663 |     "                data.loc[data[\"A\"] == 0], family=f).fit()\n",
 664 |     "\n",
 665 |     "y_a1 = fm_a1.predict(data)\n",
 666 |     "y_a0 = fm_a0.predict(data)\n",
 667 |     "risk_ratio = np.mean(y_a1) / np.mean(y_a0)\n",
 668 |     "\n",
 669 |     "print(\"RR\", np.round(risk_ratio, 4))"
 670 |    ]
 671 |   },
 672 |   {
 673 |    "cell_type": "markdown",
 674 |    "metadata": {},
 675 |    "source": [
 676 |     "## 4. Inverse Probability of Treatment Weighting\n",
 677 |     "\n",
 678 |     "### 4.1 Inverse probability of treatment weighting based on the propensity score plus regression adjustment"
 679 |    ]
 680 |   },
 681 |   {
 682 |    "cell_type": "code",
 683 |    "execution_count": 18,
 684 |    "metadata": {},
 685 |    "outputs": [
 686 |     {
 687 |      "name": "stdout",
 688 |      "output_type": "stream",
 689 |      "text": [
 690 |       "ATE 0.083294\n"
 691 |      ]
 692 |     }
 693 |    ],
 694 |    "source": [
 695 |     "# Box 17: Computation of the IPTW estimator for the ATE\n",
 696 |     "f = sm.families.family.Binomial()\n",
 697 |     "fm_pa = smf.glm(\"A ~ C + W1 + W2 + W3 + W4\", \n",
 698 |     "                data, family=f).fit()\n",
 699 |     "p_score = fm_pa.predict(data)  # Calculating propensity scores\n",
 700 |     "data['p_score'] = p_score\n",
 701 |     "\n",
 702 |     "iptw = 1 / np.where(data['A'] == 1, p_score, 1 - p_score)  # IPTW\n",
 703 |     "data['iptw'] = iptw\n",
 704 |     "\n",
 705 |     "d_a1 = data.loc[data[\"A\"] == 1].copy()\n",
 706 |     "d_a0 = data.loc[data[\"A\"] == 0].copy()\n",
 707 |     "ate = (np.average(d_a1['Y'], weights=d_a1['iptw']) - \n",
 708 |     "       np.average(d_a0['Y'], weights=d_a0['iptw']))\n",
 709 |     "print(\"ATE\", np.round(ate, 6))"
 710 |    ]
 711 |   },
 712 |   {
 713 |    "cell_type": "code",
 714 |    "execution_count": 19,
 715 |    "metadata": {},
 716 |    "outputs": [
 717 |     {
 718 |      "name": "stdout",
 719 |      "output_type": "stream",
 720 |      "text": [
 721 |       "95% Confidence limits for the ATE\n",
 722 |       "Percentile method:    [0.057431 0.106584]\n",
 723 |       "Normal approx method: [0.058198 0.10839 ]\n"
 724 |      ]
 725 |     }
 726 |    ],
 727 |    "source": [
 728 |     "# Box 18: Bootstrap computation for the IPTW estimator\n",
 729 |     "ate_rs = []\n",
 730 |     "for i in range(1000):  # Drawing 1000 bootstrapped samples\n",
 731 |     "    d_star = data.sample(n=data.shape[0], # Same size as input data\n",
 732 |     "                         replace=True)  # Draw with replacement\n",
 733 |     "    fm_pa = smf.glm(\"A ~ C + W1 + W2 + W3 + W4\", \n",
 734 |     "                    d_star, family=f).fit()\n",
 735 |     "    ps_score = fm_pa.predict(d_star)  # Calculating propensity scores\n",
 736 |     "    d_star['iptw'] = 1 / np.where(d_star['A'] == 1, \n",
 737 |     "                                  ps_score, 1 - ps_score)    \n",
 738 |     "    ds_a1 = d_star.loc[d_star[\"A\"] == 1].copy()\n",
 739 |     "    ds_a0 = d_star.loc[d_star[\"A\"] == 0].copy()\n",
 740 |     "    ate_rs.append(np.average(ds_a1['Y'], weights=ds_a1['iptw']) - \n",
 741 |     "                  np.average(ds_a0['Y'], weights=ds_a0['iptw']))\n",
 742 |     "\n",
 743 |     "print(\"95% Confidence limits for the ATE\")\n",
 744 |     "ci_perc = np.percentile(ate_rs, q=[2.5, 97.5])\n",
 745 |     "print(\"Percentile method:   \", np.round(ci_perc, 6))\n",
 746 |     "ate_se = np.std(ate_rs, ddof=1)\n",
 747 |     "print(\"Normal approx method:\", np.round([ate - 1.96*ate_se,\n",
 748 |     "                                          ate + 1.96*ate_se], 6))"
 749 |    ]
 750 |   },
 751 |   {
 752 |    "cell_type": "code",
 753 |    "execution_count": 20,
 754 |    "metadata": {},
 755 |    "outputs": [
 756 |     {
 757 |      "name": "stdout",
 758 |      "output_type": "stream",
 759 |      "text": [
 760 |       "                 RD    SE(RD)    95%LCL    95%UCL\n",
 761 |       "labels                                           \n",
 762 |       "Intercept  0.303444  0.007712  0.288328  0.318559\n",
 763 |       "A          0.083294  0.013046  0.057723  0.108864\n"
 764 |      ]
 765 |     }
 766 |    ],
 767 |    "source": [
 768 |     "# Box 19: IPTW estimator using zEpid\n",
 769 |     "ipw = zepid.causal.ipw.IPTW(data, treatment=\"A\", outcome=\"Y\")\n",
 770 |     "ipw.treatment_model(\"C + W1 + W2 + W3 + W4\", \n",
 771 |     "                    stabilized=False,  # Set to True for stabilized\n",
 772 |     "                    print_results=False)\n",
 773 |     "ipw.marginal_structural_model(\"A\")\n",
 774 |     "ipw.fit()\n",
 775 |     "print(ipw.risk_difference)"
 776 |    ]
 777 |   },
 778 |   {
 779 |    "cell_type": "code",
 780 |    "execution_count": 21,
 781 |    "metadata": {},
 782 |    "outputs": [
 783 |     {
 784 |      "name": "stdout",
 785 |      "output_type": "stream",
 786 |      "text": [
 787 |       "  Confounder       Raw  Weighted\n",
 788 |       "2          C  0.093144  0.000325\n",
 789 |       "3         W1 -0.061352 -0.003754\n",
 790 |       "4         W2  0.091364 -0.002439\n",
 791 |       "0         W3  0.035606  0.002426\n",
 792 |       "1         W4  0.071853  0.000404\n"
 793 |      ]
 794 |     },
 795 |     {
 796 |      "data": {
 797 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEGCAYAAAB1iW6ZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAeD0lEQVR4nO3de3xV5Z3v8c8vICByUUAqiiHgeA1Y5OIZLqHYqdgWS8eqdTDOlL7Gg7dTa+z0aF+ZStWTc5wjVg+2Hic4jqONo1PBVntRtEpBAsVgqVw9VgUG9KVchEIRMfA7fzwrYSdmJzvJ3tnJw/f9euWVtdde61m/Z2/4Zu1n7f1sc3dERKTrK8h3ASIikh0KdBGRSCjQRUQioUAXEYmEAl1EJBLd83XgQYMGeVFRUb4On3M7d+4EYODAgXmuRERismrVqh3ufmJT9+Ut0IuKiqipqcnX4XPukUceAWDWrFl5rUNE4mJmm9PdpyEXEZFIKNBFRCKhQBcRiUTextBFpPP75JNP2Lp1KwcOHMh3KUedXr16MXToUI455piM91Ggi0haW7dupW/fvhQVFWFm+S7nqOHu7Ny5k61btzJ8+PCM98vakIuZnWRmT5jZW2a2ysx+ZWZnpNt+zRooKICiIqiqylYVeVJVFToSTYdEggMHDjBw4ECFeQczMwYOHNjqV0ZZCXQLz/bTwGJ3P83dxwLfAz6Tbp+DB8EdNm+G2bO7cAZWVYUObN7csEPvv5/vykSyQmGeH2153LN1hn4B8Im7P1i3wt3/4O5LM9l5/34oL89SJR2tvDx0INX+/fDOO/mpR0SOWtkK9JHAqpY2MrPZZlZjZp/6RNGWLVmqpKOlK/zjjzu2DpFIbdq0iZEjRzZY94Mf/IC5c+fm5HgTJ05scZuioiJ27NjxqfWLFy+murq61cdM115rdejbFt290t3Hufu4xvcVFnZkJVmUrvCePTu2DpHOIILrSW0J5DptDfRsyVagrwPGtnXn3r2hoiJLlXS0iorQgVS9e0MrrkyLRCHd9aQchvrUqVO55ZZbOP/88znjjDNYujSM8k6fPp3XX38dgPPOO4877rgDgNtuu4358+cDcPfddzN+/HjOPfdc5syZU99mnz59ADh8+DDXX389Z511FhdeeCFf/vKXeeqpp+q3u//++xkzZgyjRo1i48aNbNq0iQcffJB7772X0aNHs3TpUrZv386ll17K+PHjGT9+PMuWLQPCXE/Tpk2juLiYq6++mmx9c1y2Av0loKeZza5bYWbnmllJuh169AAzGDYMKiuhtDRLlXS00tLQgWHDGnboM2mvB4vEKd31pBxfIKutrWXlypXcd9993H777QCUlJSwdOlS9uzZQ/fu3euDdOnSpUyZMoVFixbx5ptvsnLlSlavXs2qVatYsmRJg3YXLlzIpk2bWL9+PY899hjLly9vcP+gQYN47bXXuO6665g7dy5FRUVce+21lJWVsXr1akpKSvj2t79NWVkZr776KgsWLODqq68G4Pbbb2fy5MmsW7eOSy65hC1ZGnPOyvvQ3d3N7BLgPjO7BTgAbAJuSrfPqFEQzdxcpaWf/ouUTM4lctRIF0rtDKt07/aoW/+1r30NgLFjx7Jp0yYgBPq8efMYPnw406dP54UXXmD//v288847nHnmmcyfP59FixZx3nnnAbBv3z7efPNNpkyZUt/+K6+8wuWXX05BQQEnnXQSF1xwQYPjpx534cKFTdb44osvsn79+vrbf/rTn9i3bx9Lliyp32f69OmccMIJrX1YmpS1Dxa5+7vA17PVnoh0MYWFYZilqfXtMHDgQD788MMG63bt2lX/gZueyfWqbt26UVtbC8D48eOpqalhxIgRXHjhhezYsYP58+czdmwYGXZ3vve973HNNde0ua6mjtvY4cOHWbFiBb169WrzcVpDc7mISHaku57Uzgtkffr0YciQIbz00ktACPPnnnuOyZMnp92nR48enHrqqfz0pz9lwoQJlJSUMHfu3Poz8IsuuoiHH36Yffv2AbBt2zY++OCDBm1MmjSJBQsWcPjwYd5//30WL17cYq19+/Zl79699benTZvG/fffX3979erVAEyZMoXHH38cgF//+tef+oPVVgp0EcmOdNeTsnCB7NFHH+XOO+9k9OjRfP7zn2fOnDmcdtppze5TUlLC4MGDOfbYYykpKWHr1q2UlITLetOmTePKK69kwoQJjBo1issuu6xBEANceumlDB06lHPOOYerrrqKMWPG0L9//2aP+ZWvfIWnn366/qLovHnzqKmp4dxzz+Wcc87hwQfDR3XmzJnDkiVLKC4uZuHChRRm6W1+lq2rq601btw41xdciHRuGzZs4Oyzz853GXmzb98++vTpw86dOzn//PNZtmwZJ510Uocdv6nH38xWNfXWb9DkXCIiaV188cXs3r2bgwcP8v3vf79Dw7wtFOgiImlkMm7emWgMXUQkEgp0EZFIKNBFRCKhQBcRiYQCXUQ6tbKyMu6777762xdddFH9nCgA3/nOd/jhD3/Y5L633XYbL774YrPtp5uKd/fu3TzwwAOtrjeXU/u2RIEuIlmTi9lzJ02aVD8l7eHDh9mxYwfr1q2rv7+6ujrtHOZ33HEHX/jCF9p03LYGej4p0EUkK3I1e+7EiRPrZzpct24dI0eOpG/fvnz44Yd8/PHHbNiwATPjc5/7HGPHjuWiiy7ivffeA8IH++qmvP3Vr37FWWedxdixY7nxxhu5+OKL64+xfv16pk6dyogRI5g3bx4At956K2+99RajR4/mu9/9LpB+yt2KigrOOOMMJk+ezBtvvNG+DreD3ocuIlnR3Oy57fn0/8knn0z37t3ZsmUL1dXVTJgwgW3btrF8+XL69+/P2WefTVlZGT//+c858cQTefLJJykvL+fhhx+ub+PAgQNcc801LFmyhOHDhzNz5swGx9i4cSMvv/wye/fu5cwzz+S6667jrrvuYu3atfXzr6ROuevuzJgxgyVLlnDcccfxxBNPsHr1ampraxkzZkz9JGAdTYEuIlmRo9lzgXCWXl1dTXV1NTfffDPbtm2jurqa/v37c8opp7Bo0SIuvPBCAA4dOsSQIUMa7L9x40ZGjBhRP0PjzJkzqaysrL9/+vTp9OzZk549ezJ48GDeb+JL3hctWtTklLt79+7lkksuoXcyMdmMGTPa3+E2UqCLSFbkaPZc4Mg4+po1axg5ciSnnnoq99xzD/369WPq1Kn1Z+xt1TPlKyPTTYebbsrd1Au2+aYxdBHJihzNnguEM/Rf/OIXDBgwgG7dujFgwAB2797N8uXLmTlzJtu3b68P9E8++aTBRVOAM888k7fffrv+CzCefPLJFo/ZeCrcdFPuTpkyhZ/97Gd89NFH7N27l2effbb9HW4jnaGLSFbUjZOXl4dhlsLCEObZ+HrJUaNGsWPHDq688soG6/bt28fgwYN56qmnuPHGG9mzZw+1tbXcdNNNFBcX12977LHH8sADD/DFL36R4447jvHjx7d4zIEDBzJp0iRGjhzJl770Je6++242bNjAhAkTgDBP+09+8hPGjBnDFVdcwWc/+1kGDx6cUdu5oulzc0TT50oMYpo+t24qXHfnhhtu4PTTT6esrCzfZTWrtdPnashFRI4K8+fPZ/To0RQXF7Nnz552ff1cZ6UhFxE5KpSVlXX6M/L20hm6iDQrX8OyR7u2PO4KdBFJq1evXuzcuVOh3sHcnZ07d9KrV69W7achFxFJa+jQoWzdupXt27fnu5SjTq9evRg6dGir9lGgi0haxxxzTP2nK6Xz05CLiEgkFOgiIpFQoIuIREKBLiISCQW6iEgkFOgiIpFQoIuIREKBLiISCQW6iEgkFOgiIpFQoIuIREKBLiISCQW6iEgkFOgiIpFQoIuIREKBLiISCQW6iEgkWgx0M7vXzG5Kuf28mT2UcvseM7vZzJab2Toze93Mrmip3V27oKgICgrC76qqNvYg36qqIumIiHR1mXwF3TLg68B9ZlYADAL6pdw/EbgFeNbd3zSzk4FVZva8u+9O1+jmzXD48JHl2bPDcmlp6zuRN1VVofD9+8Pt1I6IiHSwTIZcqoEJyXIxsBbYa2YnmFlP4Gxghbu/CeDu7wIfACc212hdmNfZvx/Ky1tVe/6Vlx8J8zpdsiMiEoMWz9Dd/V0zqzWzQsLZ+HLgFELI7wHWuPvBuu3N7HygB/BW47bMbDaQnMKO/dSxtmxpSxfyKF3BXa4jIhKDTC+KVhPCvC7Ql6fcXla3kZkNAR4Dvunuhxs34u6V7j7O3cc1dZDCwtYVn3fpCu5yHRGRGGQa6MsI4T2KMOSygnCGPpEQ9phZP+CXQLm7r2jxwI2O3Ls3VFRkWnYnUVERCk/VJTsiIjFozRn6xcAudz/k7ruA4wmhXm1mPYCngUfd/alMGhw2LPyYhd+VlV3sgiiEgisrI+iIiMQgk3e5AKwhvLvl8Ubr+rj7DjO7CpgCDDSzWcn9s9x9dboGBwyAmprWF9zplJYqwEWkU8go0N39EA3fqoi7z0pZ/gnwk6xWJiIiraJPioqIREKBLiISCQW6iEgkFOgiIpFQoIuIREKBLiISCQW6iEgkFOgiIpFQoIuIREKBLiISCQW6iEgkFOgiIpFQoIuIREKBLiISCQW6iEgkFOgiIpFQoIuIREKBLiISCQW6iEgkFOgiIpFQoIuIREKBLiISCQW6iEgkFOgiIpFQoIuIREKBLiISCQW6iEgkFOgiIpFQoIuIREKBLiISCQW6iEgkFOgiIpFQoIuIREKBLiISCQW6iEgkFOgiIpFQoIuIREKBLiISCQW6iEgkFOgiIpFoMdDN7F4zuynl9vNm9lDK7XvM7OZkuZ+ZbTWzH7XU7po1UFAARUVQVdW24nOmqioU1mkLFBH5tEzO0JcBEwHMrAAYBBSn3D8RqE6W7wSWZHLggwfBHTZvhtmzO1FmVlWFgjZv7qQFiog0LZNArwYmJMvFwFpgr5mdYGY9gbOB18xsLPAZYFFri9i/H8rLW7tXjpSXh4JSdaoCRUSa1mKgu/u7QK2ZFRLOxpcDvyOE/DhgDVAL3AP8Q3NtmdlsM6sxs5rG923Z0vricyJdIZ2mQBGRpmV6UbSaEOZ1gb485fYy4HrgV+6+tblG3L3S3ce5+7jG9xUWtqbsHEpXSKcpUESkaZkGet04+ijCkMsKwhl63fj5BOC/mdkmYC7wd2Z2V6ZF9O4NFRWtqDqXKipCQak6VYEiIk1rzRn6xcAudz/k7ruA4wlBXu3upe5e6O5FhGGXR9391uYa7NEDzGDYMKishNLStnciq0pLQ0HDhnXSAkVEmtY9w+3WEN7d8nijdX3cfUdbDjxqFNR8aiS9kygtVYCLSJeTUaC7+yGgX6N1s9Js+wjwSDvrEhGRVtInRUVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUi0GOhmdq+Z3ZRy+3kzeyjl9j1mdrOZPWdmu83sF5kevKoKioqgoCD8rqpqZfXtlfcCRESyJ5Mz9GXARAAzKwAGAcUp908EqoG7gb/N9MC7dsHs2bB5M7iH37Nnd2CmVlXluQARkezKJNCrgQnJcjGwFthrZieYWU/gbOA1d/8NsDfTA2/bBvv3N1y3fz+Ul2faQjuVl+e5ABGR7Ore0gbu/q6Z1ZpZIeFsfDlwCiHk9wBr3P1gJgczs9nA7HBrbJPbbNmSSUtZkO5AHVaAiEh2ZXpRtJoQ5nWBvjzl9rJMD+bule4+zt3H9ejR9DaFhZm21k7pDtRhBYiIZFemgV43jj6KMOSygnCGXjd+3mqnnAK9ezdc17s3VFS0pbU2qKjIcwEiItnVmjP0i4Fd7n7I3XcBxxNCvU2BPmAAVFbCsGFgFn5XVkJpaVtaa4PS0jwXICKSXS2OoSfWEN7d8nijdX3cfQeAmS0FzgL6mNlW4O/d/fnmGi0tzXN+5r0AEZHsySjQ3f0Q0K/RulmNbpdkrywREWktfVJURCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSCnQRkUgo0EVEIqFAFxGJhAJdRCQSLQa6md1rZjel3H7ezB5KuX2Pmc0xs9fMbLWZrTOza1tqd9cuKCqCgoLwu6qqjT2AsHPWGhMR6ZoyOUNfBkwEMLMCYBBQnHL/ROBlYIK7jwb+C3CrmZ3cXKObN4cf9/B79uw25nBVVdg5K42JiHRdmQR6NTAhWS4G1gJ7zewEM+sJnA2scPePk216ZtLu4cMNb+/fD+XlGVadqrw87JyVxkREuq4Wg9fd3wVqzayQcDa+HPgdIeTHAWvc/aCZnWpmrwP/CfxTsl8DZjbbzGrMrKapY23Z0oYepNupTY2JiHRdmV4UrSaEeV2gL0+5vQzA3f/T3c8F/gL4hpl9pnEj7l7p7uPcfVxTByksbH0H0u7UpsZERLquTAO9bhx9FGHIZQXhDH0iIezrJWfma4GSZg/c6Mi9e0NFRYbVpKqoCDtnpTERka6rNWfoFwO73P2Qu+8CjieEerWZDTWzYwHM7ARgMvBGcw0OGxZ+zMLvykooLW1DD0pLw85ZaUxEpOvqnuF2awjvbnm80bo+7r7DzC4E7jEzBwyY6+5rmmtwwACoaXIkvQ1KSxXgInLUyyjQ3f0Q0K/Rulkpyy8A52a1MhERaRV9UlREJBIKdBGRSCjQRUQioUAXEYmEuXt+Dmy2lxbe2hiBQcCOfBeRQ7H3D+LvY+z9g/j6OMzdT2zqjkzftpgLb6T7xGgszKwm5j7G3j+Iv4+x9w+Ojj7W0ZCLiEgkFOgiIpHIZ6BX5vHYHSX2PsbeP4i/j7H3D46OPgJ5vCgqIiLZpSEXEZFIKNBFRCKR80A3sy+a2Rtm9kczu7WJ+3ua2ZPJ/b8zs6Jc15RtGfRxSvIl2rVmdlk+amyPDPp3s5mtN7PXzew3ZjYsH3W2RwZ9vNbM1iRfhP6KmZ2TjzrbqqX+pWx3qZm5mXWpt/ll8PzNMrPtyfO32syuzkedOefuOfsBugFvASOAHsAfgHMabXM98GCy/DfAk7msKU99LCLMRvkocFm+a85B/y4AeifL10X6HPZLWZ4BPJfvurPZv2S7vsASwhfYjMt33Vl+/mYBP8p3rbn+yfUZ+vnAH939bXc/CDwBfLXRNl8F/i1Zfgr4KzOzHNeVTS320d03ufvrwOGmGujkMunfy+5e903dK4ChHVxje2XSxz+l3DwO6ErvJsjk/yHAncA/AQc6srgsyLR/0ct1oJ9C+NLoOluTdU1u4+61wB5gYI7ryqZM+tiVtbZ/fw/8OqcVZV9GfTSzG8zsLeB/Azd2UG3Z0GL/zGwMcKq7/7IjC8uSTP+NXpoMCz5lZqd2TGkdSxdFJWvM7CpgHHB3vmvJBXf/sbufBtwC/GO+68kWMysAfgh8J9+15NCzQJGHL7J/gSOjAlHJdaBvA1L/Eg5N1jW5jZl1B/oDO3NcVzZl0seuLKP+mdkXgHJghrt/3EG1ZUtrn8MngL/OZUFZ1lL/+gIjgcVmtgn4S+CZLnRhtMXnz913pvy7fAgY20G1dahcB/qrwOlmNtzMehAuej7TaJtngG8ky5cBL3lyFaOLyKSPXVmL/TOz84B/JoT5B3mosb0y6ePpKTenA292YH3t1Wz/3H2Puw9y9yJ3LyJcB5nh7tn61t9cy+T5G5JycwawoQPr6zgdcAX6y8D/I1yFLk/W3UH4BwPQC/gp8EdgJTAi31eKc9DH8YRxvT8TXn2sy3fNWe7fi8D7wOrk55l815yDPv4fYF3Sv5eB4nzXnM3+Ndp2MV3oXS4ZPn//K3n+/pA8f2flu+Zc/Oij/yIikdBFURGRSCjQRUQioUAXEYmEAl1EJBIKdBGRSCjQI2Nmf53MlndWyrqpZvaLLLT9SEuzRSbHmtjKdnubWVUym+HaZDbDPmZ2vJld376qGxynyMzWtrON+sfAzB5q76yL6WpK1n9kZr83sw1mttLMZqXcP6NuVkEzOzGZqfT3ZlZiZpcn+7zcntqk6+me7wIk62YCryS/5+Th+FOBfUB1K/b5NvC+u48CMLMzgU+AQYTZOB/Ico0ZMbPuHuYXapK753oK1rfc/byklhHAQjMzd/9Xd3+GIx+e+StgTV09ZvYc8F/d/ZVMDtJSP6Xr0Bl6RMysDzCZMEHW3zS6u5+Z/TKZM/pBMysws27JGefa5Oy4LGlntJmtSCYyetrMTmjiWJvMbFCyPM7MFluYy/5aoCyZc7okOXtcYGavJj+Tmih9CCkf1Xb3Nzx8TPsu4LSkrbuTs/bfWJhbfo2ZfTU5flFyRjrfzNaZ2SIzOza5b6yZ/cHM/gDckFJ/kZktTdp6re5VRfIKY6mZPQOst+BHyeP2IjA4pY3FSd9n2JF5tt8ws3dSjv1bM1tlZs/XfVoxXU3Ncfe3gZtJJgWzML/3j8xsNGGysK8mx59D+DfwL8lj1i35/WryfF6Tpp/NbbfYwoRWGy28krLkvvFmVp30ZaWZ9U3XjnSQfH+yST/Z+wFKgX9JlquBscnyVMKUqCMIc0e/QJhmYSzwQsr+xye/Xwc+lyzfAdyXLD9CMp87sAkYlCyPAxYnyz8A/iGlzceByclyIbChibpHAx8Ay4H/AZyerC8C1qZs151kXnLC2fsfAUu2qwVGJ/f9B3BVSl+mJMt317UH9AZ6JcunAzUpj9WfgeHJ7a8lj1c34GRgd8pjsJhGn6hMjn0DcEzyHJyYrL8CeLi5mhq1U9R4PXA88FGyPItkfm8azfWdWhcwG/jHZLknUAMMb6KfzW23hzA/SkHyHE0mzDv+NjA+2adf8vw02U6+/28cLT8aconLTMJH1CFMIDUTWJXcXunhLA8z+3fCf8rfACPM7H7gl8AiM+tPCPbfJvv9G2Fqhrb6AnCOHZnivp+Z9XH3fXUr3H11MqQwLdn+VTObAHzUqC0D/qeZTSHMLX8K8JnkvnfcfXWyvAooMrPjk74sSdY/BnwpWT4GqDvDPQSckXKcle7+TrI8Bfh3dz8EvGtmL6XrqJn9d0Lg/tjMRhImvHoh6Xs34L0WampJW74nYBpwrh259tGf8AfsIA372dJ2W5M+rib8sdkDvOfur8KR+eLNLF07dceRHFKgR8LMBgCfB0aZmRMCxM3su8kmjed4cHf/0Mw+C1xEGCr5OlCW4SFrOTJk16uZ7QqAv3T3Zr80IQn4hYRx4sOEuTkWNNqsFDiR8MrjEwszA9YdO3WGx0PAsS3UX0aYf+azSY2p9f25hX0/xcJsk5cT/gBACN917j6h0XbHt7btFOfR+kmlDPiWuz/fqI6pNOxnc9s1fmyby40m25GOoTH0eFwGPObuwzzMmncq4ayoJLn/fAuz0RUQXv6/YmEMvMDdFxDm9x7j7nuAD82sbr+/BX7Lp23iyBSkl6as30uYjrXOIuBbdTeSM+IGzGySJeP0FmbLOwfY3ERb/YEPkjC/AGj2u0vdfTew28wmJ6tKG7X1nrsfTvrYLU0zS4ArkrHhIYSv22tc/zDgx8Dl7l73quIN4MTklQZmdoyZFbdQU1oWrk/MBe7PZPsUzwPXmdkxSTtnmNlx7diuzhvAEDMbn2zf18L0161tR7JIZ+jxmEn4+rBUC5L1TxKmGP0R8BeE2eaeBkYB/5qEPMD3kt/fAB40s96EcdJvNnG82wkX3u4kjNnWeRZ4ysIFy28RLuL92MxeJ/x7W0J4NZDqNOD/JhfbCgjDPwvc3c1smYW39f066d+zZraGMDa7MYPH5ZvAw8mrlkUp6x8AFpjZ3wHPkf6s/GnCK5/1wBbCGHJjswjfsvWzZHjlXXf/cjLsMC8ZxuoO3EeY8S9dTY2dZma/J7wK2QvMc/dHWupwIw8RhkheSx7f7TQ9l3um2wHg7gfN7ArgfgsXoD8iDJe1qh3JLs22KCISCQ25iIhEQoEuIhIJBbqISCQU6CIikVCgi4hEQoEuIhIJBbqISCT+P4OcpPFIPVnCAAAAAElFTkSuQmCC\n",
 798 |       "text/plain": [
 799 |        "<Figure size 432x288 with 1 Axes>"
 800 |       ]
 801 |      },
 802 |      "metadata": {
 803 |       "needs_background": "light"
 804 |      },
 805 |      "output_type": "display_data"
 806 |     }
 807 |    ],
 808 |    "source": [
 809 |     "# Box 20: Assessing IPTW balance\n",
 810 |     "rename_cols = {\"smd_w\": \"Weighted\", \"smd_u\": \"Raw\", \n",
 811 |     "               \"labels\": \"Confounder\"}\n",
 812 |     "\n",
 813 |     "smd = ipw.standardized_mean_differences().rename(columns=rename_cols)\n",
 814 |     "smd = smd.sort_values(by='Confounder')\n",
 815 |     "print(smd[['Confounder', 'Raw', 'Weighted']])\n",
 816 |     "\n",
 817 |     "# zEpid plotting functionality\n",
 818 |     "ipw.plot_love()\n",
 819 |     "plt.show()"
 820 |    ]
 821 |   },
 822 |   {
 823 |    "cell_type": "code",
 824 |    "execution_count": 22,
 825 |    "metadata": {},
 826 |    "outputs": [
 827 |     {
 828 |      "data": {
 829 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEKCAYAAADn+anLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAABCr0lEQVR4nO3dd3iT5frA8e+T7l0oZYNsCpRdZIoCDtxbxA0eURwIKs6fRzzHDSoOHOhRnDhRWbIFQTYyRFH2KrtQumee3x9PoG2a0kCbvEl7f64rV5J3JHcD7Z33GfejtNYIIYQQrtisDkAIIYTvkiQhhBCiTJIkhBBClEmShBBCiDJJkhBCCFEmSRJCCCHK5LEkoZT6SCl1SCm1sdi2mkqpuUqpLY77Gp56fyGEEBXnySuJScBAp22PA/O11i2B+Y7nQgghfJTy5GQ6pVQTYLrWOtHx/B/gPK31fqVUPWCh1rq1xwIQQghRIYFefr86Wuv9jscHgDplHaiUGgYMA4iIiOiakJDghfCEEKLqWLNmzRGtdXxFXsPbSeIkrbVWSpV5GaO1nghMBEhKStKrV6/2WmxCCFEVKKV2VfQ1vD266aCjmQnH/SEvv78QQojT4O0kMRW43fH4duAnL7+/EEKI0+DJIbCTgWVAa6XUXqXUncBLwAVKqS3A+Y7nQgghfJTH+iS01oPL2DXAU+8phPBf+fn57N27l5ycHKtD8TuhoaE0bNiQoKCgSn9tyzquhRCiuL179xIVFUWTJk1QSlkdjt/QWpOSksLevXtp2rRppb++lOUQQviEnJwc4uLiJEGcJqUUcXFxHrsCkyQhhPAZkiDOjCc/N0kSQgghyiRJQgghHAICAujUqROJiYlcfvnlpKamArBz504SExNLHDtmzBjGjRt38vm4ceNISEigU6dOdOvWjU8//fSM48jJySEhIYE//vjj5LaxY8dy9913n/FrnilJEkKcgdxc8GDZM2GRsLAw1q1bx8aNG6lZsyYTJkxw67z33nuPuXPnsnLlStatW8f8+fOpSF280NBQxo8fz7333ovWmuTkZN577z1eesn7swYkSQhxGr78Elq3htBQqFkTnnwS8vOtjkp4Qs+ePUlOTnbr2BdeeIF3332X6OhoAKKjo7n99tvLOevUBg4cSL169fj0008ZNWoUY8aMoUYN76+uIENghXCD3Q733AMffADNmsHQobB5M7z4IqxaBT//DIHy21RpRo6Edesq9zU7dYLx4907trCwkPnz53PnnXee3LZt2zY6dep08vmBAwd45JFHSEtLIz09nWbNmpX7umPHjuWLL74otb1v3768+eabpbaPHz+es88+m5YtW3Lrrbe6F3wlk//WQrjhgQdMghg0CEaNKkoIX30F48aZbW+9ZW2MouKys7Pp1KkTycnJtGnThgsuuODkvubNm7OuWOYaM2bMab/+6NGjGT16tNvH169fn/79+3PZZZed9ntVFkkSQpTju+/gnXfg2mvhkUeg+GjDG2+ELVtgwgS46Sbo2dO6OKsSd7/xV7YTfRJZWVlcdNFFTJgwgREjRpzynOjoaCIjI9m+fXu5VxOneyUBYLPZsNms6xmQPgkhTmHfPrjrLmjeHB5+uGSCOGHUKNM/8cAD0pldVYSHh/Pmm2/y6quvUlBQUO7xTzzxBPfddx9paWkAZGRkuBzdNHr0aNatW1fqVlaC8AWSJIQog91urhSysuCllyA42PVxkZGmj2LNGpg927sxCs/p3LkzHTp0YPLkyeUeO3z4cPr160e3bt1ITEzknHPOsfTbf2Xy6PKllUUWHRJWeOUVeOwxGD3a9EWciv1wClOu+4ILgxfRL/Gw6d2+9VYYIPUs3bVp0ybatGljdRh+y9Xnp5Rao7VOqsjrVo1UJ0QlW7cOnn4aevSAG244xYF2O7U/HUeXa87ixcwHaXxsHdnJR+Gbb+D88+HKKyEjw1thC1HpJEkI4SQnxzQzhYfDf/7juh8CwJaVQfNRl9HozdHkNG3L2se/IiFgC490mQ/z5pnOjOnToX9/yM727g8hRCWRJCGEk4cfhn/+gX//23RIu2LLyqDV3f2IWTqbg4MfZPfj7xDargXndUxl8uw48gNC4O674dlnzUSKf/3Luz+EEJVEkoQQxcyYYYa7Xn019O3r+hiVn0eLEZcQ/vfvJA9/jqOX3HrycuOKc45yLD2Q2UvNzFsuvhhuu81M1f7xR+/8EEJUIkkSQjjs22f6mps0MVcTZWn08v1ErVvM/qFPkt7zwhL7+nZMIzKsgC9+LnYJMny4edERIyAvzyOxC+EpkiSEAAoLTQd1Vha8/LKpzeRK3E8fEf/jB6RceCPHz7uq1P7gIE3/LqlMWxJLdo6jMyMoCB58EPbsgXff9dwPIYQHSJIQAjMR7rffzBVE8+aujwneu51Gr9xPVssOHLppZJmvdWnPo2RmBzBnRXTRxj59oF07GDsW3JicJazhK6XCT7ynUoq3itV7uf/++5k0aVKFXvd0SZIQ1d64ceYL/vXXwzXXlHGQ3U6Tf5sCa8n3vgABZVe06d42g7CQQn5cEFO0USnTN5GcDD/8UInRi8rkK6XCT6hduzZvvPEGeRY2U0qSENXaa6+ZyXJ9+pi6TGWJ//otojYs5dDgkRTUqnvK1wwO0vRom8bPS2NKluk47zyoXRtef71SYheeZXWpcID4+HgGDBjAJ598UuHXOlNS4E9USwUFJjmMH28SxNixEBDg+tjAIwdo8M5TZCZ0IbV/WZcaJfXvmsova2vw+99hdG3jmCMREABXXQUTJ8KuXXDWWZXys1RJFtcK95VS4QCPPfYYF198MUOHDnUr9somSUJUO5s2wZAhsGKFmRD9xBOnXgui4esPYcvLZf/Qp8qeWefkvE5pKKWZsiC2KEkAXHaZSRIffwxnUGpaeJavlQoHaNasGd27d+fLL7887ferDJIkRLWxa5dZJOh//4OQEFN248orT31O+PqlxM2eTMrAm8mv5/43/7iYAto1yWTmkhiev29/0Y769SExEb74Ap55xu2kU+1YVCvcF0uFAzz55JNcd911nHvuue79IJVI+iRElaU1bN8OH35oKmM0b24eX3ih6TsuL0Fgt3PWS/dSEF2TI9cMO+33752Yxvqt4Rw97tSOdfHFsHUrbNx42q8pvMPXSoUnJCTQtm1bpk2bdmY/UAVIkhBVQkEB/PknfPKJGc7apw/ExJjEcNdd8NdfppLrTz+ZekxlldsortaPHxC+ZT0HBz2APSzitGPq0zENrRVzV0SV3HGiMux33532awrv8bVS4U899RR79+6t1Nd0h5QKF35ryxb47DOYO9f0cebkmO0hIWaCc8uW0KYNdOwIrVufXsuOLSON9lc2I79mbXY+++kZNQvlF0DPezpx44VH+d8zu0vuvOkmU0Fw7drTft2qSkqFV4ynSoVLn4TwO5s3m0lv06eb5y1bwiWXmKb+du3MoKFTdUS7o/77zxB4PIU9o149436DoEDo0iqd+auiSu/s29e0fR04AHVPPaRWCCtJc5PwK5MmmZGM8+fDHXeYRDF5Mjz5JFxxhWleqmiCCN67nfhvJ3C8+wXkNE8s/4RT6JWYxq4DoexIdlrW7kSTkxT9Ez5OkoTwG6+9ZoautmgB338P99/vmS/hjV4bBTYbh24aVeHXOqdjOgA/L40uuaNFC9MxIuudluAPzd++yJOfmyQJ4Re++so0MfXuDe+/byYue0LkmkXE/jqVlIE3U1Cz4m/SrH4OtWLymLPMqclJKejSBX791SymLQgNDSUlJUUSxWnSWpOSkkJoWVUpK0j6JITP27QJhg41ndBjx0JwcPnnnBG7nUZjR5AfW4uUKypndqtS0K11OkvWRaO1U/dGz55mBbsNG0wbWjXXsGFD9u7dy+HDh60Oxe+EhobSsGFDj7y2JAnh0+x2uP1208/w6qseTBBA3NSPCd+6geS7nkGHVN63su7t0vl5ZRybdoTStllO0Y4ePcz9rFmSJICgoCCaNm1qdRjCiTQ3CZ/2zjtm9c+RIz3XxAQQcPwoDd98lOwmbUjrc2mlvnaPdqZfYt6KyJI76tSBhg3NGF4hfJQlSUIpNUop9adSaqNSarJSyjONacKvZWSYyhUdOpiRS57UcPzDBGSksv/Op6CSJ0E1qp1HfEweC1e7GArbtSssWwb5+ZX6nkJUFq8nCaVUA2AEkKS1TgQCgBu9HYfwfa++CkePmqsIT5Y4ili3hFrTJnFswHXkNkmo9NdXysyXWLI+ilJ9sklJkJ1d+RVPhagkVjU3BQJhSqlAIBzYZ1EcwkelpZkhrz16mCsJT7FlZ9L0mdvIj43n0A33e+x9urfN4HBqEJt3hZTc0aWLuf/lF4+9txAV4fUkobVOBsYBu4H9wHGt9Rzn45RSw5RSq5VSq2W0Q/UzcaJJFPfc49n3aTR2BMH7drLv7jHo0HCPvU/3E/0SK52anOrUMbdff/XYewtREVY0N9UArgSaAvWBCKXULc7Haa0naq2TtNZJ8fHx3g5TWKiwEN56C9q2NaU2PCV2zlfUmvoRxy4YRFZid8+9EdCkbi41o/JZuDqy9M727WH5ckq3RQlhPSuam84HdmitD2ut84EpQC8L4hA+aupU2L3b1MArS0BqCiE7/8aWmX5G7xH+5yqajhlCdtM2HLrx1OsFVIaT/RLrXPRLdOkCKSmwbZvH4xDidFmRJHYDPZRS4UopBQwANlkQh/BRb70FcXFw/vml94X9tZrWQ3rS6fxaJF7Xhs7nRpNwaxJxP3wABe6NEArbvJ6WDwykMCKKPQ+PRwd5cPJFMd3bpnPgaDDbnes4de1q7qVfQvggK/okVgDfAb8DfzhimOjtOIRv2rcPFi0yVV2dC/XVmPsNCUN7Ebrzbw5f+S+Sh43hyOVDCDx2mCbPD6P9Fc2I/+otVG6O6xcHopfMoNWwc9FKsfvxdyiMifPwT1Ske7sMAOY5ry/RtClERpofXAgfY8mMa631M8AzVry38G0ffWRmWV91Vcnt4euX0vT/biancSt2P/oW9siYk/sOX38vkWt/pdYPH9B43AjqffgfDl93L8cuHETOWa1Ba8I3rabOF69Rc9635NZvwp6HXie/TiOv/mzN6+cQG5nPL6uiuPvalKIdNpupcb5smVfjEcIdsuiQ8BlaQ6tWpvRG8ZUfbVkZtLumNcpeyPYXJmOPiC7zBcL/XEmtn/5HxN+/m022ANAape3Yg0I4NuA6Dl8/HB1szfzN+15txua94eyd5bR06cSJ5paS4t6yeUK4QRYdElXKqlVm6efHHiu5vd57/yb4yD52PvVB2QkCQCmyEruzO7E7gSkHiFy3hOCDe9E2RW6D5mR06Xvq872ge9t0fllXg537gmlSP69ox4naTcuWwaWVWxZEiIqQJCF8xqefmn6IgQOLtgXv30Wdr9/ieI+LyE7o7PZrFcTVJXXAdR6IsmJ6JJrRWPNXRnLnVUeLdrRrZ4ZA/fabJAnhU6TAn/AJWptV5jp3hqhi/bp1//c8AIcGe36Yqje0aJBDdHgBC5yXNI2IMOuuLl9uTWBClEGShPAJmzbBrl1w7rlF2wKPHiJuxqcc734BBTXrWBdcJbLZoHNLM1+ilLZtYc0amVQnfIokCeETvv3W3J93XtG22l+9iSrII+WqOy2JyVPObpPB7oMh7D4QVHJHx46mFsmWLdYEJoQLkiSET5g2DZo3L7Zmtd1O3LSPyWrdmbx6TawMrdKdqONUqsmpY0dzv2SJlyMSomySJITlDh+GtWtLXkVELZ9D8OF9HOt3tWVxeUpC42yiXPVLNG0KYWGm81oIHyFJQlhuyhQzga54koif8j6F4ZFkJPW3LC5PsdmgY/MMFv/uVOwvIABat4aVK60JTAgXJEkIy/34o6nVlOBY78eWcZyYJTNI634hOjjklOf6q7PbpLPzQCjJh5z6JRITTS9+VpY1gQnhRJKEsFRurilZ1KdP0epzsQt/xFaQz/FKXmval/R0zJf4xbl0eMeOplb6mjUWRCVEaZIkhKVmzTKrd/brV7St5uzJ5MfEkd2ivXWBeVjCWdlEhhUw33kRohPL8C1e7P2ghHBBkoSw1A8/QEiIWeoZwJaZTtTqX8joep5pvK+iAhz9Er+udUoScXFmpTqZVCd8RNX9LRQ+T2tzJdGtG4Q66u3FLvwRW34ex3sOPPXJVUC3hAy2J4ey/4hTdZy2bU0hKyF8gCQJYZk1a+DgwZJNTTXmfk1BdE2yW3W0LjAvOVHH6RfnobDt28OBA5CcbEFUQpQkSUJY5rvvTGd1nz7mucrLJWrVAjI69a7STU0ntG2SRXhIYen5EsUrwgphsar/myh81owZZthrnGNxuMg1CwnIzSa963mWxuUtgQGmX2KR83yJhARTDldmXgsfIElCWGLvXti4seQEutiFP2IPDCKz7dmWxeVt3RLS2bo3jIMpxfolgoOhRQvpvBY+QZKEsMT335v74lVfY5bNJrtVR3RomDVBWaBne9MvsXCN09VEYiKsWwd5eaVPEsKLJEkIS0ydakZ6Nm9unofs2UrIvh1kdDrH2sC8rG2TLMKCC5nnPF+iSxcz03DtWmsCE8JBkoTwuqws09x+zjlFs6xjFv4IQHqXc8s+sQoKCoQOzTNY/LuLJAGwcKHXYxKiOEkSwutmzDCtKMWHvsYsmUFe7Ybk12loXWAW6ZaQwT+7wzh8rFi/RK1apm66zLwWFpMkIbzuhx8gPLzoy7ItO5PIDUvJaN/T2sAsUjRfwqlfol07WLFCVqoTlpIkIbzKboc5c6B7dwhyFECNXDkfW34eGV36WhucRdo3yyQspJA5y13MlzhyBHbssCQuIUCShPCy5cshJcWpqWnZLOxBwWQldLEuMAsFBULnFuksWB1dckfXrub+11+9H5QQDpIkhFd9/72ZTN27d9G2qJXzyW6eWGXXjnBH7/bp7NgXyq79wUUbmzc37XKLFlkXmKj2JEkIr5o+3UwBiIkxzwOPHCBs92Yy21WfCXSu9OmYBsDsZcWanAICoE0bmVQnLCVJQnjNjh2weXPJWdbRy2YBkNmxt+uTqokWDXKIi85j1lKnJqeOHeGffyA11ZK4hJAkIbzmxCzrEkli+WwKw6PIOauVJTH5CqXg7IR0Fv0ehd1ebEdSkhndJE1OwiKSJITXTJ0KjRpB48aODVoTtXohWa07gy3A0th8Qe8OaRxNC2L95mJlSTp2NLWcZs+2LjBRrUmSEF6RlmYqX58oCw4QsnsLwSkHyGjfw7rAfEifDma+xM/Fm5xCQswiRL/8YlFUorqTJCG8YsYMKCiA/v2LtkX/NhOAzA7VcxKds9o18mlSN5t5K5zmS3TrBn//DYcOWROYqNYkSQivmDIFoqOhQ4eibdHL55Bfszb5tatfKY6y9GibxtINUeTkqqKNvXqZ+7lzrQlKVGuSJITHFRSYv289ephRnQAUFhK1bjGZbZKKqvwJ+nRIJzffxpJ1xUp0tGlj5ktIv4SwgCVJQikVq5T6Tin1t1Jqk1JK2huqsCVL4PjxkqOawv9aRUBWhjQ1OeneNp0Am+bn34o1OQUGmkswqQgrLGDVlcQbwCytdQLQEdhkURzCC77/3vydKz7LOmapY35Eu+4WReWbIsLstG+WwZzlMSV39OoFe/aYORNCeJHXk4RSKgboC/wPQGudp7VO9XYcwntmzDAjOSMiirZFrZxHToNmFMbUtC4wH9W343E2bg9n3+Ggoo0nil1NmWJNUKLasuJKoilwGPhYKbVWKfWhUirC+SCl1DCl1Gql1OrDhw97P0pRKf75x8y0Lr5MqcrJIuLPlWS1TbIuMB/Wr8txAGYsLjYUtl49aNYMfvrJoqhEdWVFkggEugDvaq07A5nA484Haa0naq2TtNZJ8fHx3o5RVJITs6yLV32N/P1XbAX5ZLTvZU1QPq5Voxxqx+YxbbFTk1Pv3rBqlSmjK4SXWJEk9gJ7tdYrHM+/wyQNUQVNnWq+ANerV7QtZtksdEAgWW3kn90VpaBX4nF+WR1NfkGxHQMGmAU55GpCeJHXk4TW+gCwRynV2rFpAPCXt+MQnpeSYr74Fp9lDY7S4E3boEPDrQnMD/TrcpyM7ICSQ2HbtoW4OPjmG+sCE9WOVaObHgC+UEptADoBL1gUh/CgadPMF9/is6wDUlMI2/5ntS8NXp5e7dMJDLAzdWGxJiebDc4/H+bNkyYn4TWWJAmt9TpHf0MHrfVVWutjVsQhPOunn6BGDfMF+ISolfNQWsv8iHJEhNrp3CKDn5c69UtcfjkUFsLkydYEJqodt5KEUupypZTMzhZuy8+H+fPNWta2Yv9zYpbNojAkjOxmidYF5yf6djzOP7vDSq5W17q1KaP72WfWBSaqFXf/8A8CtiilXlFKJXgyIFE1LFkC6eklRzUBRK1aQHbLjmZ2nTil/klmKOzURcWGwioFl1wCK1eaon9CeJhbSUJrfQvQGdgGTFJKLXPMY4gq51RRTf3wg8kDPYpVAQ/ev4uQA7vJTJT+CHc0qZtLw1o5/PBLbMkd114LQUHw2muWxCWqF7ebkLTWaZjhql8B9YCrgd+VUg94KDbhx2bONOWGSsyyXmGqmGZ2kPkR7lAK+nVJZfG6KI5nFPtVrVHDXKJ98YVZqEMID3K3T+JKpdQPwEIgCDhba30xpu7Sw54LT/ijbdvMrW/fktujl8+lIDKG3AbNrAnMD13UPZWCQhtTFzl1YN90E2RlwYQJ1gQmqg13rySuAV7XWrfXWo/VWh8C0FpnAXd6LDrhl374wdwXr/qK1kT9vojsVp1K9mSLU+rUIpO46Hy+n1+j5I7EROjcGcaNg4wMa4IT1YK7v60HtNa/Ft+glHoZQGs9v9KjEn5t2jSzlnXDYmsJhezeQtDRg2QmStXX02GzQd+OqcxdEV1yISKA++6Do0dh/HhLYhPVg7tJ4gIX2y6uzEBE1ZCebtay7uXU7RC9zCyYI+tZn76Lzk4lKzeAOcujS+7o1AnOPhteeUWWNhUec8okoZQarpT6A0hQSm0odtsBbPBOiMKfzJ5t5kg4D32NXjmP/Nha5NdpZE1gfqxHYjqRYQV8Oy+29M7RoyE7G0aO9HZYopoo70riS+By4CfH/YlbV8ewWCFK+PFHM6KpU6diG+12ItcuJqt1Z1mq9AwEB2p6Jx5n5pIYCgqcdjZtCoMGmRnY86XlV1S+8pKE1lrvBO4D0ovdUErJajGiBLvdXEl061ZyrlzYto0Eph+T/ogKuKh7KkfTg1iw2sXUpHvuMWV2b7sNUlO9Hpuo2ty5kgBYA6x23K8p9lyIk9asgSNHSi4wBBC1zLFUaaL0R5yp8zofJzykkM9muPhuFhYGzz0HBw7AXXd5PzhRpZ0ySWitL3PcN9VaN3Pcn7jJYHdRwg8/mNakc84puT165Xzy4utTUKuuNYFVAaHBmn6dj/HTolhy81w02XXsCHfcAd99B5984vX4RNXl7mS63ieWGFVK3aKUek0p1dizoQl/M3MmJCRAbGyxjQUFRK5fSlaCLDBUUZf1PkZ6ViAzlkS7PmDYMGjXzgyN3bbNu8GJKsvdIbDvAllKqRMzrLcBUoZSnHT0KGzYAD2dKoCHb1pNQHaGDH2tBL3bpxEbmc/nrpqcwHQEvfSSeXz99WaYmRAV5G6SKNBaa+BK4G2t9QRAivuJk2bPBq3NMszFxTjmR2S1laJ+FRUYABd0PcbMpbGkZ5bxq1uvHjz+OKxdC08+6d0ARZXkbpJIV0o9AdwCzHCsLRHkubCEv5k50wx9bdeu5PaoVfPJrd+EwhgZDFcZLu9zlNx8G98tiC37oIsvNrdXX5VhsaLCTmc9iVzgTsca1Q2BsR6LSvgVrWHBAlNKqPjQV5WXS8TGlWQmdLUuuCqmS6tM6sXl8vFPcac+8IknoH59uOUW0xYoxBlydz2JA1rr17TWix3Pd2utP/VsaMJfbN4M+/aVLsURsWEptvxcMqU/otLYbHD1OUdYvC6abXuDyz4wPBxefNGMSR461HsBiirH3dFN1yiltiiljiul0pRS6UopKWQvAJg+3dy7qteklY2sNkneD6oKu75fCjalef/7Wqc+sG1bMyz2p59M1UUhzoC7zU2vAFdorWO01tFa6yitdRnj8ER1M3u26S8tXvUVIHr1L+Q2aoE9QsY4VKY6NfPp2e44n0yPK12mw9nQoaYk7/DhZv0JIU6Tu0nioNZ6k0cjEX4pPx9++82U4ijOlp1J+N9ryGwr/RGecEP/Ixw6Fsz0JTGnPjA4GJ56CpKT4emnvROcqFLcTRKrlVJfK6UGO5qerlFKXePRyIRf+O038wW1T5+S2yN//xVVWEhmYk/XJ4oKOa/zceKi83nnm3KanACSkuDCC+Gtt2D7ds8HJ6oUd5NENJAFXEhRJdjLPBWU8B8zZpjOVOcriejls9EBgabyq6h0QYFwTd/DzF0Zy5bdIeWfMHKk+YcaNcrjsYmqxd3RTUNc3GTIhGDuXFOKI8qp2yFq9S9kN0lAh4ZZE1g1cMtFhwkMsPPqZ7XLP7h2bbMu9tSpsGSJ54MTVYa7o5taKaXmK6U2Op53UEr9n2dDE77u2DH44w/o4TTCNSDtGGFbN8qoJg+Ljy3gwqRjfDYzjtT0gPJPGDIEatQwVxVaezw+UTW429z0AfAEkA+gtd4A3OipoIR/mDXLrCHhPPQ1cvVClLaT0UH6IzztjksOkpUbwPvflzO5DszciXvuMTXdv/nG88GJKsHdJBGutV7ptK28wXeiivv5Z/N3JzGx5PboFbOxB4WQ06K9NYFVI4nNsunUIp23vqpNvju/kVdeaYbEPvUU5Y+fFcL9JHFEKdUc0ABKqeuA/R6LSvi8skpxAEStXkh283booFPMCBaV5q7LD5B8JIRJU924mggMLCol/tFHng9O+D13k8R9wPtAglIqGRgJ3OOpoITv27rVDL13Lg0emHKQsF3/kNm2m+sTRaU7r3MaCY0yef6jeu5dHAwYAK1bw5gxkJPj6fCEnztlklBKPaSUegi4CpgJPA+8B0wBrvV4dMJnnSjF4VwaPGqlqTqa2V76I7xFKRh+9X52HQjhs5/dqLarFIwYAfv3w5tvej5A4dfKu5KIctySgOFADSAWcxUhS41VY3PmQN26LkpxrJhDYWg4OU0TrAmsmjo/6Tgt6mfx3If1KCx044Tu3aFLF1MEME3KsImylbfG9bNa62cxpcG7aK0f0Vo/DHQFZPnSaio/HxYvNhPolNNyy1FrFpLdogMEBLo+WXiEUnDP1fvZnhzKZzPdXLtjxAhITYWXX/ZobMK/udsnUQfIK/Y8z7FNVEPLlkFmZummpqADuwnZv4vMdrIKnRUGnp1Kq4ZZ/Pvd+uTmqfJPSEyEc86B8ePh0CGPxyf8k7tJ4lNgpVJqjFJqDLACmOSpoIRvmznTVHg42ykXRK2YC0CmzI+whM0GowYls+dQCBPcqekE5moiJweefdazwQm/5W5ZjueBIcAxx22I1vrFiryxUipAKbVWKTW9Iq8jvG/uXGjVCqKdisXHLJ9LQUQ0uQ2bWxOYoG/HNLq0TOOFj+qVvQ52cU2bwkUXwQcfwK5dng9Q+B13ryTQWv+utX7DcVtbCe/9ICDlx/1MaiqsW2f6PUvQmqjfF5HdqpP5SissoRSMHpxMSloQL01ys0X43nvN/ZNPei4w4bcs+W1WSjUELgU+tOL9xZmbO9eU4nDujwjZs5WglANkJkp/hNU6tsyif+djvDG5DoeOujGAoF49uPpq+Oor+Osvzwco/IpVX/nGA48C9rIOUEoNU0qtVkqtPnz4sNcCE6c2cyaEhUGHDiW3Ry+fDcj8CF/x8I3J5OTZePrdeu6dMGwYhITAo496NjDhd7yeJJRSlwGHtNZrTnWc1nqi1jpJa50UHx/vpehEeebPh06dSpfiiF4+l/yYOPLqyshoX9C0fi5X9jnCR1Pj2brHjfIosbEweLBZIGT5co/HJ/yHFVcSvYErlFI7ga+A/kqpzy2IQ5ymbdtgz57SpcGx24n8fRFZbbqWnjghLPPg9fsIDLDz6BsN3DvhjjsgJgYeeURKiYuTvJ4ktNZPaK0baq2bYMqNL9Ba3+LtOMTpmznT3DsvVRr+z1oCM46TIU1NPiU+toBbLjzEDwtrsnJjePknhIebRPHbbzB7tsfjE/5BhqEIt82aZRY4a+zUohT9m8keMj/C9wy74gAxEfk8/HoD9y4OBg0y/8iPPipXEwKwOElorRdqrWWtbD9QUGBKcSQllW5Ril4+l9x6Z1EY6+YELuE1kWF27rlyP0vWRzNjSXT5JwQHw913myUHv/7a8wEKnydXEsItK1ZAenrpoa8qJ5uIP1dIaXAfNviCI9SvlcujbzTAXuZ4wmIuuwzOOksWJhKAJAnhpunTzRw55/UjItctxpafR0bHXq5PFJYLDtQ8eF0ym3aGM2maG8X/AgLMwkTbt8OHMpWpupMkIdwyZ04ZpTh++xkdEEhWG7mS8GWX9jxGQqNMnn63Pjm5boxA69cP2rQxNZ2ysz0foPBZkiREuY4dK6MUBxC9ch7ZTdugQ8O8Hpdwn80GjwxOZt+REN740o15RycWJjpwAF5/3fMBCp8lSUKUa84cU4rjnHNKbg9IPULYto1SGtxP9GqfTo+2x3npk3qkpgeUf0K3bmakwtixcPy45wMUPkmShCjX9OkQEWGWHyguetkcADI69XFxlvBFjwxO5nhmAP/9wM3ifw8+aKo6Pv+8R+MSvkuShDglrWHePOjcuXQpjphlsygMiyCnaRtrghOnrW2TbAaefZQJ39Zh78Gg8k9o08b0T7z9tixMVE1JkhCn9Pffplm6l/PgJa2JWrWArFadZKlSP/PQoH3Y7fDE2/XdO+GBByA3F55+2rOBCZ8kSUKc0nTHklDO/REhu7cQfDiZzETnQk7C1zWIz+P68w7z5aw4/tgSWv4JjRvDpZfCRx/Bjh2eD1D4FEkS4pR+/hkaNjRLDhQX8+tUADK6nOPiLOHr7r92P+EhhTwy3s3if8OHmyFSsjBRtSNJQpQpJweWLTODXJzFLJlJXu2G5Ndu6P3ARIXFRhVy56UHmLMilvkrI8s/oXZtuPZaU6pj40bPByh8hiQJUaZ580yiOO+8kttVThaRG34jo700NfmzOy45RJ0auTz0WkP3ynX8619mxanHHvN4bMJ3SJIQZZoyBUJDzVD54qJWzjelOLr0tSYwUSlCgjUPXLuPDVsj+HymG+U6YmLgpptMzfiVKz0foPAJkiSES1qb/oiuXc2qlsXF/joNe3AIWW2SXJ8s/MZV5xylVcMsnpzgZrmOW281tVlkmdNqQ5KEcGn9ejP0ta/zxYLWRC+bRVarTuggN5bFFD7NZoNHbtpL8uEQxn9Zu/wTIiLg9tth0SJYsMDzAQrLSZIQLk2ZYu6d+yNCdm0m5OAeMjrLqKaqok/7dHq2Pc6Lk+qSkupGuY4bb4RateDxx2VhompAkoRwado0aN0a4uJKbo9d9BMA6V3OtSAq4SmP3bKXzOwA94bEhoTAkCGwalXRRBpRZUmSEKUcOGCqvjqvZQ0Qs2Q6uXUbU1CrXumdwm+1apTDjf0P8cmMWizb4MZ62NdeaybPPP447g2NEv5KkoQo5SdzsUC/fiW3B6SnErlhKRkde5c+Sfi9kTfsIy46n3teaExhYTkHBwbCsGHw11+yzGkVJ0lClPLtt1CnjmluKi7612mowkLSul9gTWDCoyLC7Dx20142bI3gra/cWHPi0kvNMqf/93+yzGkVJklClHDsmBm40q+fWXemuJrzv6MguiY5zRNdnyz83iU9j3F2Qhr/924Ddu4rZ/SazQb33muWOf34Y+8EKLxOkoQoYcoU86XwootKblc52UStmGuammzy36aqUgqeu2sXdg23Pn1W+d0N/fubS85nnjGVYkWVI7/tooRvvoH4eBcLDK2YS0BuNmndz7cmMOE1DWvnMfrGPSxZH82b5TU7KQX33w/795s1J0SVI0lCnHT8OPzyC5x7bummphrzvqEwNFyWKq0mBg1IoWfb4zzxdkP+3hFy6oN79oSOHeGFF2SZ0ypIkoQ46ccfIT8fBg502lFYSMySmWQmdodAN1YzE35PKXjh7l2EBNm59tFmZOeUU7Jj1Cg4ehTGjPFKfMJ7JEmIk774wkye69Ch5PaoVQsITD9G2tnS1FSd1KmZzwt37+CvHeHc+1KjUx+cmAgXXggTJpiObFFlSJIQgFm+eMEC83vu3C8dN/Mz7CFhZHSVWdbVTb/OadwxcD+TpsfzyfRyKsWOGmX+84wc6ZXYhHdIkhAAfP45FBbCFVeU3K7ycold9BMZHXujg91Y6lJUOQ/duI+OzdO598XGbDpV/0R8PNxyi6np8ssv3gtQeJQkCQHAJ59A8+bQsmXJ7dFLfyYgM43U3pdYE5iwXGAAjB+xg+AgO1c91JzM7FP82RgyxBT/GzGC8qdtC38gSUKwaRNs2ACXuMgDcTM+pTAimswOvbwfmPAZdWrm88rwHWzZG8qQMY3LLv4aGmqGxG7cCB995NUYhWdIkhD873+mKfnSS0tut2VnErP0Z9K79DW1ekS11qdDOvdeuY9v58fx1te1yj7w0kuhTRt48kkZElsFSJKo5goKzKimrl1NK0FxsfO+wZabQ2qfy6wJTvic4VcfoFe74zwyvhHL/yijWqxSZh3slBRT10n4NUkS1dz06aY0+LXXlt4X//1E8mrVJ7tNV+8HJnySzQbj7ttBzah8rhvdrOxFihITTfvlu+/Cn396N0hRqSRJVHMTJpi5Ea5WoIvcuJzUvpeVnn4tqrXYqELGj9jOwWNBDHq8adn1nUaONH0Uw4fLCnZ+zOtJQinVSCn1i1LqL6XUn0qpB70dgzB27oT5800TsnOXQ/z376FtNo73u8aS2IRv69gii0dv2sv81TH8Z2Jd1wfVqGHWnFi82NSfF37JiiuJAuBhrXVboAdwn1KqrQVxVHvvvGMuEm64wWlHQT41Z35OZruzKYg9RQelqNZuvuAwF519lP/+rz5zlkW5PmjQIGja1Ey0y8ryboCiUng9SWit92utf3c8Tgc2AW4srCsqU34+TJoE3bpBXacvgrELfiAo9TDH+stVhCjbibLijevkMPippuw/4mIEXGCg6cTetw/+8x/vBykqzNI+CaVUE6AzsMLFvmFKqdVKqdWHDx/2emxV3eTJcPiw+aLnrM6Xr5EfV5eMLlKGQ5xaRKidNx/cTkZ2ALf/+yzXXQ9JSWbdiddfl7pOfsiyJKGUigS+B0ZqrdOc92utJ2qtk7TWSfHxbiylKNymNYwbB40aQZ8+JfeF/f07kRtXcHTAdWArY+SKEMW0aJjDg9cmM3dlLO9/H+f6oEceMUOj7rvPu8GJCrMkSSilgjAJ4gut9RQrYqjOFi2CP/6AwYNLF/Or+9k47MGhpPZ3MSZWiDLcfvEhurRM5+HxjdiR7GLZ09q1TcmOWbPMuGvhN6wY3aSA/wGbtNavefv9BbzyCkRHly7mF5hykNj533G850DsEWV0RArhgs0GLw/fCRpufbqJ62Gxt90GDRrAAw/IUqd+xIorid7ArUB/pdQ6x02qx3nJli3my9xVV5kh7MXV/eRlVGEhKZfeaklswr81iM9j9OA9/LYhire/dtFEHBQEo0ebsddjx3o9PnFmrBjdtERrrbTWHbTWnRy3md6Oo7p6+WUICICbbiq5PSA1hVpT3ie967nk1zvLmuCE37uhfwrdEtJ4ckID9h50sYphnz7Qq5dZ6jQ52fsBitMmM66rkf374bPPzMJCznWa6nzxKgE5WRy5Zpg1wYkqQSn47792UVCouPv5Mlaze/RRUzTsQZlH6w8kSVQjzz9vfjfvuqvkdlvGcWp//TbpHfuQ26il65OFcFPjOnncc+U+Zi6twTdzY0sf0LAh3HwzfP+9LE7kByRJVBOHDpmS4AMGmKGvxdWd9BIBWekcvvZua4ITVc6dlx2kZcMsHnilEanpLoZS/+tfZsTTvfeaby7CZ0mSqCZeecUMKBnm1JoUeOQAtSe/QXqXc8lt2saa4ESVExhgmp2OpAbx8GsuCiqEhsJDD8Hff8Mbb3g/QOE2SRLVwNGjpmLzueeaMjrF1X/nKWz5eRwcLO3DonJ1aJ7F4AGH+GhaPIvWRJQ+YMAAMxv7mWekE9uHSZKoBl5+2dRWu9upNSlkx9/Umv4JqX2vIL9uY2uCE1XayEH7qFsjl3/99yxycp1KzisFTz1lCondc481AYpySZKo4g4dgrfegr59oaVTn3Sj10ehg4I5fN1wa4ITVV5EqJ1nhu5m694wxkysV/qARo1g6FAzC/v7770foCiXJIkq7plnTF/EiBElt0cvnk7M0lkcufQ2CmNqWhOcqBbO7ZTGxd1TePXzuqzfHFb6gDvuMO2g990HaaXKuAmLSZKownbuhA8/hIsvhiZNirar3Bwav3w/eXUakXL5HRZFJ6qTp2/fQ2RYAUPGnEVhodPOwEB4+mlz2Tt6tCXxibJJkqjCnnjC1NS5996S2+t+9DwhB3ax//ZHIdDFrFghKllsVCFP3LKHtZsjePXz2qUP6NABrr4aPvjArGQnfIYkiSrqjz/g66/hmmugTp2i7cF7tlL307GkdT2PrPY9rQtQVDuX9TpGn/apPPN+A7btdVEp9sEHzdyJW26BjAzvByhckiRRRY0aBWFhZs7SSXY7TZ+5HR0QwMHbHrUsNlE9KQXP3rkbm9IMGeNigaKICHj2Wdizp3QnmrCMJIkqaNo0mD/fDBqJjS3aHv/120RuWMqhGx+koKaLS34hPKxeXD6jbtjL4nXRvPedi/XTk5LMousff2z+IwvLSZKoYvLyYORIU7b/5puLtgcn76DB24+T2SaJVFm7Wlho8PlH6NoqjUfGN2TzrpDSB4wYYUZaDB0KR454PT5RkiSJKubEMsIPP2zK9wNgt9PkmdsA2DfsGXPdL4RFbDZ45d6dBARoBj/ZlHzn0k0hIfDcc5Caar7puFzBSHiLJIkq5OBB87vVrZuZPHdCnU9eJmrdEg4NfpCCWi4mNAnhZfXi8nnmjl38/k8ET79bv/QBCQlw//0wZw689JL3AxQnSZKoQoYPh5wcU67/hPCNK2jw3tOkd+oj61YLn3JJz1Qu63mEsZ/WZcGqyNIH3Hyz+bbz73/LsFgLSZKoIqZOhR9+MMsInyjiZ8tIo9njgyiIqsm+u5+VZibhc54ZsocG8bkMeqIZyYec5uwoBf/5jxkWe/310j9hEUkSVUBGhrmKOOusYgsKac1Z/72T4IN7SL7vOeyRMZbGKIQrEWF23h65jcwsG1c91IzcPKcvMpGRpkJlSoqZbJefb02g1ZgkiSrgoYfM0qRPP13UWV37s3HUnP8dR64cSnZCV2sDFOIUWjbK4b//2snqvyO554VGpedPtG0Ljz0GS5aYfgrhVYFWByAqZto0U8ng+uuhUyezLWr5HBq+/QQZHXtz5GpZs1r4vkt6pvLnjv18PL0ezRrk8fRdB0oecPXVsGULTJwI7dtLsvAiSRJ+7OBBGDLE9EGMHGm2Be/ZSrPHbyCvdkOS73vBjDcUwg88fOM+9qcE8+/3G1A/Po87rzrqdMDDsGOH+c+ekADnn29JnNWN/AXxU4WFcNNNkJ4OL7xghpYHphyk1fABKLudPQ+/jj3MxWpgQvgomw1eGr6Lbq3TuPuFJkyeHVvygIAAGDsW6tc3RcnWrbMizGpHkoSfevxxWLDA1ERr2dKMZGp57/kEHT3I7odfl5XmhF8KDtS88/A2EptmcMv/NWPSNKe1TiIj4Z13zLeiCy80M0eFR0mS8ENffgnjxsGll5oyN7bMdFrefxFhO/5i730vktOqk9UhCnHGIsLs/O/xrXRumc7Q/zRh3Ke1S3Zm16sHEyaY1bT69YMDB8p8LVFxkiT8zPz5ph+ibVuzPHBgRiqt7u5HxF8rSb77WTK79C3/RYTwceGhdj54bCt9O6Qy+s1GDBnTmLz8YsNjmzeH8ePNQkV9+kBysmWxVnWSJPzIypVw5ZVQty68+SZEHN1D6zv7EL5lPXvve4H0XhdbHaIQlSY0WDPhoe0MvXg/n8yIp/ttrflnZ7GCgB06mESxbx/06gW7dlkWa1UmScJPLFwIAwaYJtl334X6u5fR5pauBCdvZ8/IcWScLSM9RNVjs8EjN+3j1fu2sS05hM43t2H8l/FFS6AmJZmmp5QU6NFDOrM9QJKEH/jmGxg4EGrUgA8n2ukw6xVaDzsXAmzs/PdHZHbqY3WIQnjUxT1SmfriX3RolsGo1xrTaXAbfv3dMXqvQwd4/30zG7tXL5gyxdpgqxhJEj4sPx8eeAAGDTJzISY/v51zn+1Pw7ceI7NtEjv++wV5jVtaHaYQXlGnZj4fPbGVcfdu49DRQM4dlsD597Rg2YYIM2/iiy+gUSO47jp45BGzuIqoMKVLzYH3PUlJSXr16tVWh+FVy5aZpUf/+gtuvDyTsbHPUf+r10DZOHjjg6Sef50U7BPVVlaOjU9+jufT2XU5nhnIuZ3TGHnzIS7vdpCAV16EGTPMFcbnn5sZ2tWUUmqN1jqpQq8hScK3bNliKiN//TU0jk3j065v0Gvl6wSmHSOtW38O3vKILD0qhENmjo3PZ8Xz5bzaHD4eTKPaudxz3WHurTGZ2Leeg8xMuPNOM+M0Ls7qcL1OkkQVUVho1lZ5913zBahbwBr+23Ai/Q58SWB2Bhltu3Hk2rvJlvkPQriUXwBzV8fyxezarN0aRYBNc3Wn7TzP/9Fy3Teo0FAzdnz0aFMuuZqQJOHHsrJg3jz46SeYMV1T99B6BgX/wM0h39E4/S/sQcFkdO5LyiW3kNM80epwhfAbW/aGMmVhHD+vqMmh1GCSQtbzWsx/6X34RxR2VM+eMHgwXH55lU8YkiT8RGoqbN4Mf/wBy5fDihWQ9ddOehcuYkDAQi4IWEC9vN1opchu2pa0nhdy/JwrsEdEWR26EH6r0A7L/4zix19rsnhDLDFZ+xiqJjE06DMa520FwN6wEbZePU0J5c6dITHR1IaqIoUx/TZJKKUGAm8AAcCHWutTLmLrq0kiI8NUYj140FQG2L/f3A4cMLeDBzRHt6cSdWwXrfmHjqyni20dXWzrqF2wH4CC8ChyWrQnvUtf0pP6UxhTs5x3FUKcroJCWLs5knmrYli8IZqIg9vpzwIGqAV0t62ibuG+k8cWBgSTX7s+9kZnYW/UBNW0CbZmZxHQtDG2Rg0JaNwAFeViuVUf5JdJQikVAGwGLgD2AquAwVrrv8o6x90koXXRzW4Hbdcnb/ZCXeq5vVCTlQVZmZqsTE1mRtEt9Zgm5Ygm/XAOGUdyyDySTc6xbHJTs8lPy6YwI4vAghwiyaAGx6jBMWpylBoco3ZACvXVfhradxNlTzsZn90WQF6dRuQ1akFWi/ZktTub3IbNq8y3FiH8xbH0AFb/E8nvf0eyeU8Yx/dnEX/sH1rzD03ZQRN2nryvw6FS56cSwz4asE81YL/N3A4ENiDNVoPMwGgyA2PJCowmOziGgPAQQiKDCI0yt7DoICKjbURHQ1QUREebW2Rk0S0qquTzsLAzG8zor0miJzBGa32R4/kTAFrrF8s6p4sK0IsJRaHLvQHY8P7VkV3ZyA+Nwh4WgT0sgvzwGPJi4smLrU1OXH1y4hqQE98IHRjs9diEEOXLybeRkhHC0YxgUtKDSc8OIq/Ahs7NJSzzMFGZh4jNOUBM9gFi8g4Rm3uIGnkHzS3/MDbsbr9XITYKCUBT9Je/sh4XF01GhZOEFYsONQD2FHu+F+jufJBSahhwYlm13EiyNnohtjOn7ZB93NxKqwX46yru/hw7SPxWk/hdsjtuHte6oi/gsyvTaa0nAhMBlFKrK5oNreTP8ftz7CDxW03it5ZSqsKduVY0hicDjYo9b+jYJoQQwsdYkSRWAS2VUk2VUsHAjcBUC+IQQghRDq83N2mtC5RS9wOzMUNgP9Ja/1nOaRM9H5lH+XP8/hw7SPxWk/itVeH4/WIynRBCCGvIAH0hhBBlkiQhhBCiTJYmCaXUQKXUP0qprUqpx09x3LVKKa2USnI8b6KUylZKrXPc3vNe1CXiOmX8Sqk7lFKHi8X5r2L7bldKbXHcbvdu5CdjqEj8hcW2WzLwwJ3/P0qpG5RSfyml/lRKfVlsu89//o5jyorf5z9/pdTrxWLcrJRKLbbP0s+/grH7w2ffWCn1i1JqrVJqg1LqkmL7nnCc949S6qJy30xrbckN02m9DWgGBAPrgbYujosCfgWWA0mObU2AjVbF7m78wB3A2y7OrQlsd9zXcDyu4S/xO/Zl+MHn3xJYe+KzBWr72efvMn5/+fydjn8AM0jF8s+/IrH7y2eP6bAe7njcFthZ7PF6IARo6nidgFO9n5VXEmcDW7XW27XWecBXwJUujvsv8DKQ483g3OBu/K5cBMzVWh/VWh8D5gIDPRRnWSoSvy9wJ/67gAmOzxit9YkiPP7y+ZcVvy843f8/g4HJjsdWf/4Vid0XuBO/BqIdj2OAExUMrwS+0lrnaq13AFsdr1cmK5OEq/IcDYofoJTqAjTSWs9wcX5Tx6XUIqXUOR6Msyzlxu9wreNy7zul1IlJhO6e60kViR8gVCm1Wim1XCl1lScDLYM78bcCWimlfnPEOfA0zvW0isQP/vH5A6CUOgvzrXXB6Z7rIRWJHfzjsx8D3KKU2gvMxFwNuXtuCT5blkMpZQNewzR5ONsPNNZapyilugI/KqXaaa3TXBxrpWnAZK11rlLqbuAToL/FMZ2OU8V/ltY6WSnVDFiglPpDa73NskhdC8Q02ZyHmdn/q1LKnxY8dhm/1joV//j8T7gR+E5rXWh1IGfAVez+8NkPBiZprV9VpqjqZ0qpM1q9zMorifLKc0QBicBCpdROoAcwVSmV5LhUSgHQWq/BtKu18krURcotL6K1TtFa5zqefgh0dfdcL6hI/Gitkx3324GFQGdPBuuCO5/hXmCq1jrfcWm9GfNH1y8+f8qO318+/xNupGRzjdWff0Vi95fP/k7gGwCt9TIgFFOs8PQ/ews7XwIxHVZNKep8aXeK4xdS1HEdj6OzBdN5kwzU9LX4gXrFHl8NLHc8rgnswHTa1XA89qf4awAhjse1gC2couPPwvgHAp8Ui3MPEOdHn39Z8fvF5+84LgHYiWPirmObpZ9/BWP3i88e+Bm4w/G4DaZPQgHtKNlxvZ1yOq699oOV8cNegvl2tA14yrHtP8AVLo5dSFGSuBb4E1gH/A5c7ovxAy864lwP/AIkFDt3KKbTaCswxJ/iB3oBfzi2/wHc6aPxK0yT5V+OOG/0s8/fZfz+8vk7no8BXnJxrqWf/5nG7i+fPWYU02+OONcBFxY79ynHef8AF5f3XlKWQwghRJlkxrUQQogySZIQQghRJkkSQgghyiRJQgghRJkkSQghhCiTJAlhGaXUR0qpQ0qpjWd4/kJHJcv1jtIVrSs7xopQSi113DdRSt10mueGK6W+UEr9oZTaqJRaopSK9EykQpRNkoSw0iQqXtjtZq11R0zJkLHOO5VSARV8/TOmte7leNgEOK0kATwIHNRat9daJ2Jm0OZXJB6llM+W4RG+S5KEsIzW+lfgaCW93K9ACwClVIZS6lWl1Hqgp1LqIce38Y1KqZGOY5oopf52fFvf5ChgGO7Y19VROHKNUmq2UqqeY/tCpdTLSqmVjjUGznFsb+fYts5RDLHliTgcsb0EnOPYP0op9atSqtOJwB1XCR2dfp56FCuXoLX+RztKpCilbnO8z3ql1GfFfp4Fju3zlVKNHdsnKaXeU0qtAF5RSjVXSs1y/GyLlVIJlfT5i6rKitmCcpPbiRsVWBuEkrPwRwNfOx5r4AbH466YmbERQCRmBnlnx/tqoLfjuI+AR4AgYCkQ79g+iKJ1EBYCrzoeXwLMczx+C3NFA6ZMQpjjcYbj/jxgerG4bwfGOx63Ala7+Nk6AYeAZcBzQEvH9naYmba1HM9rOu6nAbc7Hg8FfnQ8ngRMp6iMzfxir9UdWGD1/wG5+fZNLj+Fv/tCKZWNqbFzohxyIfC943Ef4AetdSaAUmoKcA4wFdijtf7NcdznwAhgFqaw5FylFJgFXvYXe78pjvs1mEQD5g/5U0qphsAUrfWWcmL+FnhaKTUa8wd9kvMBWut1jiqjFwLnA6sc1Tz7A99qrY84jjtxJdYTuMbx+DPgleLvp7UudPRp9AK+dfxsYGr4CFEmSRLCZzn6E9Y4nk7VWv/bxWE3a61XO23L0e6VpXauSaMx9ZL+1Fr3LOOcE1VxC3H8/mitv3Q051wKzFRK3a21XlDG+Wits5RSczELwNxAseq6TsdlYJLSFKWUHXP1kufGz+Us03FvA1K11p3O4DVENSV9EsJnaa0LtdadHDdXCcIdi4GrHKOFIjDVbBc79jV2fDsH07G8BFP0LP7EdqVUkFKq3anewPGNf7vW+k3gJ6CD0yHpmNL3xX0IvAms0o6V55xes7dSqobjcTCmYNsuzOI31yul4hz7ajpOWYopaw1wc7Gf8SRt1lvZoZS63nGuctEXIkQJkiSEZZRSkzFNNa2VUnuVUndW9ntorX/HNOesBFYAH2qt1zp2/wPcp5TahCkB/a42y0FeB7zs6Pheh2miOZUbgI1KqXWYpqpPnfZvAAodHc2jHHGtAdKAj8t4zebAIqXUH5h1rlcD32ut/wSed+xbj6kSC6apbYhSagNwK2Z0lCs3A3c6zv0T/1qyVlhAqsCKakkp1QTTmXxGq3VVwvvXx3SEJ2it7VbEIIQ75EpCCC9TSt2Guap5ShKE8HVyJSGEEKJMciUhhBCiTJIkhBBClEmShBBCiDJJkhBCCFEmSRJCCCHK9P9ma8LtkQSvrAAAAABJRU5ErkJggg==\n",
 830 |       "text/plain": [
 831 |        "<Figure size 432x288 with 1 Axes>"
 832 |       ]
 833 |      },
 834 |      "metadata": {
 835 |       "needs_background": "light"
 836 |      },
 837 |      "output_type": "display_data"
 838 |     }
 839 |    ],
 840 |    "source": [
 841 |     "# Box 21: Assessing IPTW overlap by hand\n",
 842 |     "density_t = gaussian_kde(1 - data.loc[data[\"A\"] == 1, 'p_score'])\n",
 843 |     "density_u = gaussian_kde(1 - data.loc[data[\"A\"] == 0, 'p_score'])\n",
 844 |     "\n",
 845 |     "x = np.linspace(0, 1, 10000)\n",
 846 |     "\n",
 847 |     "ax = plt.gca()\n",
 848 |     "ax.fill_between(x, density_t(x), color=\"b\", alpha=0.2, label=None)\n",
 849 |     "ax.plot(x, density_t(x), color=\"b\", label='RHC = Y')\n",
 850 |     "ax.fill_between(x, density_u(x), color=\"r\", alpha=0.2, label=None)\n",
 851 |     "ax.plot(x, density_u(x), color=\"r\", label='RHC = N')\n",
 852 |     "ax.set_ylim([0, 10])\n",
 853 |     "ax.set_ylabel(\"density\")\n",
 854 |     "ax.set_xlim([0.45, 0.8])\n",
 855 |     "ax.set_xlabel(\"1 - Propensity Score\")\n",
 856 |     "ax.legend()\n",
 857 |     "plt.show()"
 858 |    ]
 859 |   },
 860 |   {
 861 |    "cell_type": "code",
 862 |    "execution_count": 23,
 863 |    "metadata": {},
 864 |    "outputs": [
 865 |     {
 866 |      "data": {
 867 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEKCAYAAADn+anLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAABA/ElEQVR4nO3dd3hU1dbA4d9OJr0TEgIECNJ7gNCRImAHrp9gwYYNOyoqol4Vy1UsV2zXAtd67QWxI9KUDqEICAgICEF6COl9f3/sIWRSSEIyczIz632eeSbZc+bMypDM4uyyttJaI4QQQlTEx+oAhBBC1F+SJIQQQlRKkoQQQohKSZIQQghRKUkSQgghKiVJQgghRKWcliSUUm8rpQ4ppTaVamuglPpZKbXdfh/lrNcXQghRe868kngXOLdM2xRgvta6DTDf/r0QQoh6SjlzMZ1SKgH4Tmvd2f79H8AQrfV+pVRjYJHWup3TAhBCCFErNhe/XiOt9X771weARpUdqJSaAEwACAkJ6dm+fXsXhCeEEJ5jzZo1R7TWMbU5h6uTRAmttVZKVXoZo7WeAcwASEpK0snJyS6LTQghPIFS6q/ansPVs5sO2ruZsN8fcvHrCyGEqAFXJ4lvgGvsX18DfO3i1xdCCFEDzpwC+zGwHGinlEpRSl0PTANGKKW2A8Pt3wshhKinnDYmobW+vJKHhjnrNYUQ7qegoICUlBRyc3OtDsVtBQYGEh8fj5+fX52f27KBayGEAEhJSSEsLIyEhASUUlaH43a01hw9epSUlBRatmxZ5+eXshxCCEvl5uYSHR0tCeI0KaWIjo522pWYJAkhhOUkQdSOM98/SRJCCCEqJUlCCOHVjh49SmJiIomJicTFxdG0adOS7/Pz80/rnIsWLWLZsmV1Et9DDz1Es2bNCA0NrZPz1ZQMXAtRS/n5YLOBj/yXyy1FR0ezfv16AKZOnUpoaCj33ntvyeOFhYXYbDX7qFy0aBGhoaH079+/1vGNHDmS22+/nTZt2tT6XKdDfq2FOE1Ll0JSEgQEQHg4XHstHD1qdVSiLowfP56bb76ZPn36MHnyZP7880/OPfdcevbsyZlnnsnWrVsB+Pbbb+nTpw/du3dn+PDhHDx4kN27d/PGG28wffp0EhMTWbx4ca1i6du3L40bN66LH+u0yJWEEKfh669hzBiIioKrr4aDB+GDD2DuXJg/H6Qe5em56y6w/6e+ziQmwosv1vx5KSkpLFu2DF9fX4YNG8Ybb7xBmzZtWLlyJbfeeisLFixg4MCBrFixAqUU//3vf3n22Wf597//zc0331zuiuSEhQsXcvfdd5drDw4OrrMuqrokSUKIGvrjD7j8ckhIgBkzzFUEmLa77oLBg2H5cjjjDCujFLU1duxYfH19yczMZNmyZYwdO7bksby8PMAkkksvvZT9+/eTn59frXUKQ4cOLenecgeSJISoAa1Nt5KPD7z88skEAdC5M7z+OtxwA4wcCcnJEBRkXazu6HT+x+8sISEhABQXFxMZGVnhB/sdd9zBpEmTGDVqFIsWLWLq1KlVntfdriRkTEKIGvj0U3OVcNttEBtb/vE2beDxx2HzZpgwwfXxiboXHh5Oy5Yt+fzzzwGzwvm3334D4Pjx4zRt2hSA9957r+Q5YWFhZGRkVHi+E1cSZW/1MUGAJAkhqk1rkwCaNoWLL678uMGD4aqrzBjFF1+4Lj7hPB9++CFvvfUW3bp1o1OnTnz9tSlgPXXqVMaOHUvPnj1p2LBhyfEjR47kq6++qpOB68mTJxMfH092djbx8fHVulqpS07dvrSuyKZDwjJHjsBrr8HSpRzaV8CHv3cj5NpL6HlrXzjFKteCApMoUlNh61aIqdXeYJ5ty5YtdOjQweow3F5F76NSao3WOqk255UrCSEq89ln0LIlPPoo/PEHx3Ye42beYMI7/WlzyzD89+2q9Kl+fvDEE5CebsYw3OD/YkJUSJKEEBV5/3247DLTt/Tpp6S89g0dc9dy79kbOHTJ7YRuWE6HcYmErpxf6SnatIEbb4Tvv4d333Vd6ELUJUkSQpS1Zg1cfz106QJvvQWtWjFjVjTFWjH6nFyOjhzPzqc+pig0grYTzyX8128qPdX48dCpE0ycCH/VerdhIVxPkoQQpeXlwbhxEBEB06dDYCBawwc/RtOjbTrNYk0tn4JGzdj92PvkNUmg1f2XEJq8sMLT+frCv/4FRUVmHUVRkSt/GCFqT5KEEKW9+ips2wYPPGASBbBhexC7/g7kgr7HHA4tDgljzwNvUBgVQ6vJY/Dbv6fCU8bHw913m6mz02TDXuFmJEkIcUJGBjz1FHTrZuax2n3yUyQ+SjOid1q5pxSFRbJ30nRUXg6tJ41C5edVeOqLLoKBA2HqVKjljEghXEqShBAnvP66mbN6550O01u/XBBFYutMGkYUVvi0/KYt2T/hUYK3/0aT/zxU4TFKmTUWsbEmYeyp+KJDWKC+lwpfs2YNXbp0oXXr1kycOBFXL1uQJCEEmMGCV181o8xdu5Y0b94ZyPa9QZzT+9gpngwZvYeTNuB8Gn00neDfKv5wCA83wxw5OXDuuSYfCeudKBW+fv16br75Zu6+++6S7/39/SksrPg/B6dSl0nilltuYebMmWzfvp3t27czZ86cOjlvdUmSEALgu+9g714z7bWUL+ebcYlz+qRVeYqDV0+mMCKalo9cVWm3U6tW8PTTsGOH6dGSRFE/1ZdS4fv37yc9PZ2+ffuilOLqq69m9uzZdfRTVo8U+BMCYOZMU/d7xAiH5h+XRdAmPpvYqIIqT1EcHMr+6x+i+b/vIu6dp9l/09QKjxswAJ55BiZPhp49zTqKjh3r4ofwAPWoVnh9KBW+b98+4uPjS76Pj49n3759Nf5ZakOShBBpafDzz6Z0a6kdyNIzfVi9OZSrzj5Q7VNlJQ4kI/FM4t6bxtGR48lvklDhcYMGwX/+A/ffD717m6uL226T3e3qEykVbkiSEOKzz8wepBde6NA8d2UYhUWKwd3Ta3S6A9fcT6v7L6bZM7fx50vfV3pcz57wv//Bww+bxXYzZ8Jjj8Ho0V6cLOpRrfD6UCq8adOmpKSklHyfkpJSUnXWVbz1V1GIkz74AOLizIYQpXy/OILggCIS22TV6HSFDeM4MnI8kUt/IHxx5UkCoHFjkxwefRQOH4b/+z9o1gzuvReWLTOFAoW1rCwV3rhxY8LDw1mxYgVaa95//31Gjx5d1z/iKUmSEN7t+HHzaTx0qMO0V63h55Xh9Gqfjr+t5lMOUy+4hvyYpjR7fiKq4NTTKJUyPV2zZ5uigE2amFlQAwaY9Xz9+8NNN5nuqaVLIatmOUvUAStLhb/22mvccMMNtG7dmlatWnHeeefV6nw1JaXChXf7+GNThmPmTOjevaR56+4AOozpzD+v/otxI46c1qlD1v5K8+mTSLn9aQ6On1Kj56alwapVsHKlKTW+Z4+ZOgumK6plSzOWcemlcN554O9/WiHWC1IqvG44q1S4jEkI7/b11xAaaor5lfL9YrMv6eDEmo1HlJbVYxCZnXrT+K0nOXrheAobxlX7uZGRcPbZ5gZQXAwHDpj9tTdvNonj229NjmvSxHRX3XjjKbe4EOK0SHeT8F7FxWZWU+/eDrOaABasDqNJdB5NY05vxe0JB66ejE9+HvEv3Ver8/j4mGQwdKiZBfXKKyb0556DsDDTHXXOOab3TIi6JElCeK9Vq8xqtlJ1msDkjqW/hdGzXcUDjzVR0CSB1GFjiP7xA4I3rar1+Urz8zNJ43//M5VEFi409aGOHq3Tl3EJd+j2rs+c+f5JkhDe63v7zKMBAxyaN2wP4niWjb6dap8kAI6MuYXC8CiaT7vFZKA65uNjtkp99lnTHTVypJnR6y4CAwM5evSoJIrTpLXm6NGjBAYGOuX8MiYhvNeCBXDGGWYAoJSfV4QB1FmSKA4K4dCYW2ny9r9o8N17pI66tk7OW9agQfDQQ6bS7KRJphSVO4iPjyclJYXDhw9bHYrbCgwMdFiZXZckSQjvlJtrdqAbObLcQwuSw2jaMJfG0XW3SOH44NFELfiS+JfvJ+2siykODa+zc5d24YWwbh289hqMGQNDhjjlZeqUn59ftVYqC2tId5PwTsuXm13o+vZ1aC4uhuUbQklql1m3r+fjw4Gr78d2/AhNZkyt23OXcc890KiRme10GgVMhXBgSZJQSt2tlPpdKbVJKfWxUso5nWlCVGbuXDNftEcPh+b12+p2PKK03DZdON7vHGI/fYWA3Vvr/PwnBAebOnk7dsCbbzrtZYSXcHmSUEo1BSYCSVrrzoAvcNmpnyVEHVu40NTtDnfs9plnH4/o44QkAXBo3N0U+/nT/JnbzbJuJxk2zGyN8eijkJ3ttJcRXsCq7iYbEKSUsgHBwN8WxSG8UU4OrF1b7ioCYGFyGPExucQ1cE7RpKKIaI6Mvp7w1fOJ/PlTp7wGmIukiRPNdFi5mhC14fIkobXeBzwP7AH2A8e11nPLHqeUmqCUSlZKJcusB1GnVq40lfP69HFoLiqCZRtDSaqD9RGnknruFeQ2b0OLf92E34G9TnudHj2gQwdTB6qoyGkvIzycFd1NUcBooCXQBAhRSl1Z9jit9QytdZLWOikmJsbVYQpPtmiRue/WzaF5/bYg0rNsTutqKmGzkXLHM/gUFnDGlLFQ6JyrFqVg/Hiz4d5nnznlJYQXsKK7aTiwS2t9WGtdAMwC+lsQh/BWy5ZB06bl1kfMW2nGI/p1quOZTRUoiGvO/mvuJ3TTSlo8fv3pjU8UFeG/bxf+KTsrXaQ3ZIipgv7KK7WLV3gvK9ZJ7AH6KqWCgRxgGCAlXoVraA3JyaZeUxkLk8NoFptbra1K60L6mRcS8PcuGn73HoVRsey767lqVegL2LOduLf+ReQvs7FlmmJNhWFRHLzibg5c+yD4+pYc6+trNjF6802zGrtdO6f9OMJDWTEmsRL4AlgLbLTHMMPVcQgv9eefcOxYua6moiJYtiGUpLZO7moq4/Alt5M2aCRxH/6bhIevxCen8s0igrau44x7L6LTxe1p8NNHZHXsxf7xU9g//gFyW7Sl6RuP0Oa2s1H5eQ7Pu+gikyxee83ZP43wRJasuNZaPwo8asVrCy/366/mvtTeEWDqNWVk2+jt7PGIspRi/w2PUBgeTcPv3iV0/RL23/AwaUMuoigyGt/0Y4StnEfMVzMIXzWPooAgUs+5nKMXXEVR5MlNbtKGXUzUz58R9/6ztHzocnY++2XJVUnDhtCvnykE+Nxz7r33hHA9KcshvMuSJRAUZGo2lTJvlX19REfnj0eUoxSHL72drM69iXv/ORKevBGevJFimx8+9kHtwtBIDo++ntTzrqQ4JKzC0xwbcQm+mWnEzJpB9Oz/cvSiG0seu+gi86PPmQOjRrnkpxIeQpKE8C4rV5qO+VL99gC/rAmlSXSe09ZHVEd2p97snPYZQdt+I3jzanwzj1MUHkVO6y5kt+8BvlX/uR4ZfT0hvy0j/uXJpA0bQ1F4FGC2QA0NhffflyQhakaShPAemZlmS7errnJoLi6GZRvCGNA5zZq4SlOKnHaJ5LRLPL3n+/hyYPwUWj5yFY3feISUyWZak5+f2Tbjhx/MCuzg4LoLWXg2KfAnvMfKlSYjlBmP2LwzkGMZNvp0dPF4hJPkJbQnvfdwYmbPxDft5P7cF15oFpvPnm1dbML9SJIQ3uPEoHWZ/aznr7ZwPMJJjoy+Hp/8PBp9OL2krUcPaNAAPvjAwsCE25EkIbzH8uUQHw8REQ7NC1eHEhuZX+v9rOuT/Gatyezcl5gvXi+ZEuvra7Y7nT/f9LwJUR2SJIR3OLGIrlOncs1LN4TRvU1GddaxuZXUc8dhyzhG5IIvS9pGjDBbm57YuVWIqkiSEN5hx44KF9Ft+yuAI2l+9O7gef+1zurSl4KoWGJmnVyrmphoqqN//rl1cQn3IklCeIclS8x9mfLgC+zjEf06e8agtQMfH44PvIDQdb/iv/8vAGw2s7Bu7lxTCFeIqkiSEN5h6VKziK7MXsoLk0NpEFZAi7i8Sp7o3tKGXoTSmujv3itpGz4cMjJgwQILAxNuQ5KE8A4VLKLTGpas98zxiBMKYpqQk9CeyHkn+5f69YOAACkfLqpHkoTwfNnZsGULdO7s0Lxrnz/7j/p75HhEaem9RxD85yYC9u4AIDAQkpLM4HUlFcaFKCFJQni+5GRT5jUx0aF5QbIZj+jr6qJ+Lpbe7xwAGvx4coHEsGFw8CCsXm1VVMJdSJIQnm/pUnPftatD88LVoUSEFNKqaa4FQblOYcM4chLaE/XzyS6nQYPAxwdmzbIwMOEWJEkIz7dsmdmerUEDh+Yl68Po1ioDHy/4K8jodRZBuzaXzHKKjIT27U0tJyFOxQv+PITXW70aOnZ0aNp7wI89BwM8fjzihIxeZwEQsfCrkrYzz4RNm+Dvv62KSrgDSRLCs6WkmM73Ml1N3jIecUJ+XAvyY5oQ+eu3JW1Dh5r7r7+2KCjhFiRJCM+2bJm5LztovSqU0KBC2rXIcX1MVlCKzG4DCP1tSckWqa1aQUyMJAlxapIkhGdbvNhsptCunUPzwjXhdG+Tia8X/QVkJA3FpyCfsOVzAbO7ab9+8MsvkOvZY/eiFrzoT0R4pZUroXVrkyjsdu3zZ+/BAPp2TLcwMNfLbtedosBgIn+ZXdI2dKhJELL6WlRGkoTwXPn5sGFDucqvJ/azHtjNO8YjStj8yO6YRMSyH81yc6BXL7P6WjYiEpWRJCE814YNkJdXbjzi5xVhRIUW0NrD10dUJCPxTPyOHSZwx0bArL5OTIQfT+YNIRxIkhCea9Eic18qSWgNi9aGk9TOc+s1nUpWt/4ARC45uaHEoEFmEtiWLVZFJeozSRLCc/3yC8TGmoV0dlt3B3L4mB/9OnvXeMQJhQ0akRfXnPDlP5W0DR5s7r/6qpInCa8mSUJ4Jq3NdqVl9rP+abkZjxjQ1cvGI0rJ6tSbkI0rULlm+m9cnKmg/u23VTxReCVJEsIz7dgBR49C9+4OzfNXhRHXII94D9rPuqYyuw3EpyCP0LW/lLQNGGDqIB47ZmFgol6SJCE804nxiN69S5qKimDxujB6eel4xAnZHXpSbLMRufTkuMTQoeb9kb2vRVmSJIRnWrjQbOZcaie637YHcTzLRv8u3tvVBKADg8g9ozNhK+eVtHXubN6ub76xMDBRL0mSEJ5p6VKzPqLUJcOcZeEA9O/inYPWpWV27kPQ7q3YjhwAzIZ9vXubva+LiiwOTtQrkiSE59m/H/bsgR49HJrnrQwjoVEOMZGFFgVWf2R1PxOAiOVzStoGD4bjx814vxAnSJIQnmeuqU1UejwiN0+xfGMYfTp6d1fTCbnN21IYEk740h9L2gYONBsRyVRYUZokCeF55syBsDCzq47dL2tDyc33YVDicQsDq0d8fMju0JOw5AUlS63DwkwP3Y8/VvFc4VUkSQjPorUZtO7e3XS0232/OBw/32L6dPSOTYaqI7NLX/zSjhC46+RS64EDzcrrlBQLAxP1iiQJ4Vm2bjWbDPXr59D888oIurXKJDiw2KLA6p/szn0BCF92clxiyBBzL3tMiBMsSRJKqUil1BdKqa1KqS1KqX5VP0uIajixafOAASVN+w75sfWvIAZ0lVlNpRXENqUgOo7wVSenwp5xhqlkIquvxQlWXUm8BMzRWrcHugFSWkzUjZ9+giZNzM3uh6Vm6uuQ7jIeUVZW+x6E/rakZN6rbEQkynJ5klBKRQCDgLcAtNb5Wus0V8chPFBOjtmJrlcvh+Yfl4TTMCKfts3kU6+srC598c3KIHjLmpK2wYNlIyJxkhVXEi2Bw8A7Sql1Sqn/KqVCyh6klJqglEpWSiUfPnzY9VEK9zN3rvl0GzaspKmoCBYkh9OnQ7pXl+KoTFbnPgCEl1ov0bs3+PvLuIQwrEgSNqAH8LrWujuQBUwpe5DWeobWOklrnRQTE+PqGIU7+vxzCAlxuJJYvTmY41k2BiXKeERFiiKiyWucQPiq+SVtJzYimjNHNiIS1iSJFCBFa73S/v0XmKQhxOkrKjIT/Hv3dtjP+vvFESilOVMGrSuV3aEnIb+vQuXnlbQNGmQWrf/xh4WBiXrB5UlCa30A2KuUamdvGgZsdnUcwsMsWQKpqQ5dTQDfL4mgQ/NsIsOkIFFlMrv0wyc/l5ANy0raBg0y97L6Wlg1u+kO4EOl1AYgEXjKojiEp3j3XQgIOPnpBhw4YmPdthCGJKZZFpY7yO6YhFY+DuslmjSB5s3hu+8sDEzUCzYrXlRrvR5IsuK1hQfKy4NZs8xy4eDgkuZvfo0AYFgvmfp6KsXBoeQ2b0P46vn8Xap94ED49FNITzdlxIV3qtaVhFJqllLqAqWUrNAW9c9335lPslGjHJq/XhRJbGQ+7ZvnWBSY+8jumETwtvX4ZJ0sgDh4sBnqObE+UXin6n7ovwaMA7YrpaaVGk8QwnpvvglRUdCnT0lTbp5i4ZowBnVNk6mv1ZDZtT+qqIjQNYtK2rp1M5PFZCMi71atJKG1nqe1vgIzC2k3ME8ptUwpda1Syu/UzxbCif78E+bNM1cRtpO9p/NWhZGT58uwJOlqqo6ctt0otvkRsWJuSZvNZiaL/fwzFEvJK69V7e4jpVQ0MB64AViHKa3RA/jZKZEJUR3Tp5tNEC67zKF59sJIgvyL6NtJ9o+oDu0fSG7LjoStdlxmPWgQHDkCyckWBSYsV90xia+AxUAwMFJrPUpr/anW+g4g1JkBClGpY8fgvffMJ1mpBZdaww9LI+jdIZ0Af1kNVl1ZnXoTtGszvmlHStoGDDD1nGQqrPeq7pXETK11R63101rr/QBKqQAArbXMUhLWePZZyMyEG25waF73RxD7j/ozrKd0NdVEVhdTOjysVFXYBg2gXTsZvPZm1U0ST1bQJjvhCuukpsIrr8CZZ5pPsVI++SkKXx/NsKQ0a2JzUzmtOlEUEETE8p8c2gcOhI0bzTYdwvucMkkopeKUUj2BIKVUd6VUD/ttCKbrSQhrTJliqr7eeqtDs9bwxfwoerTJIEpWWdeMr42cNl0JS17k0Dx4sHlfpeCfd6rqSuIc4HkgHngB+Lf9Ngl40LmhCVGJtWvhrbdg9Gho08bhoU1/BrLr70DO7n3MouDcW1an3gTs343fgb0lbe3amRnGshGRdzplktBav6e1HgqM11oPLXUbpbWe5aIYhTipsNCMQYSFwcSJ5R7+eE4UPkpzTp8018fmAbK69gcgvNRUWB8fsxHRokVQUGBRYMIyVXU3XWn/MkEpNanszQXxCeFo6lRYtw7uucckijK+nB9FYutMGkYUuj42D5AX34rC0AiHJAGmyykz0+xYJ7xLVd1NJzYDCgXCKrgJ4TrLlsG0aTB8OJx/frmHN+8MZNveIM7uJV1Np83Hh5y2iYStXeSwmUSfPmZx3ezZlkUmLHLKAn9a6zft94+5JhwhKpGeDpdfDtHR8M9/VnjIx3OiUEpzbt8018bmYTI79yFs7S8E/LWNvAQzcyw0FLp0MVuIC+9S3cV0zyqlwpVSfkqp+Uqpw6W6ooRwLq3huusgJQUef9x8YlVwyEc/NaBbq0xio6TjvDZKxiVKbWkKZrbxjh2we7cFQQnLVHedxNla63TgQkztptbAfc4KSggHM2bAl1+aRJFU8drNVb8Hs3NfIKMHHnVxcJ6nILYpBQ1iCV/pWHHnxFYdX35pQVDCMtVNEie6pS4APtday1JW4Rq//w533gndu8ONN1Z62DtfR+PvV8z5/WQ8otaUIrtdd0LXLXao7NeiBTRuDN9/b2FswuWqmyS+U0ptBXoC85VSMUCu88ISAsjNhYsvhsBAePpp8PWt8LCCQvh8fhRndkkjLFjKldaFzC79sGWlE7RtfUmbUqaW09KlkJ1tXWzCtapbKnwK0B9I0loXAFnAaGcGJgQPPQR//AGPPgoNG1Z62PdLIkhN9+Mfg1JdGJxny7bXcSq9pSnAkCGQnw9z51bwJOGRarLTXHvgUqXU1cAY4GznhCQEsHIlvPSSmepaat/qirz7TQMiQwsY1E16QetKYWRD8ho1I3zlPIf27t3NVuKyEZH3qNYe10qp/wGtgPXAiYI4GnjfOWEJr5aXB1dfDZGRcN+p50ccSrXx/dJIxg45jJ8lO7Z7ruwOPYlY/hOqIB/t5w+YBNGjh5kKqzWy658XqO6VRBIwQGt9q9b6DvutfE0EIerCCy/Atm3wwAMVrqoubeZX0RQW+XD58COnPE7UXGaXfvjk5RCycYVD++DB8PffZk6B8HzVTRKbgDhnBiIEYDYSmjYNevUyHeCnUFwMM7+KoXvrDFrHyzyKupbdqTda+VS4XgJkIyJvUd0k0RDYrJT6SSn1zYmbMwMTXurRRyEjA+6+u8pD564I468DAVw67LALAvM+xSFh5DVrRdiq+Q7tjRrBGWfIVFhvUd1e3KnODEIIwCzlffNNOPtsaNu2ysP/81kMkSEFnCsVX50mq0MSDeZ9jk9OFsVBISXt/fvDxx/D8eMQEWFhgMLpqjsF9hfMSms/+9ergbVOjEt4o/vvN31It99e5aG7//bnx2WRjBpwFH8/2cfaWbK69kMVFRK69leH9oEDoagI5s2r5InCY1S3dtONwBfAm/ampsBsJ8UkvNG6dfDFFzB2rFnWW4Xn3o8FYPwFh5wdmVfLbteDYl9buXGJbt3MGsc5cyp5ovAY1R2TuA0YAKQDaK23A7HOCkp4oXvvhaAguP76Kg89lu7Lu9825OykVOIaSDE/Z9IBgeS27EDY6gUO7X5+0LUrzJ9fyROFx6huksjTWuef+EYpZcOskxCi9ubNgwUL4JprzNqIKrz6aUOy83y5fuRB58cmyO7Yi6Cdv+Ob5lg8sU8f2LUL9uyxKDDhEtVNEr8opR4EgpRSI4DPAdnxVtSe1uYqomFDGDeuysPz8hWvfhZL7/bH6ZiQ44IARWbX/iitCUt2vJoYMMDc//ijBUEJl6lukpgCHAY2AjcBPwAV7/wiRE188gn89hvcfLPp5K7Cq5/GcOiYPzeOkqsIV8lp1ZnigCDClzvuONSqFURFSR0nT1etKbBa62Kl1GxgttZaJqWLulFQAA8+aGpQX3hhlYenZ/rw1DtxJLVLZ0CXDBcEKACw2chu1Znw5IUOzUpBz56waJGZlOZTk0pwwm2c8p9VGVOVUkeAP4A/7LvSPeKa8IRHe+01szZi4kSzgXIVnnu/Eanpftxz2T7nxyYcZHfqRcC+nfgdTHFo79cPUlNhwwaLAhNOV1Xuvxszq6mX1rqB1roB0AcYoJSqekmsEJXJzIQnnoDOnaus8gqmkN/0jxpxVvdUurWWzQxcLdO+pWnYKseFEf36mfsffnB1RMJVqkoSVwGXa613nWjQWu8ErgSudmZgwsNNmwZHj5ryG9UoJfrYjMbk5vtw96V/uyA4UVZe87YUhoQTvsJxXCI2Fpo1g59/ruSJwu1VlST8tNblymvaxyX8nBOS8HiHDsH06WbZbrduVR6+a58/M79qyIX9jtCqaZ4LAhTl+PiQ07YbYcmLzIy0Unr3hhUrzEaCwvNUlSTyT/OxKimlfJVS65RS39XmPMINPfyw+US5665qHf7Qf5qgfODOsfudG5c4paxOffA/eoCAv7Y5tPfvb/45ly61KDDhVFUliW5KqfQKbhlAl1q+9p3AllqeQ7ibHTvg7bfhggsgIaHKwzdsD+KTnxtw2dBDxEXL6morZXY1AxBhKx37lpKSzPbjMi7hmU6ZJLTWvlrr8ApuYVrr0+5uUkrFAxcA/z3dcwg3NXmy+US57bbqHf5SE0ICi7j5ogNODkxUpSCuOQVRMUSsdFwYERIC7dtLiQ5PZdXM5heByUBxZQcopSYopZKVUsmHD8vSDI+wdq3ZqeaSS8wK6yr8ujaUn1ZEcu15B4gMLaryeOFkSpHdNpHQdYvNwohSevc202BTUy2KTTiNy5OEUupC4JDWes2pjtNaz9BaJ2mtk2JiYlwUnXCqyZMhNBSuu67KQ7WGe19sSsOIfMafL5Ve64uszn2xZaQRtG29Q/uAAebfTEqHex4rriQGAKOUUruBT4CzlFIfWBCHcKVffjH9EVdeWeW+1QCzF0WwenMot4zeT1CA1JKsL7K6mfUS4Sscu5w6dzZFfKWOk+dxeZLQWj+gtY7XWicAlwELtNZXujoO4UJaw333QYMGJklUoagIprzSlGYxuYw9q9wMbGGhwqgY8ho1I3yl4yWDzQaJiaaYr/AsUm1FON8338Dq1WaviGoU8Xv322i27QnizrH7sPm6ID5RIzntexC6YRmqwHEWfJ8+pmz4zp0WBSacwtIkobVepLWuurKbcF/FxTBlitlt7v/+r8rDc/MUj7zRhPbNszivb5rz4xM1ltm5Lz55OYRsWunQfuaZ5l6mwnoWuZIQzvXBB7B1qykF7lf1rOlXPonh7yP+TLpsX3WqdQgLZHXujVaKsDKlw5s3N2U6JEl4FkkSwnkKCszq6pYt4bzzqjz8eKYPT7/bmN7t0xkopcDrreLQCPLiWxG+ynFhhFLQqxcsXmz+6YVnkCQhnOf1100n9e23V2uzgaffjuNYhk1KgbuBrA5JBG9Jxicny6F94EBT4HfFCosCE3VOkoRwjpwcePJJ6NixWqXADx618fKnsQzvkUqXVlIKvL7L6tIXn6JCQtf+6tDet6/5/8B3UpHNY0iSEM7x/PNw+LDZUKgagwsPv96Y/AIfJl0mpcDdQXb7nmhfW7nS4WFhpkSHbGnqOSRJiLqXlmaSRK9epvpbFXbs9eedb2P4x8AjJDSWUuDuQAcGkZPQgbBV5RdG9Oljti0/IktcPIIkCVH3nnwS0tPhzjurdfiDrzbB10dzxxgpBe5OsjslEbRzE77HHQs2DRxo1k/KRkSeQZKEqFsHD5q9q4cMMf0OVVi3NYjP50czbvhBYqNkSow7yezSH6U1YWVWX3fubEp0ybiEZ5AkIerWww9DXp6Z0VQN973YlLDgQm4afdDJgYm6ltO6C8UBQUQsc1wY4esLPXqYUl1aym65PUkSou7s2gXvvmvWRFRjQ6EFq0OZnxzB9ecfIDxESoG7HZuN7HbdTbG/Mtmgf39zUfn77xbFJuqMJAlRdx580NzfckuVh2ptriJiI/O5+jwpBe6uMhLPxP/IfgJ3bnZoHzjQ3H//vQVBiTolSULUjd9/h88+M/WZ4uKqPPyTn6JY+0cot170N4H+0ifhrjJ7mjUwEb987dAeFwfNmkmJDk8gSULUjfvvh4AAuOGGKg/Ny1dMeaUprZtkc/GQoy4ITjhLYYNG5DVJIGJp+WzQvz8sW2Ymugn3JUlC1N7KlaZf4fLLISqqysNf/CiGPQcDuG9cCr7yG+j2Mjv3JXTTSnwyHbPB0KFQWAhz5lgUmKgT8icqam/yZAgPh6uvrvLQo2m+PPVOY/p2PM6Z3aSInyfI7DkEVVRYbre6xESzAvvLL62JS9QNSRKidubNg19/hWuuMZPjq/DIG43JzPZlypUpLghOuEJ220SKAoOJ/PUbh3abzay+njvXXFEI9yRJQpw+rc1YRMOGcNllVR6+cXsgM76KZdSAI7RtluuCAIVL2Gxkt+9R4VTYoUNNlZZly6wJTdSeJAlx+mbPhrVr4cYbzaD1KWgNNz3VnOCAIu4bJ6XAPU1m90H4pR4kaMdGh/YBA8wVxaxZFgUmak2ShDg9RUVmW9KmTWH06CoPf/e7BizfGMZdY1OICpOFc54ms4eZChs5/wuH9tBQ6NoVvv3WiqhEXZAkIU7PO+/Atm1w223mv4qnkJbhy30vxtMpIZNLzpIpr56oMLIhOS3aE7nwq3KPDR4MO3eaXxfhfiRJiJrLyTE1mtq1gxEjqjz89mfiScuwMfW6PdXZoE64qYyeQwj+cxP++/9yaB82zNx//LEFQYlakz9ZUXPTp8OBA3DXXVVuKDR7UQQfzmnI+HMP0KlljmviE5bI6Gv+wxA5z7HLKS7ObFD4+edWRCVqS5KEqJljx+CZZ6B3b7Op0CkcTfPlxidb0LpJNneMlb0iPF1+4xbkxTUnakH5hRHDh5vKLdLl5H4kSYiaefxxyMiockMhreGGJ5qTluHLtFt242+T+kzeILPHIEI2rcR27LBD+7nnmvuPPrIgKFErkiRE9e3dazYUGj7cjEecwksfxzD7lwbcMvpvOiZIN5O3SO9zNkoXlxvAjo01XU6ffWZRYOK0SZIQ1Xf33eYSYeLEUx62fEMI970Uz4DOabKZkJfJbdmBgqgYouaXH4AYMQK2bIGtWy0ITJw2SRKien791RThufxyaNy40sOOpPkyZvIZxEQU8Pztu2U2k7dRioykoYQlL8Q3zXG68znnmHkO//ufRbGJ0yJ/wqJqRUVw660QEwMTJpzysEuntORImo3pE3cSIbvNeaXjZ45EFRUR9fOnDu2xsdCtG3zwgWxr6k4kSYiqvfmmmZpyxx0QGFjpYY+80ZgFyRHcP24vXVtluzBAUZ/kJrQnv1Ezon8of8kwciTs2QOLF1sQmDgtkiTEqR07Bg89BF26mL2rK/H9knCefrcxF/Q9wmXDj7gwQFHvKEV6nxGEblyB34G9Dg+dfTYEBcF//2tRbKLGJEmIU3vwQbO12OTJlS6c+2u/P1f+syWtGufw+A17qlpfJ7xA2qCRADQoczURFASDBpmCf1lZVkQmakqShKjcxo0wcyZceCF06FDhIfkFiovuOYOCQsXLd+0kKEA6mwUUNGpGTot2RP/4QbnHLrrIJAhZge0eJEmIimltivcFBZ1yyuvE5+JZty2EJ27YTULjPBcGKOq74wPOJ2jXFoK2rHFo79HDlOp46y2LAhM1IklCVOzLL83o4o03QmRkhYd89GMUb86K5YrhBzm3T5pLwxP13/FBIyn28yfm89cc2n18zMXpkiVSpsMduDxJKKWaKaUWKqU2K6V+V0qdur6DcL2cHFO8LyEBLr20wkO27grgxn+1oHPLTCZfIVuRivKKQ8LJ7D6IBj9/ik+O4wDEmDHg6wsvv2xRcKLarLiSKATu0Vp3BPoCtymlOloQh6jM00/Dvn1w330V7hWRX6C4ePIZ+PlqXrpzJ36n3k5CeLHUEWPxzcki6ifHOuENG8KZZ5qFddkyW7pec3mS0Frv11qvtX+dAWwBmro6DlGJPXvguefMFJQ+fSo85ImZjdi8K5jHrttN4+gCFwco3ElOux7kNWpGzBevl3vsssvMxDlZgV2/WTomoZRKALoDKyt4bIJSKlkplXz48OFyzxVOcuedZtD63nsrfHj9H4FMe68x5/RK5ezex10cnHA7SpE25B+EbF1bbgC7Z0/To/nqq7ICuz6zLEkopUKBL4G7tNbpZR/XWs/QWidprZNiYmJcH6A3WrgQZs+GceOgSZNyDxcWwjWPJhAaVMTD4/eWf74QFUg762KKA4KIe+9Zh3alzNjEpk2wdKlFwYkqWZIklFJ+mATxodZ6lhUxiDIKC82U19hYuP76Cg+Z9k4jNuwI4cGr9tIgvNDFAQp3VRwcStqAC4ha+CV+h/Y5PDZqFISGwrRpFgUnqmTF7CYFvAVs0Vq/4OrXF5V4/XVTx/muuyqsz7R5ZwBPvt2EwYlpXNDvmOvjE24t9YIroVgT+9F0h/bgYLO47ocfZDpsfWXFlcQA4CrgLKXUevvtfAviECekpsLDD5sSnSNGlHu4uBiundoCP1sxj10nZTdEzRXExpPZbQANv5qJT3amw2NXXmkm0cnVRP1kxeymJVprpbXuqrVOtN9+cHUcopTJk82WpFOmVFif6cUPY1i1OYzJ41KIjZLZTOL0HBk5HltWOjGfvOLQHh1tCv99+CEcOmRRcKJSsuLa261eDW+/ba7527Qp9/DOFH8efqMpfTumc/HgoxWcQIjqyW3bjaz2PYn74Plyi+vGj4eCAnhBOqDrHUkS3qy42GwiFBlp9oooQ2u4dmpztIYnbvhLuplErR0eczO29FRiPnFcat2yJfTvb4bG0svNdRRWkiThzd54A9avNwX8QkPLPTxjVjS/ro/g7kv20TQm3/XxCY+T0667uZr4X/mriRtvNAli+vRKniwsIUnCWx05YvaK6NrVVFsr4+/DNu57KZ7E1hmMGyGLGUXdOTz2FmzpqcT+73mH9s6dzSL/6dPlaqI+kSThrSZNgsxMeOCBcoPVWsP1j7Ugr8CHpyb8hY/8log6lNM2kcyu/Yn733PYUh1Hqm+5BY4fl6uJ+kT+/L3RsmWmYM6YMRUOVn/4QxRzVkRy20V/yx4RwikOXjEJn7xcmrz6gEO7XE3UP5IkvE1+vllRHR0Nt95a7uEjab7c+e9mdGyRxbXnH7QgQOEN8pskkDZ4FA2/fZfAHZscHpOrifpFkoS3eeIJ2LrVrIkICSn38C1PNSM9y5enbtqNzdeC+ITXODz2NooDgmj+3B0OFf5OXE288AIck8X9lpMk4U02bTLLWocONbcyvloYwRcLornhwgO0bZZrQYDCmxSFRXLkH9cTtmYRkXM/dXhs4kSzvvOxxywKTpSQJOEtiorg6qtNsZwHHyz3cFqGL7c83ZxWTbK5+R8HLAhQeKPUc8eRG9+K5s/fiU/myUGIdu1g+HCzbmLPHgsDFJIkvMa//w3r1plZTVFR5R6+bVo8R9L8+NeEv/C3SXF/4SK+NvZf9xC2tMM0feV+h4cmTjS9UFOmWBSbACRJeIdNm0wBv7594YILyj389S8RfPRTQ6497wBdW8leksK1ctt0JW3QKGJmzSB406qS9saN4eKL4ZNP4LffLAzQy0mS8HR5eXDJJRAUZDp4y6yJSD3uy4Qnm9O6STa3j9lvUZDC2x0adzdFYZG0fOQqVP7JadcTJpj5FXffLbvXWUWShKebPNnsE/Hww2baaxm3TmvG0eN+PH3zbulmEpYpDg7l7+v+SeCebTT9z8kxs/BwuO66k5smCteTJOHJfvoJXn7ZbP81ZEi5hz/+KZJPf47mugv206lljuvjE6KUrJ6DSBtwPrEfvUjI+iUl7ePGQYsWZowiR35NXU6ShKc6cACuugqaNTNXE2Xs2ufPhCcT6JSQye3/J91Mon44eM1kCqMa0vLhK0sKANps5lc4JQWefNLiAL2QJAlPVFhoRvzS0+GZZ8ptR1pQCGMmnwFa88Idu/CzWRSnEGUUB4Xy94RH8T+wh+ZP31zS3qePWdrz/POwc6eFAXohSRKe6P77TX2myZOhbdtyD9/3YlPW/hHC1Ov+olmslAAX9Ut2pz4cPe9Kon/4gAbfvlPSft994OsLN9wgg9iuJEnC08yaZeoZjBwJo0eXe/j976N46ZM4xgw+xPn90lwfnxDVcPiS28hu3YUW024lcOdmAGJj4eabzSD2zJkWB+hFJEl4ko0bzarqNm0qXIG0alMwE55MoFurDP45PsWCAIWoJl8b++54hmK/AM647//wyc4E4PLLzRYo99xjxiiE80mS8BSHDsH554O/vymfGRDg8PCeA36MntSKqNAC/jNpp0x3FfVeYYNY/r75cQL3bOeMKZdAcTE+Pma5T0EBXHONdDu5giQJT5Cba1ZSHzpkupri4hwePpRqY+hNbcnM9uE/k/6kQXihRYEKUTNZ3QZw6PKJRCz7kfiX7gPMhL3bboMFC8xAtnAuSRLurrgYrrgCkpPh0UehSxeHh9MyfBlxa2v2HfLn1Uk76JAgE82Fe0k99wrSBo2i0Ycv0HDWm4Dpdurf32ysuHy5xQF6OEkS7kxrM5I3axbcdBOcc47Dw4dSbZx5fVs27wrihdv/pHeHrEpOJEQ9phT7r32QrA49af70rUTO/QylzJqJhg3NBotHj1odpOeSJOHO7r/fTPO47DIzL7CUnSn+9Lu2Hdv3BvLKXX8ytIfsBSncmM3G3kkvktuyA2c8fAXhv35LeLhZBnT4sOltzZUtUJxCkoQ70hoeeQSee878ddxzj0Phvp+Wh9Hzyg4cSrUxc/I2BidKghDuTwcGsef+/5DXOIFW948hYv6XdO5s/hRWrjS9rsXFVkfpeSRJuButzZ4QTzwBI0aYvxB7gigshMfejOP8iW2ICivgs8e3ktReupiE5ygODmXPg2+Q1/QMWk25hIZfvM5555l9sWfNghtvlERR16QggzspKDC1k9991yyUe+gh8DF5/vc/A7nq4QTWbQthRM9UnrrpL0KC5K9FeJ6isEj+emgm8dMnmcV2u7dy3Z3/Jj3dxttvm/8svf22WZ0tak+ShLs4ehT+8Q9YsgSuvBLuvBOUIvW4Lw+/3pgZX8USEljEc7fs5IL+snu88Gw6MIi9971C3LvTaPTJy4RsWsV9z36Jr28T3n/f1Lf84gsIC7M6UventBusRklKStLJyclWh2GdpUvNnL/9+82cv9GjScvw5aWPY5j+YSMysn0ZPfAIky79m+gIWQMhvEvEr98S9940tM2flLue5+WcG3nxJUXr1iZRlJkV7lWUUmu01km1OYeMSdRnubmmvMagQWaHuTfeIKXvGO5+vinx53dh6oymdErI4osnNvOvCXskQQivdHzQSHY++RF5TRJo8dRNPDZvAJ/ctYJDh6BnTzN8ly91LE+bXEnUR1rDl1+aWUt79lA8fARzhj7DK9+1ZO6KCACG9zzGhNEH6CiL44QwiouJnP8FMbPexJZ5nEO9L+Dh/IeZsb4PzZvDU0+Z2eLeNFZRF1cSkiTqk/x8+PRTmDYNNm8mt3FL3u/4NFPXX8T+o/5EhxcwasARLh9+hHgp8S1EhXxysmjw3Xs0mPsJvrnZ7G/Wi2eyJzLz6EVENwvh9tvh2mshJsbqSJ1PkoQnyM6GxYvhk0/QX32FOn6cw+Fn8ILvfTx37Abw8aVX+3QuOesIw3qmyQZBQlSTT04mkQu+ImreZ/gf2U+BLYi5QaOZkXEZi3yGkTgwlDFjTKGCNm0clhp5DEkS7iYjA7Zsgd9+g/Xr0WvWwpo1qMIC8nwC+d53FDMLxjOXs+naOpvz+qRyfr80GWsQojaKiwneupaIxd8StvYXfLMzKVR+rLAN5JuCc1nCQPbF9iCxbyA9ekCvXtCtGzRuXDLD3G25bZJQSp0LvAT4Av/VWk871fH1KUlobcaT8/PNsoUTt/w8TfHBw6g9f+Hz1y5se3diS/kL/wN7CDi4h8Aje/HPPl5ynlwVxO+qEwuKh7CAs1gX2I9OHYoZlJjOkO7pxEYVWPhTCuGhCgsI/mMdYWt+IWTTSgL27wagQPmxybcbywt7sYUObKEDO/3a49usCc0TfGje3BRXjoszySMuzmyCFBMDkZH1d5zDLZOEUsoX2AaMAFKA1cDlWuvNlT3nVElCa7PCsrjY8eviIo0u1hQXaYoKNdlZmqwsyMrUDrf04+aWmaHJSDe3zAxNVkYxhcez0BmZqMwMfHMy8MtJJyQvlRh9iDgO0IiDxHGAxuynOXsIxnEQOZ0w/qJFyW0PzdlGW3YFdqAgpgmt4vPp2jqL7m2yaNs8B1s9/UUTwlP5ph0haPsGgretJ2jHRvxTdmLLPVmloED5ccgnjn26CXuLm3KEhhwnotxNBQXiFxaEf3ggAeEBBDcIJLhBICHRgYTHBBAZ609UtA/RMT6ER/oQGOxDQKAiIMBsQR8QYG62Ou5Odtck0Q+YqrU+x/79AwBa66cre04P5asXE4hCV3nzwTU/T65vCOkB0WT6R5MVEE1GcCzpIY1JD2tCZlhjssIaURwYip+fxt9X42crxs9WTGx4HhHBcpUgRL2kNbbMYwQf3kvgwd0EpB3ELyMVv8xj+GWk4pOdiS0vG9+iupk4UoQPxWVumrobHAkjq9ZJwoph0KbA3lLfpwB9yh6klJoATLB/mxdK9iYXxFZ9RVmQnQXZe6pzdEPgiJMjchZ3jh0kfqtJ/KdUbL85TbvanqDezpXRWs8AZgAopZJrmw2t5M7xu3PsIPFbTeK3llKq1oO5Vozd7wOalfo+3t4mhBCinrEiSawG2iilWiql/IHLgG8siEMIIUQVXN7dpLUuVErdDvyEmQL7ttb69yqeNsP5kTmVO8fvzrGDxG81id9atY7fLRbTCSGEsIabrycUQgjhTJIkhBBCVMrSJKGUOlcp9YdSaodSakoFj09SSm1WSm1QSs1XSrUo9dg1Sqnt9ts1ro28JIbaxF+klFpvv1kycF+N+G9WSm20x7hEKdWx1GMP2J/3h1LqHNdGXhLDacWvlEpQSuWUev/fcH30Vcdf6riLlVJaKZVUqq3ev/+ljnOIvz68/9X43RmvlDpcKsYbSj3mDp89p4q/Zp89WmtLbphB6z+BMwB/4DegY5ljhgLB9q9vAT61f90A2Gm/j7J/HeUu8du/z7Tqva9B/OGlvh4FzLF/3dF+fADQ0n4eXzeKPwHYVN/ff/txYcCvwAogyZ3e/1PEb+n7X83fnfHAqxU8110+eyqM3/5YjT57rLyS6A3s0Frv1FrnA58Ao0sfoLVeqLXOtn+7ArOmAuAc4GetdarW+hjwM3Cui+I+oTbx1wfViT+91LchUFLzZDTwidY6T2u9C9hhP58r1Sb++qDK+O2eAJ4Bcku1ucX7b1dR/FarbuwVcYvPnrpkZZKoqDxH01Mcfz3w42k+1xlqEz9AoFIqWSm1Qin1DyfEV5Vqxa+Uuk0p9SfwLDCxJs91strED9BSKbVOKfWLUupM54ZaoSrjV0r1AJpprb+v6XNdoDbxg7Xvf3Xfv4vtXcVfKKVOLAB2i/ferqL4oYafPW4xcK2UuhJIAp6zOpbTUUn8LbRZ7j8OeFEp1cqS4Kqgtf6P1roVcD/wT6vjqalK4t8PNNdadwcmAR8ppcKtirEiSikf4AXgHqtjOR1VxF/v33/gWyBBa90Vc7XwnsXx1NSp4q/RZ4+VSaJa5TmUUsOBh4BRWuu8mjzXyWoTP1rrffb7ncAioLszg61ATd/DT4B/nOZzneG047d30xy1f70G07/b1jlhVqqq+MOAzsAipdRuoC/wjX3w1x3e/0rjrwfvf5Xvn9b6aKm/1/8CPav7XBeoTfw1/+xx5YBLmcETG2bQpyUnB186lTmmO+YXqE2Z9gbALszAUZT96wZuFH8UEGD/uiGwnQoG/epB/G1KfT0SSLZ/3QnHgdOduH7gtDbxx5yIFzP4t68+/v6UOX4RJwd+3eL9P0X8lr7/1fzdaVzq64uAFfav3eWzp7L4a/zZ47IfrJIf9nzMBkR/Ag/Z2x7H/K8bYB5wEFhvv31T6rnXYQbsdgDXulP8QH9go/0fdyNwfT2N/yXgd3vsC0v/ImKujv4E/gDOc6f4gYtLta8FRtbH+Mscuwj7h6y7vP+VxV8f3v9q/O48bY/xN/vvTvtSz3WHz54K4z+dzx4pyyGEEKJSbjFwLYQQwhqSJIQQQlRKkoQQQohKSZIQQghRKUkSQgghKiVJQnisUtUuNymlPldKBdfgueOVUq/W8PUyK2l/3L6oEqXUolLVUH9QSkXab7fW5LWEcBVJEsKT5WitE7XWnYF84ObSDyqlXLJ9r9b6Ea31vAraz9dapwGRgCQJUS9JkhDeYjHQWik1RCm12F5Hf7NSKlAp9Y5934l1SqmhpZ7TzP4//+1KqUdPNCqlZiul1iilfldKTSj9Ikqp6fb2+UqpGHvbu0qpMWUDUkrtVko1BKYBrexXPc8ppd4vXXhNKfWhUsppVT6FOBVJEsLj2a8YzsOsMAXoAdyptW4L3AZorXUX4HLgPaVUoP243pjVwV2Bserkpj/Xaa17Yoo2TlRKRdvbQzClPzoBvwAliaUKU4A/7Vc99wFvYfYDQCkVgVklW1ElVSGcTpKE8GRBSqn1QDKwB/PhC7BKm30YAAYCHwBorbcCf3Gy2NzP2hRKywFm2Y8Fkxh+w+wR0gxoY28vBj61f/1BqeNrRGv9C9DGfiVyOfCl1rrwdM4lRG25pE9WCIvkaK0TSzcopQCyqvn8sjVrtFJqCDAc6Ke1zlZKLQICqVhtat68D1wJXAZcW4vzCFErciUhvN1i4AoApVRboDmmaB7ACKVUA6VUEKbM+FIgAjhmTxDtMSWwT/ABTow9jAOWVDOGDExp7dLeBe4C0Fpvrv6PI0TdkiQhvN1rgI9SaiOmq2i8PlmHfxXwJbAB0+WTDMwBbEqpLZgB5xWlzpUF9FZKbQLOwlTlrJI2eysstU/Vfc7edhDYArxT2x9QiNqQKrBC1EP2NR0bgR5a6+NWxyO8l1xJCFHP2BfebQFekQQhrCZXEkIIISolVxJCCCEqJUlCCCFEpSRJCCGEqJQkCSGEEJWSJCGEEKJS/w8Xq5N2aFefPwAAAABJRU5ErkJggg==\n",
 868 |       "text/plain": [
 869 |        "<Figure size 432x288 with 1 Axes>"
 870 |       ]
 871 |      },
 872 |      "metadata": {
 873 |       "needs_background": "light"
 874 |      },
 875 |      "output_type": "display_data"
 876 |     }
 877 |    ],
 878 |    "source": [
 879 |     "# Box 22: Assessing IPTW overlap using zEpid\n",
 880 |     "ipw.plot_kde()\n",
 881 |     "plt.ylim([0, 10])\n",
 882 |     "plt.xlim([0.2, 0.55])\n",
 883 |     "plt.show()"
 884 |    ]
 885 |   },
 886 |   {
 887 |    "cell_type": "markdown",
 888 |    "metadata": {},
 889 |    "source": [
 890 |     "### 4.2 Marginal structural model with stabilised weights"
 891 |    ]
 892 |   },
 893 |   {
 894 |    "cell_type": "code",
 895 |    "execution_count": 24,
 896 |    "metadata": {},
 897 |    "outputs": [
 898 |     {
 899 |      "name": "stderr",
 900 |      "output_type": "stream",
 901 |      "text": [
 902 |       "/home/pzivich/.pyenv/versions/3.6.5/lib/python3.6/site-packages/statsmodels/genmod/generalized_estimating_equations.py:501: DomainWarning: The identity link function does not respect the domain of the Binomial family.\n",
 903 |       "  DomainWarning)\n",
 904 |       "/home/pzivich/.pyenv/versions/3.6.5/lib/python3.6/site-packages/statsmodels/genmod/generalized_linear_model.py:278: DomainWarning: The identity link function does not respect the domain of the Binomial family.\n",
 905 |       "  DomainWarning)\n"
 906 |      ]
 907 |     },
 908 |     {
 909 |      "name": "stdout",
 910 |      "output_type": "stream",
 911 |      "text": [
 912 |       "Unstabilized Weights\n",
 913 |       "ATE    0.083294\n",
 914 |       "95% CL [0.05772325 0.10886425]\n"
 915 |      ]
 916 |     },
 917 |     {
 918 |      "name": "stderr",
 919 |      "output_type": "stream",
 920 |      "text": [
 921 |       "/home/pzivich/.pyenv/versions/3.6.5/lib/python3.6/site-packages/statsmodels/genmod/generalized_estimating_equations.py:501: DomainWarning: The identity link function does not respect the domain of the Binomial family.\n",
 922 |       "  DomainWarning)\n",
 923 |       "/home/pzivich/.pyenv/versions/3.6.5/lib/python3.6/site-packages/statsmodels/genmod/generalized_linear_model.py:278: DomainWarning: The identity link function does not respect the domain of the Binomial family.\n",
 924 |       "  DomainWarning)\n"
 925 |      ]
 926 |     },
 927 |     {
 928 |      "name": "stdout",
 929 |      "output_type": "stream",
 930 |      "text": [
 931 |       "\n",
 932 |       "Stabilized Weights\n",
 933 |       "ATE    0.083294\n",
 934 |       "95% CL [0.05772325 0.10886425]\n"
 935 |      ]
 936 |     }
 937 |    ],
 938 |    "source": [
 939 |     "# Box 23: Computation of the IPTW estimator using a MSM\n",
 940 |     "\n",
 941 |     "### Unstabilized IPTW ###\n",
 942 |     "fm_pa = smf.glm(\"A ~ C + W1 + W2 + W3 + W4\", \n",
 943 |     "                data, family=f).fit()\n",
 944 |     "p_score = fm_pa.predict(data)  # Calculating propensity scores\n",
 945 |     "iptw = 1 / np.where(data['A'] == 1, p_score, 1 - p_score)  # IPTW\n",
 946 |     "# Estimating Marginal Structural Model\n",
 947 |     "f = sm.families.family.Binomial(sm.families.links.identity())\n",
 948 |     "fm = smf.gee(\"Y ~ A\", data.index, data,\n",
 949 |     "             cov_struct=sm.cov_struct.Independence(), \n",
 950 |     "             family=f, weights=iptw).fit()\n",
 951 |     "print(\"Unstabilized Weights\")\n",
 952 |     "print(\"ATE   \", np.round(fm.params['A'], 6))\n",
 953 |     "print(\"95% CL\", np.asarray(fm.conf_int().loc[\"A\"]))\n",
 954 |     "\n",
 955 |     "### Stabilized IPTW ###\n",
 956 |     "f = sm.families.family.Binomial()\n",
 957 |     "# Numerator\n",
 958 |     "fm_ma = smf.glm(\"A ~ 1\", data, family=f).fit()\n",
 959 |     "num = np.where(data['A'] == 1, fm_ma.predict(data), \n",
 960 |     "               1 - fm_ma.predict(data))\n",
 961 |     "# Denominator\n",
 962 |     "fm_pa = smf.glm(\"A ~ C + W1 + W2 + W3 + W4\", \n",
 963 |     "                data, family=f).fit()\n",
 964 |     "den = np.where(data['A'] == 1, fm_pa.predict(data), \n",
 965 |     "               1 - fm_pa.predict(data))\n",
 966 |     "# IPTW\n",
 967 |     "iptw = num / den\n",
 968 |     "# Estimating Marginal Structural Model\n",
 969 |     "f = sm.families.family.Binomial(sm.families.links.identity())\n",
 970 |     "fm = smf.gee(\"Y ~ A\", data.index, data,\n",
 971 |     "             cov_struct=sm.cov_struct.Independence(), \n",
 972 |     "             family=f, weights=iptw).fit()\n",
 973 |     "print(\"\\nStabilized Weights\")\n",
 974 |     "print(\"ATE   \", np.round(fm.params['A'], 6))\n",
 975 |     "print(\"95% CL\", np.asarray(fm.conf_int().loc[\"A\"]))"
 976 |    ]
 977 |   },
 978 |   {
 979 |    "cell_type": "markdown",
 980 |    "metadata": {},
 981 |    "source": [
 982 |     "### 4.3 IPTW with regression adjustment"
 983 |    ]
 984 |   },
 985 |   {
 986 |    "cell_type": "code",
 987 |    "execution_count": 25,
 988 |    "metadata": {},
 989 |    "outputs": [
 990 |     {
 991 |      "name": "stdout",
 992 |      "output_type": "stream",
 993 |      "text": [
 994 |       "ATE 0.083929\n",
 995 |       "ATE 0.083426\n"
 996 |      ]
 997 |     }
 998 |    ],
 999 |    "source": [
1000 |     "# Box 24: Computation of the IPTW-RA estimator\n",
1001 |     "f = sm.families.family.Binomial()\n",
1002 |     "\n",
1003 |     "fm_a1 = smf.glm(\"Y ~ C + W1 + W2 + W3 + W4\", \n",
1004 |     "                data.loc[data[\"A\"] == 1],  # Only A=1\n",
1005 |     "                weights=data.loc[data[\"A\"] == 1, 'iptw'], # Box 17\n",
1006 |     "                family=f).fit()\n",
1007 |     "y_a1 = fm_a1.predict(data)\n",
1008 |     "\n",
1009 |     "fm_a0 = smf.glm(\"Y ~ C + W1 + W2 + W3 + W4\", \n",
1010 |     "                data.loc[data[\"A\"] == 0],  # Only A=0\n",
1011 |     "                weights=data.loc[data[\"A\"] == 0, 'iptw'],  # Box 17\n",
1012 |     "                family=f).fit()\n",
1013 |     "y_a0 = fm_a0.predict(data)\n",
1014 |     "\n",
1015 |     "ate = np.mean(y_a1 - y_a0)\n",
1016 |     "print(\"ATE\", np.round(ate, 6))\n",
1017 |     "ate = (np.mean(data['iptw']*data['A']*y_a1) / np.mean(data['iptw']*data['A']) - \n",
1018 |     "       np.mean(data['iptw']*(1-data['A'])*y_a0) / np.mean(data['iptw']*(1-data['A'])))\n",
1019 |     "print(\"ATE\", np.round(ate, 6))"
1020 |    ]
1021 |   },
1022 |   {
1023 |    "cell_type": "code",
1024 |    "execution_count": 26,
1025 |    "metadata": {},
1026 |    "outputs": [],
1027 |    "source": [
1028 |     "# Box 25: IPTW-RA \n",
1029 |     "# Not supported by zEpid"
1030 |    ]
1031 |   },
1032 |   {
1033 |    "cell_type": "markdown",
1034 |    "metadata": {},
1035 |    "source": [
1036 |     "## 5. Augmented Inverse Probability Weighting"
1037 |    ]
1038 |   },
1039 |   {
1040 |    "cell_type": "code",
1041 |    "execution_count": 27,
1042 |    "metadata": {},
1043 |    "outputs": [
1044 |     {
1045 |      "name": "stdout",
1046 |      "output_type": "stream",
1047 |      "text": [
1048 |       "ATE 0.083796\n",
1049 |       "95% Confidence limits for the ATE\n",
1050 |       "Percentile method:    [0.058572 0.109738]\n",
1051 |       "Normal approx method: [0.058901 0.108691]\n"
1052 |      ]
1053 |     }
1054 |    ],
1055 |    "source": [
1056 |     "# Box 26: Computation of the AIPTW estimator\n",
1057 |     "f = sm.families.family.Binomial()\n",
1058 |     "\n",
1059 |     "# Step 1: g-computation\n",
1060 |     "fm_a1 = smf.glm(\"Y ~ C + W1 + W2 + W3 + W4\", \n",
1061 |     "                data.loc[data[\"A\"] == 1], family=f).fit()\n",
1062 |     "fm_a0 = smf.glm(\"Y ~ C + W1 + W2 + W3 + W4\", \n",
1063 |     "                data.loc[data[\"A\"] == 0], family=f).fit()\n",
1064 |     "y_a1 = fm_a1.predict(data)\n",
1065 |     "y_a0 = fm_a0.predict(data)\n",
1066 |     "\n",
1067 |     "# Step 2: propensity scores\n",
1068 |     "fm_pa = smf.glm(\"A ~ C + W1 + W2 + W3 + W4\", \n",
1069 |     "                data, family=f).fit()\n",
1070 |     "p_score = fm_pa.predict(data)\n",
1071 |     "\n",
1072 |     "# Step 3: analytic formula\n",
1073 |     "ys_a1 = ((data['A'] * data['Y'])/ (p_score) + \n",
1074 |     "         (y_a1*(p_score-data['A'])) / p_score)\n",
1075 |     "ys_a0 = (((1-data['A']) * data['Y'])/ (1-p_score) + \n",
1076 |     "         (y_a0*(data['A']-p_score)) / (1-p_score))\n",
1077 |     "ate = np.mean(ys_a1 - ys_a0)\n",
1078 |     "print(\"ATE\", np.round(ate, 6))\n",
1079 |     "\n",
1080 |     "# Step 4: bootstrap for inference\n",
1081 |     "ate_rs = []\n",
1082 |     "for i in range(1000):\n",
1083 |     "    d_star = data.sample(n=data.shape[0], # Same size as input data\n",
1084 |     "                     replace=True)  # Draw with replacement\n",
1085 |     "    fm_a1 = smf.glm(\"Y ~ C + W1 + W2 + W3 + W4\", \n",
1086 |     "                    d_star.loc[d_star[\"A\"] == 1], family=f).fit()\n",
1087 |     "    fm_a0 = smf.glm(\"Y ~ C + W1 + W2 + W3 + W4\", \n",
1088 |     "                    d_star.loc[d_star[\"A\"] == 0], family=f).fit()\n",
1089 |     "    y_a1 = fm_a1.predict(d_star)\n",
1090 |     "    y_a0 = fm_a0.predict(d_star)\n",
1091 |     "    fm_pa = smf.glm(\"A ~ C + W1 + W2 + W3 + W4\", \n",
1092 |     "                    d_star, family=f).fit()\n",
1093 |     "    p_score = fm_pa.predict(d_star)\n",
1094 |     "    ys_a1 = ((d_star['A'] * d_star['Y'])/ (p_score) + \n",
1095 |     "             (y_a1*(p_score-d_star['A'])) / p_score)\n",
1096 |     "    ys_a0 = (((1-d_star['A']) * d_star['Y'])/ (1-p_score) + \n",
1097 |     "             (y_a0*(d_star['A']-p_score)) / (1-p_score))\n",
1098 |     "    ate_rs.append(np.mean(ys_a1 - ys_a0))\n",
1099 |     "\n",
1100 |     "\n",
1101 |     "print(\"95% Confidence limits for the ATE\")\n",
1102 |     "ci_perc = np.percentile(ate_rs, q=[2.5, 97.5])\n",
1103 |     "print(\"Percentile method:   \", np.round(ci_perc, 6))\n",
1104 |     "ate_se = np.std(ate_rs, ddof=1)\n",
1105 |     "print(\"Normal approx method:\", np.round([ate - 1.96*ate_se,\n",
1106 |     "                                          ate + 1.96*ate_se], 6))"
1107 |    ]
1108 |   },
1109 |   {
1110 |    "cell_type": "code",
1111 |    "execution_count": 28,
1112 |    "metadata": {},
1113 |    "outputs": [
1114 |     {
1115 |      "name": "stdout",
1116 |      "output_type": "stream",
1117 |      "text": [
1118 |       "ATE    0.083796\n",
1119 |       "95% CL [0.058546 0.109046]\n"
1120 |      ]
1121 |     }
1122 |    ],
1123 |    "source": [
1124 |     "# Box 27: AIPTW estimator with zEpid\n",
1125 |     "aipw = zepid.causal.doublyrobust.AIPTW(data, \n",
1126 |     "                                       exposure=\"A\", \n",
1127 |     "                                       outcome=\"Y\")\n",
1128 |     "aipw.exposure_model(\"C + W1 + W2 + W3 + W4\", \n",
1129 |     "                    print_results=False)\n",
1130 |     "aipw.outcome_model(\"A + C + W1 + W2 + W3 + W4 + \"\n",
1131 |     "                   \"A:C + A:W1 + A:W2 + A:W3 + A:W4\", \n",
1132 |     "                   print_results=False)\n",
1133 |     "aipw.fit()\n",
1134 |     "\n",
1135 |     "print(\"ATE   \", np.round(aipw.risk_difference, 6))\n",
1136 |     "print(\"95% CL\", np.round(aipw.risk_difference_ci, 6))\n",
1137 |     "# zEpid calculates the variance using influence curves"
1138 |    ]
1139 |   },
1140 |   {
1141 |    "cell_type": "markdown",
1142 |    "metadata": {},
1143 |    "source": [
1144 |     "## 6. Data-Adaptive Estimation: Ensemble Learning Targeted Maximum Likelihood Estimation"
1145 |    ]
1146 |   },
1147 |   {
1148 |    "cell_type": "code",
1149 |    "execution_count": 29,
1150 |    "metadata": {},
1151 |    "outputs": [
1152 |     {
1153 |      "name": "stdout",
1154 |      "output_type": "stream",
1155 |      "text": [
1156 |       "ATE 0.083796\n",
1157 |       "95% CL [0.058546 0.109047]\n"
1158 |      ]
1159 |     }
1160 |    ],
1161 |    "source": [
1162 |     "# Box 28: Computation of TMLE by hand\n",
1163 |     "f = sm.families.family.Binomial()\n",
1164 |     "n = data.shape[0]\n",
1165 |     "\n",
1166 |     "# Step 1: g-computation\n",
1167 |     "fm_a1 = smf.glm(\"Y ~ C + W1 + W2 + W3 + W4\", \n",
1168 |     "                data.loc[data[\"A\"] == 1], family=f).fit()\n",
1169 |     "fm_a0 = smf.glm(\"Y ~ C + W1 + W2 + W3 + W4\", \n",
1170 |     "                data.loc[data[\"A\"] == 0], family=f).fit()\n",
1171 |     "y_a1 = fm_a1.predict(data)\n",
1172 |     "y_a0 = fm_a0.predict(data)\n",
1173 |     "y_a_ = np.where(data['A'] == 1, y_a1, y_a0)\n",
1174 |     "\n",
1175 |     "# Step 2: propensity scores\n",
1176 |     "fm_pa = smf.glm(\"A ~ C + W1 + W2 + W3 + W4\", \n",
1177 |     "                data, family=f).fit()\n",
1178 |     "p_score = fm_pa.predict(data)\n",
1179 |     "\n",
1180 |     "# Step 3: targeting step\n",
1181 |     "logodds_y1 = np.log(probability_to_odds(y_a1))\n",
1182 |     "logodds_y0 = np.log(probability_to_odds(y_a0))\n",
1183 |     "logodds_ya = np.log(probability_to_odds(y_a_))\n",
1184 |     "\n",
1185 |     "clever_cov_a1 = data['A']/p_score\n",
1186 |     "clever_cov_a0 = - (1-data['A'])/(1-p_score)\n",
1187 |     "\n",
1188 |     "submodel = sm.GLM(data['Y'], \n",
1189 |     "                  np.column_stack((clever_cov_a1, clever_cov_a0)), \n",
1190 |     "                  offset=logodds_ya,\n",
1191 |     "                  family=f).fit()\n",
1192 |     "epsilon = submodel.params\n",
1193 |     "\n",
1194 |     "# Step 4: calculating ATE\n",
1195 |     "ys_a1 = logistic.cdf(logodds_y1 + epsilon[0] / p_score)\n",
1196 |     "ys_a0 = logistic.cdf(logodds_y0 - epsilon[1] / (1-p_score))\n",
1197 |     "ate = np.mean(ys_a1 - ys_a0)\n",
1198 |     "print(\"ATE\", np.round(ate, 6))\n",
1199 |     "\n",
1200 |     "# Step 5: inference via influence curve\n",
1201 |     "ic = (clever_cov_a1 + clever_cov_a0) * (data['Y'] - y_a_) + (y_a1 - y_a0) - ate\n",
1202 |     "sd = np.sqrt(np.nanvar(ic, ddof=1) / n)\n",
1203 |     "cl = [ate - 1.96*sd, ate + 1.96*sd]\n",
1204 |     "print(\"95% CL\", np.round(cl, 6))"
1205 |    ]
1206 |   },
1207 |   {
1208 |    "cell_type": "code",
1209 |    "execution_count": 30,
1210 |    "metadata": {},
1211 |    "outputs": [
1212 |     {
1213 |      "name": "stdout",
1214 |      "output_type": "stream",
1215 |      "text": [
1216 |       "ATE    0.083796\n",
1217 |       "95% CL [0.058546 0.109047]\n"
1218 |      ]
1219 |     }
1220 |    ],
1221 |    "source": [
1222 |     "# Box 29: TMLE with zEpid\n",
1223 |     "tmle = zepid.causal.doublyrobust.TMLE(data, \n",
1224 |     "                                      exposure=\"A\", \n",
1225 |     "                                      outcome=\"Y\")\n",
1226 |     "tmle.exposure_model(\"C + W1 + W2 + W3 + W4\", \n",
1227 |     "                    print_results=False)\n",
1228 |     "tmle.outcome_model(\"A + C + W1 + W2 + W3 + W4 + \"\n",
1229 |     "                   \"A:C + A:W1 + A:W2 + A:W3 + A:W4\", \n",
1230 |     "                   print_results=False)\n",
1231 |     "tmle.fit()\n",
1232 |     "\n",
1233 |     "print(\"ATE   \", np.round(tmle.risk_difference, 6))\n",
1234 |     "print(\"95% CL\", np.round(tmle.risk_difference_ci, 6))"
1235 |    ]
1236 |   },
1237 |   {
1238 |    "cell_type": "markdown",
1239 |    "metadata": {},
1240 |    "source": [
1241 |     "## 7. Simulation"
1242 |    ]
1243 |   },
1244 |   {
1245 |    "cell_type": "code",
1246 |    "execution_count": 31,
1247 |    "metadata": {},
1248 |    "outputs": [],
1249 |    "source": [
1250 |     "def data_generator(n, true_ate=False):\n",
1251 |     "    \"\"\"Function to generate data consisting of `n` observations\"\"\"\n",
1252 |     "    d = pd.DataFrame()\n",
1253 |     "    # Confounders\n",
1254 |     "    d['W1'] = np.round(np.random.uniform(low=1, high=5, size=n))\n",
1255 |     "    d['W2'] = np.random.binomial(n=1, p=0.45, size=n)\n",
1256 |     "    d['W3'] = np.round(np.random.uniform(low=0, high=1, size=n)\n",
1257 |     "                       + 0.8*d['W1'] + 0.75*d['W2'])\n",
1258 |     "    d['W3'] = np.where(d['W3'] > 4, 1, d['W3'])\n",
1259 |     "    d['W4'] = np.round(np.random.uniform(low=0, high=1, size=n)\n",
1260 |     "                       + 0.2*d['W1'] + 0.75*d['W2'])\n",
1261 |     "    # Treatment\n",
1262 |     "    pr_a = logistic.cdf(-1 - 0.15*d['W4'] + 1.5*d['W2'] + \n",
1263 |     "                        0.75*d['W3'] + 0.25*d['W1'] + \n",
1264 |     "                        0.8*d['W2']*d['W4'])\n",
1265 |     "    d['A'] = np.random.binomial(n=1, \n",
1266 |     "                                p=pr_a, \n",
1267 |     "                                size=n)\n",
1268 |     "    # Potential outcomes\n",
1269 |     "    pr_y1 = logistic.cdf(-3 + 1 + 0.25*d['W4'] + 0.75*d['W3'] + \n",
1270 |     "                         0.8*d['W2']*d['W4'] + 0.05*d['W1'])\n",
1271 |     "    y1 = np.random.binomial(n=1, p=pr_y1, size=n)\n",
1272 |     "    pr_y0 = logistic.cdf(-3 + 0 + 0.25*d['W4'] + 0.75*d['W3'] + \n",
1273 |     "                         0.8*d['W2']*d['W4'] + 0.05*d['W1'])\n",
1274 |     "    y0 = np.random.binomial(n=1, p=pr_y0, size=n)\n",
1275 |     "    # Causal consistency\n",
1276 |     "    d['Y'] = np.where(d['A'] == 1, y1, y0)\n",
1277 |     "    # Return generated data\n",
1278 |     "    if true_ate:\n",
1279 |     "        return np.mean(y1 - y0), np.mean(y1) / np.mean(y0)\n",
1280 |     "    else:\n",
1281 |     "        return d\n",
1282 |     "\n",
1283 |     "\n",
1284 |     "# Generating true value from super-population\n",
1285 |     "true_ate, true_rr = data_generator(n=1000000, true_ate=True)\n",
1286 |     "\n",
1287 |     "# Setting up SuperLearner libraries\n",
1288 |     "from zepid.superlearner import SuperLearner, StepwiseSL\n",
1289 |     "from sklearn.linear_model import LogisticRegression\n",
1290 |     "from pygam import LogisticGAM, f, s\n",
1291 |     "\n",
1292 |     "import warnings\n",
1293 |     "warnings.simplefilter('ignore', RuntimeWarning)  # Hides some NumPy errors for sparse models\n",
1294 |     "\n",
1295 |     "family = sm.families.family.Binomial()\n",
1296 |     "\n",
1297 |     "sl_library_main = [LogisticRegression(penalty='none', solver='lbfgs'),\n",
1298 |     "                  StepwiseSL(family, selection=\"backward\"), \n",
1299 |     "                  StepwiseSL(family, selection=\"forward\", order_interaction=1)]\n",
1300 |     "sl_main_labs = [\"LogR\", \"Step.zero\", \"Step.one\"]\n",
1301 |     "sl_main = SuperLearner(sl_library_main, sl_main_labs, folds=5,\n",
1302 |     "                       loss_function='nloglik')\n",
1303 |     "\n",
1304 |     "sl_library_alt = [LogisticRegression(penalty='none', solver='lbfgs'),\n",
1305 |     "                  StepwiseSL(family, selection=\"backward\"), \n",
1306 |     "                  StepwiseSL(family, selection=\"forward\", order_interaction=1),\n",
1307 |     "                  LogisticGAM(f(0) + s(1) + f(2) + s(3) + s(4), \n",
1308 |     "                              lam=0.6)]\n",
1309 |     "sl_alt_labs = [\"LogR\", \"Step.zero\", \"Step.one\", \"GAM\"]\n",
1310 |     "sl_alt = SuperLearner(sl_library_alt, sl_alt_labs, folds=5,\n",
1311 |     "                      loss_function='nloglik')\n",
1312 |     "\n",
1313 |     "# Simulation\n",
1314 |     "naive_rd, gform_rd, iptw_rd, aipw_rd, tmle_rd = [], [], [], [], []\n",
1315 |     "aipw_slm_rd, aipw_sla_rd, tmle_slm_rd, tmle_sla_rd = [], [], [], []\n",
1316 |     "\n",
1317 |     "for i in range(1000):    \n",
1318 |     "    data = data_generator(n=1000)\n",
1319 |     "\n",
1320 |     "    # Naive\n",
1321 |     "    fm = smf.ols(\"Y ~ A + W1 + W2 + W3 + W4\", data).fit()\n",
1322 |     "    naive_rd.append(fm.params['A'])\n",
1323 |     "    \n",
1324 |     "    # G-formula\n",
1325 |     "    g_formula = zepid.causal.gformula.TimeFixedGFormula(data, \n",
1326 |     "                                                    exposure=\"A\", \n",
1327 |     "                                                    outcome=\"Y\")\n",
1328 |     "    g_formula.outcome_model(\"A + W1 + W2 + W3 + W4\",\n",
1329 |     "                            print_results=False)\n",
1330 |     "    g_formula.fit(\"all\")  # all sets A=1\n",
1331 |     "    y_a1 = g_formula.marginal_outcome\n",
1332 |     "    g_formula.fit(\"none\")  # none sets A=0\n",
1333 |     "    y_a0 = g_formula.marginal_outcome\n",
1334 |     "    gform_rd.append(y_a1 - y_a0)\n",
1335 |     "    \n",
1336 |     "    # IPTW\n",
1337 |     "    ipw = zepid.causal.ipw.IPTW(data, treatment=\"A\", outcome=\"Y\")\n",
1338 |     "    ipw.treatment_model(\"W1 + W2 + W3 + W4\", bound=0.01,\n",
1339 |     "                        stabilized=True, print_results=False)\n",
1340 |     "    ipw.marginal_structural_model(\"A\")\n",
1341 |     "    ipw.fit()\n",
1342 |     "    iptw_rd.append(ipw.risk_difference)\n",
1343 |     "\n",
1344 |     "    # AIPW\n",
1345 |     "    aipw = zepid.causal.doublyrobust.AIPTW(data, \n",
1346 |     "                                       exposure=\"A\", \n",
1347 |     "                                       outcome=\"Y\")\n",
1348 |     "    aipw.exposure_model(\"W1 + W2 + W3 + W4\", bound=0.01,\n",
1349 |     "                        print_results=False)\n",
1350 |     "    aipw.outcome_model(\"A + W1 + W2 + W3 + W4\", \n",
1351 |     "                       print_results=False)\n",
1352 |     "    aipw.fit()\n",
1353 |     "    aipw_rd.append(aipw.risk_difference)\n",
1354 |     "    \n",
1355 |     "    # AIPW -- Super Learner main\n",
1356 |     "    aipw = zepid.causal.doublyrobust.AIPTW(data, \n",
1357 |     "                                       exposure=\"A\", \n",
1358 |     "                                       outcome=\"Y\")\n",
1359 |     "    aipw.exposure_model(\"W1 + W2 + W3 + W4\", bound=0.01,\n",
1360 |     "                        custom_model=sl_main,\n",
1361 |     "                        print_results=False)\n",
1362 |     "    aipw.outcome_model(\"A + W1 + W2 + W3 + W4\", \n",
1363 |     "                        custom_model=sl_main,\n",
1364 |     "                       print_results=False)\n",
1365 |     "    aipw.fit()\n",
1366 |     "    aipw_slm_rd.append(aipw.risk_difference)\n",
1367 |     "    \n",
1368 |     "    # AIPW -- Super Learner alternative\n",
1369 |     "    aipw = zepid.causal.doublyrobust.AIPTW(data, \n",
1370 |     "                                       exposure=\"A\", \n",
1371 |     "                                       outcome=\"Y\")\n",
1372 |     "    aipw.exposure_model(\"W1 + W2 + W3 + W4\", bound=0.01,\n",
1373 |     "                        custom_model=sl_alt,\n",
1374 |     "                        print_results=False)\n",
1375 |     "    aipw.outcome_model(\"A + W1 + W2 + W3 + W4\", \n",
1376 |     "                        custom_model=sl_alt,\n",
1377 |     "                       print_results=False)\n",
1378 |     "    aipw.fit()\n",
1379 |     "    aipw_sla_rd.append(aipw.risk_difference)\n",
1380 |     "    \n",
1381 |     "    # TMLE\n",
1382 |     "    tmle = zepid.causal.doublyrobust.TMLE(data, \n",
1383 |     "                                          exposure=\"A\", \n",
1384 |     "                                          outcome=\"Y\")\n",
1385 |     "    tmle.exposure_model(\"W1 + W2 + W3 + W4\", bound=0.01,\n",
1386 |     "                        print_results=False)\n",
1387 |     "    tmle.outcome_model(\"A + W1 + W2 + W3 + W4\", \n",
1388 |     "                       print_results=False)\n",
1389 |     "    tmle.fit()\n",
1390 |     "    tmle_rd.append(tmle.risk_difference)\n",
1391 |     "\n",
1392 |     "    # TMLE -- Super Learner main\n",
1393 |     "    tmle = zepid.causal.doublyrobust.TMLE(data, \n",
1394 |     "                                          exposure=\"A\", \n",
1395 |     "                                          outcome=\"Y\")\n",
1396 |     "    tmle.exposure_model(\"W1 + W2 + W3 + W4\", bound=0.01,\n",
1397 |     "                        custom_model=sl_main,\n",
1398 |     "                        print_results=False)\n",
1399 |     "    tmle.outcome_model(\"A + W1 + W2 + W3 + W4\", \n",
1400 |     "                       custom_model=sl_main,\n",
1401 |     "                       print_results=False)\n",
1402 |     "    tmle.fit()\n",
1403 |     "    tmle_slm_rd.append(tmle.risk_difference)\n",
1404 |     "    \n",
1405 |     "    # TMLE -- Super Learner alternative\n",
1406 |     "    tmle = zepid.causal.doublyrobust.TMLE(data, \n",
1407 |     "                                          exposure=\"A\", \n",
1408 |     "                                          outcome=\"Y\")\n",
1409 |     "    tmle.exposure_model(\"W1 + W2 + W3 + W4\", bound=0.01,\n",
1410 |     "                        custom_model=sl_alt,\n",
1411 |     "                        print_results=False)\n",
1412 |     "    tmle.outcome_model(\"A + W1 + W2 + W3 + W4\", \n",
1413 |     "                       custom_model=sl_alt,\n",
1414 |     "                       print_results=False)\n",
1415 |     "    tmle.fit()\n",
1416 |     "    tmle_sla_rd.append(tmle.risk_difference)\n",
1417 |     "    \n"
1418 |    ]
1419 |   },
1420 |   {
1421 |    "cell_type": "code",
1422 |    "execution_count": 32,
1423 |    "metadata": {},
1424 |    "outputs": [
1425 |     {
1426 |      "name": "stdout",
1427 |      "output_type": "stream",
1428 |      "text": [
1429 |       "=============================\n",
1430 |       "Naive\n",
1431 |       "-----------------------------\n",
1432 |       "Abs. Bias: 0.048\n",
1433 |       "Rel. Bias: 26.7\n",
1434 |       "=============================\n",
1435 |       "=============================\n",
1436 |       "G-formula\n",
1437 |       "-----------------------------\n",
1438 |       "Abs. Bias: 0.002\n",
1439 |       "Rel. Bias: 0.8\n",
1440 |       "=============================\n",
1441 |       "=============================\n",
1442 |       "IPTW\n",
1443 |       "-----------------------------\n",
1444 |       "Abs. Bias: 0.106\n",
1445 |       "Rel. Bias: 58.5\n",
1446 |       "=============================\n",
1447 |       "=============================\n",
1448 |       "AIPW\n",
1449 |       "-----------------------------\n",
1450 |       "Abs. Bias: 0.003\n",
1451 |       "Rel. Bias: 1.9\n",
1452 |       "=============================\n",
1453 |       "=============================\n",
1454 |       "AIPW-SL1\n",
1455 |       "-----------------------------\n",
1456 |       "Abs. Bias: 0.003\n",
1457 |       "Rel. Bias: 1.5\n",
1458 |       "=============================\n",
1459 |       "=============================\n",
1460 |       "AIPW-SL2\n",
1461 |       "-----------------------------\n",
1462 |       "Abs. Bias: 0.003\n",
1463 |       "Rel. Bias: 1.7\n",
1464 |       "=============================\n",
1465 |       "=============================\n",
1466 |       "TMLE\n",
1467 |       "-----------------------------\n",
1468 |       "Abs. Bias: 0.009\n",
1469 |       "Rel. Bias: 4.8\n",
1470 |       "=============================\n",
1471 |       "=============================\n",
1472 |       "TMLE-SL1\n",
1473 |       "-----------------------------\n",
1474 |       "Abs. Bias: 0.006\n",
1475 |       "Rel. Bias: 3.5\n",
1476 |       "=============================\n",
1477 |       "=============================\n",
1478 |       "TMLE-SL2\n",
1479 |       "-----------------------------\n",
1480 |       "Abs. Bias: 0.008\n",
1481 |       "Rel. Bias: 4.5\n",
1482 |       "=============================\n"
1483 |      ]
1484 |     }
1485 |    ],
1486 |    "source": [
1487 |     "# Results\n",
1488 |     "result = [naive_rd, gform_rd, iptw_rd, aipw_rd, aipw_slm_rd, \n",
1489 |     "          aipw_sla_rd, tmle_rd, tmle_slm_rd, tmle_sla_rd]\n",
1490 |     "labels = [\"Naive\", \"G-formula\", \"IPTW\", \"AIPW\", \"AIPW-SL1\", \n",
1491 |     "          \"AIPW-SL2\", \"TMLE\", \"TMLE-SL1\", \"TMLE-SL2\"]\n",
1492 |     "for x, y in zip(result, labels):\n",
1493 |     "    print(\"=============================\")\n",
1494 |     "    print(y)\n",
1495 |     "    print(\"-----------------------------\")\n",
1496 |     "    print(\"Abs. Bias:\", np.round(np.mean(x - true_ate), 3))\n",
1497 |     "    rel_bias = np.abs(np.mean((x - true_ate) / true_ate)*100)\n",
1498 |     "    print(\"Rel. Bias:\", np.round(rel_bias, 1))\n",
1499 |     "    print(\"=============================\")\n"
1500 |    ]
1501 |   },
1502 |   {
1503 |    "cell_type": "markdown",
1504 |    "metadata": {},
1505 |    "source": [
1506 |     "END"
1507 |    ]
1508 |   }
1509 |  ],
1510 |  "metadata": {
1511 |   "kernelspec": {
1512 |    "display_name": "Python 3",
1513 |    "language": "python",
1514 |    "name": "python3"
1515 |   },
1516 |   "language_info": {
1517 |    "codemirror_mode": {
1518 |     "name": "ipython",
1519 |     "version": 3
1520 |    },
1521 |    "file_extension": ".py",
1522 |    "mimetype": "text/x-python",
1523 |    "name": "python",
1524 |    "nbconvert_exporter": "python",
1525 |    "pygments_lexer": "ipython3",
1526 |    "version": "3.6.5"
1527 |   }
1528 |  },
1529 |  "nbformat": 4,
1530 |  "nbformat_minor": 4
1531 | }
1532 | 


--------------------------------------------------------------------------------
/RCodeBoxes.R:
--------------------------------------------------------------------------------
  1 | ###################################################################################################
  2 | 
  3 | # Tutorial: causal inference methods made easy for applied resarchers/epidemiologists/statisticians 
  4 | 
  5 | # ICON-LSHTM, LONDON, 16th October 2020
  6 | 
  7 | # Miguel Angel Luque Fernandez, PhD
  8 | # Assistant Professor of Epidemiology and Biostatistics
  9 | # Matthew Smith, PhD
 10 | # Research Fellow
 11 | 
 12 | # Inequalities in Cancer Outcomes Network, LSHTM, London, UK
 13 | 
 14 | # Copyright (c) 2020 Permission is hereby granted, free of charge, to any person obtaining a copy 
 15 | # of this software and associated documentation files (the "Software"), to deal in the Software 
 16 | # without restriction, including without limitation the rights to use, copy, modify, merge, 
 17 | # publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to 
 18 | # whom the Software is furnished to do so, subject to the following conditions: The above 
 19 | # copyright notice and this permission notice shall be included in all copies or substantial 
 20 | # portions of the Software.
 21 | 
 22 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING 
 23 | # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON 
 24 | # INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES 
 25 | # OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 
 26 | # IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 27 | 
 28 | # Bug reports: miguel-angel.luque at lshtm.ac.uk	
 29 | 
 30 | # The rhc dataset can be dowloaded at http://biostat.mc.vanderbilt.edu/wiki/Main/DataSets
 31 | 
 32 | ###################################################################################################
 33 | 
 34 | 
 35 | # Preliminaries
 36 |   rm(list=ls())
 37 | 
 38 |   ### Box 1: Setting the data
 39 |   setwd("your path")
 40 |   #setwd("~/Dropbox/ESTIMATORSCIproject/R_Stata_master_files/Data")
 41 |   library(haven)
 42 |   #data <- read_dta("~/Dropbox/ESTIMATORSCIproject/R_Stata_master_files/Data/rhc.dta")
 43 |   data <- read_dta("rhc.dta")
 44 |   # Define the outcome (Y), exposure (A), confounder (C), and confounders (W)
 45 |   data$Y  <- data$death_d30; data$Y <- as.numeric(data$Y); Y <- data$Y 
 46 |   data$A  <- data$rhc; data$A <- as.numeric(data$A); A <- data$A
 47 |   data$C  <- data$gender; data$C <- as.numeric(data$C); C <- data$C
 48 |   data$w1 <- data$age; data$w1 <- as.numeric(data$w1); w1 <- data$w1
 49 |   data$w2 <- data$edu; data$w2 <- as.numeric(data$w2); w2 <- data$w2
 50 |   data$w3 <- data$race; data$w3 <- as.numeric(data$w3); w3 <- data$w3
 51 |   data$w4 <- data$carcinoma; data$w4 <- as.numeric(data$w4); w4 <- data$w4
 52 |   data2   <- as.data.frame(Y); data2$A <- A; data2$C <- C; data2$w1 <- w1; data2$w2 <- w2; data2$w3 <- w3; data2$w4 <- w4
 53 |     
 54 |   
 55 |   ### Box 2: Naive estimate of the ATE
 56 |   naive <- lm(Y ~ A + C, data=data); naive  # Naive estimate of the ATE is 0.07352
 57 | 
 58 |   
 59 | # 3. G-Formula
 60 |   
 61 |   ## 3.1 Non-parametric G-formula
 62 |   
 63 |     ### Box 3: Non-parametric G-formula for the ATE 
 64 |     mean(data$A[data$C==1], na.rm=TRUE)       # 
 65 |     mean(data$A[data$C==0], na.rm=TRUE)       # 
 66 |     mean(data$Y[data$A==1], na.rm=TRUE) - mean(data$Y[data$A==0],na.rm=TRUE)    # Unadjusted Estimate
 67 |     reg <- lm(Y ~ A, data=data); reg          # Unadjusted Estimate Regression 
 68 |     pr.l <- prop.table(table(data$C)); pr.l   # Marginal probability of C
 69 |     tab.out <- aggregate(Y ~ A + C, data, mean); tab.out    # Table of Means in 
 70 |     ATE <- ((mean(data$Y[data$A==1 & data$C==1]) - mean(data$Y[data$A==0 & data$C==1]))*pr.l[2]) + 
 71 |      (mean(data$Y[data$A==1 & data$C==0]) - mean(data$Y[data$A==0 & data$C==0]))*pr.l[1]   # G-formula Non-parametric ATE 
 72 |     ATE; rm(ATE)    # The ATE from the non-parametric estimator is 0.073692
 73 |     
 74 |     ### Box 4: Non-parametric G-formula for the ATT
 75 |     ATTm <- mean(data$C[data$A==1], na.rm=TRUE)       # Proportion of those who are male amongst treated
 76 |     ATTf <- 1-mean(data$C[data$A==1], na.rm=TRUE)       # Proportion of those who are female amongst treated  
 77 |     ATT <-((mean(data$Y[data$A==1 & data$C==1]) - mean(data$Y[data$A==0 & data$C==1]))*ATTm) + 
 78 |     (mean(data$Y[data$A==1 & data$C==0]) - mean(data$Y[data$A==0 & data$C==0]))*ATTf   # G-formula Non-parametric ATT
 79 |     ATT       # The ATT from the non-parametric estimator is 0.073248
 80 |     rm(ATT)
 81 |     
 82 |     ### Box 5: Bootstrap the 95% confidence intervals (CI) for the ATE/ATT estimated using the non-parametric G-Formula
 83 |     # ATE
 84 |     library(boot)
 85 |     g.comp = function(data,indices)     # Define the function to estimate the ATE
 86 |     {
 87 |       dat=data[indices,]
 88 |       pr.l <- prop.table(table(dat$C))
 89 |       
 90 |       ATE = ((mean(dat$Y[dat$A==1 & dat$C==1]) - mean(dat$Y[dat$A==0 & dat$C==1]))*pr.l[2]) + 
 91 |        (mean(dat$Y[dat$A==1 & dat$C==0]) - mean(dat$Y[dat$A==0 & dat$C==0]))*pr.l[1] ; ATE
 92 |     }
 93 |     g.comp(data,indices=1:nrow(data))     # Can get original estimate, by plugging in indices 1:n
 94 |     boot.out=boot(data,g.comp,200)      # Draw 200 bootstrap sample estimates 
 95 |     boot.ci(boot.out,type="perc",conf=0.95)     # compute confidence intervals using percentile method
 96 |     boot.ci(boot.out,type="norm",conf=0.95)
 97 |     
 98 |     # ATT
 99 |     g.comp = function(data,indices)     # Define the function to estimate the ATT
100 |     {
101 |       dat=data[indices,]
102 |       
103 |       ATTm <- mean(dat$C[dat$A==1], na.rm=TRUE)       # Proportion of those who are male among treated
104 |       ATTf <- 1-mean(dat$C[dat$A==1], na.rm=TRUE)  
105 |       
106 |       ((mean(dat$Y[dat$A==1 & dat$C==1]) - mean(dat$Y[dat$A==0 & dat$C==1]))*ATTm) + 
107 |         (mean(dat$Y[dat$A==1 & dat$C==0]) - mean(dat$Y[dat$A==0 & dat$C==0]))*ATTf 
108 |     }
109 |     g.comp(data,indices=1:nrow(data))           # Can get original estimate, by plugging in indices 1:n
110 |     boot.out=boot(data,g.comp,200)              # Draw 200 bootstrap sample estimates 
111 |     boot.ci(boot.out,type="perc",conf=0.95)     # compute confidence intervals using percentile method
112 |     boot.ci(boot.out,type="norm",conf=0.95)
113 |     
114 |   
115 |     ### Box 6: Non-parametric G-Formula using a fully saturated regression model in Stata (A)
116 |         # Method 1: conditional probabilities
117 |     data$A1 <- ifelse(data$A == 1, 1, 0)
118 |     data$A0 <- ifelse(data$A == 0, 1, 0)
119 |     data$C1 <- ifelse(data$C == 1, 1, 0)
120 |     data$C0 <- ifelse(data$C == 0, 1, 0)
121 |     reg <- glm(Y ~ -1 + (A1 + A0) + A1:(C1) + A0:(C1), data=data); summary(reg)
122 |     ATE <- mean((reg$coefficients[1] + reg$coefficients[3]*C) - (reg$coefficients[2] + reg$coefficients[4]*C)); ATE
123 |     rm(ATE)
124 |     
125 |     ### Box 7: Non-parametric G-Formula using a fully saturated regression model in Stata (B)
126 |         # Method 2: Marginal probabilities
127 |     install.packages("margins")
128 |     library(margins)
129 |     reg <- glm(Y ~ -1 + (A1 + A0) + A1:(C1) + A0:(C1), data=data); summary(reg)
130 |     Y1 <- margins(reg, variables="A1"); Y1
131 |     Y0 <- margins(reg, variables="A0"); Y0
132 |     ATE <- Y1$fitted[A==1]-Y0$fitted[A==0]; mean(ATE)
133 |     rm(ATE)
134 |     
135 |   ## 3.2 Parametric G-formula
136 |     ### Box 8: Parametric G-formula by hand
137 |     mod1  <- glm(Y ~  C, family="binomial", data=data[data$A==1,])    # Expected probability amongst those with RHC
138 |     mod0  <- glm(Y ~  C, family="binomial", data=data[data$A==0,])    # Expected probability amongst those without RHC
139 |     GcompRA  <- cbind(Y1 = predict(mod1, newdata=data.frame(A = 1, C), type="response"),
140 |                    Y0 = predict(mod0, newdata=data.frame(A = 0, C), type="response"))
141 |     GcompRA <- as.data.frame(GcompRA)
142 |     Y.1 <- GcompRA$Y1
143 |     Y.0 <- GcompRA$Y0
144 |     ATE <- mean((Y.1) - (Y.0), na.rm=TRUE); ATE     # Difference between expected probabilities (ATE) 
145 |     rm(ATE)
146 |   
147 |     
148 |     ### Box 9: Parametric regression adjustment (one confounder) using stdReg R-package
149 |     install.packages("stdReg")
150 |     library(stdReg)
151 |     reg <- glm(Y ~ A + C, data = data, family = poisson(link="log")); summary(reg)
152 |     reg.std <- stdGlm(fit=reg, data = data, X = "A", x=seq(0,1))
153 |     print(summary(reg.std, contrast = "difference", reference=0))
154 |     plot(reg.std)
155 | 
156 |     ### Box 10: Bootstrap for the parametric regression adjustment one confounder)
157 |     library(boot)           # Install the Bootstrap package
158 |     attach(data)          
159 |     g.comp=function(data,indices)       # Define the function to estimate the ATE
160 |     {
161 |       dat=data[indices,]
162 |       glm1  <- glm(Y ~ C, family="binomial", dat=dat[dat$A==1,])
163 |       glm2  <- glm(Y ~ C, family="binomial", dat=dat[dat$A==0,])
164 |       Y.1 = predict(glm1, newdata=data.frame(A = 1, C), type="response")
165 |       Y.0 = predict(glm2, newdata=data.frame(A = 0, C), type="response")
166 |       ATE <- mean((Y.1) - mean(Y.0)); ATE
167 |     }
168 |     g.comp(data,indices=1:nrow(data))     # Can get original estimate, by plugging in indices 1:n
169 |     boot.out=boot(data,g.comp,200)        # Draw 1000 bootstrap sample estimates of RD
170 |     boot.ci(boot.out,type="norm",conf=0.95)     # Bootstrapped 95% CI based on normal approximation
171 |     boot.ci(boot.out,type="perc",conf=0.95)     # Bootstrapped 95% CI based on percentiles of the bootstrap replicates
172 |     
173 |     # Now with more than one confounder
174 |     
175 |     ### Box 11: Parametric multivariate regression adjustment implementation of the G-Formula
176 |     mod1  <- glm(Y ~  C + w1 + w2 + w3 + w4, family="binomial", data=data[data$A==1,])    # Expected probability amongst those with RHC
177 |     mod0  <- glm(Y ~  C + w1 + w2 + w3 + w4, family="binomial", data=data[data$A==0,])    # Expected probability amongst those without RHC
178 |     GcompRA  <- cbind(Y1 = predict(mod1, newdata=data.frame(A = 1, C, w1, w2, w3, w4), type="response"),
179 |                    Y0 = predict(mod0, newdata=data.frame(A = 0, C, w1, w2, w3, w4), type="response"))
180 |     GcompRA <- as.data.frame(GcompRA)
181 |     Y.1 <- GcompRA$Y1
182 |     Y.0 <- GcompRA$Y0
183 |     ATE <- mean((Y.1) - (Y.0), na.rm=TRUE); ATE     # ATE 
184 |     rm(ATE)
185 |     
186 |     
187 |     ### Box 12: Parametric multivariate regression adjustment using "stdReg" R-package
188 |     install.packages("stdReg")
189 |     library(stdReg)
190 |     reg <- glm(Y ~ A + C + w1 + w2 + w3 + w4, data = data, family = poisson(link="log")); summary(reg)
191 |     reg.std <- stdGlm(fit=reg, data=data, X="A", x=seq(0,1))
192 |     print(summary(reg.std, contrast="difference", reference=0))
193 |     plot(reg.std)
194 |     
195 |     
196 |     ### Box 13: Parametric multivariate regression adjustment using "margins" R-package
197 |     reg1 <- glm(Y ~ -1 + (A1 + A0) + A1:(C1 + w1 + w2 + w3 + w4) + A0:(C0 + w1 + w2 + w3 + w4) , data=data); summary(reg1)
198 |     poY1m <- margins(reg1, variables="A1"); poY1m
199 |     poY0m <- margins(reg1, variables="A0"); poY0m
200 |     ATE2 <- poY1m$fitted[A==1] - poY0m$fitted[A==0]; mean(ATE2)
201 |     
202 |     ### Box 14 Bootstrap for the multivariate parametric regression adjustment
203 |     library(boot)           # Install the Bootstrap package
204 |     attach(data)          
205 |     g.comp=function(data,indices)       # Define the function to estimate the ATE
206 |     {
207 |       dat=data[indices,]
208 |       glm1  <- glm(Y ~ C + w1 + w2 + w3 + w4, family="binomial", dat=dat[dat$A==1,])
209 |       glm2  <- glm(Y ~ C + w1 + w2 + w3 + w4, family="binomial", dat=dat[dat$A==0,])
210 |       Y.1 = predict(glm1, newdata=data.frame(A = 1, C, w1, w2, w3, w4), type="response")
211 |       Y.0 = predict(glm2, newdata=data.frame(A = 0, C, w1, w2, w3, w4), type="response")
212 |       mean((Y.1) - mean(Y.0))
213 |     }
214 |     g.comp(data,indices=1:nrow(data))     # Can get original estimate, by plugging in indices 1:n
215 |     boot.out=boot(data,g.comp,200)        # Draw 1000 bootstrap sample estimates of RD
216 |     boot.ci(boot.out,type="norm",conf=0.95)     # Bootstrapped 95% CI based on normal approximation
217 |     boot.ci(boot.out,type="perc",conf=0.95)     # Bootstrapped 95% CI based on percentiles of the bootstrap replicates
218 |   
219 |   
220 |     ### Box 15 Computing the parametric marginal risk ratio after regression adjustment
221 |     reg <- glm(Y ~ A + C + w1 + w2 + w3 + w4, data=data2, family = binomial(link="logit")); summary(reg)
222 |     reg.std <- stdGlm(fit=reg, data=data2, X="A", x=seq(0,1))
223 |     print(summary(reg.std, contrast="ratio", reference=0))    # 27% (95% CI 1.18-1.37) increase in relative risk
224 |     plot(reg.std)
225 |   
226 | # 4. Inverse Probability of Treatment Weighting
227 |   
228 |   ## 4.1 Inverse probability of treatment weighting based on the propensity score plus regression adjustment
229 |     
230 |     # Box 16 (IPTW by hand)
231 |     p.s <- glm(A ~ as.factor(C) + w1 + w2 + w3 + w4, data=data, family=binomial)      # Propensity score mmodel for the exposure
232 |     p.score <- ifelse(data$A == 0, 1 - predict(p.s, type = "response"), predict(p.s, type = "response"))  # Assign Propensity score weights
233 |     #table(p.score)      # Table of Propensity Scores
234 |     data$w <- 1/p.score       # Generate IP Weights 
235 |     data2$w <- 1/p.score
236 |     #table(data$w); summary(data$w); sd(data$w)
237 |     
238 |     ATE <- mean(data$w*as.numeric(data$A==1)*data$Y) - mean(data$w*as.numeric(data$A==0)*data$Y);ATE  # Estimate ATE
239 |     rm(ATE)
240 |   
241 |     
242 |     # Box 17 Bootstrap computation for the IPTW estimator
243 |     library(boot)
244 |     iptw.w = function(data,indices)     # Define the function to estimate the ATE
245 |     {
246 |       dat=data[indices,]
247 |       mean(dat$w*as.numeric(dat$A==1)*dat$Y) - mean(dat$w*as.numeric(dat$A==0)*dat$Y)
248 |     }
249 |     iptw.w(data,indices=1:nrow(data))     # Can get original estimate, by plugging in indices 1:n
250 |     boot.out=boot(data,iptw.w,100)      # Draw 200 bootstrap sample estimates 
251 |     boot.ci(boot.out,type="perc",conf=0.95)     # compute confidence intervals using percentile method
252 |     boot.ci(boot.out,type="norm",conf=0.95)
253 | 
254 |   
255 |     ### Box 18: Computation of the IPTW estimator for the ATE using IPW R-package
256 |     install.packages("ipw", "survey")
257 |     library(ipw)
258 |     library(survey)
259 |     
260 |     # Univariable
261 |     ipw.ATE <- ipwpoint(exposure = A, family = "binomial", link = "logit", 
262 |                         numerator = ~ 1, 
263 |                         denominator = ~ C, 
264 |                         data = data2)
265 |     summary(ipw.ATE$ipw.weights)
266 |     ipwplot(weights = ipw.ATE$ipw.weights, logscale = FALSE, main = "Unstabilized weights", xlim = c(0.5, 2))
267 |     summary(ipw.ATE$num.mod)
268 |     summary(ipw.ATE$den.mod)
269 |     data2$usw <- ipw.ATE$ipw.weights
270 |     msm <- (svyglm(Y ~ A, design = svydesign(~ 1, weights = ~ usw, data = data2)))
271 |     coef(msm);  confint(msm)
272 |     
273 |     # Multivariable
274 |     ipw.ATE <- ipwpoint(exposure = A, family = "binomial", link = "logit", 
275 |                         numerator = ~ 1, 
276 |                         denominator = ~ C + w1 + w2 + w3 + w4, 
277 |                         data = data2)
278 |     summary(ipw.ATE$ipw.weights)
279 |     ipwplot(weights = ipw.ATE$ipw.weights, logscale = FALSE, main = "Unstabilized weights", xlim = c(0.5, 2))
280 |     summary(ipw.ATE$num.mod)
281 |     summary(ipw.ATE$den.mod)
282 |     data2$usw <- ipw.ATE$ipw.weights
283 |     msm <- (svyglm(Y ~ A, design = svydesign(~ 1, weights = ~ usw, data = data2)))
284 |     coef(msm);  confint(msm)
285 |   
286 |     
287 |     ### Box 19: Assessing IPTW balance
288 |     install.packages("twang")
289 |     library(twang)
290 |     ps.balance <- ps(A ~ C + w1 + w2 + w3 + w4, data = data2, 
291 |                      n.trees=1000, interaction.depth=2, shrinkage=0.01, perm.test.iters=0,
292 |                      stop.method=c("es.mean","ks.max"), estimand = "ATE", verbose=FALSE)
293 |     plot(ps.balance)
294 |     summary(ps.balance$gbm.obj, n.trees=ps.balance$desc$ks.max.ATE$n.trees, plot=FALSE)
295 |     data2.balance <- bal.table(ps.balance); data2.balance  
296 |     
297 |     
298 |     ### Box 20: Assessing IPTW overlap by hand
299 |     install.packages("xtable")
300 |     library(xtable)
301 |     pretty.tab <- data2.balance$ks.max.ATE[,c("tx.mn","ct.mn","ks")]
302 |     pretty.tab <- cbind(pretty.tab, data2.balance$unw[,"ct.mn"])
303 |     names(pretty.tab) <- c("E(Y1|t=1)","E(Y0|t=1)","KS","E(Y0|t=0)")
304 |     xtable(pretty.tab, caption = "Balance of the treatment and comparison groups",
305 |            label = "tab:balance", digits = c(0, 2, 2, 2, 2), align=c("l","r","r","r","r"))
306 |     plot(ps.balance, plots = 6)
307 |     
308 |     
309 |     ### Box 21: Assessing overlap using plots
310 |     # Fit a propensity score model
311 |     m_PS<-glm(A ~ C + w1 + w2 + w3 + w4, data = data2, family=binomial(link="logit"))
312 |     summary(m_PS)
313 |     
314 |     # Estimate the propensity score
315 |     data$PS<-fitted.values(m_PS)
316 |     
317 |     # Histogram of the PS
318 |     hist(data$PS[data$rhc==0])
319 |     hist(data$PS[data$rhc==1])
320 |     plot(density(data$PS[data$rhc==0]),col="red",lwd=2, xlab="PS")
321 |     lines(density(data$PS[data$rhc==1]),col="blue",lwd=2)
322 |     legend("topright", legend=c("No RHC", "RHC"), pch="--", col=c("red","blue"), bty="n", lwd=2)
323 |     
324 |     # Look at minimum and maximum PS in each exposure group
325 |     min(data$PS[data$rhc==0])
326 |     min(data$PS[data$rhc==1])
327 |     max(data$PS[data$rhc==0])
328 |     max(data$PS[data$rhc==1])
329 |     
330 |     # Investigate overlap (i.e. positivity) 
331 |     data$overlap <- ifelse(data$PS>=min(data$PS[data$rhc==1]) & data$PS<=max(data$PS[data$rhc==0]),1,0); table(data$overlap,data$rhc)
332 |     
333 |   ## 4.2 Marginal structural model with stabilised weights
334 |     ### Box 22: Computation of the IPTW estimator for the ATE using a MSM
335 |     
336 |     # Unstabilized weights 
337 |     msm <- lm(Y  ~ A + C +  w1 + w2 + w3 + w4, data = data, weights = data$w)     # MSM
338 |     library(sandwich)
339 |     SE <-sqrt(diag(vcovHC(msm, type="HC0")))       # robust standard errors
340 |     beta <- coef(msm)
341 |     lcl <- beta-1.96*SE 
342 |     ucl <- beta+1.96*SE
343 |     cbind(beta, lcl, ucl)[2,]
344 |   
345 |     # Stabilized weights
346 |     denom.fit <- glm(A ~ as.factor(C) + w1 + w2 + w3 + w4, 
347 |                      family = binomial(), data = data)
348 |     denom.p <- predict(denom.fit, type = "response")    # Stablized Weights  
349 |   
350 |     numer.fit <- glm(A ~ 1, family = binomial(), data = data)
351 |     summary(numer.fit)
352 |     numer.p <- predict(numer.fit, type = "response")    # estimation of numerator of ip weights
353 |   
354 |     data$sw <- ifelse(data$A == 0, ((1-numer.p)/(1-denom.p)), (numer.p/denom.p))
355 |   
356 |     msm <- lm(Y  ~ A, data = data, weights = sw)
357 |   
358 |     SE <-sqrt(diag(vcovHC(msm, type="HC0"))) # robust standard errors
359 |     beta <- coef(msm)
360 |     lcl <- beta-1.96*SE 
361 |     ucl <- beta+1.96*SE
362 |     cbind(beta, lcl, ucl)[2,]     
363 | 
364 |   ## 4.3 IPTW with regression adjustment  
365 |     ### Box 23: Computation of the IPTW-RA estimator for the ATE and bootstrap for statistical inference
366 |     glm1  <- glm(Y ~ C + w1 + w2 + w3 + w4,  weights = data$w[data$A==1], data=data[data$A==1,])
367 |     Y.1 = predict(glm1,  newdata=data.frame(A = 1, C, w1, w2, w3, w4), type="response")
368 |     glm2  <- glm(Y ~ C + w1 + w2 + w3 + w4,  weights = data$w[data$A==0], data=data[data$A==0,])
369 |     Y.0 = predict(glm2,  newdata=data.frame(A = 0, C, w1, w2, w3, w4), type="response")
370 |     ATE <- mean(Y.1 - Y.0); ATE
371 |     ATE2 <- mean(data$w*as.numeric(data$A==1)*Y.1)/mean( data$w*as.numeric(data$A==1)) - mean(data$w*as.numeric(data$A==0)*Y.0)/mean(data$w*as.numeric(data$A==0));ATE2
372 |     rm(ATE, ATE2)
373 |   
374 |     ### Box 24: Computation of the IPTW-RA estimator for the ATE using the ipw R-package
375 |     library(ipw)
376 |     ipw.ATE <- ipwpoint(exposure = A, family = "binomial", link = "logit", 
377 |                         numerator = ~ C, 
378 |                         denominator = ~ C + w1 + w2 + w3 + w4, 
379 |                         data = data2)
380 |     summary(ipw.ATE$ipw.weights)
381 |     ipwplot(weights = ipw.ATE$ipw.weights, logscale = FALSE, main = "Stabilized weights", xlim = c(0.5, 2))
382 |     summary(ipw.ATE$num.mod)
383 |     summary(ipw.ATE$den.mod)
384 |     
385 |     data2$sw <- ipw.ATE$ipw.weights
386 |     msm <- (svyglm(Y ~ A, design = svydesign(~ 1, weights = ~ sw, data = data2)))
387 |     coef(msm);  confint(msm)
388 |   
389 |   
390 | # 5. Augmented inverse probability weighting
391 |     
392 |     ### Box 25: Computation of the AIPTW estimator for the ATE and bootstrap for statistical inference
393 |     mod  <- glm(Y ~ A + C + w1 + w2 + w3 + w4, family="binomial", data=data)
394 |     PO   <- cbind(Yhat = predict(mod),
395 |                   Y1 = predict(mod, newdata=data.frame(A = 1, C, w1, w2, w3, w4), type="response"),
396 |                   Y0 = predict(mod, newdata=data.frame(A = 0, C, w1, w2, w3, w4), type="response"))
397 |     RA <- as.data.frame(PO)   # Potential Outcomes
398 |     Yhat <- RA$Yhat
399 |     Y.1a <- RA$Y1
400 |     Y.0a <- RA$Y0
401 |   
402 |     g <- glm(A ~ C + w1 + w2 + w3 + w4, family = binomial(), data = data)
403 |     gw <- predict(g, type = "response")
404 |     gws <- ifelse(data$A == 0, (-(1 - data$A)/(1 - gw)),(data$A/gw)); sum(gws)    # estimation of weights
405 |     AIPTW <- mean(gws*(data$Y - plogis(RA$Yhat)) + ((Y.1a) - (Y.0a))); AIPTW  # ATE 
406 |     RR <- mean(Y.1a/Y.0a); RR # RR
407 |     
408 |     IC <- (gws*(data$Y - plogis(RA$Yhat)) + ((Y.1a) - (Y.0a)))-AIPTW    # Estimate the influence function (functional Delta method)
409 |     n  <- nrow(data)
410 |     varHat.IC <- var(IC)/n; varHat.IC
411 |     lci <- AIPTW-1.96*sqrt(varHat.IC)
412 |     uci <- AIPTW+1.96*sqrt(varHat.IC)
413 |     cat(AIPTW,lci,uci)      # Inference Influence function
414 |   
415 |     AIPTW.b = function(data,indices)        # Inference using Bootstrap
416 |     {
417 |       dat=data[indices,]
418 |       mod  <- glm(Y ~ A + C + w1 + w2 + w3 + w4, family="binomial", data=data)
419 |       Yhat = predict(mod)
420 |       Y1 = predict(mod, newdata=data.frame(A = 1, C, w1, w2, w3, w4))
421 |       Y0 = predict(mod, newdata=data.frame(A = 0, C, w1, w2, w3, w4))
422 |       g <- glm(A ~ C + w1 + w2 + w3 + w4, family="binomial",  data = data)
423 |       gw <- predict(g,type="response")
424 |       gws <- ifelse(A == 0, (-(1 - A)/(1 - gw)),(A/gw))
425 |       mean(gws*(Y - plogis(Yhat)) + (plogis(Y1) - plogis(Y0)))
426 |     }
427 |     AIPTW.b(data,indices=1:nrow(data))    # Can get original estimate, by plugging in indices 1:n
428 |     boot.out=boot(data,AIPTW.b,200)     # Draw 200 bootstrap sample estimates 
429 |     boot.ci(boot.out,type="perc",conf=0.95)     # compute confidence intervals using percentile method
430 |     boot.ci(boot.out,type="norm",conf=0.95)
431 |     
432 |     ### Box 26: Computation of the AIPTW estimator for the ATE and marginal risk ratio
433 |     w <- subset(data, select=c(C, w1, w2, w3 , w4))
434 |     fit1 <- drtmle(W = w, A = A, Y = Y, # input data
435 |                    a_0 = c(0, 1), # return estimates for A = 0 and A = 1
436 |                    SL_Q = "SL.npreg", # use kernel regression for E(Y | A = a, W)
437 |                    glm_g = "C + w1 + w2 + w3 + w4", # use misspecified main terms glm for E(A | W)
438 |                    SL_Qr = "SL.npreg", # use kernel regression to guard against
439 |                    # misspecification of outcome regression
440 |                    #SL_gr = "SL.npreg", # use kernel regression to guard against
441 |                    # misspecification of propensity score
442 |                    returnModels = TRUE # for visualizing fits later
443 |     )
444 |     ATE <- ci(fit1, contrast = c(-1,1)); ATE
445 |     RR <- riskRatio <- list(f = function(eff){ log(eff) },
446 |                       f_inv = function(eff){ exp(eff) },
447 |                       h = function(est){ est[2]/est[1] },
448 |                       fh_grad =  function(est){ c(1/est[1],-1/est[2]) })
449 |     ci(fit1, contrast = riskRatio)
450 |     rm(ATE, RR)
451 |     
452 | # 6. DATA-ADAPTIVE ESTIMATION: ENSEMBLE LEARNING TARGETED MAXIMUMLIKELIHOOD ESTIMATION
453 |     
454 |     ### Box 27: Computational implementation of TMLE by hand
455 |     # Step 1
456 |     Gcomp <- glm(Y ~ A + C + w1 + w2 + w3 + w4, family="binomial", data=data2)
457 |     # Prediction for A, A=1 and, A=0
458 |     QAW <- predict(Gcomp)
459 |     Q1W = predict(Gcomp, newdata=data.frame(A = 1, data2[,c("C", "w1","w2","w3","w4")])) 
460 |     Q0W = predict(Gcomp, newdata=data.frame(A = 0, data2[,c("C", "w1","w2","w3","w4")]))
461 |     # Step 2 estimation of the propensity score (ps)
462 |     psm <- glm(A ~ C + w1 + w2 + w3 + w4, family = binomial, data=data2) 
463 |     gW = predict(psm, type = "response")
464 |     g1W = (1 / gW)
465 |     g0W = (-1 / (1-gW))
466 |     # Step 3 computation of H and estimation of epsilon
467 |     HAW <- (data2$A / gW -(1-data2$A) / (1 - gW))
468 |     H1W = (1/gW)
469 |     H0W = (-1 / (1 - gW))
470 |     epsilon <- coef(glm(data2$Y ~ -1 + HAW + offset(QAW), family = "binomial"))
471 |     # Step 4  ATE
472 |     ATE<- mean(plogis(Q1W + epsilon * H1W) - plogis(Q0W + epsilon * H0W)); ATE
473 |     # Step 5  Maringinal RR
474 |     T1.EY1 <- mean(plogis(Q1W + epsilon * H1W)) 
475 |     T1.EY0 <- mean(plogis(Q0W + epsilon * H0W))
476 |     RR <- (T1.EY1/T1.EY0); RR
477 |     rm(ATE, RR)
478 |     
479 |     ### Box 28: TMLE with data-adaptive estimation using the R package
480 |     set.seed(777)
481 |     library(tmle)
482 |     w <- subset(data, select=c(C, w1, w2, w3 , w4))
483 |     fittmle <- tmle(data$Y, data$A, W=w, family="binomial", 
484 |                     Q.SL.library = c("SL.glm","SL.glm.interaction","SL.step.interaction","SL.gam","SL.randomForest"),
485 |                     g.SL.library = c("SL.glm","SL.glm.interaction","SL.step.interaction","SL.gam","SL.randomForest"))
486 |     fittmle
487 |   
488 | #  7. Simulation
489 |     ### Box 29: Data generation for the Monte Carlo experiment
490 |     
491 |     rm(list=ls())
492 |     
493 |     # Super Learner libraries
494 |     SL.library <- c("SL.glm","SL.step","SL.step.interaction","SL.glm.interaction","SL.gam") #"SL.randomForest","SL.glmnet" 
495 |                   
496 |     # Data generation A: dual misspecification for the model of the outcome and treatment 
497 |     set.seed(7777)
498 |     generateData <- function(n){
499 |       w1 <- round(runif(n, min=1, max=5), digits=0) 
500 |       w2 <- rbinom(n, size=1, prob=0.45)
501 |       w3 <- round(runif(n, min=0, max=1), digits=0 + 0.75*w2 + 0.8*w1)
502 |       w4 <- round(runif(n, min=0, max=1), digits=0 + 0.75*w2 + 0.2*w1)
503 |       A  <- rbinom(n, size=1, prob= plogis(-1 -  0.15*w4 + 1.5*w2 + 0.75*w3 + 0.25*w1 + 0.8*w2*w4))
504 |       # Counterfactuals
505 |       Y.1 <- rbinom(n, size=1, prob = plogis(-3 + 1 + 0.25*w4 + 0.75*w3 + 0.8*w2*w4 + 0.05*w1))
506 |       Y.0 <- rbinom(n, size=1, prob = plogis(-3 + 0 + 0.25*w4 + 0.75*w3 + 0.8*w2*w4 + 0.05*w1)) 
507 |       # Observed outcome
508 |       Y <- Y.1*A + Y.0*(1 - A)
509 |       # return data.frame
510 |       data.frame(w1, w2, w3, w4, A, Y, Y.1, Y.0)
511 |       }
512 |     
513 |     # True ATE
514 |     ObsDataTrueATE <- generateData(n=5000000)
515 |     True_ATE <- mean(ObsDataTrueATE$Y.1 - ObsDataTrueATE$Y.0);True_ATE 
516 |     True_EY.1 <- mean(ObsDataTrueATE$Y.1)
517 |     True_EY.0 <- mean(ObsDataTrueATE$Y.0)
518 |     True_RR <- (True_EY.1 / True_EY.0);True_RR
519 |     
520 |     #Simulations 
521 |     library(tmle) 
522 |     library(SuperLearner) 
523 |     #install.packages("dbarts")
524 |     R <- 1000
525 |     #Empty vectors
526 |     naive_RR <- rep(NA,R) 
527 |     ATEtmle1 <- rep(NA,R) 
528 |     RRtmle1 <- rep(NA,R) 
529 |     ATE_AIPTW   <- rep(NA,R) 
530 |     RR_AIPTW <- rep(NA,R) 
531 |     ATEtmle2 <- rep(NA,R) 
532 |     RRtmle2 <- rep(NA,R) 
533 |     ATEtmle3 <- rep(NA,R) 
534 |     RRtmle3 <- rep(NA,R)
535 |     for(r in 1:R){
536 |       print(paste("This is simulation run number",r)) 
537 |       CancerData <- generateData(n=1000)
538 |       # ATE naive approach
539 |       naive_RR[r] <- exp(glm(data = CancerData, Y ~ A + w1 + w2 + w3 + w4,  family = poisson(link="log"))$coef[2])
540 |       # TMLE implementation by hand
541 |       # Step 1
542 |       gm <- glm(Y ~ A + w1 + w2 + w3 + w4, family="binomial", data=CancerData)
543 |       # Prediction for A, A=1 and, A=0
544 |       QAW <- predict(gm)
545 |       Q1W = predict(gm, newdata=data.frame(A = 1, CancerData[,c("w1","w2","w3","w4")])) 
546 |       Q0W = predict(gm, newdata=data.frame(A = 0, CancerData[,c("w1","w2","w3","w4")]))
547 |       # Step 2 estimation of the propensity score (ps)
548 |       psm <- glm(A ~ w1 + w2 + w3 + w4, family = binomial, data=CancerData) 
549 |       gW = predict(psm, type = "response")
550 |       g1W = (1 / gW)
551 |       g0W = (-1 / (1-gW))
552 |       # Step 3 computation of H and estimation of epsilon
553 |       HAW <- (CancerData$A / gW -(1-CancerData$A) / (1 - gW))
554 |       H1W = (1/gW)
555 |       H0W = (-1 / (1 - gW))
556 |       epsilon <- coef(glm(CancerData$Y ~ -1 + HAW + offset(QAW), family = "binomial"))
557 |       # Step 4 updated ATE
558 |       ATEtmle1[r] <- mean(plogis(Q1W + epsilon * H1W) - plogis(Q0W + epsilon * H0W))
559 |       # Step 5 updated MOR
560 |       T1.EY1 <- mean(plogis(Q1W + epsilon * H1W)) 
561 |       T1.EY0 <- mean(plogis(Q0W + epsilon * H0W))
562 |       RRtmle1[r] <- (T1.EY1 / T1.EY0)
563 |       
564 |       # Augmented inverse probability treatment weight (AIPTW) estimator
565 |       ATE_AIPTW[r] <- mean((HAW*(CancerData$Y - plogis(QAW)) + (plogis(Q1W)-plogis(Q0W))))
566 |       AIPTW1 <- mean(CancerData$A * (CancerData$Y - plogis(Q1W)) / gW + plogis(Q1W) )
567 |       AIPTW0 <- mean((1- CancerData$A) * (CancerData$Y - plogis(Q0W)) / (1-gW) + plogis(Q0W)) 
568 |       RR_AIPTW[r] <- mean( AIPTW1 / AIPTW0)
569 |       
570 |       # R-package tmle (base implementation includes SL.step, SL.glm and SL.glm.interaction) 
571 |       ATE2 <- tmle(Y=CancerData$Y, A=CancerData$A, W=CancerData[,c("w1","w2","w3","w4")], family="binomial")
572 |       ATEtmle2[r] <- ATE2$estimates$ATE$psi
573 |       RRtmle2[r] <- ATE2$estimates$RR$psi
574 |       
575 |       # Improved Super learner
576 |       ATE3 <- tmle(Y = CancerData$Y, A=CancerData$A, W=CancerData[,c("w1","w2","w3","w4")], family="binomial", Q.SL.library=SL.library, g.SL.library=SL.library)
577 |       ATEtmle3[r] <- ATE3$estimates$ATE$psi
578 |       RRtmle3[r] <- ATE3$estimates$RR$psi
579 |     }
580 |     # Mean naive
581 |     mean(naive_RR)
582 |     # Mean AIPTW
583 |     mean(ATE_AIPTW)
584 |     mean(RR_AIPTW)
585 |     # Estimate of TMLE by hand
586 |     mean(ATEtmle1)
587 |     mean(RRtmle1)
588 |     # Estimate of TMLE + SL default implementation
589 |     mean(ATEtmle2)
590 |     mean(RRtmle2)
591 |     # Estimate of TMLE + SL2 default plus more algorithms
592 |     mean(ATEtmle3)
593 |     mean(RRtmle3)
594 |     save.image("your path\results.RData")
595 |     
596 | # Relative Bias ATE
597 | abs(mean((True_ATE - ATE_AIPTW) / True_ATE)*100)
598 | abs(mean((True_ATE - ATEtmle1) / True_ATE)*100)
599 | abs(mean((True_ATE - ATEtmle2) / True_ATE)*100)
600 | abs(mean((True_ATE - ATEtmle3) / True_ATE)*100)
601 | 
602 | # Relative Bias RR 
603 | abs(mean((True_RR - naive_RR) / True_RR)*100)
604 | abs(mean((True_RR - RR_AIPTW) / True_RR)*100)
605 | abs(mean((True_RR - RRtmle1) / True_RR)*100)
606 | abs(mean((True_RR - RRtmle2) / True_RR)*100)
607 | abs(mean((True_RR - RRtmle3) / True_RR)*100)    
608 |     
609 |     
610 |     
611 |     
612 |     
613 |     
614 |     
615 |     
616 |     
617 |     
618 |     
619 |     
620 |     
621 | 
622 | 
623 | 
624 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Tutorial: Introduction to computational causal inference for applied researchers and epidemiologists
 2 | 
 3 | ### Matthew James Smith, Camille Maringe, Bernard Rachet, Mohammad A. Mansournia, Paul Zivich, Stephen R. Cole, Miguel Angel Luque Fernandez
 4 | 
 5 | ### This repository makes available to the scientific community the data and code used in the preprint manuscript available at
 6 | 
 7 | [Link to the preprint article](https://arxiv.org/abs/2012.09920)
 8 | 
 9 | ### CITE this repository:
10 | 
11 | [![DOI](https://zenodo.org/badge/272439035.svg)](https://zenodo.org/badge/latestdoi/272439035)  
12 | 
13 | ### Matthew James Smith, Camille Maringe, Bernard Rachet, Mohammad A. Mansournia, Paul Zivich, Stephen R. Cole, Miguel Angel Luque Fernandez
14 | 
15 | ### ABSTRACT
16 | The purpose of many health studies is to estimate the effect of an exposure on an outcome. It is not always ethical to assign an exposure to individuals in randomised controlled trials, instead observational data and appropriate study design must be used. There are major challenges with observational studies, one of which is confounding that can lead to biased estimates of the causal effects. Controlling for confounding is commonly performed by simple adjustment for measured confounders; although, often this is not enough. Recent advances in the field of causal inference have dealt with confounding by building on classical standardisation methods. However, these recent advances have progressed quickly with a relative paucity of computational-oriented applied tutorials contributing to some confusion in the use of these methods among applied researchers. In this tutorial, we show the computational implementation of different causal inference estimators from a historical perspective where different estimators were developed to overcome the limitations of the previous one. Furthermore, we also briefly introduce the potential outcomes framework, illustrate the use of different methods using an illustration from the health care setting, and most importantly, we provide reproducible and commented code in Stata, R and Python for researchers to apply in their own observational study. The code can be accessed at  
17 | 
18 | [https://github.com/migariane/TutorialCausalInferenceEstimators](https://github.com/migariane/TutorialCausalInferenceEstimators)  
19 | 
20 | KEYWORDS: Causal Inference; Regression adjustment; G-methods; G-formula; Propensity score; Inverse probability weighting; Double-robust methods; Machine learning; Targeted maximum likelihood estimation;  Epidemiology; Statistics; Tutorial
21 | 


--------------------------------------------------------------------------------
/Results.RData:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/migariane/TutorialCausalInferenceEstimators/d809b657b382e227837d3032b1517612c478818d/Results.RData


--------------------------------------------------------------------------------
/StataCodeBoxes.do:
--------------------------------------------------------------------------------
  1 | /* 
  2 | Tutorial: causal inference methods made easy for applied resarchers/epidemiologists/statisticians 
  3 | =================================================================================================
  4 | 
  5 | ICON-LSHTM, LONDON, 16th October 2020
  6 | 
  7 | Miguel Angel Luque Fernandez, PhD
  8 | Assistant Professor of Epidemiology and Biostatistics
  9 | Camille Maringe, PhD
 10 | Assistant Professor
 11 | 
 12 | Inequalities in Cancer Outcomes Network, LSHTM, London, UK
 13 | 
 14 | Copyright (c) 2020 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
 15 | 
 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 17 | 
 18 | Bug reports: miguel-angel.luque@lshtm.ac.uk	
 19 | 
 20 | The rhc dataset can be dowloaded at http://biostat.mc.vanderbilt.edu/wiki/Main/DataSets
 21 | */
 22 |  
 23 | 
 24 | 
 25 | 
 26 | *** Preliminaries
 27 | 			clear
 28 | 			set more off
 29 | 			cd "C:\Data" 					// this path should point to where the RHC data are
 30 | 			use "rhc.dta", clear
 31 | 			describe
 32 | 			count
 33 | 			* 83 variables and 5,735 observations
 34 | 			
 35 | /* Box 1: Setting the data */
 36 | 			* Define the outcome (Y), exposure (A), confounder (C), and confounders (W)
 37 | 			global Y death_d30 
 38 | 			global A rhc 
 39 | 			global C gender  
 40 | 			global W gender age edu race carcinoma 
 41 | 	
 42 | /* Box 2: Naive estimate of the ATE */
 43 | 			* Naive approach to estimate the causal effect
 44 | 			regr $Y $A $C 	
 45 | 			* The naive estimate of the causal effect is 0.07352
 46 | 
 47 | /* 3. G-formula */
 48 | /* 3.1 Non-parametric G-formula */
 49 |        
 50 | 	   * 1) ATE    
 51 | /* Box 3: Non-parametric G-Formula for the ATE */
 52 |             proportion $C
 53 |             matrix m=e(b)
 54 |             gen genderf = m[1,1]
 55 |             sum genderf
 56 |             gen genderm = m[1,2]
 57 |             sum genderm
 58 | 			* you may need to install the command sumup, type:
 59 | 			* ssc install sumup
 60 |             sumup $Y, by($A $C)
 61 | 			* from sumup command extract the conditinal means by the given A and C levels i.e. zero and one
 62 | 			* see matrix list y00: position subscript [3,1] is th one of interest
 63 |             matrix y00 = r(Stat1)
 64 |             matrix y01 = r(Stat2)
 65 |             matrix y10 = r(Stat3)
 66 |             matrix y11 = r(Stat4)
 67 |             gen EY1 = ((y11[3,1]-y01[3,1]))*genderm 
 68 | 			gen EY0 = ((y10[3,1]-y00[3,1]))*genderf
 69 |             qui: mean EY1 EY0
 70 | 			matrix ATE =  r(table) 
 71 |             display "The ATE is: "  ATE[1,1] + ATE[1,2]
 72 |             drop EY*
 73 |             * The ATE from non-parametric estimator is:  0.073692
 74 | 			// Also one can try
 75 | 			gen ATE = ((y11[3,1]-y01[3,1]))*genderm + ((y10[3,1]-y00[3,1]))*genderf
 76 | 			qui sum ATE
 77 | 			drop ATE
 78 | 			
 79 |             * Check that Stata "teffects" command obtains the same estimate
 80 |             teffects ra ($Y $C) ($A)
 81 |             * The ATE from "teffects" implementation is:  0.073692
 82 |     	
 83 |         * 2) ATT
 84 | /* Box 4: Non-parametric G-Formula for the ATT */
 85 |             * Estimate the marginal probabilities
 86 |             proportion $C if $A==1
 87 |             matrix m=e(b)
 88 |             gen genderfatet = m[1,1]
 89 |             gen gendermatet = m[1,2]
 90 | 			gen EY1 = ((y11[3,1]-y01[3,1]))*gendermatet 
 91 | 			gen EY0 = ((y10[3,1]-y00[3,1]))*genderfatet
 92 | 			qui: mean EY1 EY0
 93 | 			matrix ATT =  r(table) 
 94 | 			display "The ATT is: "  ATT[1,1] + ATT[1,2] // Applying the G-formula
 95 |             drop EY*
 96 |             * The ATT from non-parametric estimator is:  0.073248
 97 | 			// Also one can try
 98 | 			gen ATT = ((y11[3,1]-y01[3,1]))*gendermatet + ((y10[3,1]-y00[3,1]))*genderfatet
 99 | 			qui sum ATT
100 | 			drop ATT            
101 | 			
102 | 			* Check using Stata "teffects" command
103 |             teffects ra ($Y $C) ($A), atet	
104 |             * The ATT from "teffects" implementation is:  0.073248
105 |    
106 | /* Box 5: Bootstrap 95% Confidence Intervals (CI) for the ATE/ATT estimated using the Non-parametric G-Formula */
107 |         
108 | 		* 1) For the ATE
109 | 			capture program drop ATE
110 |             program define ATE, rclass
111 | 				capture drop y1
112 | 				capture drop y0
113 | 				capture drop ATE
114 | 				sumup $Y, by($A $C)
115 | 				matrix y00 = r(Stat1)
116 | 				matrix y01 = r(Stat2)
117 | 				matrix y10 = r(Stat3)
118 | 				matrix y11 = r(Stat4)
119 | 				gen ATE = ((y11[3,1]-y01[3,1]))*genderm + ((y10[3,1]-y00[3,1]))*genderf
120 | 				qui sum ATE
121 | 				return scalar ate = `r(mean)'
122 |             end
123 |             
124 |             qui bootstrap r(ate), reps(1000): ATE
125 |             estat boot, all	
126 |     	
127 |         * 2) For the ATT
128 |             capture program drop ATT
129 |             program define ATT, rclass
130 |                capture drop y1
131 |                capture drop y0
132 |                capture drop ATT
133 |                sumup $Y, by($A $C)
134 |                matrix y00 = r(Stat1)
135 |                matrix y01 = r(Stat2)
136 |                matrix y10 = r(Stat3)
137 |                matrix y11 = r(Stat4)
138 |                gen ATT = ((y11[3,1]-y01[3,1]))*gendermatet + ((y10[3,1]-y00[3,1]))*genderfatet
139 |                qui sum ATT
140 |                return scalar att = `r(mean)'
141 |             end
142 |             
143 |             qui bootstrap r(att), reps(1000): ATT
144 |             estat boot, all
145 |             	
146 |             drop ATE ATT
147 |     	
148 | /* Box 6: Non-parametric G-Formula using a fully saturated regression model in Stata (A) */
149 | 			* method 1: conditional probabilities
150 |             regress $Y ibn.$A ibn.$A#c.($C) , noconstant vce(robust) coeflegend
151 |             predictnl ATE = (_b[1.rhc] + _b[1.rhc#c.gender]*gender) - (_b[0bn.rhc] + _b[0bn.rhc#c.gender]*gender)
152 | 			qui: sum ATE
153 |             display "The ATE is:  " "`r(mean)'"
154 |             drop ATE
155 | 			
156 | /* Box 7: Non-parametric G-Formula using a fully saturated regression model in Stata (B) */
157 | 			* method 2: marginal probabilities
158 |             regress $Y ibn.$A ibn.$A#c.($C) , noconstant vce(robust) coeflegend
159 | 			
160 | 			* Marginal probability in each treatment group
161 | 			margins $A , vce(unconditional) 
162 | 			
163 | 			* Difference in marginal probability between treatment groups
164 |             margins r.$A , contrast(nowald) 
165 | 			
166 | /* 3.2 PARAMETRIC G-FORMULA */			
167 | 
168 | * One confounder
169 | 
170 | /* Box 8: Parametric G-formula */		
171 | 			* Calculations by hand
172 | 			* Expected probability amongst treated
173 | 			regress $Y $C if $A==1      
174 | 			predict double y1hat
175 | 			
176 | 			* Expected probability amongst untreated
177 | 			regress $Y $C if $A==0      
178 | 			predict double y0hat
179 | 			mean y1hat y0hat            
180 | 			
181 | 			* Difference between expected probabilities (ATE) and biased confidence interval
182 | 			lincom _b[y1hat] - _b[y0hat]  
183 | 	
184 | /* Box 9: Parametric regression adjustment using Stata's teffects (one confounder) */
185 | 			teffects ra ($Y $C) ($A) 
186 | 
187 | /* Box 10: Bootstrap for the parametric regression adjustment */
188 | 			capture program drop ATE
189 | 			program define ATE, rclass
190 | 			   capture drop y1
191 | 			   capture drop y0
192 | 			   reg $Y $C if $A==1 
193 | 			   predict double y1, xb
194 | 			   quiet sum y1
195 | 			   reg $Y $C if $A==0 
196 | 			   predict double y0, xb 
197 | 			   quiet sum y0
198 | 			   mean y1 y0 
199 | 			   lincom _b[y1]-_b[y0]
200 | 			   return scalar ace =`r(estimate)'
201 | 			end
202 | 			qui bootstrap r(ace), reps(1000): ATE
203 | 			estat boot, all
204 | 
205 | * More than one confounder
206 | 
207 | /* Box 11: Parametric multivariate regression adjustment implementation of the G-Formula */
208 | 			regress $Y $W if $A==1
209 | 			predict double y1hat 
210 | 			regress $Y $W if $A==0
211 | 			predict double y0hat
212 | 			mean y1hat y0hat
213 | 			lincom _b[y1hat] - _b[y0hat]
214 | 		
215 | /* Box 12: Parametric multivariate regression adjustment using Stata’s teffects command */	
216 | 			teffects ra ($Y $W) ($A)
217 | 
218 | /* Box 13: Parametric multivariate regression adjustment using Stata’s margins command */
219 | 			regress $Y ibn.$A ibn.$A#c.($W) , noconstant vce(robust)
220 | 			margins $A, vce(unconditional)
221 | 			margins r.$A, contrast(nowald)
222 | 		
223 | /* Box 14: Bootstrap for the multivariate parametric regression adjustment */
224 | 			capture program drop ATE
225 | 			program define ATE, rclass
226 | 			   capture drop y1
227 | 			   capture drop y0
228 | 			   reg $Y $W if $A==1 
229 | 			   predict double y1, xb
230 | 			   quiet sum y1
231 | 			   reg $Y $W if $A==0 
232 | 			   predict double y0, xb 
233 | 			   quiet sum y0
234 | 			   mean y1 y0 
235 | 			   lincom _b[y1]-_b[y0]
236 | 			   return scalar ace =`r(estimate)'
237 | 			end
238 | 			qui bootstrap r(ace), reps(1000): ATE dots
239 | 			estat boot, all
240 | 
241 | /* Box 15: Computing the parametric marginal risk ratio after regression adjustment */
242 | 			teffects  ra ($Y $W) ($A), aequations
243 | 			teffects  ra ($Y $W) ($A), coeflegend
244 | 			nlcom  100*_b[ATE:r1vs0.$A]/_b[POmean:0.$A]   
245 | 			* 27.4% increase  in  relative  risk
246 | 			teffects  ra ($Y $W) ($A), pom coeflegend
247 | 			nlcom _b[POmeans:1.rhc]/ _b[POmeans:0bn.rhc]
248 | 			* 27.4% increase  in  relative  risk
249 | 
250 | /* 4 Inverse probability of treatment weighting  */		
251 | /* 4.1 Inverse probability of treatment weighting based on the propensity score plus regression adjustment */
252 | 
253 | /* Box 16: Computation of the IPTW estimator for the ATE */
254 | 			* propensity score model for the exposure
255 | 			logit $A $W, vce(robust) nolog
256 | 			
257 | 			* propensity score predictions
258 | 			predict double ps
259 | 			
260 | 			*  Sampling  weights  for  the  treated  group
261 | 			generate double ipw1 = ($A==1)/ps
262 | 			
263 | 			*  Weighted  outcome  probability  among  treated
264 | 			regress $Y [pw=ipw1]
265 | 			scalar Y1 = _b[_cons]
266 | 			
267 | 			*  Sampling  weights  for  the  non-treated  group
268 | 			generate double ipw0 = ($A==0)/(1-ps)
269 | 			regress $Y [pw=ipw0]
270 | 			scalar Y0 = _b[_cons]
271 | 			display "ATE =" Y1 - Y0
272 | 	
273 | /* Box 17: Bootstrap computation for the IPTW estimator */
274 | 			* Bootstrap the confidence intervals
275 | 			capture program drop ATE
276 | 			program define ATE, rclass
277 | 			   capture drop y1
278 | 			   capture drop y0
279 | 			   regress $Y [pw=ipw1]
280 | 			   matrix y1 = e(b)
281 | 			   gen double y1 = y1[1,1]
282 | 			   regress $Y [pw=ipw0]
283 | 			   matrix y0 = e(b)
284 | 			   gen double y0 = y0[1,1]
285 | 			   mean y1 y0
286 | 			   lincom _b[y1]-_b[y0]
287 | 			   return scalar ace = `r(estimate)'
288 | 			end
289 | 			qui bootstrap r(ace), reps(1000): ATE
290 | 			estat boot, all
291 |     
292 | /* Box 18: Computation of the IPTW estimator for the ATE using Stata’s teffects command */ 
293 | 			teffects ipw ($Y) ($A $W, logit), nolog vsquish
294 | 
295 | /* Box 19: Assessing IPTW balance */
296 | 			* Stata teffects and tebalance commands
297 | 			qui teffects ipw ($Y) ($A $W)   
298 | 			tebalance summarize
299 | 			
300 | 			* By hand - with the example of gender
301 | 			egen  genderst = std(gender) 	//  Standardization
302 | 			logistic $A $W 				 	//  Propensity  score
303 | 			capture drop ps
304 | 			predict  double  ps
305 | 			gen  ipw = .
306 | 			replace  ipw=($A==1)/ps if $A==1
307 | 			replace  ipw=($A==0)/(1-ps) if $A==0
308 | 			regress  genderst  $A 		 	// Raw  difference
309 | 			regress  genderst  $A [pw=ipw]  // Standardized  difference
310 | 		
311 | /* Box 20: Assessing IPTW overlap by hand */
312 | 			sort $A
313 | 			by $A: summarize ps
314 | 			kdensity ps if $A==1, generate(x1pointsa d1A) nograph n(10000) 
315 | 			kdensity ps if $A==0, generate(x0pointsa d0A) nograph n(10000) 
316 | 			label variable d1A "density for RHC=1"
317 | 			label variable d0A "density for RHC=0"
318 | 			twoway (line d0A x0pointsa , yaxis(1))(line d1A x1pointsa, yaxis(2))
319 | 		
320 | /* Box 21: Assessing overlap using Stata's teffects overlap */
321 | 			qui: teffects ipw ($Y) ($A $W, logit), nolog vsquish
322 | 			teffects overlap
323 | 
324 | 			
325 | /* 4.2 Marginal structural model with stabilized weights */		
326 | /* Box 22: Computation of the IPTW estimator for the ATE using a MSM */
327 | 			* Baseline treatment probabilities
328 | 			logit $A, vce(robust) nolog
329 | 			predict double nps, pr
330 | 			
331 | 			* propensity score model
332 | 			logit $A $W, vce(robust) nolog
333 | 			predict double dps, pr
334 | 			
335 | 			* Unstabilized weight
336 | 			cap drop ipw
337 | 			gen ipw = .
338 | 			replace ipw=($A==1)/dps if $A==1
339 | 			replace ipw=($A==0)/(1-dps) if $A==0
340 | 			sum ipw 
341 | 		
342 | 			* Stabilized weight 
343 | 			gen sws = .
344 | 			replace sws = nps/dps if $A==1
345 | 			replace sws = (1-nps)/(1-dps) if $A==0
346 | 			sum sws
347 | 
348 | 			* MSM 
349 | 			reg $Y $A [pw=ipw], vce(robust) 	// MSM unstabilized weight
350 | 			reg $Y $A [pw=sws], vce(robust) 	// MSM stabilized weight
351 | 
352 | 
353 | /* 4.3 IPTW with regression adjustment */
354 | 		
355 | /* Box 23: Computation of the IPTW-RA estimator for the ATE and bootstrap for statistical inference */
356 | 			capture program drop ATE
357 | 			program define ATE, rclass
358 | 			   capture drop y1
359 | 			   capture drop y0
360 | 			   reg $Y $W if $A==1 [pw=sws]
361 | 			   predict double y1, xb
362 | 			   quiet sum y1
363 | 			   return scalar y1=`r(mean)'
364 | 			   reg $Y $W if $A==0 [pw=sws]
365 | 			   predict double y0, xb 
366 | 			   quiet sum y0
367 | 			   return scalar y0=`r(mean)'
368 | 			   mean y1 y0 
369 | 			   lincom _b[y1]-_b[y0]
370 | 			   return scalar ace =`r(estimate)'
371 | 			end
372 | 			qui bootstrap r(ace), reps(10): ATE
373 | 			estat boot, all
374 | 		
375 | /* Box 24: Computation of the IPTW-RA estimator for the ATE using Stata’s teffects */
376 | 			teffects ipwra ($Y $W) ($A $W)
377 | 			nlcom 100*_b[r1vs0.$A]/_b[POmean:0.$A]
378 | 		    teffects  ipwra ($Y $W) ($A $W), pom coeflegend
379 | 			nlcom _b[POmeans:1.rhc]/ _b[POmeans:0bn.rhc]
380 | 		    *eltmle to check marginal RR
381 | 			eltmle  $Y $A $W, tmle
382 |     
383 | /* 5. Augmented inverse probability weighting */	
384 | /* Box 25: Computation of the AIPTW estimator for the ATE and bootstrap for statistical inference */
385 | 			* Step (i) prediction model for the outcome
386 | 			qui glm $Y $A $W, fam(bin) 
387 | 			predict double QAW, mu
388 | 			qui glm $Y $W if $A==1, fam(bin) 
389 | 			predict double Q1W, mu
390 | 			qui glm $Y $W if $A==0, fam(bin)
391 | 			predict double Q0W, mu
392 | 			
393 | 			* Step (ii): prediction model for the treatment
394 | 			cap drop dps nps sws y1 y0
395 | 			qui logit $A $W
396 | 			predict double dps, pr
397 | 			qui logit $A
398 | 			predict double nps, pr
399 | 			gen sws = .
400 | 			replace sws = nps/dps if $A==1
401 | 			replace sws = (1-nps)/(1-dps) if $A==0
402 | 		
403 | 			* Step (iii): Estimation equation
404 | 			gen double y1 = (sws*($Y-QAW) + (Q1W))
405 | 			quiet sum y1
406 | 			scalar y1=`r(mean)'
407 | 			gen double y0 = (sws*($Y-QAW) + (Q0W))
408 | 			quiet sum y0
409 | 			scalar y0=`r(mean)'
410 | 			mean y1 y0
411 | 			lincom _b[y1] - _b[y0]
412 | 		
413 | 			* Step (iv): Bootstrap confidence intervals
414 | 			capture program drop ATE
415 | 			program define ATE, rclass
416 | 			capture drop y1
417 | 			capture drop y0
418 | 			capture drop Q*
419 | 			qui glm $Y $A $W, fam(bin) 
420 | 			predict double QAW, mu
421 | 			qui glm $Y $W if $A==1, fam(bin) 
422 | 			predict double Q1W, mu
423 | 			qui glm $Y $W if $A==0, fam(bin)
424 | 			predict double Q0W, mu
425 | 			gen double y1 = (sws*($Y-QAW) + (Q1W))
426 | 			quiet sum y1
427 | 			return scalar y1=`r(mean)'
428 | 			gen double y0 = (sws*($Y-QAW) + (Q0W))
429 | 			quiet sum y0
430 | 			return scalar y0=`r(mean)'
431 | 			mean y1 y0
432 | 			lincom _b[y1] - _b[y0]
433 | 			return scalar ace =`r(estimate)'
434 | 			end
435 | 			qui bootstrap r(ace), reps(1000): ATE
436 | 			estat boot, all
437 | 
438 | /* Box 26: Computation of the AIPTW estimator for the ATE and marginal risk ratio using Stata’s teffects */
439 | 			teffects aipw ($Y $W) ($A $W, logit)
440 | 			*  marginal Relative  Risk
441 | 			nlcom 100*_b[r1vs0.$A]/_b[POmean:0.$A]	
442 | 			*  another way to compute it
443 | 			teffects aipw ($Y $W) ($A $W, logit), pom coeflegend
444 | 	        nlcom _b[POmeans:1.rhc]/ _b[POmeans:0bn.rhc]
445 | 		
446 | /* 6. DATA-ADAPTIVE ESTIMATION: ENSEMBLE LEARNING TARGETED MAXIMUMLIKELIHOOD ESTIMATION*/
447 | /*Box 27: Computational implementation of TMLE by hand */
448 | 
449 | 			* Step 1: prediction model for the outcome Q0 (g-computation)
450 | 			glm $Y $A $W, fam(binomial)
451 | 			predict double QAW_0, mu
452 | 			gen aa=$A
453 | 			replace $A = 0
454 | 			predict double Q0W_0, mu
455 | 			replace $A= 1
456 | 			predict double Q1W_0, mu
457 | 			replace $A = aa
458 | 			drop aa
459 | 
460 | 			// Q to logit scale
461 | 			gen logQAW = log(QAW / (1 - QAW))
462 | 			gen logQ1W = log(Q1W / (1 - Q1W))
463 | 			gen logQ0W = log(Q0W / (1 - Q0W))
464 | 
465 | 			* Step 2: prediction model for the treatment g0 (IPTW)
466 | 			glm $A $W, fam(binomial)
467 | 			predict gw, mu
468 | 			gen double H1W = $A  / gw
469 | 			gen double H0W = (1 - $A ) / (1 - gw)
470 | 
471 | 			* Step 3: Computing the clever covariate H(A,W) and estimating the parameter (epsilon) (MLE)
472 | 			glm $Y H1W H0W, fam(binomial) offset(logQAW) noconstant
473 | 			mat a = e(b)
474 | 			gen eps1 = a[1,1]
475 | 			gen eps2 = a[1,2]
476 | 
477 | 			* Step 4: update from Q0 to Q1
478 | 			gen double Q1W_1 = exp(eps1 / gw + logQ1W) / (1 + exp(eps1 / gw + logQ1W))
479 | 			gen double Q0W_1 = exp(eps2 / (1 - gw) + logQ0W) / (1 + exp(eps2 / (1 - gw) + logQ0W))
480 | 
481 | 			* Step 5: Targeted estimate of the ATE 
482 | 			gen ATE = (Q1W_1 - Q0W_1)
483 | 			summ ATE
484 | 			global ATE = r(mean)
485 | 			drop ATE 
486 | 			
487 | 			* Step 6: Statistical inference (efficient influence curve)
488 | 			qui sum(Q1W_1)
489 | 			gen EY1tmle = r(mean)
490 | 			qui sum(Q0W_1)
491 | 			gen EY0tmle = r(mean)
492 | 
493 | 			gen d1 = (($A  * ($Y - Q1W_1)/gw)) + Q1W_1 - EY1tmle
494 | 			gen d0 = ((1 - $A ) * ($Y - Q0W_1)/(1 - gw))  + Q0W_1 - EY0tmle
495 | 
496 | 			gen IC = d1 - d0
497 | 			qui sum IC
498 | 			gen varIC = r(Var) / r(N)
499 | 			drop d1 d0 IC
500 | 			
501 | 			global LCI =  $ATE - 1.96*sqrt(varIC)
502 | 			global UCI =  $ATE + 1.96*sqrt(varIC)
503 | 			display "ATE:"  %05.4f  $ATE _col(15) "95%CI: " %05.4f  $LCI "," %05.4f  $UCI		
504 | 
505 | /* Box 28: TMLE with data-adaptive estimation using the Stata’s user writen eltmle */
506 | 			* if not already installed, type:
507 | 			* ssc install eltmle		
508 | 			preserve
509 | 			eltmle $Y $A $W, tmle   
510 | 			restore
511 | 
512 | 		
513 | /* 7. Simulation */
514 | /* Box 29: Data generation for the Monte Carlo experiment */
515 | 
516 | 			* Data generation
517 | 			clear
518 | 			set obs 1000
519 | 			set seed 777
520 | 			gen w1    = round(runiform(1, 5)) //Quintiles of Socioeconomic Deprivation
521 | 			gen w2    = rbinomial(1, 0.45) //Binary: probability age >65 = 0.45
522 | 			gen w3    = round(runiform(0, 1) + 0.75*(w2) + 0.8*(w1)) //Stage 
523 | 			recode w3 (5/6=1)       //Stage (TNM): categorical 4 levels
524 | 			gen w4    = round(runiform(0, 1) + 0.75*(w2) + 0.2*(w1)) //Comorbidites: categorical four levels
525 | 			gen A     = (rbinomial(1,invlogit(-1 -  0.15*(w4) + 1.5*(w2) + 0.75*(w3) + 0.25*(w1) + 0.8*(w2)*(w4)))) //Binary treatment
526 | 			gen Y1 = (invlogit(-3 + 1 + 0.25*(w4) + 0.75*(w3) + 0.8*(w2)*(w4) + 0.05*(w1))) // Potential outcome 1
527 | 			gen Y0 = (invlogit(-3 + 0 + 0.25*(w4) + 0.75*(w3) + 0.8*(w2)*(w4) + 0.05*(w1))) // Potential outcome 2
528 | 			gen psi = Y1-Y0  // Simulated ATE
529 | 			gen Y = A*(Y1) + (1 - A)*Y0 //Binary outcome
530 | 
531 | 	
532 | 	// Estimate the true simulated ATE
533 | 			mean psi
534 | 				
535 | 		//	ATE estimation
536 | 			* Regression adjustment
537 | 			teffects ra (Y w1 w2 w3 w4) (A)
538 | 			estimates store ra
539 | 				
540 | 			* IPTW
541 | 			teffects ipw (Y) (A w1 w2 w3 w4)
542 | 			estimates store ipw
543 | 				
544 | 			* IPTW-RA
545 | 			teffects ipwra (Y w1 w2 w3 w4) (A w1 w2 w3 w4)
546 | 			estimates store ipwra	
547 | 				
548 | 			* AIPTW
549 | 			teffects aipw (Y w1 w2 w3 w4) (A w1 w2 w3 w4)
550 | 			estimates store aipw
551 | 				
552 | 			* Results
553 | 			qui reg psi
554 | 			estimates store psi
555 | 			estout psi ra ipw ipwra aipw
556 | 		
557 | 		// Ensemble learning maximum likelihood estimation
558 | 			preserve
559 | 			eltmle Y A w1 w2 w3 w4, tmle
560 | 			restore
561 | 				
562 | 		// Relative bias of each ATE
563 | 			* Regression adjustment
564 | 			display abs(0.1787 - 0.203419)/0.1787 
565 | 			
566 | 			* IPTW
567 | 			display abs(0.1787 - 0.2776)/0.1787 
568 | 			
569 | 			* IPTW-RA
570 | 			display abs(0.1787 - .2052088)/0.1787
571 | 			
572 | 			* AIPTW
573 | 			display abs(0.1787 - 0.2030)/0.1787
574 | 			
575 | 			* ELTMLE
576 | 			display abs(0.1787 - 0.1784)/0.1787
577 | 				
578 | 


--------------------------------------------------------------------------------
/rhc.Rdata:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/migariane/TutorialCausalInferenceEstimators/d809b657b382e227837d3032b1517612c478818d/rhc.Rdata


--------------------------------------------------------------------------------
/rhc.dta:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/migariane/TutorialCausalInferenceEstimators/d809b657b382e227837d3032b1517612c478818d/rhc.dta


--------------------------------------------------------------------------------