├── Class Exercises ├── L2 - Exercises │ ├── Multiple_Testing.ipynb │ ├── exp_data.csv │ ├── potential_outcomes.xlsx │ ├── t_z_tests.ipynb │ └── ttest.xlsx ├── L3 │ ├── CI-Bootstrap.ipynb │ ├── CIMean0.png │ ├── CIMeanSmall.png │ ├── Power_SampleSize.png │ ├── power_analysis.ipynb │ └── skewness │ │ ├── exp_data_2.csv │ │ └── skew.ipynb ├── L4 │ └── s4 │ │ ├── AA.ipynb │ │ ├── SRM Tests.xlsx │ │ └── sample_data_aatest.csv ├── L5 │ ├── AA.ipynb │ ├── Heterogenous.ipynb │ ├── Interleaving.xlsx │ ├── cluster.ipynb │ ├── exp_data_cluster.csv │ ├── sample_data_aatest.csv │ ├── sample_data_segments.csv │ └── variance_lift.ipynb ├── L6 │ ├── Interleaving.xlsx │ ├── Split Traffic.ipynb │ ├── cluster.ipynb │ ├── exp_data_cluster.csv │ ├── variance_lift.ipynb │ └── ~$Interleaving.xlsx ├── L7 │ ├── Control_Variables.ipynb │ └── exp_data_3.csv └── Readme ├── README.md └── Slides ├── Guest_Lecture_KennyXie.pdf ├── Guest_Lecture_Tencent.pdf ├── JasonMa_Tencent_PresentationHK.pdf ├── L1 - AB -Testing Overview_SH.pdf ├── L2-Statistics1_SH.pdf ├── L3-Statistics2_SH.pdf ├── L4-InternalExternalValidity_SH.pdf ├── L5-Improve_SensitivityI_SH.pdf ├── L6 - ImproveSensitivity II-SH.pdf ├── L7- ObservationalCausal-SH.pdf └── Readme.md /Class Exercises/L2 - Exercises/Multiple_Testing.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### Import Data" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [ 15 | { 16 | "name": "stdout", 17 | "output_type": "stream", 18 | "text": [ 19 | "Requirement already satisfied: termcolor in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (1.1.0)\n", 20 | "Note: you may need to restart the kernel to use updated packages.\n" 21 | ] 22 | } 23 | ], 24 | "source": [ 25 | "pip install termcolor" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 2, 31 | "metadata": {}, 32 | "outputs": [], 33 | "source": [ 34 | "from termcolor import colored, cprint\n", 35 | "import itertools\n", 36 | "import numpy as np, statsmodels.stats.api as sms\n", 37 | "import matplotlib.pyplot as plt\n", 38 | "from matplotlib.ticker import NullFormatter\n", 39 | "import pandas as pd\n", 40 | "import matplotlib.ticker as ticker\n", 41 | "from sklearn import preprocessing\n", 42 | "from numpy import random" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 3, 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "df = pd.read_csv('exp_data.csv')" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 4, 57 | "metadata": {}, 58 | "outputs": [ 59 | { 60 | "data": { 61 | "text/html": [ 62 | "
\n", 63 | "\n", 76 | "\n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | "
Uingenderagedevicehas_interest_onlineinterestss_TVShowsinterests_Travelinterests_Societyinterests_Petsinterests_Natural...interests_Fashioninterests_Techinterests_Entertainmentinterests_Healthinterests_Cartooninterests_Financeinterests_Realestateinterests_Videogamesinterests_ArtGroup
0137f94de7d47cb5bbb2bf3265558f5b8241210.7964940.04.1104830.0000000.0...0.0000000.0000000.0000002.2615620.00.0000000.0000000.00.0000000
1337d0ac1a7b86b8acfb107490cfcc37b22714138.7149420.0369.5483900.0000000.0...62.609615120.487872189.237181291.6065920.01125.9210140.0000000.00.0000000
2e7065ba7a831950c4711dd3fd3904ffe237210.0000000.06.9301920.0000000.0...0.0000000.0000001.4488555.6202500.01.6176990.0000000.01.2930021
336498084c61db712ad18e0c6def8579e133113.1795790.0455.5465893.6146260.0...0.0000000.0000000.000000158.1259690.00.0000000.0000000.00.0000002
4400dac9d50c6ca15e26d67590a758426262000.0000000.05.9149610.0000000.0...0.0000000.0000000.0000001.6783890.00.0000001.1462080.00.4925510
\n", 226 | "

5 rows × 28 columns

\n", 227 | "
" 228 | ], 229 | "text/plain": [ 230 | " Uin gender age device has_interest_online \\\n", 231 | "0 137f94de7d47cb5bbb2bf3265558f5b8 2 41 2 1 \n", 232 | "1 337d0ac1a7b86b8acfb107490cfcc37b 2 27 14 1 \n", 233 | "2 e7065ba7a831950c4711dd3fd3904ffe 2 37 2 1 \n", 234 | "3 36498084c61db712ad18e0c6def8579e 1 33 1 1 \n", 235 | "4 400dac9d50c6ca15e26d67590a758426 2 62 0 0 \n", 236 | "\n", 237 | " interestss_TVShows interests_Travel interests_Society interests_Pets \\\n", 238 | "0 0.796494 0.0 4.110483 0.000000 \n", 239 | "1 38.714942 0.0 369.548390 0.000000 \n", 240 | "2 0.000000 0.0 6.930192 0.000000 \n", 241 | "3 3.179579 0.0 455.546589 3.614626 \n", 242 | "4 0.000000 0.0 5.914961 0.000000 \n", 243 | "\n", 244 | " interests_Natural ... interests_Fashion interests_Tech \\\n", 245 | "0 0.0 ... 0.000000 0.000000 \n", 246 | "1 0.0 ... 62.609615 120.487872 \n", 247 | "2 0.0 ... 0.000000 0.000000 \n", 248 | "3 0.0 ... 0.000000 0.000000 \n", 249 | "4 0.0 ... 0.000000 0.000000 \n", 250 | "\n", 251 | " interests_Entertainment interests_Health interests_Cartoon \\\n", 252 | "0 0.000000 2.261562 0.0 \n", 253 | "1 189.237181 291.606592 0.0 \n", 254 | "2 1.448855 5.620250 0.0 \n", 255 | "3 0.000000 158.125969 0.0 \n", 256 | "4 0.000000 1.678389 0.0 \n", 257 | "\n", 258 | " interests_Finance interests_Realestate interests_Videogames \\\n", 259 | "0 0.000000 0.000000 0.0 \n", 260 | "1 1125.921014 0.000000 0.0 \n", 261 | "2 1.617699 0.000000 0.0 \n", 262 | "3 0.000000 0.000000 0.0 \n", 263 | "4 0.000000 1.146208 0.0 \n", 264 | "\n", 265 | " interests_Art Group \n", 266 | "0 0.000000 0 \n", 267 | "1 0.000000 0 \n", 268 | "2 1.293002 1 \n", 269 | "3 0.000000 2 \n", 270 | "4 0.492551 0 \n", 271 | "\n", 272 | "[5 rows x 28 columns]" 273 | ] 274 | }, 275 | "execution_count": 4, 276 | "metadata": {}, 277 | "output_type": "execute_result" 278 | } 279 | ], 280 | "source": [ 281 | "df.head()" 282 | ] 283 | }, 284 | { 285 | "cell_type": "code", 286 | "execution_count": 5, 287 | "metadata": {}, 288 | "outputs": [ 289 | { 290 | "data": { 291 | "text/plain": [ 292 | "Index(['Uin', 'gender', 'age', 'device', 'has_interest_online',\n", 293 | " 'interestss_TVShows', 'interests_Travel', 'interests_Society',\n", 294 | " 'interests_Pets', 'interests_Natural', 'interests_Cars',\n", 295 | " 'interests_Foods', 'interests_Music', 'interests_Digital',\n", 296 | " 'interests_Life', 'interests_Sports', 'interests_Reading',\n", 297 | " 'interests_Childproducts', 'interests_Fashion', 'interests_Tech',\n", 298 | " 'interests_Entertainment', 'interests_Health', 'interests_Cartoon',\n", 299 | " 'interests_Finance', 'interests_Realestate', 'interests_Videogames',\n", 300 | " 'interests_Art', 'Group'],\n", 301 | " dtype='object')" 302 | ] 303 | }, 304 | "execution_count": 5, 305 | "metadata": {}, 306 | "output_type": "execute_result" 307 | } 308 | ], 309 | "source": [ 310 | "df.columns" 311 | ] 312 | }, 313 | { 314 | "cell_type": "code", 315 | "execution_count": 6, 316 | "metadata": {}, 317 | "outputs": [ 318 | { 319 | "data": { 320 | "text/plain": [ 321 | "(10000, 28)" 322 | ] 323 | }, 324 | "execution_count": 6, 325 | "metadata": {}, 326 | "output_type": "execute_result" 327 | } 328 | ], 329 | "source": [ 330 | "df.shape" 331 | ] 332 | }, 333 | { 334 | "cell_type": "markdown", 335 | "metadata": {}, 336 | "source": [ 337 | "### Compare Age between Group 0 vs. 1" 338 | ] 339 | }, 340 | { 341 | "cell_type": "code", 342 | "execution_count": 6, 343 | "metadata": {}, 344 | "outputs": [], 345 | "source": [ 346 | "age_0 = df[df['Group'] == 0]['age']\n", 347 | "age_1 = df[df['Group'] == 1]['age']" 348 | ] 349 | }, 350 | { 351 | "cell_type": "code", 352 | "execution_count": 7, 353 | "metadata": {}, 354 | "outputs": [ 355 | { 356 | "name": "stdout", 357 | "output_type": "stream", 358 | "text": [ 359 | "(-0.2258780702654209, 0.8213030931691397, 6750.0)\n" 360 | ] 361 | } 362 | ], 363 | "source": [ 364 | "cm_age = sms.CompareMeans(sms.DescrStatsW(age_0), sms.DescrStatsW(age_1))\n", 365 | "print(cm_age.ttest_ind(alternative='two-sided', usevar='pooled'))" 366 | ] 367 | }, 368 | { 369 | "cell_type": "markdown", 370 | "metadata": {}, 371 | "source": [ 372 | "### Multiple Testing with t tests" 373 | ] 374 | }, 375 | { 376 | "cell_type": "code", 377 | "execution_count": 8, 378 | "metadata": {}, 379 | "outputs": [], 380 | "source": [ 381 | "def multi_cm(x):\n", 382 | " x0 = df[df['Group'] == 0][x]\n", 383 | " x1 = df[df['Group'] == 1][x]\n", 384 | " x2 = df[df['Group'] == 2][x]\n", 385 | " cm01 = sms.CompareMeans(sms.DescrStatsW(x0), sms.DescrStatsW(x1))\n", 386 | " cm02 = sms.CompareMeans(sms.DescrStatsW(x0), sms.DescrStatsW(x2))\n", 387 | " cm12 = sms.CompareMeans(sms.DescrStatsW(x1), sms.DescrStatsW(x2))\n", 388 | " cprint(x,'red', 'on_yellow')\n", 389 | " print(cm01.ttest_ind(alternative='two-sided', usevar='pooled')) \n", 390 | " print(cm02.ttest_ind(alternative='two-sided', usevar='pooled')) \n", 391 | " print(cm12.ttest_ind(alternative='two-sided', usevar='pooled')) " 392 | ] 393 | }, 394 | { 395 | "cell_type": "code", 396 | "execution_count": 9, 397 | "metadata": {}, 398 | "outputs": [ 399 | { 400 | "name": "stdout", 401 | "output_type": "stream", 402 | "text": [ 403 | "\u001b[43m\u001b[31mgender\u001b[0m\n", 404 | "(1.338773515763321, 0.18068950041259244, 6750.0)\n", 405 | "(0.9053695111401618, 0.36530258109856395, 6626.0)\n", 406 | "(-0.41833581824238186, 0.6757151790218867, 6618.0)\n", 407 | "\u001b[43m\u001b[31mage\u001b[0m\n", 408 | "(-0.2258780702654209, 0.8213030931691397, 6750.0)\n", 409 | "(0.9497330944934246, 0.3422825277496915, 6626.0)\n", 410 | "(1.1687121493699624, 0.24256172111339383, 6618.0)\n", 411 | "\u001b[43m\u001b[31mdevice\u001b[0m\n", 412 | "(0.10889071847020237, 0.9132924051645153, 6750.0)\n", 413 | "(0.5134641434648958, 0.6076438065780314, 6626.0)\n", 414 | "(0.41388040743233595, 0.6789751051271262, 6618.0)\n", 415 | "\u001b[43m\u001b[31mhas_interest_online\u001b[0m\n", 416 | "(-1.2845094396946113, 0.19900784380497932, 6750.0)\n", 417 | "(-1.6808531965006905, 0.0928385650413649, 6626.0)\n", 418 | "(-0.40792621277474284, 0.6833410435536809, 6618.0)\n", 419 | "\u001b[43m\u001b[31minterestss_TVShows\u001b[0m\n", 420 | "(-0.5234272052816967, 0.6006941789213049, 6750.0)\n", 421 | "(0.23350129033532566, 0.8153793808681323, 6626.0)\n", 422 | "(0.6736860794427328, 0.5005344408716177, 6618.0)\n", 423 | "\u001b[43m\u001b[31minterests_Travel\u001b[0m\n", 424 | "(0.06482087845276627, 0.9483185373224183, 6750.0)\n", 425 | "(-0.5013152001110853, 0.6161659636359065, 6626.0)\n", 426 | "(-0.5394472710402294, 0.5895964190860086, 6618.0)\n", 427 | "\u001b[43m\u001b[31minterests_Society\u001b[0m\n", 428 | "(-0.6847791455098242, 0.4935068089879331, 6750.0)\n", 429 | "(1.2351380997665236, 0.2168229057175556, 6626.0)\n", 430 | "(1.9446141119994387, 0.0518637481483557, 6618.0)\n", 431 | "\u001b[43m\u001b[31minterests_Pets\u001b[0m\n", 432 | "(0.9707276306009271, 0.3317187109438976, 6750.0)\n", 433 | "(0.31475064308464806, 0.7529608999543934, 6626.0)\n", 434 | "(-0.6519002384228214, 0.5144881696069725, 6618.0)\n", 435 | "\u001b[43m\u001b[31minterests_Natural\u001b[0m\n", 436 | "(-1.0612678410854939, 0.2886061658249812, 6750.0)\n", 437 | "(-1.8427709795172764, 0.06540709922408983, 6626.0)\n", 438 | "(-1.3663540716083276, 0.17187427850386666, 6618.0)\n", 439 | "\u001b[43m\u001b[31minterests_Cars\u001b[0m\n", 440 | "(0.6187802423559168, 0.5360820204981972, 6750.0)\n", 441 | "(-1.3893444622272728, 0.1647747146564894, 6626.0)\n", 442 | "(-1.7725664974397572, 0.07634656053156527, 6618.0)\n", 443 | "\u001b[43m\u001b[31minterests_Foods\u001b[0m\n", 444 | "(1.0184637185219496, 0.30849413623581146, 6750.0)\n", 445 | "(2.0767837483140696, 0.0378599681773576, 6626.0)\n", 446 | "(1.016626487214549, 0.3093683127187254, 6618.0)\n", 447 | "\u001b[43m\u001b[31minterests_Music\u001b[0m\n", 448 | "(-0.3262239225314997, 0.7442650563623125, 6750.0)\n", 449 | "(0.715621661229616, 0.4742502108804256, 6626.0)\n", 450 | "(0.9673603471637798, 0.33339927465462404, 6618.0)\n", 451 | "\u001b[43m\u001b[31minterests_Digital\u001b[0m\n", 452 | "(-1.7427156587400412, 0.08142883367459604, 6750.0)\n", 453 | "(-2.227313101078629, 0.025959817302320727, 6626.0)\n", 454 | "(-0.45167712090086354, 0.6515164041179344, 6618.0)\n", 455 | "\u001b[43m\u001b[31minterests_Life\u001b[0m\n", 456 | "(0.27387057610143695, 0.7841924640867347, 6750.0)\n", 457 | "(1.7190916772349614, 0.08564438610483052, 6626.0)\n", 458 | "(1.4489976215244424, 0.14738562626501506, 6618.0)\n" 459 | ] 460 | } 461 | ], 462 | "source": [ 463 | "var = df.columns\n", 464 | "for i in range(14):\n", 465 | " multi_cm(var[i+1])" 466 | ] 467 | }, 468 | { 469 | "cell_type": "markdown", 470 | "metadata": {}, 471 | "source": [ 472 | "### Multiple Testing with CIs" 473 | ] 474 | }, 475 | { 476 | "cell_type": "code", 477 | "execution_count": 10, 478 | "metadata": {}, 479 | "outputs": [], 480 | "source": [ 481 | "def multi_cm(x):\n", 482 | " x0 = df[df['Group'] == 0][x]\n", 483 | " x1 = df[df['Group'] == 1][x]\n", 484 | " x2 = df[df['Group'] == 2][x]\n", 485 | " cm01 = sms.CompareMeans(sms.DescrStatsW(x0), sms.DescrStatsW(x1))\n", 486 | " cm02 = sms.CompareMeans(sms.DescrStatsW(x0), sms.DescrStatsW(x2))\n", 487 | " cm12 = sms.CompareMeans(sms.DescrStatsW(x1), sms.DescrStatsW(x2))\n", 488 | " cprint(x,'red', 'on_yellow')\n", 489 | " print(cm01.zconfint_diff(alpha=0.05, alternative='two-sided', usevar='pooled'))\n", 490 | " print(cm02.zconfint_diff(alpha=0.05, alternative='two-sided', usevar='pooled'))\n", 491 | " print(cm12.zconfint_diff(alpha=0.05, alternative='two-sided', usevar='pooled'))" 492 | ] 493 | }, 494 | { 495 | "cell_type": "code", 496 | "execution_count": 11, 497 | "metadata": {}, 498 | "outputs": [ 499 | { 500 | "name": "stdout", 501 | "output_type": "stream", 502 | "text": [ 503 | "\u001b[43m\u001b[31mgender\u001b[0m\n", 504 | "(-0.00755761848672714, 0.04013358344703349)\n", 505 | "(-0.012993350685755072, 0.03530293765051445)\n", 506 | "(-0.029182923978018634, 0.018916545982471663)\n", 507 | "\u001b[43m\u001b[31mage\u001b[0m\n", 508 | "(-0.7642663066007759, 0.6063125348501205)\n", 509 | "(-0.3598776790635133, 1.0365304029973976)\n", 510 | "(-0.28252633538176, 1.1171328310662996)\n", 511 | "\u001b[43m\u001b[31mdevice\u001b[0m\n", 512 | "(-0.0779825235473426, 0.08715727980668687)\n", 513 | "(-0.06048222514523573, 0.1034209839991136)\n", 514 | "(-0.06306407475614212, 0.09682807735067572)\n", 515 | "\u001b[43m\u001b[31mhas_interest_online\u001b[0m\n", 516 | "(-0.03812695208476372, 0.007937504703966848)\n", 517 | "(-0.04324119211221372, 0.0033149379949782252)\n", 518 | "(-0.028259631892169006, 0.018522825155730387)\n", 519 | "\u001b[43m\u001b[31minterestss_TVShows\u001b[0m\n", 520 | "(-2.2203402873774665, 1.2843729567695452)\n", 521 | "(-1.4700744002895791, 1.867724775833508)\n", 522 | "(-1.2731471241684726, 2.606764830320323)\n", 523 | "\u001b[43m\u001b[31minterests_Travel\u001b[0m\n", 524 | "(-0.9902440329339546, 1.0579840235354163)\n", 525 | "(-1.4302725552067663, 0.8476345702936463)\n", 526 | "(-1.5066922381941543, 0.8563142626795726)\n", 527 | "\u001b[43m\u001b[31minterests_Society\u001b[0m\n", 528 | "(-18.002333285345074, 8.679974327869173)\n", 529 | "(-4.839505220035202, 21.333003608691918)\n", 530 | "(-0.10188914020129225, 25.91774648633391)\n", 531 | "\u001b[43m\u001b[31minterests_Pets\u001b[0m\n", 532 | "(-0.5937913501693202, 1.7591542447413473)\n", 533 | "(-1.1483538385538703, 1.5877437949394773)\n", 534 | "(-1.4543197197440776, 0.7283467815576574)\n", 535 | "\u001b[43m\u001b[31minterests_Natural\u001b[0m\n", 536 | "(-0.01627155560873128, 0.00484013975675153)\n", 537 | "(-0.04742375429868649, 0.0014615092369716746)\n", 538 | "(-0.042031755560590536, 0.007500926350855475)\n", 539 | "\u001b[43m\u001b[31minterests_Cars\u001b[0m\n", 540 | "(-1.2575971667146193, 2.418029186772535)\n", 541 | "(-4.837229964177681, 0.824115752653428)\n", 542 | "(-5.447022449426756, 0.2734762178445882)\n", 543 | "\u001b[43m\u001b[31minterests_Foods\u001b[0m\n", 544 | "(-1.0414254980968631, 3.2945402844391736)\n", 545 | "(0.11601246330335302, 4.008851183165975)\n", 546 | "(-0.8684068866688182, 2.740155746795835)\n", 547 | "\u001b[43m\u001b[31minterests_Music\u001b[0m\n", 548 | "(-2.809851661049954, 2.007957094322481)\n", 549 | "(-1.4951288755817855, 3.2148270481067356)\n", 550 | "(-1.2936968795040942, 3.815289618756517)\n", 551 | "\u001b[43m\u001b[31minterests_Digital\u001b[0m\n", 552 | "(-2.8099268125223746, 0.1648675971060436)\n", 553 | "(-3.342235657330044, -0.21339494186324837)\n", 554 | "(-2.430908360130023, 1.520336976353062)\n", 555 | "\u001b[43m\u001b[31minterests_Life\u001b[0m\n", 556 | "(-3.7439323420447472, 4.9601792027096145)\n", 557 | "(-0.5141038063684737, 7.852361862272196)\n", 558 | "(-1.0794157796756902, 7.201426974914545)\n" 559 | ] 560 | } 561 | ], 562 | "source": [ 563 | "for i in range(14):\n", 564 | " multi_cm(var[i+1])" 565 | ] 566 | }, 567 | { 568 | "cell_type": "code", 569 | "execution_count": null, 570 | "metadata": {}, 571 | "outputs": [], 572 | "source": [] 573 | }, 574 | { 575 | "cell_type": "code", 576 | "execution_count": null, 577 | "metadata": {}, 578 | "outputs": [], 579 | "source": [] 580 | } 581 | ], 582 | "metadata": { 583 | "kernelspec": { 584 | "display_name": "Python 3", 585 | "language": "python", 586 | "name": "python3" 587 | }, 588 | "language_info": { 589 | "codemirror_mode": { 590 | "name": "ipython", 591 | "version": 3 592 | }, 593 | "file_extension": ".py", 594 | "mimetype": "text/x-python", 595 | "name": "python", 596 | "nbconvert_exporter": "python", 597 | "pygments_lexer": "ipython3", 598 | "version": "3.8.5" 599 | } 600 | }, 601 | "nbformat": 4, 602 | "nbformat_minor": 4 603 | } 604 | -------------------------------------------------------------------------------- /Class Exercises/L2 - Exercises/potential_outcomes.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Class Exercises/L2 - Exercises/potential_outcomes.xlsx -------------------------------------------------------------------------------- /Class Exercises/L2 - Exercises/t_z_tests.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Compare Means\n", 8 | "\n", 9 | "#### A/B Test Example \n", 10 | "\n", 11 | "Considering WeChat wants to use algorithms to rank the feeds on WeChat Moments instead of showing the organic feeds chronologically.\n", 12 | "\n", 13 | "**We are testing the two approaches to rank the feeds:** \n", 14 | "\n", 15 | "Control Group: show feeds chronologically\n", 16 | "\n", 17 | "Treatment Group: Rank feeds with algorithms\n", 18 | "\n", 19 | "Metric (#days): the number of days that a user clicks any feeds on WeChat Moments during the recent 30 days. \n", 20 | "\n", 21 | "**Objective**: Find whether $\\mu_1$ is signfiicantly different from $\\mu_0$, \n", 22 | "\n", 23 | "Ho: $\\delta$ = $\\mu_1$ - $\\mu_0$ = 0 " 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 2, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "import numpy as np, statsmodels.stats.api as sms\n", 33 | "import itertools\n", 34 | "import matplotlib.pyplot as plt\n", 35 | "from matplotlib.ticker import NullFormatter\n", 36 | "import pandas as pd\n", 37 | "import matplotlib.ticker as ticker\n", 38 | "from sklearn import preprocessing\n", 39 | "import scipy.stats as st\n", 40 | "import random\n", 41 | "import pandas as pd" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "## t tests" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 3, 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "test0_s = [1,2,1,2,1,4,3,2,4,3,2,3]\n", 58 | "ctrl0_s = [2,3,2,4,3,5,4,3,5,5,4,3]" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": 4, 64 | "metadata": {}, 65 | "outputs": [ 66 | { 67 | "name": "stdout", 68 | "output_type": "stream", 69 | "text": [ 70 | "-1.25\n" 71 | ] 72 | } 73 | ], 74 | "source": [ 75 | "mean_d=np.mean(test0_s) - np.mean(ctrl0_s)\n", 76 | "print(mean_d)" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 5, 82 | "metadata": {}, 83 | "outputs": [ 84 | { 85 | "name": "stdout", 86 | "output_type": "stream", 87 | "text": [ 88 | "(-2.8393466239285283, 0.009542324891921597, 21.997899382545775)\n" 89 | ] 90 | } 91 | ], 92 | "source": [ 93 | "#//usevar='unequal' or 'pooled'\n", 94 | "\n", 95 | "cm0 = sms.CompareMeans(sms.DescrStatsW(test0_s), sms.DescrStatsW(ctrl0_s))\n", 96 | "print(cm0.ttest_ind(alternative='two-sided', usevar='unequal'))" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "metadata": {}, 102 | "source": [ 103 | "**The WeChat Experiment Example**" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": 3, 109 | "metadata": {}, 110 | "outputs": [], 111 | "source": [ 112 | "#Population Characeristics\n", 113 | "lift = 1.001\n", 114 | "ctr0=0.5\n", 115 | "ctr1=ctr0*lift\n", 116 | "m0=30*ctr0\n", 117 | "m1=30*ctr1\n", 118 | "delta_p = m1-m0\n", 119 | "se_p_0 = np.sqrt(30*ctr0*(1-ctr0))\n", 120 | "se_p_1 = np.sqrt(30*ctr0*lift*(1-ctr0*lift))" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": 59, 126 | "metadata": {}, 127 | "outputs": [ 128 | { 129 | "name": "stdout", 130 | "output_type": "stream", 131 | "text": [ 132 | "0.014999999999998792 2.7386127875258306 2.7386114182190946\n" 133 | ] 134 | } 135 | ], 136 | "source": [ 137 | "print(delta_p,se_p_0,se_p_1)" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": 9, 143 | "metadata": {}, 144 | "outputs": [], 145 | "source": [ 146 | "#DRAW SAMPLES from Treatment and Control Populations, k=1000\n", 147 | "ctrl = np.random.binomial(30, p=ctr0, size=50) * 1.0\n", 148 | "test = np.random.binomial(30, p=ctr1, size=50) * 1.0" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 10, 154 | "metadata": {}, 155 | "outputs": [ 156 | { 157 | "name": "stdout", 158 | "output_type": "stream", 159 | "text": [ 160 | "-0.620000000000001\n" 161 | ] 162 | } 163 | ], 164 | "source": [ 165 | "#Sample characteristics\n", 166 | "delta_s = np.mean(test)-np.mean(ctrl)\n", 167 | "se0 = np.std(ctrl)\n", 168 | "se1 = np.std(test)\n", 169 | "print(delta_s)" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": 11, 175 | "metadata": {}, 176 | "outputs": [ 177 | { 178 | "data": { 179 | "text/plain": [ 180 | "" 181 | ] 182 | }, 183 | "execution_count": 11, 184 | "metadata": {}, 185 | "output_type": "execute_result" 186 | }, 187 | { 188 | "data": { 189 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAU5UlEQVR4nO3df5DddX3v8eebJcmWJDUYVgxEXOw1VyDEACtUfiUBvQpIAY0dYmilUydlphriXH/kXu4o6lUzTBtjtLdOrKBWUCzyy9L2ytTkBmwKyULEhFCVmsJKgBAlkCAxCe/7x/mGhmSze3bP2T37SZ6PmZ09+/35/pzv7Gs/+znfH5GZSJLKc1irC5AkDY4BLkmFMsAlqVAGuCQVygCXpEIZ4JJUKANckgplgEt9iIiNEfG2Vtch9cYA15CLiG17fb0UEb/Z6+e5g9jeioj4QC/Tz4yIf6leR0TMj4h1EbE9Inoi4u8i4uRq/tcj4n832K6vR8RvI+L56mtdRHw+Il41gG34B0KDZoBryGXmuD1fwGPAxXtNu7GJu7oQ+Ifq9ReBq4H5wKuBKcDtwEX1bCgiDq9zn9dl5nigA/gT4PeBH0XE2PrLlgbHAFfLRMRhEbEwIh6NiC0R8d2IeHU1rz0ivlVNfzYiVkfE0RHxWeAc4MtVD/7Le23yQuAfIuKNwJ8DczLzh5m5IzNfyMwbM3NRRMwD5gIfq7bx/WqfGyPi4xHxELB9ACFOZr6YmauBPwAmUgtzIuL3IuKHVTueiYgbI2JCNe9vgeOA71d1fKya/ncR8WREbI2IlRFxUgNvsw5iBrhaaT5wKTADOAb4NfBX1bz3A68CXkctEK8CfpOZ1wD3AB+sevAfBIiIScDRwIPA+UBPZt7f204zcxlwI7Xe87jMvHiv2XOo9dInZOaugTYoM58H7qb2RwYggM9X7Tuhas+11bJ/xCv/I7muWucfgTcCrwEeqGqV9mOAq5X+DLgmM3sycwe1YJtd9Xx3Ugvu/5KZuzOzOzOf62NbFwL/lLW7s00ENg2ypqWZ+Xhm/maQ6wM8QW3Yhsz8eWbeXf0XsBlYTO0P1gFl5vWZ+fxe78mbBzKurkNH3f8iSkPg9cBtEfHSXtN2U+tJ/y213up3qiGHb1EL+50H2NaFwE3V6y3ApEHW9Pgg19vbscCvACLiNcBSaj3y8dQ6Tb8+0IoR0QZ8FngvtXH1Pe/NUcDWJtSmg4g9cLXS48AFmTlhr6/2zPxlZu7MzE9l5onAmcC7gD+u1nvFPZAjYhS1Xu3d1aR/BiZHRFcf+z7QfZQbur9yRIwD3kZtmAdqwycJTMvM3wWuoDascqD9vQ+4pNrGq4DOPZtupC4dnAxwtdJXgM9GxOsBIqIjIi6pXs+KiJOrHulz1IZUdlfrPQW8Ya/tnAM8tGeIJTN/Bvwf4NsRMTMiRlcfil4eEQsPsI2GRMSYiDiN2pkuvwZuqGaNB7YBz0bEscBH91l13zrGAzuo/RdxBPC5ZtWog48Brlb6InAn8IOIeB74V+CMat5rgVuohfcG4P9RG0bZs97siPh1RCzllacP7jEf+DK1D0WfBR4FLgO+X83/GnBidYbL7Q204WNV7b8Cvgl0A2dm5vZq/qeAU6kNf9wF3LrP+p8H/ldVx0eqbfwH8EvgYWrvidSr8Ik8Kl1EPAzMzsyHW12LNJzsgatoETEa+KbhrUORPXBJKpQ9cEkq1LCeB37UUUdlZ2fncO5SkorX3d39TGZ27Dt9WAO8s7OTNWvWDOcuJal4EfEfvU13CEWSCmWAS1KhDHBJKpQ3s5I0rHbu3ElPTw8vvvhiq0sZcdrb25k8eTKjRo2qa3kDXNKw6unpYfz48XR2dhLhPbr2yEy2bNlCT08Pxx9/fF3rOIQiaVi9+OKLTJw40fDeR0QwceLEAf1n0m+AR8T1EfF0RKzba9qrI+LuiPhZ9f3IQdYs6RBkePduoO9LPT3wrwPv3GfaQuCfM/ON1O69vHDflSRJQ6vfMfDMXBkRnftMvgSYWb3+BrAC+HgzC5N0aOhceFdTt7dx0UV9zt+yZQvnn38+AE8++SRtbW10dNQucrz//vsZPXp0n+uvWLGC0aNHc+aZZzan4AYM9kPMozNzE0BmbqoeG9Wr6gng8wCOO+64Qe5Oh6qB/HJvnPau5hfwPm/2drCZOHEia9euBeDaa69l3LhxfOQjH6l7/RUrVjBu3LgREeBD/iFmZi7LzK7M7NrzV06SRpLu7m5mzJjBaaedxjve8Q42bao9E3vp0qWceOKJTJs2jcsvv5yNGzfyla98hS984QtMnz6de+65p58tD63B9sCfiohJVe97EvB0M4uSpOGSmXzoQx/ijjvuoKOjg5tvvplrrrmG66+/nkWLFvGLX/yCMWPG8OyzzzJhwgSuuuqqAffah8pgA/xO4P3Aour7HU2rSJKG0Y4dO1i3bh1vf/vbAdi9ezeTJk0CYNq0acydO5dLL72USy+9tIVV9q7fAI+Ib1P7wPKoiOgBPkktuL8bEX8KPAa8dyiLlKShkpmcdNJJrFq1ar95d911FytXruTOO+/kM5/5DOvXr29BhQdWz1kocw4w6/wm1yJJw27MmDFs3ryZVatW8da3vpWdO3fy05/+lBNOOIHHH3+cWbNmcfbZZ3PTTTexbds2xo8fz3PPPdfqsgEvpZfUYv2d9jfUDjvsMG655Rbmz5/P1q1b2bVrFwsWLGDKlClcccUVbN26lczkwx/+MBMmTODiiy9m9uzZ3HHHHXzpS1/inHPOaVntBrikQ9a111778uuVK1fuN//ee+/db9qUKVN46KGHhrKsunkvFEkqlAEuSYUywCWpUAa4JBXKAJekQhngklQoTyOU1Fo3NfnhDnXcQfLJJ59kwYIFrF69mjFjxtDZ2cmSJUuYMmXKgHa1ZMkS5s2bxxFHHDGg9caNG8e2bdsGtE5v7IFLOqRkJpdddhkzZ87k0Ucf5eGHH+Zzn/scTz311IC3tWTJEl544YVe5+3evbvRUvtlgEs6pCxfvpxRo0Zx1VVXvTxt+vTpnH322Xz0ox9l6tSpnHzyydx8881A7f7fM2fOZPbs2bzpTW9i7ty5ZCZLly7liSeeYNasWcyaNQuo9aw/8YlPcMYZZ7Bq1SoWL17M1KlTmTp1KkuWLGl6WxxCkXRIWbduHaeddtp+02+99VbWrl3Lj3/8Y5555hne8pa3cO655wLw4IMPsn79eo455hjOOussfvSjHzF//nwWL17M8uXLOeqoowDYvn07U6dO5dOf/jTd3d3ccMMN3HfffWQmZ5xxBjNmzOCUU05pWlvsgUsStcvm58yZQ1tbG0cffTQzZsxg9erVAJx++ulMnjyZww47jOnTp7Nx48Zet9HW1sZ73vOel7d32WWXMXbsWMaNG8e73/3upj8AwgCXdEg56aST6O7u3m965oE//BwzZszLr9va2ti1a1evy7W3t9PW1tbv9prFAJf6clPU96VinHfeeezYsYOvfvWrL09bvXo1Rx55JDfffDO7d+9m8+bNrFy5ktNPP73PbY0fP57nn3++13nnnnsut99+Oy+88ALbt2/ntttua/qdCx0Dl9Raw/zg6IjgtttuY8GCBSxatIj29vaXTyPctm0bb37zm4kIrrvuOl772tfyyCOPHHBb8+bN44ILLmDSpEksX778FfNOPfVUrrzyypf/CHzgAx9o6vg3QAxHN3+Prq6uXLNmzbDtT+Vr+VPp6+XT6+u2YcMGTjjhhFaXMWL19v5ERHdmdu27rEMoklQoA1ySCmWASxp2wzl0W5KBvi8GuKRh1d7ezpYtWwzxfWQmW7Zsob29ve51PAtF0rCaPHkyPT09bN68udWljDjt7e1Mnjy57uUNcEnDatSoURx//PGtLuOg4BCKJBXKAJekQhngklQoA1ySCmWAS1KhDHBJKpQBLkmFMsAlqVAGuCQVqqEAj4gPR8T6iFgXEd+OiPov4pckNWTQAR4RxwLzga7MnAq0AZc3qzBJUt8aHUI5HPidiDgcOAJ4ovGSJEn1GHSAZ+Yvgb8AHgM2AVsz8wf7LhcR8yJiTUSs8e5jktQ8jQyhHAlcAhwPHAOMjYgr9l0uM5dlZldmdnV0dAy+UknSKzQyhPI24BeZuTkzdwK3Amc2pyxJUn8aCfDHgN+PiCMiIoDzgQ3NKUuS1J9GxsDvA24BHgB+Um1rWZPqkiT1o6En8mTmJ4FPNqkWSdIAeCWmJBXKAJekQhngklQoA1ySCmWAS1KhDHBJKpQBLkmFMsAlqVAGuCQVygCXpEIZ4JJUKANckgplgEtSoQxwSSqUAS5JhWrofuCSajoX3lX3shsXXTSElehQYg9ckgplgEtSoQxwSSqUAS5JhTLAJalQBrgkFcoAl6RCGeCSVCgDXJIKZYBLUqEMcEkqlAEuSYUywCWpUAa4JBXKAJekQhngklSohgI8IiZExC0R8UhEbIiItzarMElS3xp9Is8XgX/KzNkRMRo4ogk1SZLqMOgAj4jfBc4FrgTIzN8Cv21OWZKk/jQyhPIGYDNwQ0Q8GBF/ExFjm1SXJKkfjQT44cCpwF9n5inAdmDhvgtFxLyIWBMRazZv3tzA7iRJe2skwHuAnsy8r/r5FmqB/gqZuSwzuzKzq6Ojo4HdSZL2NugAz8wngccj4r9Wk84HHm5KVZKkfjV6FsqHgBurM1D+HfiTxkuSJNWjoQDPzLVAV3NKkSQNhFdiSlKhDHBJKpQBLkmFMsAlqVAGuCQVygCXpEIZ4JJUKANckgplgEtSoQxwSSqUAS5JhTLAJalQBrgkFcoAl6RCGeCSVKhGH+ggDUrnwrtaXULr3BT1Lfe+HNo6VDx74JJUKANckgplgEtSoQxwSSqUAS5JhTLAJalQBrgkFcoAl6RCGeCSVCgDXJIKZYBLUqEMcEkqlAEuSYUywCWpUAa4JBXKAJekQhngklSohgM8Itoi4sGI+PtmFCRJqk8zeuBXAxuasB1J0gA0FOARMRm4CPib5pQjSapXoz3wJcDHgJcOtEBEzIuINRGxZvPmzQ3uTpK0x6ADPCLeBTydmd19LZeZyzKzKzO7Ojo6Brs7SdI+GumBnwX8QURsBL4DnBcR32pKVZKkfg06wDPzf2Tm5MzsBC4HfpiZVzStMklSnzwPXJIKdXgzNpKZK4AVzdiWJKk+9sAlqVAGuCQVygCXpEIZ4JJUKANckgplgEtSoQxwSSqUAS5JhTLAJalQBrgkFcoAl6RCGeCSVCgDXJIKZYBLUqEMcEkqVFPuBy4NlY3T3tXqEupSSp06uNgDl6RCGeCSVCgDXJIKZYBLUqEMcEkqlAEuSYUywCWpUAa4JBXKAJekQhngklQoA1ySCmWAS1KhDHBJKpQBLkmFMsAlqVAGuCQVatABHhGvi4jlEbEhItZHxNXNLEyS1LdGnsizC/jvmflARIwHuiPi7sx8uEm1SZL6MOgeeGZuyswHqtfPAxuAY5tVmCSpb015JmZEdAKnAPf1Mm8eMA/guOOOa8buNNxuiroW63zo7+vepM+QrEOd7zvU/95vXHTRYKvRCNTwh5gRMQ74HrAgM5/bd35mLsvMrszs6ujoaHR3kqRKQwEeEaOohfeNmXlrc0qSJNWjkbNQAvgasCEzFzevJElSPRrpgZ8F/BFwXkSsrb4ubFJdkqR+DPpDzMy8F6j/UxZJUlN5JaYkFcoAl6RCGeCSVCgDXJIKZYBLUqEMcEkqlAEuSYUywCWpUAa4JBXKAJekQhngklQoA1ySCmWAS1KhDHBJKpQBLkmFaspDjTVydC68q67lhuKhwj6ouHXqf++z/o3W+1Dl9w1gm2oqe+CSVCgDXJIKZYBLUqEMcEkqlAEuSYUywCWpUAa4JBXKAJekQhngklQoA1ySCmWAS1KhDHBJKpQBLkmFMsAlqVAGuCQVygCXpEIZ4JJUqIYCPCLeGRH/FhE/j4iFzSpKktS/QQd4RLQBfwVcAJwIzImIE5tVmCSpb430wE8Hfp6Z/56ZvwW+A1zSnLIkSf1p5KHGxwKP7/VzD3DGvgtFxDxgXvXjtoj4t0Hu7yjgmUGuO9K0vC11Pq62Hi1vS5McLO2Avtoyt4lHfii3+Z8OjePSv9f3NrGRAO/tqO33eOrMXAYsa2A/tZ1FrMnMrka3MxLYlpHnYGkH2JaRaija0sgQSg/wur1+ngw80Vg5kqR6NRLgq4E3RsTxETEauBy4szllSZL6M+ghlMzcFREfBP4v0AZcn5nrm1bZ/hoehhlBbMvIc7C0A2zLSNX0tkTmfsPWkqQCeCWmJBXKAJekQhUR4AfTJfsRsTEifhIRayNiTavrqVdEXB8RT0fEur2mvToi7o6In1Xfj2xljfU6QFuujYhfVsdlbURc2Moa6xERr4uI5RGxISLWR8TV1fTijksfbSnxuLRHxP0R8eOqLZ+qpjf9uIz4MfDqkv2fAm+nduriamBOZj7c0sIGKSI2Al2ZWdTFCRFxLrAN+GZmTq2mXQf8KjMXVX9Yj8zMj7eyznocoC3XAtsy8y9aWdtARMQkYFJmPhAR44Fu4FLgSgo7Ln205Q8p77gEMDYzt0XEKOBe4Grg3TT5uJTQA/eS/REgM1cCv9pn8iXAN6rX36D2CzfiHaAtxcnMTZn5QPX6eWADtSukizsufbSlOFmzrfpxVPWVDMFxKSHAe7tkv8gDW0ngBxHRXd1moGRHZ+YmqP0CAq9pcT2N+mBEPFQNsYz4YYe9RUQncApwH4Ufl33aAgUel4hoi4i1wNPA3Zk5JMelhACv65L9gpyVmadSu4vjn1f/zqv1/hr4PWA6sAn4y5ZWMwARMQ74HrAgM59rdT2N6KUtRR6XzNydmdOpXaF+ekRMHYr9lBDgB9Ul+5n5RPX9aeA2akNEpXqqGrvcM4b5dIvrGbTMfKr6pXsJ+CqFHJdqjPV7wI2ZeWs1ucjj0ltbSj0ue2Tms8AK4J0MwXEpIcAPmkv2I2Js9QENETEW+G/Aur7XGtHuBN5fvX4/cEcLa2nInl+symUUcFyqD8u+BmzIzMV7zSruuByoLYUel46ImFC9/h3gbcAjDMFxGfFnoQBUpw4t4T8v2f9saysanIh4A7VeN9RuY3BTKW2JiG8DM6ndEvMp4JPA7cB3geOAx4D3ZuaI/3DwAG2ZSe3f9AQ2An+2Z7xypIqIs4F7gJ8AL1WT/ye1seOijksfbZlDecdlGrUPKduodZK/m5mfjoiJNPm4FBHgkqT9lTCEIknqhQEuSYUywCWpUAa4JBXKAJekQhngklQoA1ySCvX/AZWxONYaFOpVAAAAAElFTkSuQmCC\n", 190 | "text/plain": [ 191 | "
" 192 | ] 193 | }, 194 | "metadata": { 195 | "needs_background": "light" 196 | }, 197 | "output_type": "display_data" 198 | } 199 | ], 200 | "source": [ 201 | "bins = np.linspace(0, 30, 30)\n", 202 | "plt.hist(test, bins=bins, label='Test')\n", 203 | "plt.hist(ctrl, bins=bins, label='Control', color='orange')\n", 204 | "plt.title('Test/Ctrl Data')\n", 205 | "plt.legend()" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": 12, 211 | "metadata": {}, 212 | "outputs": [], 213 | "source": [ 214 | "cm = sms.CompareMeans(sms.DescrStatsW(test), sms.DescrStatsW(ctrl))\n", 215 | "se_s = cm.std_meandiff_separatevar" 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": 13, 221 | "metadata": {}, 222 | "outputs": [ 223 | { 224 | "name": "stdout", 225 | "output_type": "stream", 226 | "text": [ 227 | "(-1.2026452604350089, 0.23201576184314465, 97.84757101932192)\n" 228 | ] 229 | } 230 | ], 231 | "source": [ 232 | "print(cm.ttest_ind(alternative='two-sided', usevar='unequal'))" 233 | ] 234 | }, 235 | { 236 | "cell_type": "markdown", 237 | "metadata": {}, 238 | "source": [ 239 | "# z tests" 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "execution_count": 15, 245 | "metadata": {}, 246 | "outputs": [ 247 | { 248 | "name": "stdout", 249 | "output_type": "stream", 250 | "text": [ 251 | "(-1.2026452604350089, 0.22911362509585143)\n" 252 | ] 253 | } 254 | ], 255 | "source": [ 256 | "print(cm.ztest_ind(alternative='two-sided', usevar='unequal'))" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": null, 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [] 265 | }, 266 | { 267 | "cell_type": "code", 268 | "execution_count": null, 269 | "metadata": {}, 270 | "outputs": [], 271 | "source": [] 272 | } 273 | ], 274 | "metadata": { 275 | "kernelspec": { 276 | "display_name": "Python 3", 277 | "language": "python", 278 | "name": "python3" 279 | }, 280 | "language_info": { 281 | "codemirror_mode": { 282 | "name": "ipython", 283 | "version": 3 284 | }, 285 | "file_extension": ".py", 286 | "mimetype": "text/x-python", 287 | "name": "python", 288 | "nbconvert_exporter": "python", 289 | "pygments_lexer": "ipython3", 290 | "version": "3.8.5" 291 | } 292 | }, 293 | "nbformat": 4, 294 | "nbformat_minor": 4 295 | } 296 | -------------------------------------------------------------------------------- /Class Exercises/L2 - Exercises/ttest.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Class Exercises/L2 - Exercises/ttest.xlsx -------------------------------------------------------------------------------- /Class Exercises/L3/CI-Bootstrap.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Confidence Interval" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "#### A/B Test Example \n", 15 | "\n", 16 | "Considering WeChat wants to use algorithms to rank the feeds on WeChat Moments instead of showing the organic feeds chronologically.\n", 17 | "\n", 18 | "**We are testing the two approaches to rank the feeds:** \n", 19 | "\n", 20 | "Control Group: show feeds chronologically\n", 21 | "\n", 22 | "Treatment Group: Rank feeds with algorithms\n", 23 | "\n", 24 | "Metric (#days): the number of days that a user clicks any feeds on WeChat Moments during the recent 30 days. \n", 25 | "\n", 26 | "**Objective**: Find the confidence interval of $\\delta$ = $\\mu_1$ - $\\mu_0$ and decide whether it is significantly larger than 0.\n", 27 | "\n", 28 | "**CI**: CI represents how often (e.g., 95%) the CI should contain the true Treatment Effects (difference between treatment and control)." 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 10, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "import numpy as np, statsmodels.stats.api as sms\n", 38 | "import itertools\n", 39 | "import matplotlib.pyplot as plt\n", 40 | "from matplotlib.ticker import NullFormatter\n", 41 | "import pandas as pd\n", 42 | "import numpy as np\n", 43 | "import matplotlib.ticker as ticker\n", 44 | "import random\n", 45 | "from sklearn import preprocessing" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 11, 51 | "metadata": {}, 52 | "outputs": [ 53 | { 54 | "name": "stdout", 55 | "output_type": "stream", 56 | "text": [ 57 | "1.5\n" 58 | ] 59 | } 60 | ], 61 | "source": [ 62 | "#population characteristics\n", 63 | "lift = 1.1\n", 64 | "ctr0=0.5\n", 65 | "ctr1=lift*ctr0\n", 66 | "# mean = n*p (n=30)\n", 67 | "mu0=30*ctr0\n", 68 | "mu1=30*ctr1\n", 69 | "delta_p = mu1-mu0\n", 70 | "print(delta_p)" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 12, 76 | "metadata": {}, 77 | "outputs": [ 78 | { 79 | "name": "stdout", 80 | "output_type": "stream", 81 | "text": [ 82 | "1.3790000000000013\n" 83 | ] 84 | } 85 | ], 86 | "source": [ 87 | "#sample characteristics\n", 88 | "# we draw the sample\n", 89 | "ctrl = np.random.binomial(30, p=ctr0, size=1000) * 1.0\n", 90 | "test = np.random.binomial(30, p=ctr1, size=1000) * 1.0\n", 91 | "delta_s = np.mean(test)-np.mean(ctrl)\n", 92 | "print(delta_s)" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 13, 98 | "metadata": {}, 99 | "outputs": [ 100 | { 101 | "name": "stdout", 102 | "output_type": "stream", 103 | "text": [ 104 | "2.720374790355182 2.675518641310503\n" 105 | ] 106 | } 107 | ], 108 | "source": [ 109 | "se0 = np.std(ctrl)\n", 110 | "se1 = np.std(test)\n", 111 | "print(se0,se1)" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": 14, 117 | "metadata": {}, 118 | "outputs": [], 119 | "source": [ 120 | "cm = sms.CompareMeans(sms.DescrStatsW(test), sms.DescrStatsW(ctrl))" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": 15, 126 | "metadata": {}, 127 | "outputs": [ 128 | { 129 | "name": "stdout", 130 | "output_type": "stream", 131 | "text": [ 132 | "(1.1422489254698085, 1.6157510745301942)\n", 133 | "(1.1423923846126793, 1.6156076153873233)\n" 134 | ] 135 | } 136 | ], 137 | "source": [ 138 | "print(cm.tconfint_diff(alpha=0.05, alternative='two-sided', usevar='unequal'))\n", 139 | "print(cm.zconfint_diff(alpha=0.05, alternative='two-sided', usevar='unequal'))" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "### Compare CIs, k=100, 1000, 2000, 5000" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": 7, 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "def ci_sample_size(n):\n", 156 | " ctrl = np.random.binomial(30, p=ctr0, size=n) * 1.0\n", 157 | " test = np.random.binomial(30, p=ctr0*lift, size=n) * 1.0\n", 158 | " cm = sms.CompareMeans(sms.DescrStatsW(test), sms.DescrStatsW(ctrl))\n", 159 | " x,y = cm.tconfint_diff(alpha=0.05, alternative='two-sided', usevar='unequal')\n", 160 | " return (x,y)" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 8, 166 | "metadata": {}, 167 | "outputs": [], 168 | "source": [ 169 | "size = [50,1000,2000,100000]\n", 170 | "ci_lower=[]\n", 171 | "ci_upper=[]" 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": 9, 177 | "metadata": {}, 178 | "outputs": [ 179 | { 180 | "name": "stdout", 181 | "output_type": "stream", 182 | "text": [ 183 | "(0.0447351492565613, 2.155264850743438)\n", 184 | "(1.0227458222717214, 1.5092541777282786)\n", 185 | "(1.0939559051256076, 1.4320440948743887)\n", 186 | "(1.4748116305281684, 1.5226283694718326)\n" 187 | ] 188 | } 189 | ], 190 | "source": [ 191 | "for i in size:\n", 192 | " print(ci_sample_size(i))\n", 193 | " ci_lower.append(ci_sample_size(i)[0])\n", 194 | " ci_upper.append(ci_sample_size(i)[1])" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 10, 200 | "metadata": {}, 201 | "outputs": [], 202 | "source": [ 203 | "data_dict = {}\n", 204 | "data_dict['lower'] = [x for x in ci_lower]\n", 205 | "data_dict['upper'] = [x for x in ci_upper]\n", 206 | "data_dict['size']=[x for x in size]\n", 207 | "dataset = pd.DataFrame(data_dict)" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": 11, 213 | "metadata": {}, 214 | "outputs": [ 215 | { 216 | "data": { 217 | "text/plain": [ 218 | "Text(0, 0.5, 'Sample Size')" 219 | ] 220 | }, 221 | "execution_count": 11, 222 | "metadata": {}, 223 | "output_type": "execute_result" 224 | }, 225 | { 226 | "data": { 227 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEWCAYAAABbgYH9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAd90lEQVR4nO3de7xVZb3v8c9XNJO84QHJUISMLurxSkRphke2ejyVdsqgOFva2zY7d2UXu2CeMivLLrvOy167C5aJhRfSvOTWEjXSjItgIl4yMRRJtoIXRDFU/J0/nmfpXJM515oL17Pmhe/79VqvMeYzbr85WMzvesYYcwxFBGZmZv1tq2YXYGZmnckBY2ZmRThgzMysCAeMmZkV4YAxM7MiHDBmZlaEA8asIElflvSLfljP2yXd0x81mQ0UB4x1JEmHSvqjpLWSHpN0s6Q3N7uunkjaR9K1kh6X9ISkxZKOAYiImyLiDQW2OVrS7yWtk3S/pBMaWOYVOTjvlfR0Xu5cSaPy9LmSPtzftVr7ccBYx5G0I3AV8H1gF2AEcAawoZl1NeDXwBxgOLArcDLwZOFtfh24n7SfxgN3NbDMJcC7gQ8COwH7A4uBI8qUaO3KAWOd6PUAEXFhRGyMiGci4tqIuB1A0l6SbpD0qKQ1kmZJ2rlr4fwX+Wcl3Z7/Qv+ppOGSrsl/6V8naUied5SkkDRN0kOSVkk6pV5hksbnntUTkpZImpDbhwKjgXMi4tn8c3NE/CFPnyBpZR6fJOmpip8NkubmadtK+o6kFZIelvQjSdv1sK+eB1ZGxHMR8V8RsainHStpIvAPwLERcUtEPB8RayPiPyLipzXmHyrpqvx+H5N0kyR/7mwh/A9tnegvwEZJMyX9z64wqCDgG8BrgDcBewBfrprnvaQP0tcD7wKuAb4ADCX9vzm5av7DgTHAkcD0/EHcfaPSCOA/ga+RegyfAS6VNAx4FFgG/ELScZKG13tzEXFxRGwfEdvn9/BX4MI8+Zu55gOA15F6b1+qty5gIfAZSUf3ME+licDCiHiwwflPAVYCw0g9sy8Avj/VFsIBYx0nIp4EDiV9kJ0DrJZ0ZdeHdkQsi4g5EbEhIlYD3wXeUbWa70fEwxHxN+AmYEFE/CkiNgCXAQdWzX9GRDwdEUuBnwEfqFHa/wGujoirI+KFiJgDLAKOiXRTwMNJh6v+HVgl6UZJY+q9z9wTuACYGxE/liTgX4BPRcRjEbGOdAhscp3lDwE+TQrFn0g6KrePyT071VjsvwGr6tVUw3PAbsCeuZd0U/gGiFsMB4x1pIi4OyI+FBG7A/uS/tL/fwCSdpV0kaS/SXoS+AWpZ1Lp4YrxZ2q83r5q/sq/6B/I26u2J3B8Plz0hKQnSEG4W655ZUR8LCL2yvM+DZzfw9s8E9iBl3pTw4DBwOKK9f8mt9fyMeDnEfF74D3Az3PIvA24vk4QPNpVb4O+TeqZXSvpr5Km92FZa3MOGOt4EfFn4DxS0EA6PBbAfhGxI6lnUeuv9b7Yo2J8JPBQjXkeJH2g71zx86qIOKtGzQ8C/1FRczeSJpN6Se+LiOdy8xpS+O1Tsf6d8qG0WrYmnYMhIm4h9XQuJh0u/FqdZa4Dxknavc706vexLiJOiYjXkg41flqSLwbYQjhgrONIeqOkU7o+BCXtQfownp9n2QF4Cnginxf5bD9s9ouSBkvaB/gn0gd1tV8A75J0lKRBkl6ZT97vLmmIpDMkvU7SVvmk/z9X1Fz5/g4kXSF3XD7EB0BEvEA6JPg9SbvmeUd0Hfqq4ZfAyZIOy4fbVpEO0Q0Htqm1QERcR7rS7TJJB0vaWtIOkj4i6Z9r1PrO/J5EuiJuY/6xLYADxjrROuAtwAJJT5M+pO8gnXCGdMnyQcBa0kn3X/XDNn9POhR0PfCdiLi2eobcKzmWdKJ7NalH81nS/8NngVGkHsKTud4NwIdqbOtYYAjwh4orya7J0z6f65ifD/9dB9T8/kxEzAamAzOAJ0gXCnwv13SVpJF13uv7gKtJIbo21zo2b6vamNz+FDAP+EFEzK2zXusw8vk2s82n9OXC5cA2EfF8k8sxaynuwZiZWREOGDMzK8KHyMzMrAj3YMzMrIitm11Aqxg6dGiMGjWq2WWY9dlfVz8NwGuHvarJldiWaPHixWsiouaXeR0w2ahRo1i0qMf7/Jm1pEk/ngfAxf/61iZXYlsiSQ/Um+ZDZGZmVoQDxszMinDAmJlZEQ4YMzMrwgFjZmZFFAsYSedKekTSHRVtu0iaI+nePBxSMe1UScsk3VN599d8x9aledrZXQ9Byo+GvTi3L8j3hOpaZmrexr2SppZ6j2ZNsXwWXD4KLtgqDdfdC2vmv/R6+awmF2iWlOzBnAdUP4Z1OulBRmNId52dDiBpb9KzKPbJy/xA0qC8zA+BaaS7so6pWOeJwOMR8TrSHWC/mde1C3A66W6644DTazwy16w9LZ8FC6fB+geASMNnHoIXNrz0euE0h4y1hGIBExE3Ao9VNR8LzMzjM4HjKtovyo+wXU663fg4SbsBO0bEvPx0vfOrlula1yXAEbl3cxQwJz8y9nHSsysafd64WWtbchpsXN/zPBvXp/nMmmygz8EMj4hVAHm4a24fQfdHzq7MbSPyeHV7t2XybdLXkp4XXm9dm5A0TdIiSYtWr15daxaz1rJ+Rf/OZ1ZQq5zkr/W42uihfXOX6d4YMSMixkbE2GHD6j223KyFDK73DLDNnM+soIEOmIfzYS/y8JHcvpLuzzTfnfRM85V5vLq92zKStgZ2Ih2Sq7cus/a3/5kwaHDP8wwanOYza7KBDpgrga6ruqYCV1S0T85Xho0mncxfmA+jrZM0Pp9fOaFqma51vQ+4IZ+n+S1wZH7G+RDgyNxm1v5GT4FxM2DwnoDScLvXwFbbvvR63Iw0n1mTFbvZpaQLgQnAUEkrSVd2nQXMlnQisAI4HiAi7pQ0G7gLeB74aERszKs6iXRF2nbANfkH4KfAzyUtI/VcJud1PSbpq8Ateb6vRET1xQZm7Wv0lO4B8uN5sMMY+OALzavJrIZiARMRH6gz6Yg6858JbNKvj4hFwL412v9ODqga084Fzm24WDMz63etcpLfzMw6jAPGzMyKcMCYmVkRDhgzMyvCAWNmZkU4YMzMrAgHjJmZFeGAMTOzIhwwZmZWhAPGzMyKcMCYmVkRDhgzMyvCAWNmZkU4YMzMrAgHjJmZFeGAMTOzIhwwZmZWhAPGzMyKcMCYmVkRDhgzMyvCAWNmZkU4YMzMrAgHjJmZFeGAMTOzIhwwZmZWhAPGzMyKcMCYmVkRDhgzMyvCAWNmZkU4YMzMrAgHjJmZFeGAMTOzIhwwZmZWhAPGzMyKcMCYmVkRDhgzMyvCAWNmZkU4YMzMrAgHjJmZFeGAMTOzIhwwZmZWhAPGzMyKcMCYmVkRDhgzMyvCAWNmZkU4YMzMrAgHjJmZFeGAMTOzIhwwZmZWhAPGzMyKcMCYmVkRDhgzMyvCAWNmZkU4YMzMrAgHjJmZFeGAMTOzIhwwZmZWREsFjKQ9JP1O0t2S7pT0idy+i6Q5ku7NwyEVy5wqaZmkeyQdVdF+sKSledrZktSM92TWzfJZcPkouGCrNFw+qzO2ZVZDSwUM8DxwSkS8CRgPfFTS3sB04PqIGANcn1+Tp00G9gGOBn4gaVBe1w+BacCY/HP0QL4Rs00snwULp8H6B4BIw4XTynzwD+S2zOrYutkFVIqIVcCqPL5O0t3ACOBYYEKebSYwF/h8br8oIjYAyyUtA8ZJuh/YMSLmAUg6HzgOuGag3ovZJpacBhvXd2/buB4WnAj3nbP56318Uhped+pLbWvmwwsbNt3WktNg9JTN35ZZH7RaD+ZFkkYBBwILgOE5fLpCaNc82wjgwYrFVua2EXm8ur16G9MkLZK0aPXq1f3+Hsy6Wb+idnt1EPSHeuusV4NZAS3Vg+kiaXvgUuCTEfFkD6dPak2IHtq7N0TMAGYAjB07dpPpZv1q8Mh8yKq6fU+YOHfz13vfvDSceNJLbZePqrOtkZu/HbM+arkejKRtSOEyKyJ+lZsflrRbnr4b8EhuXwnsUbH47sBDuX33Gu1mzbP/mTBocPe2QYNTeztvy6yOlgqYfKXXT4G7I+K7FZOuBKbm8anAFRXtkyVtK2k06WT+wnwYbZ2k8XmdJ1QsY9Yco6fAuBmpx4LScNyMMudEBnJbZnU0dIhM0nbAyIi4p3A9hwD/CCyVdFtu+wJwFjBb0onACuB4gIi4U9Js4C7SFWgfjYiNebmTgPOA7Ugn932C35pv9JSB+5AfyG2Z1dBrwEh6F/Ad4BXAaEkHAF+JiHf3dzER8Qdqnz8BOKLOMmcCm/T7I2IRsG//VWdmZn3RyCGyLwPjgCcAIuI2YFSpgszMrDM0EjDPR8Ta4pWYmVlHaeQczB2SPggMkjQGOBn4Y9myzMys3TXSg/k46VYsG4ALgLXAJwvWZGZmHaCRHszBwJci4rSuBkkHAbcWq8rMzNpeIz2Y3wI3SBpe0faTQvWYmVmHaCRg7gG+DcyV9Lbc5lvfm5lZjxo5RBYRcZWke4CLJZ1Ljft6mZmZVWqkByOAiLgXeDtwGLBfyaLMzKz99dqDiYgDK8afBt4vybdkNTOzHtUNGEmfi4hvSTq7ziwnF6rJzMw6QE89mLvzcPFAFGJmZp2lbsBExK/zcGZXm6QhwBMR4ZP8ZmbWo7on+SV9SdIb8/i2km4A7iM9/GviQBVoZmbtqaeryCaRvgMD6SFfAoYB7wC+XrguMzNrcz0FzLMVh8KOAi6KiI0RcTcNPqjMzMy2XD0FzAZJ+0oaBhwOXFsxbXCdZczMzICeeyKfAC4hHRb7XkQsB5B0DPCnAajNzMzaWE9XkS0A3lij/Wrg6pJFmZlZ+2vkVjFmZmZ95oAxM7MiHDBmZlZErwEjabCkL0o6J78eI+md5UszM7N21kgP5mfABuCt+fVK4GvFKjIzs47QSMDsFRHfAp4DiIhn8BMtzcysF40EzLOStiM/xVLSXqQejZmZWV2N3PLldOA3wB6SZgGHAB8qWZSZmbW/Rp5oOUfSrcB40qGxT0TEmuKVmZlZW+vpiZYHVTWtysORkkZGxK3lyjIzs3bXUw/m33uYFsD/6OdazMysg/R0L7LDB7IQMzPrLL2eg5H0SuDfgENJPZebgB9FxN8L12ZmZm2skavIzgfWAd/Prz8A/Bw4vlRRZmbW/hoJmDdExP4Vr38naUmpgszMrDM08kXLP0ka3/VC0luAm8uVZGZmnaCRHsxbgBMkrcivRwJ3S1oKRETsV6w6MzNrW40EzNHFqzAzs47TyDf5H5A0BNijcn5/0dLMzHrSyGXKXyXde+w+8g0v8RctzcysF40cIns/6Zb9z5YuxszMOkcjV5HdAexcuA4zM+swjfRgvkG6VPkOKp4DExHvLlaVmZm1vUYCZibwTWAp8ELZcszMrFM0EjBrIuLs4pWYmVlHaSRgFkv6BnAl3Q+R+TJlMzOrq5GAOTAPx1e0+TJlMzPrUSNftPRzYczMrM8a6cEg6X8B+wCv7GqLiK+UKsrMzNpfr9+DkfQjYBLwcUCk58DsWbguMzNrc4180fJtEXEC8HhEnAG8lXRfMjMzs7oaCZhn8nC9pNcAzwGjy5VkZmadoJFzMFdJ2hn4NnAr6Qqyc0oWZWZm7a+Rq8i+mkcvlXQV8MqIWFu2LDMza3d1D5FJerOkV1e8PgGYDXxV0i4DUZyZmbWvns7B/Bh4FkDSYcBZwPnAWmBG+dLMzKyd9XSIbFBEPJbHJwEzIuJS0qGy24pXZmZmba2nHswgSV0BdARwQ8W0hr6gaWZmW66eAuZC4PeSriBdqnwTgKTXkQ6T9TtJ50p6JD97pqttF0lzJN2bh0Mqpp0qaZmkeyQdVdF+sKSledrZklSiXrM+WT4LLh8FF2yVhstnNbui8rbE92wvqhswEXEmcApwHnBoRETFMh8vVM95wNFVbdOB6yNiDHB9fo2kvYHJpFvYHA38QNKgvMwPgWnAmPxTvU6zgbV8FiycBusfACINF07r7A/cLfE9Wzc9HuqKiPk12v5SqpiIuFHSqKrmY4EJeXwmMBf4fG6/KCI2AMslLQPGSbof2DEi5gFIOh84DrimVN1mvVpyGmxc371t43pYcCLc9zK/Vvb4pDS87tSXt57+tmY+vLChe9vG9WlfjJ7SnJpsQDXyTf5mGx4RqwDycNfcPgJ4sGK+lbltRB6vbt+EpGmSFklatHr16n4v3OxF61fUbq/+AO4k9d5bvX1hHaedT9bXOq8SPbRv2hgxg3zJ9dixY2vOY9YvBo/Mh4qq2/eEiXNf3rrvm5eGE096eevpb5ePqvOeRw54KdYc7dCDeVjSbgB5+EhuX0n3m27uDjyU23ev0W7WPPufCYMGd28bNDi1d6ot8T1bN+0QMFcCU/P4VOCKivbJkraVNJp0Mn9hPoy2TtL4fPXYCRXLmDXH6CkwbkbqsaA0HDejs89FbInv2bppqUNkki4kndAfKmklcDrpDgKzJZ0IrCA9j4aIuFPSbOAu4HngoxGxMa/qJNIVaduRTu77BL813+gpW96H65b4nu1FLRUwEfGBOpOOqDP/mcAm/e2IWATs24+lmZlZH7XDITIzM2tDDhgzMyvCAWNmZkU4YMzMrAgHjJmZFeGAMTOzIhwwZmZWhAPGzMyKcMCYmVkRDhgzMyvCAWNmZkU4YMzMrAgHjJmZFeGAMTOzIhwwZmZWhAPGzMyKcMCYmVkRDhgzMyvCAWNmZkU4YMzMrAgHjJmZFeGAMTOzIhwwZmZWhAPGzMyKcMCYmVkRDhgzMyvCAWNmZkU4YMzMrAgHjJmZFeGAMTOzIhwwZmZWhAPGzMyKcMCYmVkRDhgzMyvCAWNmZkU4YMzMrAgHjJmZFeGAMTOzIhwwZmZWhAPGzMyKcMCYmVkRDhgzMyvCAWNmZkU4YMzMrAgHjJmZFeGAMTOzIhwwZmZWhAPGzMyKcMCYmVkRDhgzMyvCAWNmZkU4YMzMrAgHjJmZFeGAMTOzIhwwZmZWhAPGzMyKcMCYmVkRHREwku6XtFTSbZIW5bZdJM2RdG8eDimy8eWz4PJRcMFWabh8VpHNmJn1u8KfXx0RMNnhEXFARIzNr6cD10fEGOD6/Lp/LZ8FC6fB+geASMOF0xwyZtb6BuDza+t+W1PrORaYkMdnAnOBz/frFpacBhvXd2/buB4WnAj3ndOvmzKr6/FJaXjdqc2tw9rLmvnwwobubRvXp8+10VP6ZROd0oMJ4FpJiyVNy23DI2IVQB7uWr2QpGmSFklatHr16r5vdf2K2u3V/2hmZq2m3udUvc+1zdApPZhDIuIhSbsCcyT9uZGFImIGMANg7Nix0eetDh6Zu5fV7XvCxLl9Xp3ZZrlvXhpOPKm5dVh7uXxUnc+vkf22iY7owUTEQ3n4CHAZMA54WNJuAHn4SL9veP8zYdDg7m2DBqd2M7NWNgCfX20fMJJeJWmHrnHgSOAO4Epgap5tKnBFv2989BQYNyP1WFAajpvRb8cvzcyKGYDPr044RDYcuEwSpPdzQUT8RtItwGxJJwIrgOOLbH30FAeKmbWnwp9fbR8wEfFXYP8a7Y8CRwx8RWZmBh1wiMzMzFqTA8bMzIpwwJiZWREOGDMzK0IRff9+YSeStBqo8a2jHg0F1hQopz+1eo2tXh+4xv7S6jW2en3QmjXuGRHDak1wwLwMkhZV3FyzJbV6ja1eH7jG/tLqNbZ6fdAeNVbyITIzMyvCAWNmZkU4YF6eGc0uoAGtXmOr1weusb+0eo2tXh+0R40v8jkYMzMrwj0YMzMrwgFjZmZFOGD6QNIukuZIujcPh9SZ735JSyXdJmnRANR1tKR7JC2TNL3GdEk6O0+/XdJBpWvajBonSFqb99ltkr40wPWdK+kRSXfUmd4K+7C3Gpu9D/eQ9DtJd0u6U9InaszT1P3YYI3N3o+vlLRQ0pJc4xk15mn672NDIsI/Df4A3wKm5/HpwDfrzHc/MHSAahoE3Ae8FngFsATYu2qeY4BrAAHjgQUDvN8aqXECcFUT/20PAw4C7qgzvan7sMEam70PdwMOyuM7AH9pwd/FRmps9n4UsH0e3wZYAIxvpf3Y6I97MH1zLDAzj88EjmteKS8aByyLiL9GxLPARaQ6Kx0LnB/JfGDnrqd9tlCNTRURNwKP9TBLs/dhIzU2VUSsiohb8/g64G5gRNVsTd2PDdbYVHnfPJVfbpN/qq/GavrvYyMcMH0zPCJWQfpFBXatM18A10paLGla4ZpGAA9WvF7Jpv9hGpmnpEa3/9Z8WOAaSfsMTGkNa/Y+bFRL7ENJo4ADSX99V2qZ/dhDjdDk/ShpkKTbSI96nxMRLbsfe9L2Dxzrb5KuA15dY9JpfVjNIRHxkKRdgTmS/pz/+ixBNdqq/9ppZJ6SGtn+raR7Gj0l6RjgcmBM6cL6oNn7sBEtsQ8lbQ9cCnwyIp6snlxjkQHfj73U2PT9GBEbgQMk7Ux6Yu++EVF57q0l9mNv3IOpEhETI2LfGj9XAA93dUPz8JE663goDx8BLiMdIiplJbBHxevdgYc2Y56Set1+RDzZdVggIq4GtpE0dOBK7FWz92GvWmEfStqG9ME9KyJ+VWOWpu/H3mpshf1YUcsTwFzg6KpJTd+PjXDA9M2VwNQ8PhW4onoGSa+StEPXOHAkUPOqn35yCzBG0mhJrwAm5zorXQmckK88GQ+s7TrUN0B6rVHSqyUpj48j/W4+OoA19qbZ+7BXzd6Heds/Be6OiO/Wma2p+7GRGltgPw7LPRckbQdMBP5cNVvL/z6CD5H11VnAbEknAiuA4wEkvQb4SUQcAwwndWkh7d8LIuI3pQqKiOclfQz4LelqrXMj4k5JH8nTfwRcTbrqZBmwHvinUvW8jBrfB5wk6XngGWBy5MtlBoKkC0lXDw2VtBI4nXRytSX2YYM1NnUfAocA/wgszecPAL4AjKyosdn7sZEam70fdwNmShpECrfZEXFVK/2fbpRvFWNmZkX4EJmZmRXhgDEzsyIcMGZmVoQDxszMinDAmJlZEQ4Y61j5+wwXSbpP0l2Srpb0+s1c19vznW1vkzRC0iV15psraezLq3yz6jtP0vt6mWeCpLcNQC0TJF1VejvW+hww1pHyF+UuA+ZGxF4RsTfp+w7DN3OVU4DvRMQBEfG3iOjxw7xFTQD6FDCS/F0522wOGOtUhwPP5S+lARARt0XETfnbz9+WdIfSc3smwYt/ec+VdImkP0ualef9MPB+4Eu5bZTyM1kkbZd7SbdLuhjYrmt7ko6UNE/SrZJ+qXT/q67nBZ2R25dKemNu317Sz3Lb7ZLe29N66qm1fqUbO34E+FTuhb09f2P8Ukm35J9D8vJfljRD0rXA+ZIWqOKGj3kfHSxpnKQ/SvpTHr6hRi3v0EvPVfmT8l0ubMvgv06sU+0LLK4z7X8DBwD7A0OBWyR13Yz0QGAf0n2dbibduPQnkg4lPSPkkvxh3eUkYH1E7CdpP9KNElG6d9X/BSZGxNOSPg98GvhKXm5NRBwk6d+AzwAfBr5IuuXHf8/rGNLAeurptv6I+LCkHwFPRcR38vovAL4XEX+QNJJ0p4U35eUPBg6NiGckfYoUsKcr3YPvNRGxWNKOwGH5Tg0Tga8D762q4zPARyPi5hyMf++lbusgDhjbEh0KXJjvWPuwpN8DbwaeBBZGxEqAfCuRUcAfeljXYcDZABFxu6Tbc/t4YG/g5nS0jlcA8yqW67rJ4mJS4EG659Tkrhki4nFJ7+xlPfXUWn+1icDeeb0AO1b0MK6MiGfy+GxgDunWNO8HfpnbdyLd0mQM6U6+29TYxs3AdyXNAn7VtW9ty+CAsU51J+meUrXUutV5lw0V4xtp7P9IrfstifQcjw/0sp3KbajGunpbTz211l9tK+CtFUGSNpgC5+mu1xHxN0mP5h7aJOBf86SvAr+LiPfkXt3c6g1ExFmS/pN036z5kiZGRPWNG61D+RyMdaobgG0l/UtXg6Q3S3oHcCMwSemhTsNIvZCFm7mdG0kXACBpX2C/3D4fOETS6/K0wer9CrZrgY9V1DtkM9dTzzrSY4Lrbe+AHpa9CPgcsFNELM1tOwF/y+MfqrWQpL0iYmlEfBNYBLxxsyq3tuSAsY6U7377HuAflC5TvhP4MuncymXA7cASUhB9LiL+azM39UNg+3xo7HPkoIqI1aQP3QvztPn0/uH6NWCI0sUHS4DDN3M99fwaeE/XSX7gZGBsvqDgLtJFAPVcQjp8N7ui7VvANyTdTLpLdi2frHg/z5CeI29bCN9N2czMinAPxszMinDAmJlZEQ4YMzMrwgFjZmZFOGDMzKwIB4yZmRXhgDEzsyL+P8NlxasObJemAAAAAElFTkSuQmCC\n", 228 | "text/plain": [ 229 | "
" 230 | ] 231 | }, 232 | "metadata": { 233 | "needs_background": "light" 234 | }, 235 | "output_type": "display_data" 236 | } 237 | ], 238 | "source": [ 239 | "for lower,upper,y in zip(dataset['lower'],dataset['upper'],range(len(dataset))):\n", 240 | " plt.plot((lower,upper),(y,y),'ro-',color='orange')\n", 241 | "plt.axvline(x=delta_p)\n", 242 | "plt.yticks(range(len(dataset)),list(dataset['size']))\n", 243 | "plt.title('SampleSize & CIs')\n", 244 | "plt.xlabel('Confidence Intervals')\n", 245 | "plt.ylabel('Sample Size')" 246 | ] 247 | }, 248 | { 249 | "cell_type": "markdown", 250 | "metadata": {}, 251 | "source": [ 252 | "### CI and Type I Error" 253 | ] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "metadata": {}, 258 | "source": [ 259 | "CIs for testing the same variable ($\\delta$) based on different samples. Draw 25 samples from the population. Show the CIs and interpret Type I error by the results." 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": 12, 265 | "metadata": {}, 266 | "outputs": [ 267 | { 268 | "name": "stdout", 269 | "output_type": "stream", 270 | "text": [ 271 | "0.5000000000000004\n" 272 | ] 273 | } 274 | ], 275 | "source": [ 276 | "#population \n", 277 | "lift2 = 1.1\n", 278 | "std = 0.2\n", 279 | "mean = 5\n", 280 | "delta_p2=5*(lift2-1)\n", 281 | "print(delta_p2)" 282 | ] 283 | }, 284 | { 285 | "cell_type": "code", 286 | "execution_count": 20, 287 | "metadata": {}, 288 | "outputs": [], 289 | "source": [ 290 | "ci=[]\n", 291 | "for i in range(20):\n", 292 | " ctrl = np.random.normal(5,std,1000)\n", 293 | " test = np.random.normal(5*lift2,std, 1000)\n", 294 | " cm = sms.CompareMeans(sms.DescrStatsW(test), sms.DescrStatsW(ctrl))\n", 295 | " x,y = cm.tconfint_diff(alpha=0.05, alternative='two-sided', usevar='unequal')\n", 296 | " ci.append((x,y))" 297 | ] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "execution_count": 21, 302 | "metadata": {}, 303 | "outputs": [ 304 | { 305 | "name": "stdout", 306 | "output_type": "stream", 307 | "text": [ 308 | " Number lower upper\n", 309 | "0 0 0.480556 0.515585\n", 310 | "1 1 0.492531 0.527923\n", 311 | "2 2 0.487024 0.521639\n", 312 | "3 3 0.490352 0.525666\n", 313 | "4 4 0.496746 0.532386\n", 314 | "5 5 0.457623 0.492577\n", 315 | "6 6 0.482577 0.517819\n", 316 | "7 7 0.501737 0.536818\n", 317 | "8 8 0.480479 0.515723\n", 318 | "9 9 0.482137 0.517534\n", 319 | "10 10 0.483448 0.518725\n", 320 | "11 11 0.493172 0.527980\n", 321 | "12 12 0.471214 0.507093\n", 322 | "13 13 0.475970 0.512398\n", 323 | "14 14 0.465279 0.501072\n", 324 | "15 15 0.480083 0.515361\n", 325 | "16 16 0.487442 0.522472\n", 326 | "17 17 0.478996 0.514368\n", 327 | "18 18 0.491380 0.526660\n", 328 | "19 19 0.469840 0.504135\n" 329 | ] 330 | } 331 | ], 332 | "source": [ 333 | "data_dict = {}\n", 334 | "data_dict['Number'] = [x for x in range(20)]\n", 335 | "data_dict['lower'] = [x[0] for x in ci]\n", 336 | "data_dict['upper'] = [x[1] for x in ci]\n", 337 | "dataset = pd.DataFrame(data_dict) \n", 338 | "print(dataset)" 339 | ] 340 | }, 341 | { 342 | "cell_type": "code", 343 | "execution_count": 22, 344 | "metadata": {}, 345 | "outputs": [ 346 | { 347 | "data": { 348 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEICAYAAAC6fYRZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAdj0lEQVR4nO3de5RcVZ328e9DACEgggSUWwgqw/KVkYgQLyiDchGRF5QBhUEFYURdgiA6CjIDojKDtxFedcZBgjBLCEK4jCIgcRQYZ0kwQCIJiSKESyCQRLkahCT83j/O6VDddFVXn7Or6tSp57NWr66qc6rqV927d5/adZ69FRGYmVk9rdPrAszMrHPcyZuZ1Zg7eTOzGnMnb2ZWY+7kzcxqzJ28mVmNuZO3rpK0oaSfSHpC0uWSjpR0Q4v9b5T0992ssS78szNwJ29NSPo7SXMkPS1pqaTrJL0twUMfCrwC2DwiDouIiyNivwSP2xWSLpT0lTb3/aKkH3a6JrNW3Mnbi0g6GTgH+GeyDnky8G/AwQkefnvg9xGxOsFj1ZqkdXtdg/U/d/I2jKSXAV8CPhkRV0bEnyNiVUT8JCL+Id/nJZLOkfRw/nWOpJfk2/aStETSZyQty98FfCTfdiZwOvCB/B3CsZKOlvSrhuffV9KifDjnO4BG1HeMpIWSHpP0M0nbN2wLSR+XdHe+/buS1LD9o/l9n5J0l6Rd89u3lnSFpOWSFkv6VJs/qyn5cx4l6QFJKySdlm/bH/hCw2udN/TzlTQ9/7k8JOkrkibk246W9L+SviXpT8CXJT0uaeeG59xC0jOStpS0maRr8rofyy9v26TW10i6Kf+5rpD0o3Zeo/U/d/I20luADYCrWuxzGvBmYCqwCzAN+MeG7a8EXgZsAxwLfFfSZhFxBtm7gx9FxMYRMb3xQSVNAq7IH2sScA+wR8P295J1nIcAWwD/A8wYUduBwO55Xe8H3pXf9zDgi8CHgU2Ag4A/SloH+AkwL693b+AkSe9q8fpHehuwU37f0yW9NiKuH/Fad8n3vQhYDbwGeAOwH9A4bv4m4F5gS7J/tlcCRzRsfz9wU0QsI/v7/QHZu6PJwDPAd5rU+GXgBmAzYFvg2+N4fdbH3MnbSJsDK8YYTjkS+FJELIuI5cCZwIcatq/Kt6+KiGuBp8k6wbEcANwVETMjYhXZkNEjDds/BvxLRCzM6/tnYGrj0TxwdkQ8HhEPAL8k+0cEWUf6tYj4TWT+EBH3k/1D2CIivhQRz0XEvcD3gcPbqHfImRHxTETMI/tnsctoO0l6BfBu4KT8HdIy4FsjnuvhiPh2RKyOiGeASxjeyf9dfhsR8ceIuCIiVkbEU8BZwN80qXEV2T+DrSPiLxHxqyb7Wc24k7eR/ghMGmM8eGvg/obr9+e3rX2MEf8kVgIbt/HcWwMPDl2JbPa8Bxu2bw+cmw9hPA78iWw4Z5uGfRr/KTQ+73Zk7wxG2h7Yeugx88f9AtlnEe1q9pyjPdd6wNKG5/oPsqP2IQ+OuM8vgA0lvSn/ZzaV/F2WpImS/kPS/ZKeBG4GNh0a/hnhc2Q/q1slLZB0zDhen/Uxf7BjI/0a+AvwXmBmk30eJuuwFuTXJ+e3lbWUrDMGIB9P365h+4PAWRFxcYHHfhB4dZPbF0fEjgUecywjp3h9EHgWmNTindKw+0TE85IuIzuafxS4Jj9qB/gM2TukN0XEI5KmAncw4nOM/HEeAT4KkJ8l9XNJN0fEHwq9MusbPpK3YSLiCbIPR78r6b350eJ6kt4t6Wv5bjOAf8w/BJyU75/iVMGfAq+TdEj+TuJTZOP7Q74HnCrpdbD2Q8zD2nzs84HPSnqjMq/Jj4xvBZ6U9Hll5/BPkLSzpN0TvJ5HgSn5uD8RsZRsXPybkjaRtI6kV0tqNsQy5BLgA2TDZJc03P5SsnH4xyW9HDij2QNIOqzhQ9nHyP6ZrCnyoqy/uJO3F4mIfwVOJvsAdDnZEejxwNX5Ll8B5gC/Be4Ebs9vK/u8K4DDgLPJho12BP63YftVwFeBS/PhiflkY9ztPPblZGPWlwBP5a/l5RGxBvi/ZMMgi4EVZP8QXlb29QCX59//KOn2/PKHgfWBu8g625nAVmPUPhv4M9lw1nUNm84BNsxrvgW4vsXD7A7MlvQ08GPgxIhYPJ4XY/1JXjTEzKy+fCRvZlZj7uTNzGrMnbyZWY25kzczq7Gunic/adKkmDJlSjef0iypOx96AoC/3ibFyTdm7bnttttWRMQWRe7b1U5+ypQpzJkzp5tPaZbUlFN+CsCcs9/T40pskEi6f+y9RufhGjOzGnMnb2ZWY+7kzcxqzJ28mVmNuZM3M6uxMTt5SRcoW8ZtfsNtu0j6taQ7Jf1E0iadLbNPLb4Yrp4Cl6yTfV9cZIZcqw23B+uBdo7kLwT2H3Hb+cApEfHXZAsY/EPiuvrf4ovh1uNg5f1AZN9vPc5/2IPK7cF6ZMzz5CPiZklTRty8E9kqNACzgJ8B/5S2tD437zRYs3L4bWtWwuxj4Z7v96YmK+21G/xtduHne43vjitugeefHX7bmpVZO9nhyCS1mY2m6Jj8fLKFkCGb/3u7ZjtKOk7SHElzli9fXvDp+tDKB0a/feQfug2GZr/3Zu3ELJGiiddjgP8n6XSyBQiea7ZjRJwHnAew2267Dc7k9RMn52/NR96+PexzY9fLsTQW/jxLvI77d3j1lCbtYXLZksxaKnQkHxGLImK/iHgj2VJwoy2QPNh2OQsmTBx+24SJ2e02eNwerEcKdfKStsy/r0O2RNz3UhZVCzscCdPOy47cUfZ92nkefx1Ubg/WI2MO10iaAewFTJK0hGyx4I0lfTLf5UrgBx2rsJ/tcKT/iO0Fbg/WA+2cXXNEk03nJq7FzMwSc+LVzKzGiiZep0q6RdLc/PTIaZ0t08zMiiiaeP0acGZETAVOz6+bdY6nBLBm3DZaKpp4DWBovpqXAQ8nrsvsBUNTAgwliIemBAB/kDno3DbGVDQMdRLwM0nfIHs38NZkFZmNVKEpIibq+OzCeKc1sM7wdBFjKvrB6yeAT0fEdsCngenNdhzYaQ0sHU8RYc14uogxFT2SPwo4Mb98OdmslKMa2GkNLJ0KTRGxsui0BtYZni5iTEWP5B8G/ia//E7g7jTlmI3CUwJYM24bYyqaeP0ocK6kdYG/AMd1skgbcENjq/NOy96GT5yc/RF7zNXcNsZUJvH6xsS1mDXnKQGsGbeNlpx4NTOrMXfyZmY1VnRagx/lUxrMlXSfpLkdrdLKcyrQWnH7qK12TqG8EPgO8J9DN0TEB4YuS/om8ETyyiwdpwKtFbePWis6rQEAkgS8n+w0SquqCiVG+906nJxdqFPi1anRWis7Jv924NGIaHqevBOvFeDEqLXi1GitFU28DjmCbI3Xppx4rYAKJUb73fN1TLw6NVprhY/k8yDUIcCP0pVjHeFUoLXi9lFrZYZr9gEWRcSSVMVYh3gRaWvF7aPWCk1rEBHTgcMZY6jGKsSpQGvF7aO2Ck9rEBFHJ6/GzMyScuLVzKzGCiVe89tPkPQ7SQskeY1XG5tTlTYat4uOKpR4lfQO4GDg9RHxrKQtO1Oe1YZTlTYat4uOK5p4/QRwdkQ8m++zrAO1WZ3UJnX72exbnRKvveS0bccVHZP/K+DtkmZLuknS7s12dOLVAKdubXRO23Zc0cTrusBmwJuB3YHLJL0qIl6UaHXi1YD6pG7rmHjtJadtO67okfwS4MrI3Ao8D0xKV5bVjlOVNhq3i44r2slfTT7zpKS/AtYHViSqyerIqUobjdtFxxVdyPsC4IL8tMrngKNGG6oxG8apShuN20VHlVnI+4OJazEzs8SceDUzqzF38mZmNVZ0Ie8vSnqoYTHvAzpbpiXnKLm14vZRG+0cyV8I7D/K7d+KiKn517Vpy7KOGoqSr7wfiBei5P5DNnD7qJlSC3lbn6rNFAO9MADTGniqgVopMyZ/vKTf5sM5mzXbydMaVJCnGLBWPNVArRSd1uDfgS8DkX//JnDMaDt6WoMKqssUA70wCNMaeKqBWil0JB8Rj0bEmoh4Hvg+MC1tWdZRjpJbK24ftVKok5e0VcPV9wHzm+1rFeQoubXi9lErRac12EvSVLLhmvuAj3WuROsIR8mtFbeP2ig6rcH0DtRiZmaJOfFqZlZjhRfyzrd9VlJIqt9c8k782VjcRqwPFE68StoO2Beo38mzTvzZWNxGrE+USbx+C/gc8F+pi+o5J0KtqTzxOvtYp0KtLxQ9hfIg4KGImNfGvv2XeHUi1MbiVKj1iXEnXiVNBE4D9mtn/75MvDoRas0MJV4nbu9UqPWFIkfyrwZ2AOZJug/YFrhd0itTFtZTTvzZWNxGrE+M+0g+Iu4Ethy6nnf0u0VEfRbyHhpTnXda9vZ74uTsj9djrTbEbcT6RKHEa0TUPwzlxJ+NxW3E+kCZhbyHtk9JVo2ZmSXlxKuZWY25kzczq7GiC3l/OV8Vaq6kGyRt3dkybVSO1dt4uL0MpKLTGnw9Il4fEVOBa4DTE9dlY3Gs3sbD7WVgFZrWICKebLi6Edm88tZNnnqhR/p0IW8vzj2wiq7xiqSzgA8DTwDvaLHfccBxAJMnOw2YjKdesPHwNAwDq3AnHxGnAadJOhU4nmzFqNH2679pDfqBp17ojX5dyNuLcw+sFGfXXAL8bYLHsfFwrN7Gw+1lYBWdhXLHhqsHAYvSlGNt82LLNh5uLwOr6ELeB0jaCXgeuB/4eCeLtCYcq7fxcHsZSF7I28ysxpx4NTOrsaKJ169LWpSnXq+StGlHq6wTpw6tHW4nlkjRxOssYOeIeD3we+DUxHXVk1OH1g63E0uoaOL1hoartwCHJq6rnpxSrYEuJF6dTrWEUozJHwNc12xjXy7k3SlOqVo7nE61hAonXgEknQasBpq+j3TitYFTqv2vG4lXp1MtocJH8pKOAg4EjoyIwe682+XUobXD7cQSKpp43R/4PHBQRKwca3/LOXVo7XA7sYSKJl5PBV4CzJIEcEtEOPXaDqcOrR1uJ5aIE69mZjXmxKuZWY0VTbweJmmBpOcl7dbZEs3G4HSojcbtAiieeJ0PHALcnLogs3FxOtRG43axVtHE60KA/ENXs97peoq4T9d4HTRODa/V8TF5J16to5wittE4NbxWqcRrO5x4tY7qdoq4X9d4HTRODa/ls2usvzkdaqNxu1jLnbz1N6dDbTRuF2sVTbz+Cfg2sAXwU0lzI+JdnSzUrCmnQ200bhdA8cQrwFWJazEzs8Q8XGNmVmPu5M3MaqzotAYvlzRL0t359806W6ZVgmPi1orbRyUVndbgFOC/I2JH4L/z61ZnjolbK24flVVoWgPgYLIzbgAuAm4kW0TE6sqLkOc8rcGoPI1AZRUdk39FRCwFyL9v2WxHT2tQE54+wFrxNAKV5WkNrD1ehDzjaQ1G52kEKqvokfyjkrYCyL8vS1eSVZJj4taK20dlFe3kfwwclV8+CvivNOVYZTkmbq24fVRW0WkNzgYuk3Qs8ABwWCeLtIpwTNxacfuopDLTGuyduBYzM0vMiVczsxor1clLOlHS/HxR75MS1WRV4PSiteL20TcKn0IpaWfgo8A04Dngekk/jYi7UxVnPTKUXhwKPw2lF8Fjrub20WfKnCf/WuCWiFgJIOkm4H3A11IUZj3kdGsLTrw63dpfygzXzAf2lLS5pInAAcB2I3dy4rUPOd1qrTjd2lcKH8lHxEJJXwVmAU8D84DVo+znxGu/cbq1OSdenW7tM6U+eI2I6RGxa0TsSbYkoMfj68DpRWvF7aOvlD27Zsv8+2TgEGBGiqKsx5xetFbcPvpK2QnKrpC0ObAK+GREPJagJqsCpxetFbePvlGqk4+It6cqxMzM0nPi1cysxtzJm5nVWNkPXj+dT2kwX9IMSRukKsy6xPF0a8ZtoxYKd/KStgE+BewWETsDE4DDUxVmXeDFl60Zt43aKHt2zbrAhpJWAROBh8uXZF3j6QsKGJBpDTx1QW0UPpKPiIeAb5AtGrIUeCIibhi5n6c1qDBPX2DNeOqC2igzC+VmwMHADsDjwOWSPhgRP2zcz9MaVJinLxi/QZnWwFMX1EaZD173ARZHxPKIWAVcCbw1TVnWFY6nWzNuG7VRppN/AHizpImSRLYc4MI0ZVlXOJ5uzbht1EaZWShnS5oJ3E42++Qd5MMy1kccT7dm3DZqoey0BmcAZySqxczMEnPi1cysxsqEoXaSNLfh60kv5m0Dy+lQa6bHbaPMmPzvgKkAkiYADwFXpSnLrI94YWtrpgJto2zidcjewD0RMcqJtWY15+SwNVOB5HCqMfnDabIqlBOvVntODlszFUgOlz6Sl7Q+cBBw6mjbnXi12nNy2JqpQHI4xZH8u4HbI+LRBI9l1n+cDrVmKtA2UnTyR+AFvG2QOR1qzVSgbZQarpE0EdgX+Fiacsz6lNOh1kyP20bZxOtKYPNEtZiZWWJOvJqZ1VjZNV43lTRT0iJJCyW9JVVhVhFOclorbh+VV/YUynOB6yPi0PxUyolj3cH6SAXSelZhbh99oczKUJsAewJHA0TEc8BzacqySnCScxQDssZrOyqQ5rSxlRmueRWwHPiBpDsknS9po5E7OfHax5zktFYqkOa0sZUZrlkX2BU4IV9A5FzgFOCfGndy4rWPOcn5YoOyxms7KpDmtLGVOZJfAiyJiNn59Zlknb7VRQXSelZhbh99oXAnHxGPAA9K2im/aW/griRVWTVUIK1nFeb20RfKnl1zAnBxfmbNvcBHypdkleIkp7Xi9lF5ZROvc4Hd0pRiZmapOfFqZlZj7uTNzGqs7LQG90m6M1/Ie06qooZxbNqacdswG1OKNV7fERErEjzOizk2bc24bZi1JdVC3p3hWL0140i9WVvKjskHcIOk2yQdN9oOpaY1cKzemnGk3qwtZY/k94iIhyVtCcyStCgibm7codS0Bo7VWzOO1Ju1pdSRfEQ8nH9fBlwFTEtR1FqOTVszbhtmbSncyUvaSNJLhy4D+wHzUxUGODZtzbltmLWlzHDNK4CrJA09ziURcX2Sqho5Nm3NuG2YjalwJx8R9wK7JKzFzMwSc+LVzKzGSnfykibkK0Ndk6Igs65xYtZaqUn7SBGGOhFYCGyS4LHMusOJWWulRu2jVCcvaVvgPcBZwMlJKjLrhsJpai/kPRBqlKguO1xzDvA54PlmO3ghb6skp6mtlRolqgsfyUs6EFgWEbdJ2qvZfl7I2yqpaJraC3kPhholqsscye8BHCTpPuBS4J2SfpikKrNOc2LWWqlR+yizkPepEbFtREwBDgd+EREfTFaZWSc5MWut1Kh9VHuqYbNOcmLWWqlJ+0jSyUfEjcCNKR7LzMzSceLVzKzG3MmbmdVYmamGN5B0q6R5khZIOjNlYTbgahIptw5x+2hbmTH5Z4F3RsTTktYDfiXpuoi4JVFtNqhqFCm3DnD7GJcyUw0H8HR+db38y2EnK6/SC7h7WoOeq9GUA91Qakw+n4FyLrAMmBURs0fZx9Ma2Ph4ygFrpUZTDnRDqVMoI2INMFXSpmSrRO0cEfNH7ONpDWx8qryAu6c16L0aTTnQDUnOromIx8nOk98/xePZgKtRpNw6wO1jXMqcXbNFfgSPpA2BfYBFieqyQVajSLl1gNvHuJQZrtkKuEjSBLJ/FpdFhFeHsjRqEim3DnH7aFuZs2t+C7whYS1mZpaYE69mZjVWZkx+O0m/lLQwT7yemLIwqxGnE20kt4muKTMmvxr4TETcLumlwG2SZkXEXYlqszpwOtFGcpvoqjJj8kuBpfnlpyQtBLYB3MnbCyqdXi3CidfSnFjtqiRj8pKmkH0I68SrDef0qo3kxGpXlV40RNLGwBXASRHx5MjtTrwOuCqnV4tw4rU8J1a7quzcNeuRdfAXR8SVaUqyWnE60UZym+iqMmfXCJgOLIyIf01XktWK04k2kttEV5UZrtkD+BBwZz4TJcAXIuLa0lVZvTidaCO5TXRNmbNrfgUoYS1mZpaYE69mZjVW9oPXCyQtkzR/7L3NzKzbyh7JX4jnkLdecCzeWnH7WKvsylA350Eos+5xLN5acfsYpnQYyqzrejpVgqc1qDxPmzBMxz949bQGlpynSrBWPG3CMB0/kve0BpZcL6dK8LQG1edpE4bxKZTWfxyLt1bcPoYpewrlDODXwE6Slkg6Nk1ZZi04Fm+tuH0MU/bsmiNSFWI2Lo7FWytuH2t5uMbMrMbcyZuZ1VjZMfn9Jf1O0h8knZKqKOsiJwOtGbeNWig8Ji9pAvBdYF9gCfAbST/2Qt59xMlAa8ZtozbKfPA6DfhDRNwLIOlS4GC8kHf/qN0i290wIIlXp0Zro8xwzTbAgw3Xl+S3DePEa4U5OWrNODVaG2WO5EdbMORFiVYnXiusbotsd8OgJF6dGq2NMkfyS4DtGq5vCzxcrhzrKicDrRm3jdoo08n/BthR0g6S1gcOB36cpizrCicDrRm3jdoos8braknHAz8DJgAXRMSCZJVZdzgZaM24bdRC2WkNrgWuTVSLmZkl5sSrmVmNuZM3M6sxd/JmZjXmTt7MrMYU0b18kqTlwCgJi2QmASs6+PhlVLW2qtYF1a3NdY1fVWural0wvLbtI2KLIg/S1U6+0yTNiYjdel3HaKpaW1XrgurW5rrGr6q1VbUuSFebh2vMzGrMnbyZWY3VrZM/r9cFtFDV2qpaF1S3Ntc1flWtrap1QaLaajUmb2Zmw9XtSN7MzBq4kzczq7G+6eTbXTRc0u6S1kg6tOG2TSXNlLRI0kJJb+l1XZJ2kjS34etJSSelqqtMbfltn5a0QNJ8STMkbVCRuk7Ma1rQ7Z+XpL0kPdHwOzt9vK+pR7VdIGmZpPlVqUvSdpJ+mf89LpB0YoVq20DSrZLm5bWdWYW6GrZPkHSHpGvaesKIqPwX2VTG9wCvAtYH5gH/p8l+vyCbGfPQhtsvAv4+v7w+sGkV6hqx/RGywEPPf2ZkyzguBjbMr18GHF2BunYG5gMTyWZQ/TmwY7fqAvYCrin6mnpRW75tT2BXYH6qmhL8zLYCds0vvxT4fVV+ZmSr3m2cX14PmA28udd1NWw/Gbik1T6NX/1yJL920fCIeA4YWjR8pBOAK4BlQzdI2oSskU8HiIjnIuLxXtc1wt7APRGRMg1ctrZ1gQ0lrUvWqaZa9atMXa8FbomIlRGxGrgJeF+X60p9344/fkTcDPwpYT2l64qIpRFxe375KWAho6wR3aPaIiKezq+ul3+lOkOl1O9S0rbAe4Dz271Pv3TyYy4aLmkbsj/4742476uA5cAP8rc450vaqAJ1NTocmJGoptK1RcRDwDeAB4ClwBMRcUOv6yI7it9T0uaSJgIHMHwJyo7WlXtL/jb+OkmvG+d9e1FbJyWpS9IU4A1kR8yVqC0fEplLdpAxKyJS1Vb2Z3YO8Dng+XafsF86+XYWDT8H+HxErBlx+7pkb1X/PSLeAPwZSDVmWqau7AGypRMPAi5PVNPahx7ltrZqk7QZ2dHFDsDWwEaSPtjruiJiIfBVYBZwPdlb3dVdrOt2siG1XYBvA1eP475llKmtk0rXJWljsndsJ0XEk1WpLSLWRMRUsrWrp0naudd1SToQWBYRt43nCfulk29n0fDdgEsl3QccCvybpPfm913S8J94Jlmn3+u6hrwbuD0iHk1UU4ra9gEWR8TyiFgFXAm8tQJ1ERHTI2LXiNiTbAji7m7VFRFPDr2Nj2xVtPUkTWrzNfWqtk4qVZek9cg6+Isj4soq1dawz+PAjcD+FahrD+Cg/O/iUuCdkn445jOm+DCh019kR+P3kh1ZDn1Y8boW+1/I8A9e/wfYKb/8ReDrVagrv+1S4CNV+pkBbwIWkI3Fi+yD6xN6XVd+fcv8+2RgEbBZt+oCXskLAcJpZMNZGu9r6mZtDdunkP6D1zI/MwH/CZyTuu0nqG0L8pMzgA3z/uPAXtc1Yp+9aPOD11JrvHZLNFk0XNLH8+2txrsh+xDv4nxo5F7gI1WoKx9X3hf4WIp6UtUWEbMlzSR727gauINEEesEv8srJG0OrAI+GRGPdbGuQ4FPSFoNPAMcHtlfXEcXtS9ZG5JmkHUKkyQtAc6IiOm9rEvS24APAXfmY98AX4jsyLW0krVtBVwkaQLZaMdlEdHe6YodrKvoc3paAzOzGuuXMXkzMyvAnbyZWY25kzczqzF38mZmNeZO3sysxtzJm5nVmDt5M7Ma+/8F5fZxJRGo7QAAAABJRU5ErkJggg==\n", 349 | "text/plain": [ 350 | "
" 351 | ] 352 | }, 353 | "metadata": { 354 | "needs_background": "light" 355 | }, 356 | "output_type": "display_data" 357 | } 358 | ], 359 | "source": [ 360 | "for lower,upper,y in zip(dataset['lower'],dataset['upper'],range(len(dataset))):\n", 361 | " plt.plot((lower,upper),(y,y),'ro-',color='orange')\n", 362 | " plt.yticks(range(len(dataset)),list(dataset['Number']))\n", 363 | " plt.axvline(delta_p2)\n", 364 | "plt.title('Confidence Intervals')\n", 365 | "plt.savefig(\"CIMean0.png\")" 366 | ] 367 | }, 368 | { 369 | "cell_type": "markdown", 370 | "metadata": {}, 371 | "source": [ 372 | "### Bootstrap CIs" 373 | ] 374 | }, 375 | { 376 | "cell_type": "markdown", 377 | "metadata": {}, 378 | "source": [ 379 | "https://github.com/facebookarchive/bootstrapped" 380 | ] 381 | }, 382 | { 383 | "cell_type": "code", 384 | "execution_count": 16, 385 | "metadata": {}, 386 | "outputs": [ 387 | { 388 | "name": "stdout", 389 | "output_type": "stream", 390 | "text": [ 391 | "Requirement already satisfied: bootstrapped in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (0.0.2)\n", 392 | "Requirement already satisfied: matplotlib>=1.5.3 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from bootstrapped) (3.3.2)\n", 393 | "Requirement already satisfied: pandas>=0.18.1 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from bootstrapped) (1.1.3)\n", 394 | "Requirement already satisfied: numpy>=1.11.1 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from bootstrapped) (1.19.2)\n", 395 | "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from matplotlib>=1.5.3->bootstrapped) (2.4.7)\n", 396 | "Requirement already satisfied: pillow>=6.2.0 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from matplotlib>=1.5.3->bootstrapped) (8.0.1)\n", 397 | "Requirement already satisfied: cycler>=0.10 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from matplotlib>=1.5.3->bootstrapped) (0.10.0)\n", 398 | "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from matplotlib>=1.5.3->bootstrapped) (1.3.0)\n", 399 | "Requirement already satisfied: certifi>=2020.06.20 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from matplotlib>=1.5.3->bootstrapped) (2020.6.20)\n", 400 | "Requirement already satisfied: python-dateutil>=2.1 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from matplotlib>=1.5.3->bootstrapped) (2.8.1)\n", 401 | "Requirement already satisfied: pytz>=2017.2 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from pandas>=0.18.1->bootstrapped) (2020.1)\n", 402 | "Requirement already satisfied: six in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from cycler>=0.10->matplotlib>=1.5.3->bootstrapped) (1.15.0)\n", 403 | "Note: you may need to restart the kernel to use updated packages.\n" 404 | ] 405 | } 406 | ], 407 | "source": [ 408 | "pip install bootstrapped" 409 | ] 410 | }, 411 | { 412 | "cell_type": "code", 413 | "execution_count": 2, 414 | "metadata": {}, 415 | "outputs": [], 416 | "source": [ 417 | "import bootstrapped.bootstrap as bs\n", 418 | "import bootstrapped.compare_functions as bs_compare\n", 419 | "import bootstrapped.stats_functions as bs_stats" 420 | ] 421 | }, 422 | { 423 | "cell_type": "code", 424 | "execution_count": 17, 425 | "metadata": {}, 426 | "outputs": [], 427 | "source": [ 428 | "ctrl = np.array(ctrl)\n", 429 | "test = np.array(test)" 430 | ] 431 | }, 432 | { 433 | "cell_type": "markdown", 434 | "metadata": {}, 435 | "source": [ 436 | "**Compare Mean Difference: value change & percent change**" 437 | ] 438 | }, 439 | { 440 | "cell_type": "code", 441 | "execution_count": 18, 442 | "metadata": {}, 443 | "outputs": [ 444 | { 445 | "name": "stdout", 446 | "output_type": "stream", 447 | "text": [ 448 | "1.3790000000000013 (1.1380000000000035, 1.6240000000000006)\n" 449 | ] 450 | } 451 | ], 452 | "source": [ 453 | "print(bs.bootstrap_ab(\n", 454 | " test, \n", 455 | " ctrl, \n", 456 | " stat_func=bs_stats.mean,\n", 457 | " compare_func=bs_compare.difference,\n", 458 | " alpha=0.05))" 459 | ] 460 | }, 461 | { 462 | "cell_type": "code", 463 | "execution_count": 19, 464 | "metadata": {}, 465 | "outputs": [ 466 | { 467 | "name": "stdout", 468 | "output_type": "stream", 469 | "text": [ 470 | "9.14395597108946 (7.492608721343869, 10.771821298348328)\n" 471 | ] 472 | } 473 | ], 474 | "source": [ 475 | "print(bs.bootstrap_ab(\n", 476 | " test, \n", 477 | " ctrl, \n", 478 | " stat_func=bs_stats.mean,\n", 479 | " compare_func=bs_compare.percent_change,\n", 480 | " alpha=0.05))" 481 | ] 482 | }, 483 | { 484 | "cell_type": "markdown", 485 | "metadata": {}, 486 | "source": [ 487 | "**Compare Median Difference: value change & percent change**" 488 | ] 489 | }, 490 | { 491 | "cell_type": "code", 492 | "execution_count": 20, 493 | "metadata": {}, 494 | "outputs": [ 495 | { 496 | "name": "stdout", 497 | "output_type": "stream", 498 | "text": [ 499 | "2.0 (2.0, 3.0)\n" 500 | ] 501 | } 502 | ], 503 | "source": [ 504 | "print(bs.bootstrap_ab(\n", 505 | " test, \n", 506 | " ctrl, \n", 507 | " stat_func=bs_stats.median,\n", 508 | " compare_func=bs_compare.difference,\n", 509 | " alpha=0.05))" 510 | ] 511 | }, 512 | { 513 | "cell_type": "code", 514 | "execution_count": 21, 515 | "metadata": {}, 516 | "outputs": [ 517 | { 518 | "name": "stdout", 519 | "output_type": "stream", 520 | "text": [ 521 | "13.333333333333334 (13.333333333333334, 20.0)\n" 522 | ] 523 | } 524 | ], 525 | "source": [ 526 | "print(bs.bootstrap_ab(\n", 527 | " test, \n", 528 | " ctrl, \n", 529 | " stat_func=bs_stats.median,\n", 530 | " compare_func=bs_compare.percent_change,\n", 531 | " alpha=0.05))" 532 | ] 533 | }, 534 | { 535 | "cell_type": "markdown", 536 | "metadata": {}, 537 | "source": [ 538 | "**Compare Standard Errors Difference: value change & percent change**" 539 | ] 540 | }, 541 | { 542 | "cell_type": "code", 543 | "execution_count": 22, 544 | "metadata": {}, 545 | "outputs": [ 546 | { 547 | "name": "stdout", 548 | "output_type": "stream", 549 | "text": [ 550 | "-0.044856149044678606 (-0.21680099414147444, 0.12374627150013212)\n" 551 | ] 552 | } 553 | ], 554 | "source": [ 555 | "print(bs.bootstrap_ab(\n", 556 | " test, \n", 557 | " ctrl, \n", 558 | " stat_func=bs_stats.std,\n", 559 | " compare_func=bs_compare.difference,\n", 560 | " alpha=0.05))" 561 | ] 562 | }, 563 | { 564 | "cell_type": "code", 565 | "execution_count": 23, 566 | "metadata": {}, 567 | "outputs": [ 568 | { 569 | "name": "stdout", 570 | "output_type": "stream", 571 | "text": [ 572 | "-1.6488959243304129 (-8.015358620926206, 4.234015978126974)\n" 573 | ] 574 | } 575 | ], 576 | "source": [ 577 | "print(bs.bootstrap_ab(\n", 578 | " test, \n", 579 | " ctrl, \n", 580 | " stat_func=bs_stats.std,\n", 581 | " compare_func=bs_compare.percent_change,\n", 582 | " alpha=0.05))" 583 | ] 584 | }, 585 | { 586 | "cell_type": "markdown", 587 | "metadata": {}, 588 | "source": [ 589 | "**Compare SUM Difference: value change & percent change**" 590 | ] 591 | }, 592 | { 593 | "cell_type": "code", 594 | "execution_count": 24, 595 | "metadata": {}, 596 | "outputs": [ 597 | { 598 | "name": "stdout", 599 | "output_type": "stream", 600 | "text": [ 601 | "1379.0 (1142.9750000000004, 1614.0)\n" 602 | ] 603 | } 604 | ], 605 | "source": [ 606 | "print(bs.bootstrap_ab(\n", 607 | " test, \n", 608 | " ctrl, \n", 609 | " stat_func=bs_stats.sum,\n", 610 | " compare_func=bs_compare.difference,\n", 611 | " alpha=0.05))" 612 | ] 613 | }, 614 | { 615 | "cell_type": "code", 616 | "execution_count": 25, 617 | "metadata": {}, 618 | "outputs": [ 619 | { 620 | "name": "stdout", 621 | "output_type": "stream", 622 | "text": [ 623 | "9.14395597108945 (7.531300354398173, 10.795311231116987)\n" 624 | ] 625 | } 626 | ], 627 | "source": [ 628 | "print(bs.bootstrap_ab(\n", 629 | " test, \n", 630 | " ctrl, \n", 631 | " stat_func=bs_stats.sum,\n", 632 | " compare_func=bs_compare.percent_change,\n", 633 | " alpha=0.05))" 634 | ] 635 | }, 636 | { 637 | "cell_type": "code", 638 | "execution_count": null, 639 | "metadata": {}, 640 | "outputs": [], 641 | "source": [] 642 | } 643 | ], 644 | "metadata": { 645 | "kernelspec": { 646 | "display_name": "Python 3", 647 | "language": "python", 648 | "name": "python3" 649 | }, 650 | "language_info": { 651 | "codemirror_mode": { 652 | "name": "ipython", 653 | "version": 3 654 | }, 655 | "file_extension": ".py", 656 | "mimetype": "text/x-python", 657 | "name": "python", 658 | "nbconvert_exporter": "python", 659 | "pygments_lexer": "ipython3", 660 | "version": "3.8.5" 661 | } 662 | }, 663 | "nbformat": 4, 664 | "nbformat_minor": 4 665 | } 666 | -------------------------------------------------------------------------------- /Class Exercises/L3/CIMean0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Class Exercises/L3/CIMean0.png -------------------------------------------------------------------------------- /Class Exercises/L3/CIMeanSmall.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Class Exercises/L3/CIMeanSmall.png -------------------------------------------------------------------------------- /Class Exercises/L3/Power_SampleSize.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Class Exercises/L3/Power_SampleSize.png -------------------------------------------------------------------------------- /Class Exercises/L4/s4/SRM Tests.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Class Exercises/L4/s4/SRM Tests.xlsx -------------------------------------------------------------------------------- /Class Exercises/L5/Interleaving.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Class Exercises/L5/Interleaving.xlsx -------------------------------------------------------------------------------- /Class Exercises/L5/cluster.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Regression with Clustered Standard Errors" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import numpy as np, statsmodels.stats.api as sms\n", 17 | "import itertools\n", 18 | "import matplotlib.pyplot as plt\n", 19 | "from matplotlib.ticker import NullFormatter\n", 20 | "import pandas as pd\n", 21 | "import numpy as np\n", 22 | "import matplotlib.ticker as ticker\n", 23 | "import random\n", 24 | "from sklearn import preprocessing" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 2, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "infile = 'exp_data_cluster.csv' \n", 34 | "data = pd.read_csv(infile)" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 3, 40 | "metadata": {}, 41 | "outputs": [ 42 | { 43 | "data": { 44 | "text/plain": [ 45 | "(16000, 4)" 46 | ] 47 | }, 48 | "execution_count": 3, 49 | "metadata": {}, 50 | "output_type": "execute_result" 51 | } 52 | ], 53 | "source": [ 54 | "data.shape" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 4, 60 | "metadata": {}, 61 | "outputs": [ 62 | { 63 | "data": { 64 | "text/html": [ 65 | "
\n", 66 | "\n", 79 | "\n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | "
useradidexpidif_click
0920921e4fb0d22340fdf9ee02b6ae4a7d9a8310
187850197fc187415fe1785d5ee02bfd348968e00
22663670b0fa14b56d3741178196daaa92e6a1e10
33293191f1fe825014d9e9a0881233d9950bd4310
4646725b2ae85128137e449eb015f6de78add500
\n", 127 | "
" 128 | ], 129 | "text/plain": [ 130 | " user adid expid if_click\n", 131 | "0 92092 1e4fb0d22340fdf9ee02b6ae4a7d9a83 1 0\n", 132 | "1 878501 97fc187415fe1785d5ee02bfd348968e 0 0\n", 133 | "2 266367 0b0fa14b56d3741178196daaa92e6a1e 1 0\n", 134 | "3 329319 1f1fe825014d9e9a0881233d9950bd43 1 0\n", 135 | "4 64672 5b2ae85128137e449eb015f6de78add5 0 0" 136 | ] 137 | }, 138 | "execution_count": 4, 139 | "metadata": {}, 140 | "output_type": "execute_result" 141 | } 142 | ], 143 | "source": [ 144 | "data.head()" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": 6, 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "import statsmodels.formula.api as smf\n", 154 | "model = smf.ols(formula='if_click ~ expid', data=data)" 155 | ] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "metadata": {}, 160 | "source": [ 161 | "#### OLS with Clustered SE" 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": 15, 167 | "metadata": {}, 168 | "outputs": [ 169 | { 170 | "name": "stdout", 171 | "output_type": "stream", 172 | "text": [ 173 | "Result with cluster\n", 174 | " Results: Ordinary least squares\n", 175 | "==================================================================\n", 176 | "Model: OLS Adj. R-squared: 0.000 \n", 177 | "Dependent Variable: if_click AIC: 1467.5819\n", 178 | "Date: 2023-04-04 15:53 BIC: 1482.9425\n", 179 | "No. Observations: 16000 Log-Likelihood: -731.79 \n", 180 | "Df Model: 1 F-statistic: 6.710 \n", 181 | "Df Residuals: 15998 Prob (F-statistic): 0.00960 \n", 182 | "R-squared: 0.000 Scale: 0.064166 \n", 183 | "--------------------------------------------------------------------\n", 184 | " Coef. Std.Err. z P>|z| [0.025 0.975]\n", 185 | "--------------------------------------------------------------------\n", 186 | "Intercept 0.0638 0.0027 23.3379 0.0000 0.0584 0.0691\n", 187 | "expid 0.0104 0.0040 2.5904 0.0096 0.0025 0.0182\n", 188 | "------------------------------------------------------------------\n", 189 | "Omnibus: 10828.253 Durbin-Watson: 2.012 \n", 190 | "Prob(Omnibus): 0.000 Jarque-Bera (JB): 91921.719\n", 191 | "Skew: 3.401 Prob(JB): 0.000 \n", 192 | "Kurtosis: 12.572 Condition No.: 3 \n", 193 | "==================================================================\n", 194 | "\n" 195 | ] 196 | } 197 | ], 198 | "source": [ 199 | "result_cluster = model.fit(cov_type='cluster', cov_kwds = {'groups': data.user})\n", 200 | "print('Result with cluster')\n", 201 | "print(result_cluster.summary2())" 202 | ] 203 | }, 204 | { 205 | "cell_type": "markdown", 206 | "metadata": {}, 207 | "source": [ 208 | "#### OLS without Clustered SE" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": 16, 214 | "metadata": {}, 215 | "outputs": [ 216 | { 217 | "name": "stdout", 218 | "output_type": "stream", 219 | "text": [ 220 | "Result without cluster\n", 221 | " Results: Ordinary least squares\n", 222 | "==================================================================\n", 223 | "Model: OLS Adj. R-squared: 0.000 \n", 224 | "Dependent Variable: if_click AIC: 1467.5819\n", 225 | "Date: 2023-04-04 15:54 BIC: 1482.9425\n", 226 | "No. Observations: 16000 Log-Likelihood: -731.79 \n", 227 | "Df Model: 1 F-statistic: 6.710 \n", 228 | "Df Residuals: 15998 Prob (F-statistic): 0.00960 \n", 229 | "R-squared: 0.000 Scale: 0.064166 \n", 230 | "--------------------------------------------------------------------\n", 231 | " Coef. Std.Err. t P>|t| [0.025 0.975]\n", 232 | "--------------------------------------------------------------------\n", 233 | "Intercept 0.0638 0.0028 22.5098 0.0000 0.0582 0.0693\n", 234 | "expid 0.0104 0.0040 2.5904 0.0096 0.0025 0.0182\n", 235 | "------------------------------------------------------------------\n", 236 | "Omnibus: 10828.253 Durbin-Watson: 2.012 \n", 237 | "Prob(Omnibus): 0.000 Jarque-Bera (JB): 91921.719\n", 238 | "Skew: 3.401 Prob(JB): 0.000 \n", 239 | "Kurtosis: 12.572 Condition No.: 3 \n", 240 | "==================================================================\n", 241 | "\n" 242 | ] 243 | } 244 | ], 245 | "source": [ 246 | "result_ols = model.fit()\n", 247 | "print('Result without cluster')\n", 248 | "print(result_ols.summary2())" 249 | ] 250 | }, 251 | { 252 | "cell_type": "markdown", 253 | "metadata": {}, 254 | "source": [ 255 | "#### Compare Means with t tests" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": 12, 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [ 264 | "y0 = data[data['expid'] == 0]['if_click']" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": 13, 270 | "metadata": {}, 271 | "outputs": [ 272 | { 273 | "name": "stdout", 274 | "output_type": "stream", 275 | "text": [ 276 | "0.010374999999999995\n" 277 | ] 278 | } 279 | ], 280 | "source": [ 281 | "y1 = data[data['expid'] == 1]['if_click']\n", 282 | "mean_d = np.mean(y1)-np.mean(y0)\n", 283 | "print(mean_d)" 284 | ] 285 | }, 286 | { 287 | "cell_type": "code", 288 | "execution_count": 14, 289 | "metadata": {}, 290 | "outputs": [ 291 | { 292 | "name": "stdout", 293 | "output_type": "stream", 294 | "text": [ 295 | "(-2.590388082735368, 0.009595515482880781, 15920.640661495236)\n" 296 | ] 297 | } 298 | ], 299 | "source": [ 300 | "cm = sms.CompareMeans(sms.DescrStatsW(y0), sms.DescrStatsW(y1))\n", 301 | "print(cm.ttest_ind(alternative='two-sided', usevar='unequal'))" 302 | ] 303 | }, 304 | { 305 | "cell_type": "code", 306 | "execution_count": null, 307 | "metadata": {}, 308 | "outputs": [], 309 | "source": [] 310 | }, 311 | { 312 | "cell_type": "code", 313 | "execution_count": null, 314 | "metadata": {}, 315 | "outputs": [], 316 | "source": [] 317 | } 318 | ], 319 | "metadata": { 320 | "kernelspec": { 321 | "display_name": "Python 3", 322 | "language": "python", 323 | "name": "python3" 324 | }, 325 | "language_info": { 326 | "codemirror_mode": { 327 | "name": "ipython", 328 | "version": 3 329 | }, 330 | "file_extension": ".py", 331 | "mimetype": "text/x-python", 332 | "name": "python", 333 | "nbconvert_exporter": "python", 334 | "pygments_lexer": "ipython3", 335 | "version": "3.8.5" 336 | } 337 | }, 338 | "nbformat": 4, 339 | "nbformat_minor": 4 340 | } 341 | -------------------------------------------------------------------------------- /Class Exercises/L5/variance_lift.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np, statsmodels.stats.api as sms\n", 10 | "import itertools\n", 11 | "import matplotlib.pyplot as plt\n", 12 | "from matplotlib.ticker import NullFormatter\n", 13 | "import pandas as pd\n", 14 | "import numpy as np\n", 15 | "import matplotlib.ticker as ticker\n", 16 | "import random\n", 17 | "from sklearn import preprocessing" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 4, 23 | "metadata": {}, 24 | "outputs": [ 25 | { 26 | "name": "stdout", 27 | "output_type": "stream", 28 | "text": [ 29 | "1.5000000000000013\n" 30 | ] 31 | } 32 | ], 33 | "source": [ 34 | "#Population Characteristics\n", 35 | "lift = 1.1\n", 36 | "ctr0=0.5\n", 37 | "n0=1000\n", 38 | "n1=1000\n", 39 | "ctrl = np.random.binomial(30, p=ctr0, size=n0) * 1.0\n", 40 | "test = np.random.binomial(30, p=ctr0*lift, size=n1) * 1.0\n", 41 | "delta_p = 30*ctr0*(lift-1)\n", 42 | "print(delta_p)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 5, 48 | "metadata": {}, 49 | "outputs": [ 50 | { 51 | "name": "stdout", 52 | "output_type": "stream", 53 | "text": [ 54 | "1.0918612408272181\n" 55 | ] 56 | } 57 | ], 58 | "source": [ 59 | "# Y1 bar and Y0 bar\n", 60 | "m1=np.mean(test)\n", 61 | "m0=np.mean(ctrl) \n", 62 | "lift=m1/m0\n", 63 | "# variance of Y1 and Yo\n", 64 | "var0 = np.var(ctrl,ddof=1)\n", 65 | "var1 = np.var(test,ddof=1)\n", 66 | "print(lift)" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 6, 72 | "metadata": {}, 73 | "outputs": [ 74 | { 75 | "name": "stdout", 76 | "output_type": "stream", 77 | "text": [ 78 | "0.007885785785785786 0.007203514514514514\n" 79 | ] 80 | } 81 | ], 82 | "source": [ 83 | "# variance of Y1 bar and Y0 bar\n", 84 | "var_m0=var0/n0\n", 85 | "var_m1=var1/n1\n", 86 | "print(var_m0, var_m1)" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 7, 92 | "metadata": {}, 93 | "outputs": [], 94 | "source": [ 95 | "var_lift = (1/m0**2)*var_m1+(m1**2/m0**4)*var_m0" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": 8, 101 | "metadata": {}, 102 | "outputs": [], 103 | "source": [ 104 | "se_lift=np.sqrt(var_lift)" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 9, 110 | "metadata": {}, 111 | "outputs": [ 112 | { 113 | "name": "stdout", 114 | "output_type": "stream", 115 | "text": [ 116 | "0.008596330540126679\n" 117 | ] 118 | } 119 | ], 120 | "source": [ 121 | "print(se_lift)" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": 10, 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [ 130 | "ci = (lift-1.96*se_lift,lift+1.96*se_lift)" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 11, 136 | "metadata": {}, 137 | "outputs": [ 138 | { 139 | "name": "stdout", 140 | "output_type": "stream", 141 | "text": [ 142 | "(1.0750124329685697, 1.1087100486858665)\n" 143 | ] 144 | } 145 | ], 146 | "source": [ 147 | "print(ci)" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 10, 153 | "metadata": {}, 154 | "outputs": [ 155 | { 156 | "name": "stdout", 157 | "output_type": "stream", 158 | "text": [ 159 | "Requirement already satisfied: bootstrapped in /Users/shanh18/opt/anaconda3/lib/python3.7/site-packages (0.0.2)\n", 160 | "Requirement already satisfied: numpy>=1.11.1 in /Users/shanh18/opt/anaconda3/lib/python3.7/site-packages (from bootstrapped) (1.18.1)\n", 161 | "Requirement already satisfied: matplotlib>=1.5.3 in /Users/shanh18/opt/anaconda3/lib/python3.7/site-packages (from bootstrapped) (3.1.3)\n", 162 | "Requirement already satisfied: pandas>=0.18.1 in /Users/shanh18/opt/anaconda3/lib/python3.7/site-packages (from bootstrapped) (1.0.1)\n", 163 | "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/shanh18/opt/anaconda3/lib/python3.7/site-packages (from matplotlib>=1.5.3->bootstrapped) (1.1.0)\n", 164 | "Requirement already satisfied: cycler>=0.10 in /Users/shanh18/opt/anaconda3/lib/python3.7/site-packages (from matplotlib>=1.5.3->bootstrapped) (0.10.0)\n", 165 | "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /Users/shanh18/opt/anaconda3/lib/python3.7/site-packages (from matplotlib>=1.5.3->bootstrapped) (2.4.6)\n", 166 | "Requirement already satisfied: python-dateutil>=2.1 in /Users/shanh18/opt/anaconda3/lib/python3.7/site-packages (from matplotlib>=1.5.3->bootstrapped) (2.8.1)\n", 167 | "Requirement already satisfied: pytz>=2017.2 in /Users/shanh18/opt/anaconda3/lib/python3.7/site-packages (from pandas>=0.18.1->bootstrapped) (2019.3)\n", 168 | "Requirement already satisfied: setuptools in /Users/shanh18/opt/anaconda3/lib/python3.7/site-packages (from kiwisolver>=1.0.1->matplotlib>=1.5.3->bootstrapped) (46.0.0.post20200309)\n", 169 | "Requirement already satisfied: six in /Users/shanh18/opt/anaconda3/lib/python3.7/site-packages (from cycler>=0.10->matplotlib>=1.5.3->bootstrapped) (1.14.0)\n", 170 | "Note: you may need to restart the kernel to use updated packages.\n" 171 | ] 172 | } 173 | ], 174 | "source": [ 175 | "pip install bootstrapped" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": 15, 181 | "metadata": {}, 182 | "outputs": [], 183 | "source": [ 184 | "import bootstrapped.bootstrap as bs\n", 185 | "import bootstrapped.compare_functions as bs_compare\n", 186 | "import bootstrapped.stats_functions as bs_stats" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 16, 192 | "metadata": {}, 193 | "outputs": [], 194 | "source": [ 195 | "ctrl = np.array(ctrl)\n", 196 | "test = np.array(test)" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": 17, 202 | "metadata": {}, 203 | "outputs": [ 204 | { 205 | "name": "stdout", 206 | "output_type": "stream", 207 | "text": [ 208 | "9.18612408272182 (7.508417171051148, 10.811030595629683)\n" 209 | ] 210 | } 211 | ], 212 | "source": [ 213 | "print(bs.bootstrap_ab(\n", 214 | " test, \n", 215 | " ctrl, \n", 216 | " stat_func=bs_stats.mean,\n", 217 | " compare_func=bs_compare.percent_change,\n", 218 | " alpha=0.05))" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": null, 224 | "metadata": {}, 225 | "outputs": [], 226 | "source": [] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": null, 231 | "metadata": {}, 232 | "outputs": [], 233 | "source": [] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": null, 238 | "metadata": {}, 239 | "outputs": [], 240 | "source": [] 241 | } 242 | ], 243 | "metadata": { 244 | "kernelspec": { 245 | "display_name": "Python 3", 246 | "language": "python", 247 | "name": "python3" 248 | }, 249 | "language_info": { 250 | "codemirror_mode": { 251 | "name": "ipython", 252 | "version": 3 253 | }, 254 | "file_extension": ".py", 255 | "mimetype": "text/x-python", 256 | "name": "python", 257 | "nbconvert_exporter": "python", 258 | "pygments_lexer": "ipython3", 259 | "version": "3.8.5" 260 | } 261 | }, 262 | "nbformat": 4, 263 | "nbformat_minor": 4 264 | } 265 | -------------------------------------------------------------------------------- /Class Exercises/L6/Interleaving.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Class Exercises/L6/Interleaving.xlsx -------------------------------------------------------------------------------- /Class Exercises/L6/Split Traffic.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np, statsmodels.stats.api as sms\n", 10 | "import matplotlib.pyplot as plt\n", 11 | "import pandas as pd" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 15, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "lift = 1.01\n", 21 | "p0 = 0.5\n", 22 | "#Given a fixed size of sample\n", 23 | "totalsamplesize=20000" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 16, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "#population mean difference = p1-p0=p0*(lift-1)\n", 33 | "delta=p0*(lift-1)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 17, 39 | "metadata": {}, 40 | "outputs": [ 41 | { 42 | "name": "stdout", 43 | "output_type": "stream", 44 | "text": [ 45 | "0.0050000000000000044 0.5\n" 46 | ] 47 | } 48 | ], 49 | "source": [ 50 | "#p(1-p)\n", 51 | "variance=p0*(1-p0)\n", 52 | "std=np.sqrt(variance)\n", 53 | "print(delta,std)" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "**Using Simulation**" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 6, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "power=[]\n", 70 | "n=100\n", 71 | "m=500\n", 72 | "for s in range(500):\n", 73 | " k =2500+25*(s+1)-100\n", 74 | " ci=[]\n", 75 | " for i in range(n):\n", 76 | " ctrl = np.random.binomial(1, p0, totalsamplesize-k)\n", 77 | " test = np.random.binomial(1, p0*lift, k)\n", 78 | " cm = sms.CompareMeans(sms.DescrStatsW(test), sms.DescrStatsW(ctrl))\n", 79 | " a,b = cm.tconfint_diff(alpha=0.05, alternative='two-sided', usevar='unequal')\n", 80 | " ci.append((a,b))\n", 81 | " t2=sum((x[0]<=0 and x[1]>=0) for x in ci)/n\n", 82 | " pw=1-t2\n", 83 | " # k1/k0\n", 84 | " r=(15000-k)/k\n", 85 | " power.append((r,pw))\n", 86 | " " 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 7, 92 | "metadata": {}, 93 | "outputs": [ 94 | { 95 | "data": { 96 | "text/plain": [ 97 | "Text(0, 0.5, 'Power')" 98 | ] 99 | }, 100 | "execution_count": 7, 101 | "metadata": {}, 102 | "output_type": "execute_result" 103 | }, 104 | { 105 | "data": { 106 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABBpElEQVR4nO3deZhcZZX48e+p6up9T3cnnXT2vRMSIknYBEF2QUBlQFwRFR1BnXFlHNBRQVHH0fkNKDKKCwOoKKKy72AIEAIhhOydvbP0vu/d9f7+uEvfqq6url6qq9N1Ps/DQy23br2dTu65513OK8YYlFJKJS9fohuglFIqsTQQKKVUktNAoJRSSU4DgVJKJTkNBEopleQ0ECilVJLTQKBUGBGZKiIvikiLiPxYLL8WkQYR2SAiZ4jIznFu0zUiss7zvFVE5o1nG9TklZLoBig1FkSk1fM0E+gC+uznnzHG3DuM010H1AK5xhgjImcA5wFlxpg2+5jFI2hjPvBfwHuALOAo8CtjzA+Gey5jTLbnvL8BKo0xNw33PEqBBgI1SYRdGPcDnzLGPB1+nIikGGN6hzjdbGCb6V9tORvY7wkCI/UTrACwFGgCFgHLR3lOpUZNu4bUpCYiZ4lIpYh8XUSOAb8WkQIReVhEauzunodFpMw+/jfAx4Gv2d0vnwF+CZxqP/+2c07Pd8wUkQft89WJyO2DNGcNcJ8xpsEYEzTG7DDG/MlzHiMiXxCRvSJSKyI/EpGI/0btYxeIyHXAhz3t/ftY/Lmp5KIZgUoG04BCrDt7H1bX0a+BKwE/cDdwO3C5MeYaEQFPV4uIdGFlGO+0n5/lnFhE/MDDwLPAR7G6o1YP0o5XgFtFpABYZ4zZHeGY99mfzwaeBnZiBaKIjDF3ichpaNeQGgXNCFQyCALfMsZ0GWM6jDF1xpg/G2PajTEtwK3Au0Z47rXAdOCrxpg2Y0ynMWbdIMd+HrgXuAHYJiIVInJR2DE/MMbUG2MOAj8Frh5hu5SKmQYClQxqjDGdzhMRyRSRX4jIARFpBl4E8u27++GaCRyIYdwBOwh9zxhzEjAF+CPwgIgUeg475Hl8ACvIKBVXGghUMggvsftlrFk/JxtjcoEz7ddlBOc+BMwSkWF1sxpjmoHvYQ0ez/W8NdPzeBZwJJbTDee7lQqngUAloxygA2i078a/NYpzbcCaBnqbiGSJSLqInB7pQBG5WUTWiEiqiKQDXwQascYBHF+1B7Nn2u//IYY2VAG6pkCNmAYClYx+CmRgrRV4BXh8pCcyxvQB7wUWAAeBSuCqwQ7HGqSuxbrTPw+42BjjXQPxV+B14E3gEeBXMTTjV0C5iDSKyEPD/ylUshPdmEapiUFEDLDQGFOR6Lao5KIZgVJKJTkNBEopleS0a0gppZKcZgRKKZXkjrsSE0VFRWbOnDmJboZSSh1XXn/99VpjTHGk9467QDBnzhw2btyY6GYopdRxRUQODPaedg0ppVSS00CglFJJTgOBUkolOQ0ESimV5DQQKKVUktNAoJRSSU4DgVJKJTkNBJNIVXMnT22rSnQzlFLHGQ0Ek8hVv3iZT/9uI31BrR+llIqdBoIEaO/upbVryC1uh9TU0UNXb5/7fH9dOwAdPX0RjzfGUN3SGfG9aGpaukbWQKXUcUEDQQKc9N2nWf6tJ0Z9npXffpIP3vXKgNc7uiMHgv975QBrb32GiuqWmL/jkbeOsubWp9mwr37E7VRKTWwaCBJgsDv2kdh0sHHg+QcJBI9uOQbAsabY7/A3HrACwFuVA79HKTU5aCCYwOrbull9y1NsqWwCYEtlE+/47lPUt3UPOPag3S0E0N5jdTtdevs6/vR6pft6Y0cPAP/8f6/zoyd2cN3vhi7el+q3/or0ho07NLX3cOr3n+HNQ43D+6GUUhOOBoIJ7NW9ddS2dnPHc9YWtj97voL6tm5e2Vs34Ni/vnnYfdzR3UdnTx9vVTbxlQc2u683tlsBpKWrlzue28OTMcwwCtiBoKc3GPL66wfrOdrUyU+e2jX8H0wpNaFoIIiTNw428LG7N7Cvtm3E50gLWL+eTntA+LG3ra4dn0jIcS/uqmGzp+vm2t+8xst2sAj4+49tbO+J+D3Hmjq5+aG3eeNgg/tadUsnP3u+ghT78z19oYEg1e+32ubp5vrjxkPsrop9/EEpNTFoIIiT9/9sPS/uquHyO14a8TnSU6yLbVdPEO+Woj4h5PnH7t7A09ur3ecN7T184tevAZCbHnBfH2xs4omtx7jnlQPc/myF+9p9rx7kh4/vpNqeMdTdF9o11N1nnavLkyl87U9vcd5PXhzeD6mUSjgNBHHWZPfLH27soCFC334kG/bVU9/WTWqK9et542DDgLUBz++qielcOekpHKpv54VBjj/a1MG2I80AtHb2T2l9wx6E7rMDQG9YRtDaZQWCzgjB5WBd+5CZQX1bN4cbOwCobu6kunngtNbmzp6QsQ+lVHwcdzuUHa9Ov+1ZslL9bP3OhVGPO1DXxpW/eJkzFxXzhXcvAKy77gc39Y8BvLK3nrtf2hfT9+ZmBDjjh88N+v6p33/WfVxlrzEIBg2b7G4iZ5A4vGuozV4H4WQE3gzlzB9Z37fp5vMoyEqN+L3v/vHzNLb3sP+2i1n7vWcA2H/bxSHHXH77S+ytbRvwulJqbGlGMA6C9sW0bZBpnV4t9l35y3tqQ7KAncf677CbOyP39YOVAXhNGeRCHElVcyfGGPbUtLrtcBashXcNOYHAmaoaPqsIoK6t2/3ZwznjFd4AEm7vKMZXlFKx00AwRu54roI5Nz4S8b01tz4d83n678ANu6tb3dd/ta4/A/BOCQ13yrwpIc97+mIvN9HZE6S5ozdk0Pjht47a5wnvGrICgTOQ3Rvhe879rxdYfPNjzLnxEdZX1Eb8zrn/9mjM7VNKxYcGgjHyoyd2Alb5iHB1MY4NQGhf/PM7YxsH8Lr54vKQ594SFOcunTro52bkZwBQ397NGwcaB7w/WNdQe5eTEQQHfMb6nBUgfvHi3iFaPnAcwjFYVqGUGhsaCMZIrt0lU9ca/aJ/7n+94D5usrtHalu7uP6+N3jzUGPIHfzT24dfSTQrzc+lK6e7z1/b3393f+7SkkE/t7Q0F7DWGmw61MCswsyQ98Pv+J3B4u6+IN29wYgZgVd1SxdvH27i2t+8xoNvRM5onAVvtz+7m9f295e0aOvu5Zt/fZuqCAPKSqnR00AwRjJSrame7UOMA1R4unt+8rS1GOuVvXU88tZRfrt+/6B31rFK8fvISos8ByA7ffC5AdecNgewLsaH6js4YUZeyPvdg2QEzuNIYwReNS2dPLmtimd3VHPvqwfdGVFe9W3dGGP4zyd38U93vuy+/vbhZn738gHW7Y7cvaSUGh0NBGPg7cNNVDVb8+2DUQY/wzV19LC+opYj9jTKrUeahryzHkrAL2TZQSlc9iAB4l/PXcSMAqtrqK61m46ePopz0kKO2V/bxv0bDroBwBsI3jzUGBLgIqlt7abFHuRu6ughLyMw4Ji61u6QdQkOp2JqpPfGysG6drYfbY7b+ZWayHT66Bi45H/WuY+HsxfAXzYd5i+bDnPxilIAdlW1jrr7w++TwTOCQV5PC/jIty/MlQ3WvP3wQLC7upV/e3ALAb+PK04qc2cVAXziN6/F1DanK6yhrRsJWx0NVkbgPa+j2g6ykdYsjJXvP7adnVUtPPvls+L2HUpNVJoRjLHhZASOds/d9Whr/wd8PrLSImcE6YFBXk/xkesGAis7KcoOnXb63FfOAqy7+e7eIHVtXYMGlsE4527s6KGzp8/tjnLUt3VF3KfBCY4jzQi6e62V2b19wUEHnmtbu9hf2xYyuB5v8QxsSg2HBoIx9vn7Nw37M97+9x+PsoibzydkDHLBT4vQLw9WgPD7hLyMAPvtufve0hTQP6voZ89VsOimx6hs6GBm2IDyUA7Z2UZf0NDa1TsgMB1p6gxZ3eyosoPjSC7SxhgW3fQYX35gMwv+/TFuuP+NiMc1dfQQNHBgnFYyv7a/nhX/8eSQXWpKjQcNBKMUfoc5kgtJW9fAC9xg/fyR/PCKFSHPU+yKoeGzhNJS+s959dpZ/a/bxe0WT81h4wFrllH4wHJqio9Uv8+dCtvVG2R2lEDg9wn3ffrkkNeONoV2e6UHQv/6Pbu92h1H8HLKT3T2DD8jOGZ/9sE3rJXZzp4M4ZxSION1Yd5S2UR3X5DHthwdl+9TKhoNBMP0wq4arr+v/66yM8Jd6nDnvUfaSGawfv5IFpZkhzz32/3veRmh3TtpngvvqfP7F545xe1Wzcp3X4vU7ZMZ1uU0e8rggWDOlEwWluQMeD3XE2C8GcFFy6exs6qFNyNsgOMUvnupopavPrA56mpkx8+f38M9L+9ns71fwszCDPe9rt4+bnl4G4+81X8RdgLBnnEKBE6dpadGMEVYqbGmgWCYPn73hpALSHeEfuueYU4BbYuwCO3z5yyM+pkPrpnJoqnZfOm8RWSmhl60/T4rEPSFtSMtxcctly/ngc+eSsDXP1jrXJAXTu2/cOekp3D92fNDPp8V9j1DdQ1FCiazp2T1f6+nq+r97ygD4IGNA9cYOBnBlsNNPPB6ZUyZwQ8e38HNf93Km4esTX1mF/Z/78b9Dfxy3T43oHf19rnn3FMzToHAHi95q7KJo00d4/KdSg1GA8EIOXelkW7+nbvLWDmDqF4Xn1DqPr5g2cAVwafOn8KT//ouvnDOwgFjAk4gCJ/bn5ri4yOnzGbNnEK3+wj6axdNy013X8tOC/CxU+eEfD58EDp80RnAR0+ZDYCIDOj6Cf9MWsCPE4/mFmWxenZBxP0bwms0tXf38tiWo4N243gzhnUV1ups7/jCr1/aD+B+t/f3tacmen2jquZOHg3rztlT08rzO6sHHOe9YQhX2djuZilPx7BBkFLxpIFghJxrTaTponc+P3Q5haGkeDaUCd+IJlx6auiv0ak35FyUHd4xAu/53zGrAIBpef2BID8zQIov9Hu9mcfsKZksLc1lybTQ7h9v91GkKaLe7qT0gM89JivNzzmeEhjh3+21r7aN6+97g2//fWvE91s8M4/ePmytDWju6H/NWbE9rzjbfs8KBIVZqe5g+WDue/Ug19/3Rkhg+fnze/jSHzeHHHevfVykMQ+wMoIzFhZTVpDBukHqMCk1XjQQjFDQzQgGBoLGjthrCw0m4Ov/1fiiXBSBARnBtLx09t92MSd7CtD99frT3UzBe/41cwrcLh5vIEgP+AmEzTJyunqWlubywlfPpjgnjcf/5Uy+974T3GPCu4/Cleb399Wnp/RnBJmpKSwp7Q8qGVEGy+979SBBA+s8i/HAmo5Z39ZNVdigdGleupv1LPZ0fzmZg5MRLC3NoaWrN2pGZ61+hs7u/u6putYu6tu6Q+ox1bZa4xrhA+RgZTQN7T2UFWSwalYBb9l7UiuVKBoIRsi5/EfKCEa7OhisO3bnhvrEsnz39bKCjAHHOnfq0eb1p4V10zgXLe9dvvP5FWVWeYlUf+hnMu2Lc2bYRdrbZRRpkLswK9XtBinxLFRLD/g5v3yae07voHf4d3v95c3DzCzMwJj+7TsBPnPP67zju0+5M4UA8jIClJfmunf9Hzl1tjtg3WRnCc6Fv9yut+QNLuEa7H2fvbu91dsL5bx1ppxNiCKdyxkfmJGfwcqyPI42Rd6YR6nxoiuLRyhaRjCc1cWDSfEJPhH6jOHK1TM5e4k1FfS/n9k9YEzB7xOe+tczB10wBtbF2MsZoA7v93/pxne7q4wDYRdj5yIfnoF4s4Dwaa8bvnEOaQE/rV29HKpvZ82cQve9tICPH1+5khsvWkLA73PXKkT6bi9jYNXMAnp6DVs8s4ycXdg22bur/f66U1g6LZd/+8tb7jhDUVYqj37xDO58YQ9/tAem+zMCKxAcbuhwH4dz9lHwBoJGOzjUtHS5WZUzzTZSRuD8/soKMphu/8ybK5s4rzx9wLFKjYe4ZgQicqGI7BSRChG5McL7eSLydxHZLCJbReQT8WzPWDIGPvDz9SH7/DoeGYO54SLidpv4fLCgJJsFJdnuRTr8or9wak7UWTyFmaGBQLBOPi03NMOYkZ/hXvD9YV1STtAI77bxjgtk2p91PlmSm05eRoAZ+RmcMm+Ku3DN+RnSA3633d4xBe8YRiS5GSksm57L1iP99YGc8zpjACvL8snLDISMjaQFfJQVZDI9P4Pu3iCdPX1u6Yvy6XZG0NTB3ppWTv3+M6y99Wl3Cir0ZwTNHT1ceefLvLq3jnr7ou90BwHua5EygspGJyPIZNn0XPw+4a0I02Yjufmht7nbszeFUmMhboFARPzAHcBFQDlwtYiUhx12PbDNGLMSOAv4sYjEvqVWAhkDrx9o4PevHYrbdzgXRu8F+caLlnDTxUs5L8reApGkhN1hX7h8Gv/+nqV89YLFMZ+jKNvq1vGHDQJ7u6ScaanRciLngh1ppfMfrjuF//vkyQO6hsLHnXPTAyybnsuemlZ3HUapfTf+VmUTuekpbsDyfo8TFJyV080dPW4X0byibFL9Pg43dPDK3nqry6alK6QktpMRVDZ0sGF/PS/sqnHrI9V4AkF/19DAjOBwQwcBv1CSk0ZmagoLS7LZHOM4wZPbjrF+T11MxyoVq3hmBGuBCmPMXmNMN/B74LKwYwyQI9YVLxuoBwZOqp+Abn9u95id650LiiK+7lz/hf6rYFZaCp86Y96QA8hD8fuET585L+qgbDin6yZ8MNw7zhAecCJxAkGk2kEnz5vCOxcWDegacoKQIzcjQPn0PIIGdhyzsgJv7Z4Sz1TY1JBA4AtpQ1NHD00dPWSl+klN8VGan87hxg4qqlvJCPjJSvWHdMU53UDO3f8OzxaiTp2oYNC4mUOkNQKHGzuYnp/h/g5PnJnP5kONQy6UM8bQ0NYzrvWQVHKIZyCYAXhvlyvt17xuB5YCR4AtwBeNMQOuDiJynYhsFJGNNTXD37UrHu54bs+YncvnEz531vwBr//P1e/gxJn5g9YIisUXzlnIJ985dzTNczmlquvDdlzzjjPMK85i2fRcvnvZ8kHPc8vlyzlhRt6AqadegZTQQOdd4wDWgrdldleO0z3k9PXPyM/gwmXT3GMjZgR2IGju7Akpiz0jP4PDjR3srm5hfkkWMwsz3Yqs3b1Bd6zBCQTe0tXOa4123SIY2DVkjGHTwYaQVder5xTS1NHjTnUdTHt3H919wYgr0ZUajXgGgki3rOG3PBcAbwLTgROB20VkwCidMeYuY8xqY8zq4uLisW5nwjg1gvwCX7twyYD3zyufykPXnz6qu/8vnbeImy8J75EbmZkFVl9+fVvo9EpvRpCZ6ueRL5wRUsIi3MqZ+fz98+8csCLaKzwjmJoblhGkBygryCAvI8DWI80YY2ju7OWGsxfw0o3v5iueLq/wMQLozwg2H2qioqbVDQzT8zM40tjBnupWFpbkUFaQ2V81tb0/ANa2DhwMdl6rb7MCQmleOkeaOkPu9HdVtVLZ0ME5njpQ715Sgk/gia2R6yA5nADcMYyqpW9VNg6r5IkxJmRMRCWHeAaCSmCm53kZ1p2/1yeAB42lAtgHDLwiTlLzi62yB+fZUyjXzClIZHOG5MyIee/K0pDXvTOFUnxj81fqkhXTQ55H6hoSEcpLc9l2pInWrl76gobcjIHBJS1C15AzhfQ7D29j86FGyuwgN7coi6rmLo40dbKgJJuyggwqGzqsbpn2/gDoHRgGa5ykxt5AxwmUy6bn2iW7+wOIM5B9zpL+QFCYlcrauYU8PkQgiDRjKZqK6lYuvf2lYW15uq6ilsvueEmDQZKJZyB4DVgoInPtAeAPAn8LO+YgcA6AiEwFFgOjX5Z7nJhXlM3mb53P1WuteHnfp09JcIuiC/h9bPmP87np4tAMI8Xvcy+w0VYED8e1p89h87fOZ3peOqkpvgHdY86F3Jk55NyNR9r5zLuGwskOvMddd+Y8/ufqVQBccVKZO6bgBIJWe5FZgycjqAsLBAunZg/ICJZNt9ZjHKhrp6u3j67ePp7eXsWKsryQMQyAC5ZNo6K61a11FGm8oN7+/s4Yu4YO1LW53x+rY3aG452NpSa/uAUCY0wvcAPwBLAd+KMxZquIfFZEPmsf9l3gNBHZAjwDfN0YkzTr7VP81lRKZ3ZQtLnzE0VOemDAtFLoX2Mw1LTPWIlYfzapKdbuaX1hF0anK2f5jDx6g4az//N5YJBA4O0asi/yOZ79Fs5dOtUdNJ+am86Vq60CeAvtQADWLKFIXUOORSU5bpbgZAArZ1qB4AM/X8/imx5n8U2Ps+lgI+csGTjj6wJ7TONPr1dS3dzJ6bc9y4NvhBbga4ywmC0aZ2FdpLUMg3HGWXZVtUQ97kBdG6tveYq941SkT8VXXBeUGWMeBR4Ne+1Oz+MjwPnxbMNE8qfPnkpnT5CP/OpVIPRiNFE9dP3p5ETZ9N6Rleanvm3suoYcaSnWbJ7wRXpOmy5YNo3zy6fypF24LTdCIPDOGnIee19zVlI7vnrBEtbOncK84mza7bvvyoaOQbuGMlP9zCjIoLHd2r2t3g4Sp80v4tb3LXe7dMDaU/rK1d4eU8v0/AwuP3E6v/zHXjbur+dIUyfPbK92q7LC8McInLv7Y82xVzeNNRDsrmqltrWbjQca3JpN6vilK4uH8NzOag7WtfPxsG0VY7VkWg47jrVQVpDB6jmFMdXSn0hOnJkf03HO6uKxyggcqSk+MgL+AYHAWQeQkernBx9YwZPbngIGywgGjhF4hS/Oy8sIcOlKa4yiPyNod3eS8/vEDRAABZmp7hhGXVsX9e3d5KSlkB7w8+GTQwv/RXPzJeW8sKuG1/Y3kJOWwqaDDSHvO4Gos8faejNSUT8vJxMYTkbgBK1dVaF3+pUN7dzxXAXfvnQ5qSk+N2DsHaJaq4L7NxwkKy3F/Ts1EU38vogE+8SvX+Nbf4tc5XIol6wo5Yv2vgLOv9mh/vEer5z6Q2M1RuC4as1Mrlwzk76wScXei3eBp3xGtK4hv09C1jl87cLF3P6hVVG/Py8jQHFOGlsON9HU3kNqis/9DudHLcxKdRezHWnsoL6tO6RNsZqSncZPP7iKj5wyiy+cs5AjTZ3uXT30L1KD2PZvdvZ6PjacQGBf4GvtQnqOhzYd5v4Nh9hbawUIp4ifdg0N7fZnK/i/lw8kuhlRaSCIo9s/9A6K7CJrEnE27eSRlZaC3ydjHug+cspsrjipLGJNp0iiZQTh2cDnzlowYHZSOBHhlHlTeHlPHQ3t3RRkBtxaSwWZqeSkp1CQleqW1z5Q1059W/eA2k6xeteiYm65/ATWzLVqMr15qD8r8A5Wx7KWwMkEqlu66A2PpINobO92b1q83UNO/SYnY3AzgiHKdoNVbfWZJN2Jra2rl8ONHSGrziciDQRxNtheAtcM0tW0sCTbnVZ6PMlKTRnzbMDLSasXTc0eUPQOcHdTi1SB1Zk1NNKFeafOm0J1SxfP7qimrCCzvwprmp9VswpYNj2XsoJMfAL769o50thBcU7aEGeNrrw0l9QUH2/YF2AICwQxjBNUNXW63Wrhg9uDaerocUt177YDgTGGTfZ0UicAOPs7HKhrGzLI3PXiXj75240cHMF+3sc7ZxZYbcvEDgQ6RhCFdyFOb18wpvIJ4ZyLj7ePe/9tFw96/FNfetewv2MiyEzzxzUQnL2kJOqf21fOX8xXL4i8BMWpW+SdPTQcp8yz7s5rW7u55fLl/Ox5a1V5VmoKv7t2rXtcaV4Gu461sK+2LWSHuZFITfGxfHpuyDhBg2chnzcQtHX1khHwhyw8bOnsoaWrl7VzC9mwr56jTR0h+00MpqmjhxNn5nO4oYOddiA4WN/udhM5gcD5f0+fobKhgzlFg9+8PG6XCt9Z1cKsKPtcT0bOLnotXb10dPcNq6TLeNKMIIpL/med+3jFt58c0TmcvuzuGFPz49WUrFSyY5hdFC/RuqTS7N9B+J4MsZpblMX0vHTKS3M5v3ya+zsN33thTlEmL+6uIWj6K5mOxokzrU1rnBuShvZud/2E0zXU0d3HmT98jm/8ZUvIZ53xgVX2YH+s4wSN7T0UZKayaFqOO2C8yZOVOJVamzt73C4kZ9wgkn21bW49pt3V0WciTUa7Pduphi9CnEg0EESxzVNHpn2IPtn/eG85z33lrAGvOxlBrH20x6vPnbWA3117cqKbEdFgYwSxEhF+e+1afnXNanw+cbumwjfomVWY5f49KS/NG3Ce4ZpZmEFXb9AdwG1o73b3L3AK7D29vYq6tm5+/9oh/r65f+H+sSbrorNqVj4Q28yhvqChubOH3IwAi6Zms6uqxa2NlJlq7SbnzQgW2fWSos0ccrKBnLQUKqqSb2DZu6929QTuHtJAMEZWzylkboT02LkL7RmDXcsmsoKsVBZHKSKXSP2BYORp+cKpOZTmWRdhJwCEb8s5x+72yElLibiT3HA54ww1LV10dPfR2RN0A4HTNfTXNw8zLTedVbPy+caDW9wVz07V0yXTcklL8YXs2jaYls4ejIH8jACLpubQ2N7Dnpo2Nh1qtPZ2yAi4lWebO3qYNSWTgswAe6IFgq3HWFGWx6rZBexKwoygorrVnUhQo4Fg8gsfFHamJTrdCL3ByZ0RTGRO11DqKKq4erkZQdjubs4/+KWluaMuEw5QkmP16Ve3dLoDxdPzrdc6uq39mZ/fWcOlJ07nP967jJauXv6x21qY73QFTctLt4rfRdl+0+HMCMrPDHBe+VTyMgJ85p6NbDvSzKpZViBw9m5otiu2zivOHnQK6ZHGDjYfauTC5dNYWJJNRXXrsArgHe+6evs4UNfGqfbe4RN55pAGgmFwardEEr6g1pmW2N81lDz/ACaa0XYNhUu3M4LwGUqzCq2McCzGByA0I3AGa52spLM3yCNvHaE3aLj8xBksn5FHTnoKr+6zNq051txJQWaA9ICfaXnpIWMEL+yqYcO+esI53T75mQHKCjK58yMncbC+nd6gYdWsAvIyU90yF82dveSmB5hXlDXoFFKnmuqFy6xA0NkT5HAMAWmy2FfbRtDA2rmFiGhGcNyaHjbLwqlnE4mza9dNFy+lILN/Lnuq30d2WgrfunRZXNqohpY6xoGgf4wgNBDMK86ivDQ3pMT0aJTYgaC6pcu9W3c2B+rs7mPjgQZK89JZWpqD3yesmVPIq/YF/lhTJ1PtwnaleRkhYwTf/ttWPn73hpC9FKB/MVlehrUG4tT5U/jRFSuZW5TFmjkF5GUEaO7oobcvSGtXr5sR1LR00dIZWpoc4LG3j7F4ag7zirNZONUqQ5FMA8a77TGRJdNymZKVqoPFx6t3LQ7d+yBaVut0BXzqjHls+mZ/+SQR4e1vX8BHT4m91IAaW2MxRuDVP0YQer70gJ9Hv3gGZywcmz0zstJSyEz1WxlBu5MR2F1DPX00tvdQnJPmzpg6eW4he2vaqG7p5Fhzp3vstLx0qpo7CQYNxhgON3bQ0dPHdfdsDFmt7NztexflXb5qBs995SzyM1PJzwjQ2NHjbs2Zm5HCPHvNS/iAcY29xecFy61iegvsgWVnJlJ9W3fE4AHWdNjw6q7Ho4rqVkSsG4Si7DTNCI5X4fVtognfx1dNHO46ghFOHw2X7o4RxH+6bElOGtUtXe6CJGeXuI6ePho9O6uBtc0nwAMbK6mobmVmoTVmUZqXTm/QUNtqZRZdvUEuXTmdqqYuPn//JndGm7drKBJrjKDHLS+Rmx5gQYl1p78tLLt4alsVxsBFdiDIywgwNTeN3VWtdPcGufyOl7jpobcjfs93H97GZXe8NKx/fxNRRXUrswozSQ/4Kc7RQHDcGs6Mz8FWEKvEExHSIuxpMFJO11B2WvwXB1kXkE52V7eSnxlguj1G0NHdR1N7N/mZ/aUslk/PJTPVz4+e2El6wM9n3mWttnZmMB1qaOeIPZvoouXTuOV9y1lXUcsPHt8B9A8WRyrTAVaAaOroCTluXlEWM/IzBpSQeHzrMWZPyQzZjnRhSQ4V1S386fVKDta3h+wF7bW72trF7dW9dcP7w5pgKqpbWWBXZi3WjOD4FUt9m5mFo58mqOIvI9U/oMroaM4FA8cI4sG5k9xd1cKikhx8PiuoddoZQb7nop3i93HSbGuXux//00p3PMHZYvRQfQdHG62xgtL8DK5cPZOPnzqb//3HPv6xu4amjh6y01IG3RcjLyOAMbgDvs4uceeVT+Ufu2tp77a6jJrae1hfUcuFy6eFLPRbUJLN7upW7niuAggtouflzHB66M3D7ms/fHwHX7h/0zD/9IZ2za83cNeLo99//MVdNbz3f9a52VJvX5C9ta0ssMdGinPSqGntmrDVhzUQRHG8p6aq323vP4GPnTpnTM7ljhGMQ0ZQkpNOdUsXO6ta3AHXjFQ/bd3Wrmnh3Thfu2AJ//3BEzm3vH/zmzI3ELS76wuciRD/fnE5aSk+XtxVQ2N7z6DZAPRnCgfr20Oen18+la7eoDt19ZkdVfQGDRfam+04Fk619nc43NjBgpJsd9zDq6cvSFVzJz6Bx7Yco7Onj4rqFu58YQ9/23yEfTEUuYtVY7s1/faHj+9kx7HR7cj2wq4athxu4h67yujB+nZ6+kx/RpCTRndvkGZ7fGWi0UAQgTGG6+99g79tDt9i2TLPs3BsggZ4FebC5aVuf/ZopQ8yaygeinPSaOnspaWzl0V2MbiMgJ/q5i6MGdiNc0JZHpedOCPktYxUq4/a6hrqJMUn7v4JqSk+yqfnsrmyiaaO7kHHB2BgIHD2h14zt5Dc9BR7XMDwt81HKM1LZ2VZfsjnnfa/Y1Y+7zmhlCZ7BpJXVXMnQQPvOaGUlq5ent1RzW2P7SQzNQWfELJr29uHm7jzhZHfzTulL4LGcOOft4zqxs8pLnf3un10dPe5pSUW2j+zMxV4os4c0kAQwebKJh7ZcnTQ9yOlzjpEkDxWlOVx+oIpIf3f8eKtYuoNBE4tIe8YQTQzCzLsrqEOpuamhyx4WzEjj7cPN1HX1h01I3C+61BYRhDw+3j3khKe2V7FT57axfM7a7h67awBi+qWTc/ltPlTuOmScqZkpWJM/wC144jddfWBk8ooyUnjR0/s5OntVXzu7PmcvqCIB984TDBo6O0L8q9/eJPbHtsRsm/CcOywB7j/7aKlvHmokXte3j+i84AVCOZMybTLfRx0S0s4Nx/F2f1rQiYiDQQRDHVnMNa7cKnjS2leBvd+6pSYL8KjERoIrItKesDvrgvIj3Lh9ppZmOlmBM7qZMeKsnzau/vYeqQ55owgxVNzCeD8ZdNoaO/h/z1bwRUnlXHD2QsGfD4zNYX7Pn0K75hV4G7c0xDWPeSMD8wsyOTSldPZV9vGtNx0rj19LlecVMbhxg5e3VfP/RsOunfdW480xfRnEG7HsRYKs1L51BlzOWtxMT98YueIFrx19vRR2dDB+1aVsXZOIXe9uJdtR5spzUt3Fx0W5WggOO4MdXfvLUf9/lVWGp4X5R+QUiPlLCqbkpXKFPuuMiPV75YriHbh9ppZkMnRpk4q69vd1cmOlTOtAnndvUF3MVkkzncdbuggzx4odpy5qJjCrFQuPqGU295/wpAlNgrtIFrfFpoROBfi6fnpfOCkMvw+4asXLCY94Of88mlkp6Xwm/X7+MnTuzlhhtXurUdG1r+//VgLS6blICLccvlyeoOGu9ftG/Z59tW2YQzML8nic2fP52hTJ49tORrSFTnRMwLdjyCCoaaCBjx/yf/l3EV89qz549JfrJKPkxE4A8VgdQ05Y1MxB4LCDPqChiNNnZSGZQTzirLJSvXT1t0XU0bQGzTkhmUi2WkprPv62WQE/DHtUleQZX2+vi30wniksYOCzACZqSksLc3l9ZvOdTOvjFQ/F59Qyh82HkIEfnftWj5zz+sjCgR9QcOuYy1cvXYWYA2or51TyEsVtcM+l7OYbn5xNkum5bBsei5bjzSHBIK8jAABv0zYekOaEUQw1F9jZ4wgNz0Fn080CKi4mZKVRqrfx5Jp/fWLvNNgo93BezlTSAF3LYLD5xOW23fX0bqa0gN+dy1GeCAAq+sn1q1Kna08wzOCI40dboVVGDgG8oGTygC44h1lLJ+RR/n03BF1DR2sb6ejp48lpf3jPKfOn8KOYy3DHtDdU2OtIJ5blIWIcL3dLeaM6YD1Zzzc1cXf/OvbfOq3rw2rLSOlgSCCof4uB1J83H3Nav7wmVPHp0Eqafl9wq8/sYbPnTXffS3ds0I62uCul7PKGPrLVHittDewGSrDcL4vd5SbEBVkDjZG0BkSCMKtmVPA7R9axc3vLQesAeh9tW20dQ1vWqYzULzUE2BPX1AEwMt7hreQbU9NKzPyM9wAfeGyadz+oVVcdmLoftjDWV3c1dvHg28c5qWKunGp2KqBIAJv11BqhBlCAZ/w7iVTWVo6NlUmlYrm9AVFlOT2X7ydQdqsVH/MpbVL89Lx212akS60K8qsjGCowOIEilgD0GDSA34yU/0DZvwcaexwF8JFIiJcsmI6uenW9y+bnocxDHsdwPZjLfgktMvtBLuC6/o9w+se2lPTyvzi/vP4fFYbw3sKirLTYs421u+po7Wrl46ePnc1eDxpIBhCpBlCY1FrXqmRclY2D2fWUorf584WipQRnLmomPetmsGaOYVRz+NmBKMMBGBlBd7Vxc32PsvRAkG4ZXbJ77cPDy8Q7DjazNyirJBuNr9POGXeFNYPIyMIBg17qttCAsFghlNm4km7hDeE7nIWLxoIIvBmBP4IF30tMKcSyckIYh0odswsyCQ1xef2z3vlpgf4yVUnujOTBuOMSYw2IwCYkp0asrr4iDtjKPZAUJqXTkFmYNjjBDuOtbAkQkZ/2vwpHKhrp7KhPabzHGvupKOnj/klA3cnDFeck0ZdW/eQ09P7goantlVx+gKriKAGggTxXudbIiwJD9+ERqnxlD7CQLB2biGrZxfEPKAbSf8YwdhnBEc8U0djJSIsm543rJlDbV29HKxvZ2mEBYHOOMH6itiyAmdFcUwZQU4afUEzYFwk3BsHG6ht7eaqNbMozErVQDCeOnv66Asaunr7hhwsHs0/JKVGy+0ainHGkONfzl3EfZ8+ZVTf7QQfp7zEaBRmpVLnCQSH7VXFw+kaAqt7aFdVC929sZUL3llllZbwzsRyLCzJpig7jZdiHCfYUz28QABDryV44u1jpPp9nL24mAX2Fp/xpoEAqGvtYsnNjzP/G4+y+KbHh6wfFOlOQqnx4nQNJWIRo5MRjEXXUKSMIODvr4MUq/LpufT0mZh3P9tx1A4EpQP/HYsIp823xgliqRS6p6aN3PQUirKHDspFMSwqM8bwxLZjnLZgCjn2fg+7q1vjXrVUAwFWP5/XYH/mv75mDX/+59P457MGLp9Xary4YwRjcDEeLjcjGIOuocKsAG3dfXT29AFWICjNyxj2ZIxl04e3wnjHsWZy0lIGzTxOXzCFmpaumO7E99S0Mr8kO6ZeglgKz20/2sKh+g4usCu3LijOpqmjh9rWkdVTipUGAgZe+G96aEvE4+YWZXHS7IKIA8hKjRdnp7XhjhGMhTHNCOxBa2ejm8MNHcMaH3DMLcoiI+Dn6W1VMVUQ3X60mSWlOYNevE+bb40TPLOjeshzhU8djSaWrqEnth5DBM5dapURd6a3xrt7SANBBG8cbIz4uu5CpiaC/owg/kXvwp21qITPnTWf8umjX0PTX2/IutsNX1UcK79P+PQZc3lyWxWfuWdj1MVlXb19vFXZxIqwEtleMwszOX3BFG5/tiJqEbqWzh6qmrtiDgRZqX4yAv4hA8Hq2QVu0HDKVFTUaCCIu1i73zQOqInAGSxOyBhBZoCvXbhk0F3MhsNbgbS3L8ix5s5hDxQ7vnT+Yr5z2TKe3VHNB36+nlse3sYtD2/jl//YG9K/vvlQE129QU6eG329xG3vX2HvU/DWoP3z/TWGhp46Ctb4g7NTWSQH69rZcazF7RYCmJZrVTCtqIpt/GOkNBAAhtgigQYCNREsLMlh0dRsdzHV8aq/3lA3u6tbCRqYMyW2i2okHzt1Dr+6Zg2N7T3cv+Eg97xygFse2c72o/0X0Vf31iFiTaWNZmZhJt94z1L+sbuW+zcciniMO3V0GBseFWWnDpoROFtzegOBiDC/OEszgvEQe0agkUAl3rS8dJ7813e5W1Aerwo9GYGzmvfU+VNGdc6zF5fwyjfOYet3LmTd19+NCDyzvcp9/9V99SyemhPTquwPnzyL0xdM4dZHtrmb8XjtqWklxSfMKoz99zBYvaFg0PDHjYc4bf6UkLpQAAtKcnSMYDzEOjFLx4iVGjvOrKe61m7WV9QytyhrRGMEgynOSePEmfk8bQeC7t4gGw/Uc8q82IKNiPCDD6wA4Ot/fiuk+Ft3b5Bntlczvzh7WN1kg3UNvby3jsqGDq5aM3PAewtKsqlq7qK5s2fAe2NFAwHEPEdXhixQrZSKVYrfR15GgJrWLl7dVz/qbCCSc5dOZXNlE9XNnWw53EhnT5BT5kXvFvIqK8jk3y8uZ/2eOu7dcNB9/b+f2cWOYy185YLFw2pPcXY6je09Axa//eG1Q+RlBEK6hRzugHEcs4K4BgIRuVBEdopIhYjcOMgxZ4nImyKyVUReiGd7BhNrRtAbjG3lolIqNoVZqby4q4bWrl5Ot6dtjiVnGuYzO6p5ZW89AGvnDi/gXL12JmcsLOL7j27nUH07rx9o4OfP7+HK1WWcVz51WOdyZgPt8gz+NrZ38/jWY1x+4vSQIniOhcdzIBARP3AHcBFQDlwtIuVhx+QDPwMuNcYsA/4pXu2JJtYxgljmKCulYleQGaCywZqiOZw79VgtmppNWUEGz2yv4pW9dSyamh2x6F40IsJtH1iBT4QvP7CZL//xTUrzMrj5kvKhPxzm3KUlFGal8uU/bqaj21pI99Cmw3T3BrkyQrcQWAPXqSk+t5xFPMQzI1gLVBhj9hpjuoHfA5eFHfMh4EFjzEEAY8zQKzjiIrYLfK8GAqXGlHNRXjItZ8jKpyMhIpy7dCr/2F3L6wcaOHmY2YBjRn4GN1+ylA376jlQ386Pr1xJzghWV5fkpvPTq05kV3UL3/zr2xhj+MPGSpbPyHVXSIfz+4R5RVnsPk4DwQzAO++q0n7NaxFQICLPi8jrIvKxSCcSketEZKOIbKypqRnzhkbLCD508iz38czjfJaGUhONs1OZU/UzHs4rn0pXb5D27j5OHkXWceXqmXzs1Nn8+3uWxjzgHMmZi4r5/NkLeOD1Sr79921sP9rMVasjZwOOeBefi2cgiDSyGn7JTQFOAi4GLgBuFpFFAz5kzF3GmNXGmNXFxcVj3tBo9/nn2X2MhVmpMe8GpZSKjZMRnBaHgWLHmjmF5KRZ1VKHWj8QjYjwncuW86kz5o26TV88dxGnzZ/Cb9bvJy3Fx6Unht8jh1pQks2hhna3LtNYi+eVrRLwhrky4EiEYx43xrQZY2qBF4GVcWxTRNEyAmfpQLyr/ymVjOYXZ5OXERjVBXooqSk+LjphGivK8ijJGX4to3jw+4T//uAqZuRncMVJZUPWblpQko0x/YvYxtroi4oP7jVgoYjMBQ4DH8QaE/D6K3C7iKQAqcDJwE/i2KaIol3kdRGZUvFzxUllXLKydMD+vmPt1vedMOEmexTnpPHsV95FSgw7XXmnkA42ljAacfvTN8b0isgNwBOAH7jbGLNVRD5rv3+nMWa7iDwOvAUEgV8aY96OV5sGbWuU9wL2nsULS3QPAqXGms8ncQ8CAAG/jwgzMxMuLSW2Rs0tysInxG3mUFx/A8aYR4FHw167M+z5j4AfxbMdQ4nW65OTFuDeT53M8jhEYaWUikVaip+bLi5n5cz4XIfiH4ongXjOaFBKqVhc+865cTu3ToMhevVRHSJQSk12Gggg9hoTSik1CWkgIHoc0EVkSqnJTscIiDxYfN+nT3b3LlVKqclsyIxARHwiMu5TOsdTxDEC7S5SSiWJIQOBMSYIbBaRWUMde7yKlBFoHFBKJYtYu4ZKga0isgFoc140xlwal1aNM73oK6WSWayB4NtxbUWCRSoxoaWFlFLJIqZAYIx5QURmAwuNMU+LSCZW2YhJIdI1P9raAqWUmkximj4qIp8G/gT8wn5pBvBQnNo0/vSar5RKYrGuI7geOB1oBjDG7AZK4tWo8Xa0qXPAa9o1pJRKFrEGgi57u0kA7LLRk+ZS+Y2/bEl0E5RSKmFiDQQviMg3gAwROQ94APh7/JqVeJMmyiml1BBiDQQ3AjXAFuAzWKWlb4pXo8bTh/73lUQ3QSmlEirW6aNnAfcaY/43jm1JiPV76iK+rltTKqWSRayB4BrgThGpA/5h/7fOGNMQr4YlmoYBpVSyiHUdwccARGQ6cAVwBzA91s8fj06O42baSik1kcR0IReRjwBnACcAtcDtWFnBpJSbnjIu+6gqpdREEOvV7qfAHuBO4DljzP54NUgppdT4imnWkDGmCLgWSAduFZENInJPXFumlFJqXMRaYiIXmAXMBuYAeUAwfs1KLB0oVkolk1i7htZ5/rvdGFMZvyZNABoJlFJJJNZZQysARCSHJLhM+nyS6CYopdS4iXXW0HLgHqDQeio1wMeNMZNuC8uPnjKbD58yaTdjU0qpAWLtGroL+JIx5jkAETnLfu20+DQrcb564WJy0wOJboZSSo2bWGsNZTlBAMAY8zyQFZcWJZhWllBKJZtYM4K9InIzVvcQwEeAffFpUmIFgxoJlFLJJdaM4FqgGHjQ/q8I+ES8GpVIgZRY/0iUUmpyiJoRiEg68FlgAVYJ6i8bY3rGo2GJkp2mpSWUUsllqNvf3wKrsYLARcCP4t4ipZRS42qo299yY8wJACLyK2BD/JuklFJqPA2VEbjdQMaY3ji3Zdw1tHUPfZBSSk1yQ2UEK0Wk2X4sWHsWN9uPjTEmN66ti7PTf/BsopuglFIJFzUQGGP849WQRGjv7kt0E5RSKuF0rqRSSiU5DQRKKZXk4hoIRORCEdkpIhUicmOU49aISJ+IXBHP9iillBooboFARPxYm9xfBJQDV4tI+SDH/QB4Il5tUUopNbh4ZgRrgQpjzF5jTDfwe+CyCMd9HvgzUB3HtgxgtLqcUkoB8Q0EM4BDnueV9msuEZkBvA+4M9qJROQ6EdkoIhtramrGpHEH69vH5DxKKXW8i2cgiLTNV/ht+E+Brxtjos7jNMbcZYxZbYxZXVxcPCaN84nuQqaUUhB7GeqRqARmep6XAUfCjlkN/F6si3IR8B4R6TXGPBTHdgGQ4tdAoJRSEN9A8BqwUETmAoeBDwIf8h5gjJnrPBaR3wAPj0cQAJCICYtSSiWfuAUCY0yviNyANRvID9xtjNkqIp+13486LhBvH/3VqwNemzMlMwEtUUqpxIpr8X1jzKPAo2GvRQwAxphr4tmWcLurW0Oe/+VzpzF7yqTcfVMppaLSXVhsq2YVJLoJSimVEElZYiJ8DcHd16xOUEuUUirxkjIQ9IVtUL927pQEtUQppRIvKQNBT19oIND5Q0qpZJaUgaC7LxjyXNeWKaWSWVIGgp7wQKA5gVIqiSVlIAgfI9CMQCmVzDQQKKVUktNAgGYESqnklpSBIGjCZw1pJFBKJa+kDASaESilVL+kDAQ1LV0hz3VvAqVUMkvKQHDVXa+EPNcwoJRKZkkZCMJpQqCUSmYaCADRSKCUSmJJHwh++TGtPKqUSm5JHwjOLZ+a6CYopVRCJX0gUEqpZKeBQCmlkpwGAqWUSnIaCJRSKsklXSDQyqNKKRUq6QKBd1OaX3z0pAS2RCmlJoakDgSleekJbIlSSk0MSRgI+ruGirLTEtgSpZSaGJIuEBxu6HAfZ6WmJLAlSik1MSRdIPinX6x3H/uS7qdXSqmBku5S2NnTP0bg92mxOaWUSrpA4KUb0iilVJIHAs0IlFIq2QOBZgRKKZXcgcCnGYFSSiV3IFBKKaWBQCmlkl7SBoLvXr480U1QSqkJIWkDAUarkCqlFCRxINBq1EopZYlrIBCRC0Vkp4hUiMiNEd7/sIi8Zf+3XkRWxrM9XkHNCJRSCohjIBARP3AHcBFQDlwtIuVhh+0D3mWMWQF8F7grXu0Jp3FAKaUs8cwI1gIVxpi9xphu4PfAZd4DjDHrjTEN9tNXgLI4tieExgGllLLEMxDMAA55nlfarw3mk8Bjkd4QketEZKOIbKypqRlxg7p6+9zH/7R63GKOUkpNaPEMBJGW7Ua8EReRs7ECwdcjvW+MucsYs9oYs7q4uHjEDTra2Ok+zk0PjPg8Sik1mcRzZ5ZKYKbneRlwJPwgEVkB/BK4yBhTF8f2cNZ/Ph/P0yul1HEpnhnBa8BCEZkrIqnAB4G/eQ8QkVnAg8BHjTG74tgWpZRSg4hbRmCM6RWRG4AnAD9wtzFmq4h81n7/TuCbwBTgZ2JVAu01xqyOV5uUUkoNFNdNe40xjwKPhr12p+fxp4BPxbMNSimlokvalcVKKaUsGgiUUirJaSBQSqkkp4FAKaWSnAYCpZRKckkTCDp7+oY+SCmlklDSBIKObg0ESikVSdIEAt1/QCmlIkuiQJDoFiil1MSUNIHAaEaglFIRJU8gSHQDlFJqgkqaQKBjBEopFVnSBII+HSRQSqmIkiYQBIOJboFSSk1MSRMI+rRrSCmlIkqeQOBJCdbMKUhgS5RSamJJokDQ//ik2YWJa4hSSk0wSRQItGtIKaUiSZpAsKem1X1sdFWBUkq5kiYQbD3S3P9E44BSSrmSJhCkpvT/qBoHlFKqX9IEggUl2e7joI4XKKWUK2kCwXtXlLqPNQwopVS/pAkEIuI+DviT5sdWSqkhJdUV8eZLyjlt/hRuePeCRDdFKaUmjJREN2A8ffKdc/nkO+cmuhlKKTWhJFVGoJRSaiANBEopleQ0ECilVJLTQKCUUklOA4FSSiU5DQRKKZXkNBAopVSS00CglFJJTsxxtpeviNQAB0b48SKgdgybM5Hpzzr5JMvPCfqzxsNsY0xxpDeOu0AwGiKy0RizOtHtGA/6s04+yfJzgv6s4027hpRSKslpIFBKqSSXbIHgrkQ3YBzpzzr5JMvPCfqzjqukGiNQSik1ULJlBEoppcJoIFBKqSSXFIFARC4UkZ0iUiEiNya6PfEkIneLSLWIvJ3otsSTiMwUkedEZLuIbBWRLya6TfEiIukiskFENts/67cT3aZ4EhG/iGwSkYcT3ZZ4E5H9IrJFRN4UkY0Ja8dkHyMQET+wCzgPqAReA642xmxLaMPiRETOBFqB3xljlie6PfEiIqVAqTHmDRHJAV4HLp+Mv1exNtzOMsa0ikgAWAd80RjzSoKbFhci8iVgNZBrjLkk0e2JJxHZD6w2xiR08VwyZARrgQpjzF5jTDfwe+CyBLcpbowxLwL1iW5HvBljjhpj3rAftwDbgRmJbVV8GEur/TRg/zcp7+BEpAy4GPhlotuSTJIhEMwADnmeVzJJLxjJSkTmAKuAVxPclLixu0veBKqBp4wxk/Vn/SnwNSCY4HaMFwM8KSKvi8h1iWpEMgQCifDapLybSkYikg38GfgXY0xzotsTL8aYPmPMiUAZsFZEJl23n4hcAlQbY15PdFvG0enGmHcAFwHX21274y4ZAkElMNPzvAw4kqC2qDFk95f/GbjXGPNgotszHowxjcDzwIWJbUlcnA5caveb/x54t4j8X2KbFF/GmCP2/6uBv2B1ZY+7ZAgErwELRWSuiKQCHwT+luA2qVGyB1B/BWw3xvxXotsTTyJSLCL59uMM4FxgR0IbFQfGmH8zxpQZY+Zg/Tt91hjzkQQ3K25EJMue6ICIZAHnAwmZ7TfpA4Exphe4AXgCa0Dxj8aYrYltVfyIyP3Ay8BiEakUkU8muk1xcjrwUay7xjft/96T6EbFSSnwnIi8hXVj85QxZtJPrUwCU4F1IrIZ2AA8Yox5PBENmfTTR5VSSkU36TMCpZRS0WkgUEqpJKeBQCmlkpwGAqWUSnIaCJRSKslpIFAqChHps6emvi0if3fm80c5/kTvNFYRuXSyV7xVxz+dPqpUFCLSaozJth//FthljLk1yvHXYFWTvGGcmqjUqGlGoFTsXsYuWCgia0VkvV03f72ILLZXrn8HuMrOIq4SkWtE5Hb7M7NF5BkRecv+/6wE/ixKuTQQKBUDe1+Lc+gvT7IDONMYswr4JvA9u8z5N4E/GGNONMb8Iew0t2PtE7ECuBf4f+PTeqWiS0l0A5Sa4DLs8s9zsDa/ecp+PQ/4rYgsxKpmG4jhXKcC77cf3wP8cExbqtQIaUagVHQddvnn2UAqcL39+neB5+xd4N4LpI/g3DpApyYEDQRKxcAY0wR8AfiKXf46Dzhsv32N59AWIGeQ06zHqqoJ8GGsLSeVSjgNBErFyBizCdiMdTH/IfB9EXkJ8HsOew4odwaLw07xBeATdhXRjwJfHIdmKzUknT6qlFJJTjMCpZRKchoIlFIqyWkgUEqpJKeBQCmlkpwGAqWUSnIaCJRSKslpIFBKqST3/wGoraS3q0/kKAAAAABJRU5ErkJggg==\n", 107 | "text/plain": [ 108 | "
" 109 | ] 110 | }, 111 | "metadata": { 112 | "needs_background": "light" 113 | }, 114 | "output_type": "display_data" 115 | } 116 | ], 117 | "source": [ 118 | "l_y=[x[1] for x in power]\n", 119 | "s_x=[x[0] for x in power]\n", 120 | "\n", 121 | "plt.plot(s_x,l_y)\n", 122 | "plt.title('Traffic Split')\n", 123 | "plt.xlabel('Ratio')\n", 124 | "plt.ylabel('Power')" 125 | ] 126 | }, 127 | { 128 | "cell_type": "markdown", 129 | "metadata": {}, 130 | "source": [ 131 | "**Using the function**" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": 18, 137 | "metadata": {}, 138 | "outputs": [], 139 | "source": [ 140 | "from statsmodels.stats.power import zt_ind_solve_power" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": 19, 146 | "metadata": {}, 147 | "outputs": [ 148 | { 149 | "name": "stdout", 150 | "output_type": "stream", 151 | "text": [ 152 | "0.0050000000000000044 0.010000000000000009\n" 153 | ] 154 | } 155 | ], 156 | "source": [ 157 | "effectsize=delta/std\n", 158 | "print(delta,effectsize)" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": 20, 164 | "metadata": {}, 165 | "outputs": [ 166 | { 167 | "name": "stdout", 168 | "output_type": "stream", 169 | "text": [ 170 | "0.4492753623188406 0.08749203476222268\n", 171 | "0.47058823529411764 0.08815202921008315\n", 172 | "0.4925373134328358 0.08877665993026321\n", 173 | "0.5151515151515151 0.08936587305120942\n", 174 | "0.5384615384615384 0.08991961833450068\n", 175 | "0.5625 0.09043784907560412\n", 176 | "0.5873015873015873 0.09092052201105622\n", 177 | "0.6129032258064516 0.09136759723198883\n", 178 | "0.639344262295082 0.09177903810393061\n", 179 | "0.6666666666666666 0.09215481119281485\n", 180 | "0.6949152542372882 0.09249488619713445\n", 181 | "0.7241379310344828 0.09279923588618337\n", 182 | "0.7543859649122807 0.0930678360443371\n", 183 | "0.7857142857142857 0.09330066542132252\n", 184 | "0.8181818181818182 0.09349770568843596\n", 185 | "0.8518518518518519 0.09365894140067338\n", 186 | "0.8867924528301887 0.09378435996474113\n", 187 | "0.9230769230769231 0.09387395161291817\n", 188 | "0.9607843137254902 0.09392770938275073\n", 189 | "1.0 0.09394562910255964\n", 190 | "1.0408163265306123 0.09392770938275073\n", 191 | "1.0833333333333333 0.09387395161291817\n", 192 | "1.127659574468085 0.09378435996474113\n", 193 | "1.1739130434782612 0.09365894140067338\n", 194 | "1.2222222222222223 0.09349770568843596\n", 195 | "1.2727272727272727 0.09330066542132252\n", 196 | "1.3255813953488371 0.0930678360443371\n", 197 | "1.380952380952381 0.09279923588618337\n", 198 | "1.4390243902439024 0.09249488619713445\n", 199 | "1.5 0.09215481119281485\n", 200 | "1.564102564102564 0.09177903810393061\n", 201 | "1.631578947368421 0.09136759723198883\n", 202 | "1.7027027027027026 0.09092052201105622\n", 203 | "1.7777777777777777 0.09043784907560412\n", 204 | "1.8571428571428572 0.08991961833450068\n", 205 | "1.9411764705882353 0.08936587305120942\n", 206 | "2.0303030303030303 0.08877665993026321\n", 207 | "2.125 0.08815202921008315\n", 208 | "2.225806451612903 0.08749203476222268\n", 209 | "2.3333333333333335 0.0867967341971172\n" 210 | ] 211 | } 212 | ], 213 | "source": [ 214 | "power=[]\n", 215 | "lpw=[]\n", 216 | "for s in range(30,70):\n", 217 | " k1 =((s+1)/100)*15000\n", 218 | " k0 = 15000-k1\n", 219 | " r = k1/k0\n", 220 | " pw=zt_ind_solve_power(effect_size=effectsize, nobs1=k0, alpha=0.05, ratio=r, alternative='two-sided')\n", 221 | " power.append((r,pw))\n", 222 | " lpw.append(pw)\n", 223 | " print(r,pw) " 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": 21, 229 | "metadata": {}, 230 | "outputs": [ 231 | { 232 | "name": "stdout", 233 | "output_type": "stream", 234 | "text": [ 235 | "0.09394562910255964\n" 236 | ] 237 | } 238 | ], 239 | "source": [ 240 | "#Find the max power\n", 241 | "maxpw=np.max(lpw)\n", 242 | "print(maxpw)" 243 | ] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "execution_count": 22, 248 | "metadata": {}, 249 | "outputs": [ 250 | { 251 | "name": "stdout", 252 | "output_type": "stream", 253 | "text": [ 254 | "[1.0]\n" 255 | ] 256 | } 257 | ], 258 | "source": [ 259 | "# Find ratio of power = maxpw\n", 260 | "index_=[power.index(x) for x in power if x[1]==maxpw]\n", 261 | "max_r=[power[i][0] for i in index_]\n", 262 | "print(max_r)" 263 | ] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": 23, 268 | "metadata": {}, 269 | "outputs": [ 270 | { 271 | "data": { 272 | "text/plain": [ 273 | "Text(0, 0.5, 'Power')" 274 | ] 275 | }, 276 | "execution_count": 23, 277 | "metadata": {}, 278 | "output_type": "execute_result" 279 | }, 280 | { 281 | "data": { 282 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA3oElEQVR4nO3deXxU1fn48c+TnYSQsIR9CUpAwg4hICq1Wq3gQsVapSogKKWCW1u/tdbaX2ur1e7UhVIBwQWtO63UpdZaEVkCsoU17DEBAoEkELI/vz/mpo0xkJmQmzuTPO/Xa16ZuffcO88dhjw559xzjqgqxhhjjL/CvA7AGGNMaLHEYYwxJiCWOIwxxgTEEocxxpiAWOIwxhgTEEscxhhjAmKJw5hGICKdROQ/IlIkIr8Vn4UickxEVovIRSKyvYljmioiy2u8PiEi5zRlDKZ5ivA6AGO8IiInaryMBUqBSuf1d1T1hQBONwM4ArRRVRWRi4DLgO6qetIp068BMSYCvwPGA3FALjBfVR8L9Fyq2rrGeZ8FslX1wUDPY4wlDtNi1fpFuhe4TVX/WbuciESoakU9p+sFbNH/jajtBeytkTQa6vf4EkZ/oADoCww8y3Mac1asqcqYWkTkYhHJFpEfishBYKGItBWRv4tIntP89HcR6e6UfxaYAvyf0xz0HeAZ4Hzn9c+qz1njPXqIyOvO+Y6KyBOnCWck8KKqHlPVKlXdpqqv1jiPishdIrJbRI6IyK9FpM7/107ZPiIyA7ipRrx/a4zPzbQcVuMwpm6dgXb4ag5h+JqyFgLfAsKBBcATwDdUdaqIQI2mHxEpxVeDudB5fXH1iUUkHPg78C/gFnzNY2mniWMl8EsRaQssV9WddZS51jm+NfBPYDu+xFUnVZ0nImOwpirTQFbjMKZuVcBPVbVUVU+p6lFVfU1Vi1W1CPgl8JUGnjsd6Arcp6onVbVEVZefpuydwAvAbGCLiGSJyLhaZR5T1XxV3Q/8AZjUwLiM8YslDmPqlqeqJdUvRCRWRP4sIvtEpBD4D5Do1B4C1QPY50e/CU7SekRVRwDtgb8Cr4hIuxrFDtR4vg9fUjLGNZY4jKlb7Wmjv4/vrqhRqtoGGOtslwac+wDQU0QCaipW1ULgEXyd5b1r7OpR43lPIMef0wXy3sbUZInDGP/EA6eA485f+z89i3Otxndb7a9EJE5EYkTkgroKishPRGSkiESJSAxwN3AcXz9Gtfuczvsezv6X/YjhEGBjOkyDWOIwxj9/AFrhG6uxEninoSdS1UrgaqAPsB/IBm44XXF8nfJH8NUkLgOuVNWaY1DeAtYC64G3gfl+hDEfSBWR4yLyZuBXYVoysYWcjAldIqJAiqpmeR2LaTmsxmGMMSYgljiMMcYExJqqjDHGBMRqHMYYYwLSIqYc6dChgyYnJ3sdhjHGhJS1a9ceUdWk2ttbROJITk4mIyPD6zCMMSakiMi+urZbU5UxxpiAWOIwxhgTEEscxhhjAmKJwxhjTEBcTRwicoWIbHfWELi/jv0iInOc/RtFZHiNfXeLyGYRyRSRe+o49gfOimYd3LwGY4wxX+Ra4nDWKXgSGAekApNEJLVWsXFAivOYATztHDsQuB3fgjdDgKtEJKXGuXvgm+xtv1vxG2OMqZubNY50IEtVd6tqGfASMKFWmQnAYvVZiW9hnC5Af2Cls9paBfARvuUxq/0e+D9sTQFjjGlybo7j6MYXVybLBkb5UaYbsBnfOsvt8a2BMB7IABCRa4DPVXWDs85znURkBr5aDD179jyrCzFnr6pKKSwpJ/9kGceKy8g/Wc6xk2XkF5dRUVlFQmwU7WKjaBsbSWJsFG3jImkbG0VMZEMW2DPGuMnNxFHXb/XaNYQ6y6jqVhF5DHgfOAFsACpEJBb4MXB5fW+uqvOAeQBpaWlWM2lipRWVbMouYNWefFbtyWft3nxOllUGfJ6e7WIZ2iORYT0TGdojkdSubYiOsGRijJfcTBzZfHFJy+58eUnL05ZR1fk4C9KIyCNO2XPxLZlZXdvoDqwTkXRVPejCNZgAHCos4dW12SzfeYR1+49RWlEFQN9Orbl2eDfO6dCadnFRtI1zahdxkbSLiyI8TCgoLudYsa9Gcry4jGPF5Rw5UcrW3EJW78ln6QbfVycqPIzUrm0Y1jORr/XvxOhz2hMe1pDVW40xDeVm4lgDpIhIb+Bz4Ebg27XKLAVmi8hL+JqxClQ1F0BEOqrqYRHpCUwEzlfVY0DH6oNFZC+QpqpHXLwOcwaqSsa+YyxasZd3Nh+kokoZ0LUNN43qRXrvdqT3bke7uKh6z9OxTTgd28Scdn9uwSnW7z/O+gPH+ezAcZas3s/CT/aSFB/NlYO6cPWQrgzvmciZmi+NMY3DtcShqhUiMht4FwgHFqhqpojMdPbPBZbh67/IAoqBW2uc4jWnj6McmOUkDRMkTpVV8tb6z1n06T625hbSJiaCqWOSuXl0L5I7xDX6+3VJaEWXQa0YN6jLf9//X9sO87cNOby4ej/PrthLt8RWXD2kK9cM6Ur/LvGWRIxxSYtYjyMtLU1tksPGoaq8tT6HX7y9lSMnSjmvczxTxiQzYWhXYqO8mTOzsKSc9zMPsXRDDsuzjlBZpQztkcjMr5zL5amdCLOmLGMaRETWqmral7Zb4jD+2pV3gp+8uZkVu44ypEciPxp3HqN6twuqv+zzT5bxtw05zF++h/35xZzTIY4ZY8/h2uHdrFPdmABZ4rDE0WAl5ZU8+WEWf/5oN9GRYfzwivOYlN4zqDulKyqr+Mfmg8z9aBeZOYV0jI9m2oW9+faonrSJifQ6PGNCgiUOSxwNsmr3Ue57dSP784u5dlg3Hhjfn6T4aK/D8puq8knWUeZ+tIvlWUeIj47glvN7MfPicy2BGFMPSxyWOAL23Mp9/GxpJt3btuKRawcxpk9oTwu2+fMC5n60i7c35dIuNorvX96PG0b2COqakzFessRhicNv5ZVV/L+lmbywaj9f7ZfEHycNa1Z/nW/KLuDhv29h9d58zuscz0NXpzLm3NBOisa44XSJw6ZVN1+Qf7KMm59ZxQur9jPzK+fyzJSRzSppAAzqnsDL3xnNUzcN50RpBd/+yypmLM5g75GTXodmTEiwGof5r625hdy+OIPDRaU8ft1gvjGsm9chua6kvJL5y/fw1IdZlFVWMe2C3sy6pE+zS5bGNITVOMwZrT9wnOvnfkp5ZRWvfOf8FpE0AGIiw5n11T58+IOLuXZYN+Z9vJvLfvcR/9p2yOvQjAlaljgMmTkFTJ6/inZxUbw56wKG9Ej0OqQm17FNDI9/cwhvzbqAtrFRTHs2g+/9dT0FxeVeh2ZM0LHE0cLtPFTELfNX0zo6ghdvH0WXhFZeh+Spwd0TWTr7Qu66pA9vrc/h8j9Y7cOY2ixxtGB7jpzk28+sIiJMePH20XRvG+t1SEEhKiKM713e7wu1j+//dYPVPoxxWOJooQ7kF3PTX1ZSWaW8cNsoVyYmDHUDuyWwdPaF3HlJH95c/7nVPoxxWOJogfKKSrnpmVWcKK3g+emjSOkU73VIQSsqIozvX96PN++4gMRWvtrHfa9s4GRphdehGeMZSxwtTHllFbNeXMfhohIWTx9Fatc2XocUEgZ1T2DpnRcw+6t9eHVdNhOe/IQdh4q8DssYT1jiaGEeXbaN1Xvy+dXEwQxtgXdPnY3oiHB+8PV+vDB9FMeLy7nmieW8ujbb67CMaXKWOFqQt9Z/zoJP9jB1THKLGafhhjF9OrDs7gsZ2iORH7yygfte2cCpBqynbkyossTRQmw7WMj9r21iZHJbfnxlf6/DCXkd42N44bbR3HVJddPVcrIOW9OVaRkscbQABafKmfncWlrHRPDkt4cTGW7/7I0hPEz43uX9WHRrOkdOlHHNE5/wxmfWdGWaP1d/g4jIFSKyXUSyROT+OvaLiMxx9m8UkeE19t0tIptFJFNE7qmx/WGn7HoReU9Eurp5DaGuqkr53svryT52iqdvGk7HNjFeh9TsjO2bxLK7LmJg1wTufXkDP3p9IyXl1nRlmi/XEoeIhANPAuOAVGCSiKTWKjYOSHEeM4CnnWMHArcD6cAQ4CoRSXGO+bWqDlbVocDfgYfcuobm4NkVe/lg22F+clUqacntvA6n2eqcEMOLt4/iuxefy5LVB/jWnz/lUGGJ12EZ4wo3axzpQJaq7lbVMuAlYEKtMhOAxeqzEkgUkS5Af2ClqharagXwEXAtgKoW1jg+Dmj+0/s20L6jJ3n83W18tV8Sk8/v5XU4zV5EuG9Z3T/fMoKswye45onlbMw+7nVYxjQ6NxNHN+BAjdfZzjZ/ymwGxopIexGJBcYDPaoLicgvReQAcBOnqXGIyAwRyRCRjLy8vLO+mFBTVaX88LWNRIaF8cjEQYjYKndN5esDOvPqzDFEhIVx/dxP+duGHK9DMqZRuZk46vpNVbt2UGcZVd0KPAa8D7wDbAAqahT4sar2AF4AZtf15qo6T1XTVDUtKSmpIfGHtBdX72fl7nx+fGX/Fj9xoRdSu7bhrdkXMKhbAncu+Yzfvb+DqiqrHJvmwc3EkU2NWgLQHaj9p9dpy6jqfFUdrqpjgXxgZx3v8SJwXaNF3ExkHyvm0WVbubBPB24Y2aP+A4wrOrSO5oXbR/HNEd2Z88FOZi9ZZ+M9TLPgZuJYA6SISG8RiQJuBJbWKrMUmOzcXTUaKFDVXAAR6ej87AlMBJY4r1NqHH8NsM3Fawg5qsqPXt+EAo9aE5XnoiPC+fU3B/Pj8f35x+aDXP/nFeQWnPI6LGPOSoRbJ1bVChGZDbwLhAMLVDVTRGY6++cCy/D1X2QBxcCtNU7xmoi0B8qBWap6zNn+KxHpB1QB+4CZbl1DKHolI5uPdx7h4QkD6NHOpkkPBiLC7WPP4dyOcdy1ZD3XPPEJ824ZwbCebb0OzZgGsTXHm5HDRSVc+tuPSO3ShiW3jyYszGobwWbHoSKmL1rDocJSfvetIVw12IYhmeBla463AL97bwcl5ZU8OnGQJY0g1bdTPG/NupAh3X2d5gs/2eN1SMYEzBJHM7E1t5CXMw4w+fxkzklq7XU45gzaxUXx3PRRXJ7aiZ/9bQuP/mOr3XFlQooljmZAVfnl21tpExPJnZf08Toc44eYyHCeumkEN4/uyZ8/2s0PXtlAeWWV12EZ4xfXOsdN0/n39jyWZx3hoatSSYyN8joc46fwMOHhCQPp3CaG37y3g7wTpTx98whaR9t/SxPcrMYR4ioqq/jlsq0kt4/l5tE2rUioERFmX5LCY9cNYsWuo0yat5K8olKvwzLmjCxxhLglaw6QdfgEPxrfn6gI++cMVTeM7MlfJo9g5+Eivjl3BfuOnvQ6JGNOy37ThLDCknJ+//4ORvVux+WpnbwOx5ylS87rxIu3j6bwVDkTn1phEySaoGWJI4Q99eEu8k+W8eCVqTZCvJkY3rMtr353DDGR4dw4byUf7Wh5E3Sa4GeJI0QdLiph4Sd7uHZYNwZ1T/A6HNOIzk1qzRt3jKFX+zhuW7SGdzbneh2SMV9giSNEzf94D+WVVdx1aUr9hU3I6dgmhpdmjGZQtwRmvfiZLUlrgooljhCUf7KM51bu4+ohXendIc7rcIxLElpF8tz0UaQnt+N7f93Ai6v2ex2SMYAljpC08JM9FJdVMvurNtivuYuLjmDhrSO5uG8SD7yxiWc+3u11SMZY4gg1BafKefaTvYwb2JmUTvFeh2OaQExkOH++JY3xgzrzi7e38qcPdtISJic1wcuGqIaY5z7dS1FpBbOsttGiREWEMefGYcREbuS37++guLyS//t6P7ubznjCEkcIOVlawfzle7jkvI4M7GZ3UrU0EeFh/OabQ2gVGc7T/95FcWkFP716gM2EbJqcJY4Q8sKqfRwrLme2TWTYYoWFCb/4xkBio8L5y8e+vq5fXTeYcEsepglZ4ggRJeWVzPvPHi7s04HhtnJciyYiPDC+P3HREfzhnzs5VV7J728YSmS4dVmapmGJI0S8knGAIydKmX3JMK9DMUFARLjna32JjQrnkWXbKK2o4slvD7f5ykyTsG9ZCFBVnl2xlyE9Ehl9TnuvwzFBZMbYc/n5hAG8v+UQs19cR1mFrelh3Odq4hCRK0Rku4hkicj9dewXEZnj7N8oIsNr7LtbRDaLSKaI3FNj+69FZJtT/g0RSXTzGoLB8qwj7Mo7ydQxNm26+bLJ5yfz8wkDeG/LIe5css4WhDKucy1xiEg48CQwDkgFJolIaq1i44AU5zEDeNo5diBwO5AODAGuEpHquTXeBwaq6mBgB/Ajt64hWCxasZcOraMYP6iL16GYIDX5/GR+ds0A3s301TwseRg3uVnjSAeyVHW3qpYBLwETapWZACxWn5VAooh0AfoDK1W1WFUrgI+AawFU9T1nG8BKoLuL1+C5A/nFfLDtMJPSexIdEe51OCaITRmTzP+7OpV3Mw9x54ufWfIwrnEzcXQDDtR4ne1s86fMZmCsiLQXkVhgPNCjjveYBvyjrjcXkRkikiEiGXl5oTs19XMr9xEmwk2jrJnK1G/qBb356dWpvJN5kLuWWPIw7nAzcdR1Y3nteRLqLKOqW4HH8DVLvQNsACq+cKDIj51tL9T15qo6T1XTVDUtKSkp0NiDwqmySl5ec4ArBnSmc0KM1+GYEHHrBb156KpU/rHZkodxh5uJI5sv1hK6Azn+llHV+ao6XFXHAvnAzupCIjIFuAq4SZvxpD1vrv+cglPlTBmT7HUoJsRMu/B/yePulyx5mMblZuJYA6SISG8RiQJuBJbWKrMUmOzcXTUaKFDVXAAR6ej87AlMBJY4r68Afghco6rFLsbvKVVl0Yq99O/ShpHJNuDPBG7ahb35yVWpLNt0kHteWm/JwzQa1wYAqmqFiMwG3gXCgQWqmikiM539c4Fl+PovsoBi4NYap3hNRNoD5cAsVT3mbH8CiAbedyZ4W6mqM926Dq+s3pPPtoNF/GriIJvIzjTY9At7o6r84u2tAPzxxqFE2Ahzc5ZcHTmuqsvwJYea2+bWeK7ArNMce9FptreIiZoWfbqXhFaRTBha+34CYwJz20XnAPiSh8Afb7DkYc6OTTkShI6cKOW9zENMHZNMqyi7BdecvdsuOgdV+OWyrUSHh/Gb64fYrLqmwSxxBKG31udQUaVcn1bXHcjGNMztY8+hpLyS376/g1ZR4fziGwOtGdQ0iCWOIPTq2mwGd0+gX2db4c80rtmX9OFUeSVP/XsXMZHhPHhlf0seJmCWOIJMZk4BW3MLeXjCAK9DMc2QiHDf1/tRXFbJ/OV7iI0K5/uX9/M6LBNiLHEEmVfXZhMVHsbVQ7p6HYpppkSEn16dSkl5JX/6VxatosK54+IWcc+JaSSWOIJIWUUVb63P4bLUTiTGRnkdjmnGRIRfXjuIU+WVPP7OdlpFhnPrBb29DsuECEscQeTD7YfJP1nGN0c063kbTZAIDxN+e/0QSsur+NnfttAqMpwb03t6HZYJAXYzdxB5dW02SfHRXJTSwetQTAsRER7GnEnDuLhfEj96YxNvfva51yGZEGCJI0gcOVHKh9sOM3FYNxucZZpUVEQYc28eweje7fn+Kxt4Z3Ou1yGZIGe/oYLEm599TkWVWjOV8URMZDjPTEljSPcE7lzyGR9uO+x1SCaIWeIIAqrKq2uzGdIjkZRONnbDeCMuOoKFt6bTr3M8M59fy4qsI16HZIKUJY4gsO1gEdsOFvHN4TYvlfFWQqtIFk8bRa/2sdy2OIPP9h+r/yDT4ljiCAJvb8wlPExsTXETFNrFRfH89FF0aB3N1IVr2HGoyOuQTJCxxOExVeXtTbmcf0572reO9jocYwDo2CaG56ePIjoijFvmr+JAfrNd+sY0gCUOj23JLWTPkZNcOdhqGya49Gwfy3PTR1FSXsXN81dxuKjE65BMkLDE4bFlm3zNVF8f0NnrUIz5kn6d41l460jyikqZPH81BcXlXodkgoAlDg+pKm9vzGXMue1pF2dTjJjgNLxnW+bdksbuvJNMW7SG4rIKr0MyHrPE4aHMnEL2Hi3mSusUN0HuwpQOzJk0lM/2H2Pm8+soq7D1y1sySxweqm6mutyaqUwIuGJgFx6dOIj/7Mjj3r+up7JKvQ7JeMTVxCEiV4jIdhHJEpH769gvIjLH2b9RRIbX2He3iGwWkUwRuafG9uudbVUikuZm/G6qvpvKmqlMKLlhZE8eGH8eb2/M5cE3N6NqyaMlci1xiEg48CQwDkgFJolIaq1i44AU5zEDeNo5diBwO5AODAGuEpEU55jNwETgP27F3hQycwrZZ81UJgTNGHsud1x8LktW7+fxd7d7HY7xgJvTqqcDWaq6G0BEXgImAFtqlJkALFbfny0rRSRRRLoA/YGVqlrsHPsRcC3wuKpudba5GLr73ra7qUwIu+/r/Th+qpyn/72LhFaRzPzKuV6HZJqQm01V3YADNV5nO9v8KbMZGCsi7UUkFhgP9AjkzUVkhohkiEhGXl5ewMG7qfpuqgv6dKCtNVOZECQiPDxhIFcN7sKv/rGNl1bv9zok04TcTBx1VQlqN4jWWcapVTwGvA+8A2wAAroHUFXnqWqaqqYlJSUFcqjrMnMK2Z9fzJWDrLZhQld4mPC7bw3lK32TeOCNTfxjk03H3lK4mTiy+WItoTuQ428ZVZ2vqsNVdSyQD+x0MdYm9d6WQ4QJXJZqicOEtuq1PIb2SOTul9azYpfNqNsSuJk41gApItJbRKKAG4GltcosBSY7d1eNBgpUNRdARDo6P3vi6wxf4mKsTeqDrYcY3rOt3U1lmoVWUeEsmDqSXu1jmbF4LZs/L/A6JOMy1xKHqlYAs4F3ga3AX1U1U0RmishMp9gyYDeQBfwFuKPGKV4TkS3A34BZqnoMQESuFZFs4HzgbRF5161rcENuwSkycwq5tH8nr0MxptEkxkaxeHo6bWIimLpwDfuOnvQ6JOMiqe8+bBEJAzaq6sCmCanxpaWlaUZGhtdhAPD8yn08+OZm3r93rC3aZJqdrMNFfHPupyS0iuTVmWNIircZn0OZiKxV1S+Nl6u3xqGqVcAGp8nInKV/bTtMz3ax9OnY2utQjGl0fTrGs3DqSA4XljJ14WqKSmxSxObI36aqLkCmiHwgIkurH24G1hydKqvkk6wjXNq/Y8iPQzHmdIb1bMtTNw9n+8EivvPcWkorKr0OyTQyfwcA/szVKFqI5VlHKK2o4mvWv2Gaua/268ivrx/MvS9v4N6X1/OnScMJD7M/lpoLvxKHqn4kIr2AFFX9pzMoL9zd0JqfD7YeIj46gpHJ7bwOxRjXXTusO0dPlPGLt7fSPi6Tn08YYDXtZsKvxCEit+ObS6odcC6+0d1zgUvdC615qapSPth2mLF9k4iKsEmJTctw20XnkFdUyp//s5sOraO5+2sp9R9kgp6/TVWz8M09tQpAVXdWj7Mw/tn0eQF5RaVc2t8+NtOy3D/uPI6cKOP3/9xBh/gobhrVy+uQzFnyN3GUqmpZdTVTRCL48vQh5gw+2HaYMPG1/RrTkogIv7puEMeKy/jJm5tpHxfFFQNtVuhQ5m+byUci8gDQSkQuA17BNzDP+OmDrYcY0autTWpoWqTI8DCe/PZwhvZI5K4l6/l011GvQzJnwd/EcT+QB2wCvoNvxPeDbgXV3NhocWNqT02SQWaOTU0SqvxNHBcDL6jq9ar6TVX9i9rSX377aLtvWvdLzrNmKtOyVU9NEu9MTXIgv9jrkEwD+Js4pgLrReRTEXlcRK4WkbYuxtWsfLzzCJ3bxJBio8WNoUtCK56dlk5peSVTFq7m2Mkyr0MyAfIrcajqZFXtC1yHbyr0J/E1XZl6VFYpn+w6woUpHewedmMcfTvFM3/qSLKPnWLaojWcKrPR5aHEr8QhIjeLyJ+BV4GvAU8AF7kZWHORmVPA8eJyLkrp4HUoxgSVkcntmHPjUNYfOM7sF9dRUVnldUjGT/42Vf0BGIpv6vO7VPVxVf3UraCak493+ha2uaCPJQ5jartiYBd+fs0APth2mAff3Ix1nYYGf6cc6SAiA4CxwC9FJAXYrqq3uBpdM/DxzjxSu7ShQ2ubXtqYutxyfjIHC0t48sNddGoTw72X9fU6JFMPf6ccaQP0BHoByUACYPXKehSXVbB23zGmXdjb61CMCWo/uLwfhwpL+eMHO+nUJoZvj7JVHIKZvyPHl9d4PKGq2e6F1Hys2p1PeaVyUZ8kr0MxJqiJCI9OHMSRE6U8+OYmkuKjuSzVxj0FK3/vqhqsqnfgGy1+3NWImpGPdx4hOiKMtGS7c9mY+lSPLh/YLYE7l6xj7b5jXodkTsPfu6oGishnwGZgi4isFZF6l5IVkStEZLuIZInI/XXsFxGZ4+zfKCLDa+y7W0Q2i0imiNxTY3s7EXlfRHY6P4P2t/LyrDzSe7cjJtJmoDfGH3HRESyYOpLObWKYvmgNWYdPeB2SqYO/d1XNA76nqr1UtSfwfWfbaYlIOL7xHuOAVGCSiKTWKjYOSHEeM4CnnWMHArfjm5F3CHCV0yEPvulPPlDVFOAD53XQOVhQwo5DJ+w2XGMC1KF1NIumpRMRJkxZsJpDhSVeh2Rq8TdxxKnqh9UvVPXfQFw9x6QDWaq6W1XLgJeACbXKTAAWq89KIFFEugD9gZWqWqyqFcBHwLU1jlnkPF8EfMPPa2hSy7N8t+FeaP0bxgSsV/s4FkwdybHiMqYuXEOhrV0eVPxNHLtF5Ccikuw8HgT21HNMN+BAjdfZzjZ/ymwGxopIe2e1wfFAD6dMJ1XNBXB+1jkBlIjMEJEMEcnIy2v6Qe4f78yjQ+tozusc3+TvbUxzMLh7Ik/fPIKdh4qYaWuXBxV/E8c0IAl43Xl0AG6t55i65teoPbqnzjKquhV4DHgfeAfYAFT4GWv1SeapapqqpiUlNe1f/VVVyidZR7iwT3vCbJ1lYxrsK32TeOy6wazYdZQfvLKRqiobIBgMzng7rojEADOBPvimVP++qvpbZ8zmf7UEgO5Ajr9lVHU+MN+J4xGnLMAhEemiqrlOs9ZhP+NpMtsPFXHkRBkXplgzlTFn67oR3TlUVMLj72ync5tofnxl7a5S09Tqq3EsAtLwJY1xwK8DOPcaIEVEeotIFHAjsLRWmaXAZOfuqtFAQXUzVPXStCLSE5gILKlxzBTn+RTgrQBiahIrd/sWqTn/3PYeR2JM8/Ddr5zLlPN78ZeP97BgeX2t5MZt9Q0ATFXVQQAiMh9Y7e+JVbVCRGYD7wLhwAJVzRSRmc7+ufgWhBoPZAHFfLH56zURaQ+UA7NUtfqm7l8BfxWR6cB+4Hp/Y2oqq3bn071tK7oltvI6FGOaBRHhoasHcLCwhIff3kLnhBjGD7LlZ71SX+L4b7OUkwgCOrmqLsOXHGpum1vjuQKzTnNsnbPvqupR4NKAAmlCqsrqvfm2trgxjSw8TPjjjcO46ZlV3PPyejq0jia9dzuvw2qR6muqGiIihc6jCBhc/VxECpsiwFCTdfgE+SfLGHWOfaGNaWwxkeE8MzmN7m1bcduiNew8VOR1SC3SGROHqoarahvnEa+qETWet2mqIEPJyj35AIzubf0bxrihbVwUi25NJyoinKkL19gAQQ/4ezuu8dOq3UfpkhBDj3bWv2GMW3q0i+XZW0dyvLiMKQtWU2QDBJuUJY5GpKqs2pPPqN7tbJlYY1w2sFsCT908gqzDJ5j5/FrKKmylh6ZiiaMR7TlykryiUkadY81UxjSFr/RN4tGJg/gk6yg/fG2jrSDYRPxdj8P4YZXTvzHK7vQwpslcn9aDgwUl/Pb9HXROiOGHV5zndUjNniWORrRq91GS4qPp3aG++R+NMY1p9iV9yCko4el/76JrQgy3nJ/sdUjNmiWORmL9G8Z4R0R4eMIA8opKeGhpJh3bxPD1AZ29DqvZsj6ORnIg/xS5BSXWTGWMRyLCw5gzaRiDuydy15LPWLsv3+uQmi1LHI1k5R7f/FTWMW6Md2KjIlgwJY0uCTFMX5TBrjxbQdANljgayard+bSLiyKlY2uvQzGmRWvvrCAYLr4VBA8X2QDBxmaJo5Gs3nuU9GTr3zAmGFSvIHj0RBnTnl3DidKAlvMx9bDE0QgOFZZwIP8UI61/w5igMaRHIk/eNIytuUXMemEd5ZU2QLCxWOJoBOv2+WZ8H94z0dtAjDFfcMl5nfjFNwby0Y48fvzGJhsg2EjsdtxGsG7/MaIiwhjQNcHrUIwxtUxK70nu8VPM+VcWXRJace9lfb0OKeRZ4mgEa/cdY3C3BKIirAJnTDC697K+5BSU8McPdtI1MYYbRvb0OqSQZr/pzlJpRSWbPy9keK+2XodijDkNEeHRiYO4KKUDD7yxmQ+3H/Y6pJBmieMsbf68kLLKKob3tMRhTDCLDA/j6ZtH0K9TPLNeWMem7AKvQwpZljjO0mf7nY7xXoneBmKMqVfr6AievXUkbWOjuPXZNRzIL/Y6pJDkauIQkStEZLuIZInI/XXsFxGZ4+zfKCLDa+y7V0QyRWSziCwRkRhn+xAR+VRENonI30TE05UI1+47Ro92regYH+NlGMYYP3VsE8OiaSMpr6xiyoLVHDtZ5nVIIce1xCEi4cCTwDggFZgkIqm1io0DUpzHDOBp59huwF1AmqoOBMKBG51jngHuV9VBwBvAfW5dQ31UlXX7j1kzlTEhpk/HeJ6Zkkb28VPctjiDkvJKr0MKKW7WONKBLFXdraplwEvAhFplJgCL1WclkCgiXZx9EUArEYkAYoEcZ3s/4D/O8/eB61y8hjP6/PgpDhWWWuIwJgSNTG7HH24Yyrr9x7j7pc+orLIxHv5yM3F0Aw7UeJ3tbKu3jKp+DvwG2A/kAgWq+p5TZjNwjfP8eqBHXW8uIjNEJENEMvLy8s7qQk5n3f7jAIywO6qMCUnjB3XhwStTeTfzEA//fYsNEPSTm4mjrkmbav+r1FlGRNriq430BroCcSJys7N/GjBLRNYC8UCdDZSqOk9V01Q1LSkpqUEXUJ91+47RKjKc8zrHu3J+Y4z7pl/Ym+kX9ubZFXv5y8e7vQ4nJLg5ADCbL9YGuvO/5qb6ynwN2KOqeQAi8jowBnheVbcBlzvb+wJXuhK9H9btP8aQHglEhNvNacaEsh+P78/BghIeWbaNzgmtuGZIV69DCmpu/sZbA6SISG8RicLXub20VpmlwGTn7qrR+JqkcvE1UY0WkVjxTTd7KbAVQEQ6Oj/DgAeBuS5ew2mdKqtkS06h9W8Y0wyEhQm//dYQ0pPb8YO/buDTXUe9DimouZY4VLUCmA28i++X/l9VNVNEZorITKfYMmA3kAX8BbjDOXYV8CqwDtjkxDnPOWaSiOwAtuGrnSx06xrOZGP2cSqq1BKHMc1ETGQ48yaPoGf7WGY8l8H2g0VehxS0pCV0BqWlpWlGRkajnvPpf+/isXe2se4nl9EuLqpRz22M8U72sWKufWoFEWHCG3dcQOeEljtGS0TWqmpa7e3WON9A6w8cI7l9rCUNY5qZ7m1jefbWkRSeKmfqwtUUlpR7HVLQscTRQBuzCxjcPdHrMIwxLhjQNYG5t4wg6/AJZj63lrIKWwSqJkscDXC4qITcghIGd7f1N4xpri5KSeKx6wazYtdR/u/VDVTZAMH/svU4GqB6Vk2rcRjTvF03oju5Baf4zXs76JzQivvHned1SEHBEkcDbMguIExgQFdP51c0xjSBWV/tQ05BCXM/2kXXxBgmn5/sdUies8TRAJuyj9OnY2viou3jM6a5ExF+fs0ADheW8NOlmXSMj+GKgZ29DstT1scRIFW1jnFjWpiI8DD+NGk4Q7oncvdLn7F2X77XIXnKEkeAcgpKOHqyzDrGjWlhWkWFM39KGl0SYpi+KINdeSe8DskzljgCtPHAccA6xo1pidq3jmbRtHTCRZiyYDWHi0q8DskTljgCtPHzAiLCxGbENaaF6tU+jgVTR3L0RBnTnl3DydIKr0NqcpY4ArQx+zjndYknJjLc61CMMR4Z0iORJ28axpacQu54YR3llS1rgKAljgBUd4wP6pbodSjGGI9dcl4nfnntID7akccDr29qUYtA2f2kAdh7tJiikgqGWMe4MQaYlN6T3OOnmPOvLLoktuJ7l/X1OqQmYYkjABuzjwMwyBKHMcZx72V9yS0oYc4HO+mSEMOk9J5eh+Q6SxwB2JhdQHREGH07Wce4McZHRHhk4iAOFZXy4Jub6dQmmkvO6+R1WK6yPo4AbMouYEDXNkTaUrHGmBoiw8N46qbh9O8Sz6wXPmODc9t+c2W/Af1UWaVszrER48aYurWOjmDB1JG0bx3FtGfXsO/oSa9Dco0lDj/tzjtBcVklg7pZ/4Yxpm4d42NYNC2dSlWmLFjN0ROlXofkClcTh4hcISLbRSRLRO6vY7+IyBxn/0YRGV5j370ikikim0VkiYjEONuHishKEVkvIhkiku7mNVTLzCkEYKAlDmPMGZyb1Jr5U9LILShh2rNrKC5rfgMEXUscIhIOPAmMA1KBSSKSWqvYOCDFecwAnnaO7QbcBaSp6kAgHLjROeZx4GeqOhR4yHntui25hURFhHFOUlxTvJ0xJoSN6NWOP00axqbPC5j1wjoqmtkAQTdrHOlAlqruVtUy4CVgQq0yE4DF6rMSSBSRLs6+CKCViEQAsUCOs12B6oUwEmpsd9WWnEL6dYq3jnFjjF8uH9CZh78xkA+35/HjNzY3qwGCbt6O2w04UON1NjDKjzLdVDVDRH4D7AdOAe+p6ntOmXuAd539YcAYF2L/AlUlM6eAy1Nb9hz8xpjA3DSqFwcLSvjTv7LolBDTbAYIuvnns9SxrXbKrbOMiLTFVxvpDXQF4kTkZmf/d4F7VbUHcC8wv843F5nh9IFk5OXlNegCqh0sLOFYcTkDutmKf8aYwHzvsr5cP6I7cz7YyYur9nsdTqNwM3FkAz1qvO7Ol5uVTlfma8AeVc1T1XLgdf5Xs5jivAZ4BV+T2Jeo6jxVTVPVtKSkpLO6kC1Ox3hqF0scxpjAVA8QvLhfEg++uYn3txzyOqSz5mbiWAOkiEhvEYnC17m9tFaZpcBk5+6q0UCBqubia6IaLSKxIiLApcBW55gc4CvO80uAnS5eA+BLHCJwniUOY0wDVA8QHNQtgTuXrGPtvmNeh3RWXEscqloBzAbexfdL/6+qmikiM0VkplNsGbAbyAL+AtzhHLsKeBVYB2xy4pznHHM78FsR2QA8gu9uLFdl5hSS3D6O1rbGuDGmgWKjIpg/dSSd2sRw26I1Ib2CoDSnnv7TSUtL04yMjAYfP/bxDxnULYEnbxpef2FjjDmDfUdPMvGpFbSKCuf1746hY5sYr0M6LRFZq6pptbfbvaX1KCwpZ39+MaldrZnKGHP2erWPY+GtI8k/WcbUhWsoKin3OqSAWeKox9bqjnFLHMaYRjK4eyJP3jSc7YeK+O7z6yirCK0BgpY46rEl15c4BljHuDGmEX21X0d+NXEQy7OO8MPXNlJVFTrdBtbbW48tOYV0aB1FUny016EYY5qZ69N6cKiwhN+8t4NObWK4f9x5XofkF0sc9diSW0hq1wR8dwUbY0zjmvXVPhwsLGHuR7vo3CaaqRf09jqkellT1RmUVVSx41CRDfwzxrhGRPjZNQO5PLUTP/v7FpZtyvU6pHpZ4jiDrMMnKK9U6xg3xrgqPEyYM2kYw3u25Z6X17Nq91GvQzojSxxn8N+OcUscxhiXxUSG88zkNHq0bcXtizPYcajI65BOyxLHGWTmFNAqMpzk9rYGhzHGfW3jolg0LZ2YyHCmLFhNzvFTXodUJ0scZ/D1AZ154Mr+hIdZx7gxpml0bxvLs7emc6KkgskLVnO8uMzrkL7EEscZjD6nPbeM7uV1GMaYFia1axvmTU5j/9Fipi/K4FRZpdchfYElDmOMCULnn9ueP9w4lHX7j3HnkuBaftYShzHGBKnxg7rw82sG8M+th3ngjU1Bs/ysDQA0xpggdsv5yeQVlTLnX1l0jI/hB1/v53VIljiMMSbY3XtZX/JOlPLEh1kkxUczZUyyp/FY4jDGmCAnIjw8YSB5RWX8v79l0r51FFcN7upZPNbHYYwxISAiPIwnvj2MtF5t+d7LG1ix64hnsVjiMMaYEOEbXT6S5A6xzFi8lsycAk/isMRhjDEhJCE2kkXT0mkTE8HUhWvYf7S4yWNwNXGIyBUisl1EskTk/jr2i4jMcfZvFJHhNfbdKyKZIrJZRJaISIyz/WURWe889orIejevwRhjgk2XhFYsnp5OWUUVkxes4siJ0iZ9f9cSh4iEA08C44BUYJKIpNYqNg5IcR4zgKedY7sBdwFpqjoQCAduBFDVG1R1qKoOBV4DXnfrGowxJlj16RjPgqkjOVhYwrRn13CytKLJ3tvNGkc6kKWqu1W1DHgJmFCrzARgsfqsBBJFpIuzLwJoJSIRQCyQU/NA8a2s9C1giYvXYIwxQWtEr7Y8+e3hZOYUMvP5tU22drmbiaMbcKDG62xnW71lVPVz4DfAfiAXKFDV92odexFwSFV31vXmIjJDRDJEJCMvL+8sLsMYY4LXpf078ejEQXy88wj3vbqhSdYudzNx1DWlbO0rqrOMiLTFVxvpDXQF4kTk5lrlJnGG2oaqzlPVNFVNS0pKCiBsY4wJLd9K68F9X+/HW+tzeGTZVtffz80BgNlAjxqvu1OruekMZb4G7FHVPAAReR0YAzzvvI4AJgIjXIncGGNCzB0Xn0teUSnPLN9DxzbRzBh7rmvv5WaNYw2QIiK9RSQKX+f20lpllgKTnburRuNrksrF10Q1WkRinb6MS4GaafRrwDZVzXYxfmOMCRkiwkNXpXLl4C48smwbr69z79ejazUOVa0QkdnAu/juilqgqpkiMtPZPxdYBowHsoBi4FZn3yoReRVYB1QAnwHzapz+RqxT3BhjviAsTPjdt4Zw7GQZ//fqRtrFRXFxv46N/j4SLNP0uiktLU0zMjK8DsMYY5pEUUk5N/x5JXuOnGTJjNEM7ZHYoPOIyFpVTau93UaOG2NMMxMfE8mz00aSltyWxFaRjX5+mx3XGGOaoY7xMTw3fZQr57YahzHGmIBY4jDGGBMQSxzGGGMCYonDGGNMQCxxGGOMCYglDmOMMQGxxGGMMSYgljiMMcYEpEVMOSIiecA+l07fATji0rkbS7DHGOzxQfDHaPGdvWCP0Yv4eqnql9alaBGJw00iklHXXC7BJNhjDPb4IPhjtPjOXrDHGEzxWVOVMcaYgFjiMMYYExBLHGdvXv1FPBfsMQZ7fBD8MVp8Zy/YYwya+KyPwxhjTECsxmGMMSYgljiMMcYExBLHGYjIFSKyXUSyROT+OvZfLCIFIrLeeTzk77FNFN99NWLbLCKVItLO2bdXRDY5+1xZV1dEFojIYRHZfJr9IiJznPg3ishwf6+tieK7yYlro4isEJEhNfa5/vn5GaPX38H64vP6O9hDRD4Uka0ikikid9dRxuvvoT8xev5d/AJVtUcdDyAc2AWcA0QBG4DUWmUuBv7ekGObIr5a5a8G/lXj9V6gg8uf4VhgOLD5NPvHA/8ABBgNrGqqz8/P+MYAbZ3n46rja6rPz88YPfsO+hNfEHwHuwDDnefxwI46/h97/T30J0bPv4s1H1bjOL10IEtVd6tqGfASMKEJjnUrvknAkkaO4YxU9T9A/hmKTAAWq89KIFFEutA0n1+98anqClU95rxcCXRv7Bjq48dneDpB8RnW4sV3MFdV1znPi4CtQLdaxbz+HtYbYzB8F2uyxHF63YADNV5n8+UvHMD5IrJBRP4hIgMCPLYp4kNEYoErgNdqbFbgPRFZKyIzGjk2f53uGpri8wvUdHx/lVYLhs+vmlffQb8Fw3dQRJKBYcCqWruC5nt4hhhr8vy7GNEUbxKipI5tte9dXodvLpcTIjIeeBNI8fPYsxXIe1wNfKKqNf8yvEBVc0SkI/C+iGxz/npsSqe7hqb4/PwmIl/F95/1whqbg+HzA2+/g4Hw9DsoIq3xJa17VLWw9u46Dmny72E9MVaXCYrvotU4Ti8b6FHjdXcgp2YBVS1U1RPO82VApIh08OfYpoivhhup1USgqjnOz8PAG/iq5U3tdNfQFJ+fX0RkMPAMMEFVj1ZvD5LPz+vvYCA8+w6KSCS+X8gvqOrrdRTx/HvoR4zB9V1syg6VUHrgq43tBnrzv46xAbXKdOZ/gyjTgf34/kqp99imiM8pl4CvDTquxrY4IL7G8xXAFS59jsmcvmP3Sr7YKbk6kGtrgvh6AlnAmFrbm+zz8yNGz76D/sTn9XfQ+SwWA384QxlPv4d+xhgU38XqhzVVnYaqVojIbOBdfHdXLFDVTBGZ6eyfC3wT+K6IVACngBvV9y9Y57EexAdwLfCeqp6scXgn4A0RAd9/jhdV9Z3GjA9ARJbgu+ung4hkAz8FImvEtwzfHS1ZQDFw65muzYP4HgLaA085n1WF+mYnbZLPz88YPfsO+hkfePgdBC4AbgE2ich6Z9sD+H4RB8X30M8YPf8u1mRTjhhjjAmI9XEYY4wJiCUOY4wxAbHEYYwxJiCWOIwxxgTEEocxxpiAWOIwppE5M8BWzwb7NxFJrKf8UGfUd/Xra9yaidWYxmC34xrTyETkhKq2dp4vAnao6i/PUH4qkKaqs5soRGPOitU4jHHXpzgT44lIurOWwmfOz34iEgX8HLjBqaXcICJTReQJ55heIvKBsw7DByLS08NrMQawxGGMa0QkHLgUWOps2gaMVdVh+EYCP6K+6bofAl5W1aGq+nKt0zyBb8rvwcALwJymid6Y07MpR4xpfK2cqSOSgbXA+872BGCRiKTgm2U10o9znQ9MdJ4/BzzeqJEa0wBW4zCm8Z1S1aFAL3yT481ytj8MfKiqA/FNMx7TgHNbp6TxnCUOY1yiqgXAXcAPnGmzE4DPnd1TaxQtwrdkaF1W4JuSHOAmYHnjR2pMYCxxGOMiVf0M33TcN+JrZnpURD7BN9tqtQ+B1OrO8VqnuAu4VUQ24ptB9e4mCNuYM7LbcY0xxgTEahzGGGMCYonDGGNMQCxxGGOMCYglDmOMMQGxxGGMMSYgljiMMcYExBKHMcaYgPx/phbvHqRkmuoAAAAASUVORK5CYII=\n", 283 | "text/plain": [ 284 | "
" 285 | ] 286 | }, 287 | "metadata": { 288 | "needs_background": "light" 289 | }, 290 | "output_type": "display_data" 291 | } 292 | ], 293 | "source": [ 294 | "l_y=[x[1] for x in power]\n", 295 | "s_x=[x[0] for x in power]\n", 296 | "\n", 297 | "plt.plot(s_x,l_y)\n", 298 | "plt.title('Traffic Split')\n", 299 | "plt.xlabel('Ratio')\n", 300 | "plt.ylabel('Power')" 301 | ] 302 | }, 303 | { 304 | "cell_type": "code", 305 | "execution_count": null, 306 | "metadata": {}, 307 | "outputs": [], 308 | "source": [] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "execution_count": null, 313 | "metadata": {}, 314 | "outputs": [], 315 | "source": [] 316 | } 317 | ], 318 | "metadata": { 319 | "kernelspec": { 320 | "display_name": "Python 3", 321 | "language": "python", 322 | "name": "python3" 323 | }, 324 | "language_info": { 325 | "codemirror_mode": { 326 | "name": "ipython", 327 | "version": 3 328 | }, 329 | "file_extension": ".py", 330 | "mimetype": "text/x-python", 331 | "name": "python", 332 | "nbconvert_exporter": "python", 333 | "pygments_lexer": "ipython3", 334 | "version": "3.7.6" 335 | } 336 | }, 337 | "nbformat": 4, 338 | "nbformat_minor": 4 339 | } 340 | -------------------------------------------------------------------------------- /Class Exercises/L6/cluster.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Regression with Clustered Standard Errors" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 16, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import numpy as np, statsmodels.stats.api as sms\n", 17 | "import itertools\n", 18 | "import matplotlib.pyplot as plt\n", 19 | "from matplotlib.ticker import NullFormatter\n", 20 | "import pandas as pd\n", 21 | "import numpy as np\n", 22 | "import matplotlib.ticker as ticker\n", 23 | "import random\n", 24 | "from sklearn import preprocessing" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 17, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "infile = 'exp_data_cluster.csv' \n", 34 | "data = pd.read_csv(infile)" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 18, 40 | "metadata": {}, 41 | "outputs": [ 42 | { 43 | "data": { 44 | "text/plain": [ 45 | "(16000, 4)" 46 | ] 47 | }, 48 | "execution_count": 18, 49 | "metadata": {}, 50 | "output_type": "execute_result" 51 | } 52 | ], 53 | "source": [ 54 | "data.shape" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 19, 60 | "metadata": {}, 61 | "outputs": [ 62 | { 63 | "data": { 64 | "text/html": [ 65 | "
\n", 66 | "\n", 79 | "\n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | "
useradidexpidif_click
0920921e4fb0d22340fdf9ee02b6ae4a7d9a8310
187850197fc187415fe1785d5ee02bfd348968e00
22663670b0fa14b56d3741178196daaa92e6a1e10
33293191f1fe825014d9e9a0881233d9950bd4310
4646725b2ae85128137e449eb015f6de78add500
\n", 127 | "
" 128 | ], 129 | "text/plain": [ 130 | " user adid expid if_click\n", 131 | "0 92092 1e4fb0d22340fdf9ee02b6ae4a7d9a83 1 0\n", 132 | "1 878501 97fc187415fe1785d5ee02bfd348968e 0 0\n", 133 | "2 266367 0b0fa14b56d3741178196daaa92e6a1e 1 0\n", 134 | "3 329319 1f1fe825014d9e9a0881233d9950bd43 1 0\n", 135 | "4 64672 5b2ae85128137e449eb015f6de78add5 0 0" 136 | ] 137 | }, 138 | "execution_count": 19, 139 | "metadata": {}, 140 | "output_type": "execute_result" 141 | } 142 | ], 143 | "source": [ 144 | "data.head()" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": 20, 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "import statsmodels.formula.api as smf" 154 | ] 155 | }, 156 | { 157 | "cell_type": "markdown", 158 | "metadata": {}, 159 | "source": [ 160 | "#### OLS with Clustered SE" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 24, 166 | "metadata": {}, 167 | "outputs": [ 168 | { 169 | "name": "stdout", 170 | "output_type": "stream", 171 | "text": [ 172 | "Result with cluster\n", 173 | " Results: Ordinary least squares\n", 174 | "==================================================================\n", 175 | "Model: OLS Adj. R-squared: 0.000 \n", 176 | "Dependent Variable: if_click AIC: 1467.5819\n", 177 | "Date: 2023-02-06 20:39 BIC: 1482.9425\n", 178 | "No. Observations: 16000 Log-Likelihood: -731.79 \n", 179 | "Df Model: 1 F-statistic: 6.710 \n", 180 | "Df Residuals: 15998 Prob (F-statistic): 0.00960 \n", 181 | "R-squared: 0.000 Scale: 0.064166 \n", 182 | "--------------------------------------------------------------------\n", 183 | " Coef. Std.Err. z P>|z| [0.025 0.975]\n", 184 | "--------------------------------------------------------------------\n", 185 | "Intercept 0.0638 0.0027 23.3379 0.0000 0.0584 0.0691\n", 186 | "expid 0.0104 0.0040 2.5904 0.0096 0.0025 0.0182\n", 187 | "------------------------------------------------------------------\n", 188 | "Omnibus: 10828.253 Durbin-Watson: 2.012 \n", 189 | "Prob(Omnibus): 0.000 Jarque-Bera (JB): 91921.719\n", 190 | "Skew: 3.401 Prob(JB): 0.000 \n", 191 | "Kurtosis: 12.572 Condition No.: 3 \n", 192 | "==================================================================\n", 193 | "\n" 194 | ] 195 | } 196 | ], 197 | "source": [ 198 | "# users' behaviors of the same ads are correlated.\n", 199 | "model = smf.ols(formula='if_click ~ expid', data=data)\n", 200 | "result = model.fit(cov_type='cluster', cov_kwds = {'groups': data.user})\n", 201 | "print('Result with cluster')\n", 202 | "print(result.summary2())" 203 | ] 204 | }, 205 | { 206 | "cell_type": "markdown", 207 | "metadata": {}, 208 | "source": [ 209 | "#### OLS without Clustered SE" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": 23, 215 | "metadata": {}, 216 | "outputs": [ 217 | { 218 | "name": "stdout", 219 | "output_type": "stream", 220 | "text": [ 221 | "Result without cluster\n", 222 | " Results: Ordinary least squares\n", 223 | "==================================================================\n", 224 | "Model: OLS Adj. R-squared: 0.000 \n", 225 | "Dependent Variable: if_click AIC: 1467.5819\n", 226 | "Date: 2023-02-06 20:39 BIC: 1482.9425\n", 227 | "No. Observations: 16000 Log-Likelihood: -731.79 \n", 228 | "Df Model: 1 F-statistic: 6.710 \n", 229 | "Df Residuals: 15998 Prob (F-statistic): 0.00960 \n", 230 | "R-squared: 0.000 Scale: 0.064166 \n", 231 | "--------------------------------------------------------------------\n", 232 | " Coef. Std.Err. t P>|t| [0.025 0.975]\n", 233 | "--------------------------------------------------------------------\n", 234 | "Intercept 0.0638 0.0028 22.5098 0.0000 0.0582 0.0693\n", 235 | "expid 0.0104 0.0040 2.5904 0.0096 0.0025 0.0182\n", 236 | "------------------------------------------------------------------\n", 237 | "Omnibus: 10828.253 Durbin-Watson: 2.012 \n", 238 | "Prob(Omnibus): 0.000 Jarque-Bera (JB): 91921.719\n", 239 | "Skew: 3.401 Prob(JB): 0.000 \n", 240 | "Kurtosis: 12.572 Condition No.: 3 \n", 241 | "==================================================================\n", 242 | "\n" 243 | ] 244 | } 245 | ], 246 | "source": [ 247 | "result = model.fit()\n", 248 | "print('Result without cluster')\n", 249 | "print(result.summary2())" 250 | ] 251 | }, 252 | { 253 | "cell_type": "markdown", 254 | "metadata": {}, 255 | "source": [ 256 | "#### Compare Means with t tests" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": 7, 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [ 265 | "y0 = data[data['expid'] == 0]['if_click']" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": 8, 271 | "metadata": {}, 272 | "outputs": [ 273 | { 274 | "name": "stdout", 275 | "output_type": "stream", 276 | "text": [ 277 | "0.010374999999999995\n" 278 | ] 279 | } 280 | ], 281 | "source": [ 282 | "y1 = data[data['expid'] == 1]['if_click']\n", 283 | "mean_d = np.mean(y1)-np.mean(y0)\n", 284 | "print(mean_d)" 285 | ] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "execution_count": 9, 290 | "metadata": {}, 291 | "outputs": [ 292 | { 293 | "name": "stdout", 294 | "output_type": "stream", 295 | "text": [ 296 | "(-2.590388082735368, 0.009595515482880781, 15920.640661495236)\n" 297 | ] 298 | } 299 | ], 300 | "source": [ 301 | "cm = sms.CompareMeans(sms.DescrStatsW(y0), sms.DescrStatsW(y1))\n", 302 | "print(cm.ttest_ind(alternative='two-sided', usevar='unequal'))" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": null, 308 | "metadata": {}, 309 | "outputs": [], 310 | "source": [] 311 | }, 312 | { 313 | "cell_type": "code", 314 | "execution_count": null, 315 | "metadata": {}, 316 | "outputs": [], 317 | "source": [] 318 | } 319 | ], 320 | "metadata": { 321 | "kernelspec": { 322 | "display_name": "Python 3", 323 | "language": "python", 324 | "name": "python3" 325 | }, 326 | "language_info": { 327 | "codemirror_mode": { 328 | "name": "ipython", 329 | "version": 3 330 | }, 331 | "file_extension": ".py", 332 | "mimetype": "text/x-python", 333 | "name": "python", 334 | "nbconvert_exporter": "python", 335 | "pygments_lexer": "ipython3", 336 | "version": "3.8.5" 337 | } 338 | }, 339 | "nbformat": 4, 340 | "nbformat_minor": 4 341 | } 342 | -------------------------------------------------------------------------------- /Class Exercises/L6/variance_lift.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 60, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np, statsmodels.stats.api as sms\n", 10 | "import itertools\n", 11 | "import matplotlib.pyplot as plt\n", 12 | "from matplotlib.ticker import NullFormatter\n", 13 | "import pandas as pd\n", 14 | "import numpy as np\n", 15 | "import matplotlib.ticker as ticker\n", 16 | "import random\n", 17 | "from sklearn import preprocessing" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 75, 23 | "metadata": {}, 24 | "outputs": [ 25 | { 26 | "name": "stdout", 27 | "output_type": "stream", 28 | "text": [ 29 | "2.999999999999999\n" 30 | ] 31 | } 32 | ], 33 | "source": [ 34 | "#Population Characteristics\n", 35 | "lift = 1.2\n", 36 | "ctr0=0.5\n", 37 | "#30*ctr0*lift-30*ctr0\n", 38 | "delta_p = 30*ctr0*(lift-1)\n", 39 | "print(delta_p) " 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 76, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "#Sample Characteristics\n", 49 | "n0=1000\n", 50 | "n1=1000\n", 51 | "ctrl = np.random.binomial(30, p=ctr0, size=n0) * 1.0\n", 52 | "test = np.random.binomial(30, p=ctr0*lift, size=n1) * 1.0" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 77, 58 | "metadata": {}, 59 | "outputs": [ 60 | { 61 | "name": "stdout", 62 | "output_type": "stream", 63 | "text": [ 64 | "1.2041074881642995\n" 65 | ] 66 | } 67 | ], 68 | "source": [ 69 | "# Y1 bar and Y0 bar\n", 70 | "m1=np.mean(test)\n", 71 | "m0=np.mean(ctrl) \n", 72 | "lift_ob=m1/m0\n", 73 | "print(lift_ob)" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 78, 79 | "metadata": {}, 80 | "outputs": [ 81 | { 82 | "name": "stdout", 83 | "output_type": "stream", 84 | "text": [ 85 | "7.204195195195196 7.113749749749751\n" 86 | ] 87 | } 88 | ], 89 | "source": [ 90 | "# variance of Y1 and Yo\n", 91 | "var0 = np.var(ctrl,ddof=1)\n", 92 | "var1 = np.var(test,ddof=1)\n", 93 | "print(var0,var1)" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 79, 99 | "metadata": {}, 100 | "outputs": [ 101 | { 102 | "name": "stdout", 103 | "output_type": "stream", 104 | "text": [ 105 | "0.007204195195195196 0.007113749749749751\n" 106 | ] 107 | } 108 | ], 109 | "source": [ 110 | "# variance of Y1 bar,m1 and Y0 bar,m0\n", 111 | "var_m0=var0/n0\n", 112 | "var_m1=var1/n1\n", 113 | "print(var_m0, var_m1)" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 80, 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "#var(lift)\n", 123 | "var_lift = (1/m0**2)*var_m1+(m1**2/m0**4)*var_m0" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": 81, 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "se_lift=np.sqrt(var_lift)" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 82, 138 | "metadata": {}, 139 | "outputs": [ 140 | { 141 | "name": "stdout", 142 | "output_type": "stream", 143 | "text": [ 144 | "0.008835774960112465\n" 145 | ] 146 | } 147 | ], 148 | "source": [ 149 | "print(se_lift)" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": 83, 155 | "metadata": {}, 156 | "outputs": [], 157 | "source": [ 158 | "ci = (lift-1.96*se_lift,lift+1.96*se_lift)" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": 84, 164 | "metadata": {}, 165 | "outputs": [ 166 | { 167 | "name": "stdout", 168 | "output_type": "stream", 169 | "text": [ 170 | "(1.1826818810781796, 1.2173181189218203)\n" 171 | ] 172 | } 173 | ], 174 | "source": [ 175 | "print(ci)" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": 85, 181 | "metadata": {}, 182 | "outputs": [ 183 | { 184 | "name": "stdout", 185 | "output_type": "stream", 186 | "text": [ 187 | "Requirement already satisfied: bootstrapped in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (0.0.2)\n", 188 | "Requirement already satisfied: matplotlib>=1.5.3 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from bootstrapped) (3.3.2)\n", 189 | "Requirement already satisfied: pandas>=0.18.1 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from bootstrapped) (1.1.3)\n", 190 | "Requirement already satisfied: numpy>=1.11.1 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from bootstrapped) (1.19.2)\n", 191 | "Requirement already satisfied: cycler>=0.10 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from matplotlib>=1.5.3->bootstrapped) (0.10.0)\n", 192 | "Requirement already satisfied: certifi>=2020.06.20 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from matplotlib>=1.5.3->bootstrapped) (2020.6.20)\n", 193 | "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from matplotlib>=1.5.3->bootstrapped) (2.4.7)\n", 194 | "Requirement already satisfied: pillow>=6.2.0 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from matplotlib>=1.5.3->bootstrapped) (8.0.1)\n", 195 | "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from matplotlib>=1.5.3->bootstrapped) (1.3.0)\n", 196 | "Requirement already satisfied: python-dateutil>=2.1 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from matplotlib>=1.5.3->bootstrapped) (2.8.1)\n", 197 | "Requirement already satisfied: pytz>=2017.2 in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from pandas>=0.18.1->bootstrapped) (2020.1)\n", 198 | "Requirement already satisfied: six in /Users/shan_huang/opt/anaconda3/lib/python3.8/site-packages (from cycler>=0.10->matplotlib>=1.5.3->bootstrapped) (1.15.0)\n", 199 | "Note: you may need to restart the kernel to use updated packages.\n" 200 | ] 201 | } 202 | ], 203 | "source": [ 204 | "pip install bootstrapped" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": 86, 210 | "metadata": {}, 211 | "outputs": [], 212 | "source": [ 213 | "import bootstrapped.bootstrap as bs\n", 214 | "import bootstrapped.compare_functions as bs_compare\n", 215 | "import bootstrapped.stats_functions as bs_stats" 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": 87, 221 | "metadata": {}, 222 | "outputs": [], 223 | "source": [ 224 | "ctrl = np.array(ctrl)\n", 225 | "test = np.array(test)" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": 88, 231 | "metadata": {}, 232 | "outputs": [ 233 | { 234 | "name": "stdout", 235 | "output_type": "stream", 236 | "text": [ 237 | "20.410748816429955 (18.735809923471688, 22.158128040221964)\n" 238 | ] 239 | } 240 | ], 241 | "source": [ 242 | "print(bs.bootstrap_ab(\n", 243 | " test, \n", 244 | " ctrl, \n", 245 | " stat_func=bs_stats.mean,\n", 246 | " compare_func=bs_compare.percent_change,\n", 247 | " alpha=0.05))" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": null, 253 | "metadata": {}, 254 | "outputs": [], 255 | "source": [] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": null, 260 | "metadata": {}, 261 | "outputs": [], 262 | "source": [] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": null, 267 | "metadata": {}, 268 | "outputs": [], 269 | "source": [] 270 | } 271 | ], 272 | "metadata": { 273 | "kernelspec": { 274 | "display_name": "Python 3", 275 | "language": "python", 276 | "name": "python3" 277 | }, 278 | "language_info": { 279 | "codemirror_mode": { 280 | "name": "ipython", 281 | "version": 3 282 | }, 283 | "file_extension": ".py", 284 | "mimetype": "text/x-python", 285 | "name": "python", 286 | "nbconvert_exporter": "python", 287 | "pygments_lexer": "ipython3", 288 | "version": "3.8.5" 289 | } 290 | }, 291 | "nbformat": 4, 292 | "nbformat_minor": 4 293 | } 294 | -------------------------------------------------------------------------------- /Class Exercises/L6/~$Interleaving.xlsx: -------------------------------------------------------------------------------- 1 | Microsoft Office User Microsoft Office User -------------------------------------------------------------------------------- /Class Exercises/L7/Control_Variables.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np\n", 10 | "import statsmodels.stats.api as sms\n", 11 | "import matplotlib.pyplot as plt\n", 12 | "import pandas as pd\n", 13 | "import statsmodels.formula.api as smf\n", 14 | "import random" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 2, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "infile = 'exp_data_3.csv'\n", 24 | "df = pd.read_csv(infile)" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 3, 30 | "metadata": {}, 31 | "outputs": [ 32 | { 33 | "data": { 34 | "text/html": [ 35 | "
\n", 36 | "\n", 49 | "\n", 50 | " \n", 51 | " \n", 52 | " \n", 53 | " \n", 54 | " \n", 55 | " \n", 56 | " \n", 57 | " \n", 58 | " \n", 59 | " \n", 60 | " \n", 61 | " \n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | "
userclicklikeagegenderfriend_cntsns_like_cntsns_comment_cntpre_clicktreat
011034223020932010
14002621684611300
250024210573001
3600391167233500
47003924432415501
.................................
9513700272485615301
96138003423871764198301
97139003025553031800
981400028220210532800
99141001912629716701
\n", 211 | "

100 rows × 10 columns

\n", 212 | "
" 213 | ], 214 | "text/plain": [ 215 | " user click like age gender friend_cnt sns_like_cnt sns_comment_cnt \\\n", 216 | "0 1 1 0 34 2 230 209 320 \n", 217 | "1 4 0 0 26 2 168 46 113 \n", 218 | "2 5 0 0 24 2 105 7 30 \n", 219 | "3 6 0 0 39 1 167 23 35 \n", 220 | "4 7 0 0 39 2 443 24 155 \n", 221 | ".. ... ... ... ... ... ... ... ... \n", 222 | "95 137 0 0 27 2 485 61 53 \n", 223 | "96 138 0 0 34 2 387 1764 1983 \n", 224 | "97 139 0 0 30 2 555 30 318 \n", 225 | "98 140 0 0 28 2 202 105 328 \n", 226 | "99 141 0 0 19 1 262 97 167 \n", 227 | "\n", 228 | " pre_click treat \n", 229 | "0 1 0 \n", 230 | "1 0 0 \n", 231 | "2 0 1 \n", 232 | "3 0 0 \n", 233 | "4 0 1 \n", 234 | ".. ... ... \n", 235 | "95 0 1 \n", 236 | "96 0 1 \n", 237 | "97 0 0 \n", 238 | "98 0 0 \n", 239 | "99 0 1 \n", 240 | "\n", 241 | "[100 rows x 10 columns]" 242 | ] 243 | }, 244 | "execution_count": 3, 245 | "metadata": {}, 246 | "output_type": "execute_result" 247 | } 248 | ], 249 | "source": [ 250 | "df.head(100)" 251 | ] 252 | }, 253 | { 254 | "cell_type": "code", 255 | "execution_count": 4, 256 | "metadata": {}, 257 | "outputs": [ 258 | { 259 | "data": { 260 | "text/plain": [ 261 | "(7999, 10)" 262 | ] 263 | }, 264 | "execution_count": 4, 265 | "metadata": {}, 266 | "output_type": "execute_result" 267 | } 268 | ], 269 | "source": [ 270 | "df.shape" 271 | ] 272 | }, 273 | { 274 | "cell_type": "markdown", 275 | "metadata": {}, 276 | "source": [ 277 | "### **1. CUPED**\n", 278 | "Controlled Experiments by Utilizing Pre-Experiment Data (Deng, Xu, Kohavi, & Walker, 2013) \n", 279 | "Remove variance in a metric that can be accounted for by pre-experiment information.\n", 280 | "Control Variates: Pre-experiment information.\n" 281 | ] 282 | }, 283 | { 284 | "cell_type": "markdown", 285 | "metadata": {}, 286 | "source": [ 287 | "#### 1.1 Define CUPED" 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": 5, 293 | "metadata": {}, 294 | "outputs": [ 295 | { 296 | "name": "stdout", 297 | "output_type": "stream", 298 | "text": [ 299 | "0.05154226423839803\n" 300 | ] 301 | } 302 | ], 303 | "source": [ 304 | "#x is preclick\n", 305 | "var_x=np.var(df.pre_click, ddof=1)\n", 306 | "print(var_x)" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": 7, 312 | "metadata": {}, 313 | "outputs": [ 314 | { 315 | "name": "stdout", 316 | "output_type": "stream", 317 | "text": [ 318 | "0.03190474453217632\n" 319 | ] 320 | } 321 | ], 322 | "source": [ 323 | "#np.cov returns a var-cov metrix\n", 324 | "cov_xy = np.cov(df.pre_click,df.click, ddof=1)[0][1]\n", 325 | "print(cov_xy)" 326 | ] 327 | }, 328 | { 329 | "cell_type": "code", 330 | "execution_count": 8, 331 | "metadata": {}, 332 | "outputs": [ 333 | { 334 | "name": "stdout", 335 | "output_type": "stream", 336 | "text": [ 337 | "0.6190016097198905\n" 338 | ] 339 | } 340 | ], 341 | "source": [ 342 | "theta = cov_xy/var_x\n", 343 | "print(theta)\n", 344 | "df['theta']=theta " 345 | ] 346 | }, 347 | { 348 | "cell_type": "code", 349 | "execution_count": 15, 350 | "metadata": {}, 351 | "outputs": [ 352 | { 353 | "name": "stdout", 354 | "output_type": "stream", 355 | "text": [ 356 | "0.0010019387709386388\n" 357 | ] 358 | } 359 | ], 360 | "source": [ 361 | "# different X\n", 362 | "var_x=np.var(df.age, ddof=1)\n", 363 | "cov_xy = np.cov(df.age,df.click, ddof=1)[0][1]\n", 364 | "theta = cov_xy/var_x\n", 365 | "print(theta)" 366 | ] 367 | }, 368 | { 369 | "cell_type": "code", 370 | "execution_count": 9, 371 | "metadata": {}, 372 | "outputs": [ 373 | { 374 | "data": { 375 | "text/html": [ 376 | "
\n", 377 | "\n", 390 | "\n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | " \n", 400 | " \n", 401 | " \n", 402 | " \n", 403 | " \n", 404 | " \n", 405 | " \n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | " \n", 410 | " \n", 411 | " \n", 412 | " \n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | " \n", 477 | " \n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | " \n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | " \n", 533 | " \n", 534 | " \n", 535 | " \n", 536 | " \n", 537 | " \n", 538 | " \n", 539 | " \n", 540 | " \n", 541 | " \n", 542 | " \n", 543 | " \n", 544 | " \n", 545 | " \n", 546 | " \n", 547 | " \n", 548 | " \n", 549 | " \n", 550 | " \n", 551 | " \n", 552 | " \n", 553 | " \n", 554 | " \n", 555 | " \n", 556 | " \n", 557 | " \n", 558 | " \n", 559 | " \n", 560 | " \n", 561 | " \n", 562 | " \n", 563 | " \n", 564 | " \n", 565 | " \n", 566 | " \n", 567 | " \n", 568 | " \n", 569 | " \n", 570 | " \n", 571 | " \n", 572 | " \n", 573 | " \n", 574 | " \n", 575 | "
userclicklikeagegenderfriend_cntsns_like_cntsns_comment_cntpre_clicktreatthetaclick_cuped
0110342230209320100.6190020.380998
140026216846113000.6190020.000000
2500242105730010.6190020.000000
36003911672335000.6190020.000000
470039244324155010.6190020.000000
.......................................
95137002724856153010.6190020.000000
961380034238717641983010.6190020.000000
971390030255530318000.6190020.000000
9814000282202105328000.6190020.000000
991410019126297167010.6190020.000000
\n", 576 | "

100 rows × 12 columns

\n", 577 | "
" 578 | ], 579 | "text/plain": [ 580 | " user click like age gender friend_cnt sns_like_cnt sns_comment_cnt \\\n", 581 | "0 1 1 0 34 2 230 209 320 \n", 582 | "1 4 0 0 26 2 168 46 113 \n", 583 | "2 5 0 0 24 2 105 7 30 \n", 584 | "3 6 0 0 39 1 167 23 35 \n", 585 | "4 7 0 0 39 2 443 24 155 \n", 586 | ".. ... ... ... ... ... ... ... ... \n", 587 | "95 137 0 0 27 2 485 61 53 \n", 588 | "96 138 0 0 34 2 387 1764 1983 \n", 589 | "97 139 0 0 30 2 555 30 318 \n", 590 | "98 140 0 0 28 2 202 105 328 \n", 591 | "99 141 0 0 19 1 262 97 167 \n", 592 | "\n", 593 | " pre_click treat theta click_cuped \n", 594 | "0 1 0 0.619002 0.380998 \n", 595 | "1 0 0 0.619002 0.000000 \n", 596 | "2 0 1 0.619002 0.000000 \n", 597 | "3 0 0 0.619002 0.000000 \n", 598 | "4 0 1 0.619002 0.000000 \n", 599 | ".. ... ... ... ... \n", 600 | "95 0 1 0.619002 0.000000 \n", 601 | "96 0 1 0.619002 0.000000 \n", 602 | "97 0 0 0.619002 0.000000 \n", 603 | "98 0 0 0.619002 0.000000 \n", 604 | "99 0 1 0.619002 0.000000 \n", 605 | "\n", 606 | "[100 rows x 12 columns]" 607 | ] 608 | }, 609 | "execution_count": 9, 610 | "metadata": {}, 611 | "output_type": "execute_result" 612 | } 613 | ], 614 | "source": [ 615 | "df['click_cuped']=df.click - df.pre_click*theta\n", 616 | "df.head(100)" 617 | ] 618 | }, 619 | { 620 | "cell_type": "markdown", 621 | "metadata": {}, 622 | "source": [ 623 | "#### 1.2 T-Test Without CUPED" 624 | ] 625 | }, 626 | { 627 | "cell_type": "code", 628 | "execution_count": 12, 629 | "metadata": {}, 630 | "outputs": [ 631 | { 632 | "name": "stdout", 633 | "output_type": "stream", 634 | "text": [ 635 | "-0.003601024491467819\n" 636 | ] 637 | } 638 | ], 639 | "source": [ 640 | "d_0 = df[df['treat'] == 0]['click']\n", 641 | "d_1 = df[df['treat'] == 1]['click']\n", 642 | "diff = np.mean(d_1) - np.mean(d_0)\n", 643 | "print(diff)" 644 | ] 645 | }, 646 | { 647 | "cell_type": "code", 648 | "execution_count": 13, 649 | "metadata": {}, 650 | "outputs": [ 651 | { 652 | "name": "stdout", 653 | "output_type": "stream", 654 | "text": [ 655 | "0.004052064006037295 (-0.8886889462018719, 0.3741970443208038, 7984.625655071232)\n" 656 | ] 657 | } 658 | ], 659 | "source": [ 660 | "cm = sms.CompareMeans(sms.DescrStatsW(d_1), sms.DescrStatsW(d_0))\n", 661 | "ttest = cm.ttest_ind(alternative = 'two-sided', usevar = 'unequal')\n", 662 | "se = cm.std_meandiff_separatevar\n", 663 | "print(se,ttest)" 664 | ] 665 | }, 666 | { 667 | "cell_type": "markdown", 668 | "metadata": {}, 669 | "source": [ 670 | "#### 1.3 T-Test With CUPED" 671 | ] 672 | }, 673 | { 674 | "cell_type": "code", 675 | "execution_count": 11, 676 | "metadata": {}, 677 | "outputs": [ 678 | { 679 | "name": "stdout", 680 | "output_type": "stream", 681 | "text": [ 682 | "-0.003997530453238325\n" 683 | ] 684 | } 685 | ], 686 | "source": [ 687 | "d_0_cuped = df[df['treat'] == 0]['click_cuped']\n", 688 | "d_1_cuped = df[df['treat'] == 1]['click_cuped']\n", 689 | "diff = np.mean(d_1_cuped) - np.mean(d_0_cuped)\n", 690 | "print(diff)" 691 | ] 692 | }, 693 | { 694 | "cell_type": "code", 695 | "execution_count": 13, 696 | "metadata": {}, 697 | "outputs": [ 698 | { 699 | "name": "stdout", 700 | "output_type": "stream", 701 | "text": [ 702 | "0.0025601980289050017 (-1.5614145500096597, 0.1184656562164157, 7973.716532346689)\n" 703 | ] 704 | } 705 | ], 706 | "source": [ 707 | "cm = sms.CompareMeans(sms.DescrStatsW(d_1_cuped), sms.DescrStatsW(d_0_cuped))\n", 708 | "ttest = cm.ttest_ind(alternative = 'two-sided', usevar = 'unequal')\n", 709 | "se = cm.std_meandiff_separatevar\n", 710 | "print(se,ttest)" 711 | ] 712 | }, 713 | { 714 | "cell_type": "markdown", 715 | "metadata": {}, 716 | "source": [ 717 | "### **2. Regression**" 718 | ] 719 | }, 720 | { 721 | "cell_type": "code", 722 | "execution_count": 16, 723 | "metadata": {}, 724 | "outputs": [ 725 | { 726 | "name": "stdout", 727 | "output_type": "stream", 728 | "text": [ 729 | " OLS Regression Results \n", 730 | "==============================================================================\n", 731 | "Dep. Variable: click R-squared: 0.000\n", 732 | "Model: OLS Adj. R-squared: -0.000\n", 733 | "Method: Least Squares F-statistic: 0.7893\n", 734 | "Date: Thu, 16 Feb 2023 Prob (F-statistic): 0.374\n", 735 | "Time: 20:05:01 Log-Likelihood: 2312.1\n", 736 | "No. Observations: 7999 AIC: -4620.\n", 737 | "Df Residuals: 7997 BIC: -4606.\n", 738 | "Df Model: 1 \n", 739 | "Covariance Type: nonrobust \n", 740 | "==============================================================================\n", 741 | " coef std err t P>|t| [0.025 0.975]\n", 742 | "------------------------------------------------------------------------------\n", 743 | "Intercept 0.0358 0.003 12.526 0.000 0.030 0.041\n", 744 | "treat -0.0036 0.004 -0.888 0.374 -0.012 0.004\n", 745 | "==============================================================================\n", 746 | "Omnibus: 7871.185 Durbin-Watson: 1.990\n", 747 | "Prob(Omnibus): 0.000 Jarque-Bera (JB): 234297.485\n", 748 | "Skew: 5.142 Prob(JB): 0.00\n", 749 | "Kurtosis: 27.438 Cond. No. 2.61\n", 750 | "==============================================================================\n", 751 | "\n", 752 | "Notes:\n", 753 | "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n" 754 | ] 755 | } 756 | ], 757 | "source": [ 758 | "mod = smf.ols(formula='click ~ treat', data=df)\n", 759 | "res = mod.fit()\n", 760 | "print(res.summary())" 761 | ] 762 | }, 763 | { 764 | "cell_type": "markdown", 765 | "metadata": {}, 766 | "source": [ 767 | "#### 2.1 Regression + pre_click (control variable)" 768 | ] 769 | }, 770 | { 771 | "cell_type": "code", 772 | "execution_count": 17, 773 | "metadata": {}, 774 | "outputs": [ 775 | { 776 | "name": "stdout", 777 | "output_type": "stream", 778 | "text": [ 779 | " OLS Regression Results \n", 780 | "==============================================================================\n", 781 | "Dep. Variable: click R-squared: 0.601\n", 782 | "Model: OLS Adj. R-squared: 0.601\n", 783 | "Method: Least Squares F-statistic: 6029.\n", 784 | "Date: Thu, 16 Feb 2023 Prob (F-statistic): 0.00\n", 785 | "Time: 20:06:30 Log-Likelihood: 5989.1\n", 786 | "No. Observations: 7999 AIC: -1.197e+04\n", 787 | "Df Residuals: 7996 BIC: -1.195e+04\n", 788 | "Df Model: 2 \n", 789 | "Covariance Type: nonrobust \n", 790 | "==============================================================================\n", 791 | " coef std err t P>|t| [0.025 0.975]\n", 792 | "------------------------------------------------------------------------------\n", 793 | "Intercept 0.0023 0.002 1.230 0.219 -0.001 0.006\n", 794 | "treat -0.0040 0.003 -1.562 0.118 -0.009 0.001\n", 795 | "pre_click 0.6190 0.006 109.799 0.000 0.608 0.630\n", 796 | "==============================================================================\n", 797 | "Omnibus: 4168.834 Durbin-Watson: 1.993\n", 798 | "Prob(Omnibus): 0.000 Jarque-Bera (JB): 142812.742\n", 799 | "Skew: -1.877 Prob(JB): 0.00\n", 800 | "Kurtosis: 23.357 Cond. No. 5.05\n", 801 | "==============================================================================\n", 802 | "\n", 803 | "Notes:\n", 804 | "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n" 805 | ] 806 | } 807 | ], 808 | "source": [ 809 | "mod = smf.ols(formula='click ~ treat + pre_click', data=df)\n", 810 | "res = mod.fit()\n", 811 | "print(res.summary())" 812 | ] 813 | }, 814 | { 815 | "cell_type": "markdown", 816 | "metadata": {}, 817 | "source": [ 818 | "### 2.2 + Gender" 819 | ] 820 | }, 821 | { 822 | "cell_type": "code", 823 | "execution_count": 18, 824 | "metadata": {}, 825 | "outputs": [ 826 | { 827 | "name": "stdout", 828 | "output_type": "stream", 829 | "text": [ 830 | " OLS Regression Results \n", 831 | "==============================================================================\n", 832 | "Dep. Variable: click R-squared: 0.003\n", 833 | "Model: OLS Adj. R-squared: 0.003\n", 834 | "Method: Least Squares F-statistic: 12.22\n", 835 | "Date: Thu, 16 Feb 2023 Prob (F-statistic): 5.01e-06\n", 836 | "Time: 20:07:41 Log-Likelihood: 2323.9\n", 837 | "No. Observations: 7999 AIC: -4642.\n", 838 | "Df Residuals: 7996 BIC: -4621.\n", 839 | "Df Model: 2 \n", 840 | "Covariance Type: nonrobust \n", 841 | "==============================================================================\n", 842 | " coef std err t P>|t| [0.025 0.975]\n", 843 | "------------------------------------------------------------------------------\n", 844 | "Intercept 0.0667 0.007 9.580 0.000 0.053 0.080\n", 845 | "treat -0.0037 0.004 -0.918 0.359 -0.012 0.004\n", 846 | "gender -0.0198 0.004 -4.864 0.000 -0.028 -0.012\n", 847 | "==============================================================================\n", 848 | "Omnibus: 7846.678 Durbin-Watson: 1.990\n", 849 | "Prob(Omnibus): 0.000 Jarque-Bera (JB): 231655.229\n", 850 | "Skew: 5.119 Prob(JB): 0.00\n", 851 | "Kurtosis: 27.295 Cond. No. 7.75\n", 852 | "==============================================================================\n", 853 | "\n", 854 | "Notes:\n", 855 | "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n" 856 | ] 857 | } 858 | ], 859 | "source": [ 860 | "mod = smf.ols(formula='click ~ treat + gender', data=df)\n", 861 | "res = mod.fit()\n", 862 | "print(res.summary())" 863 | ] 864 | }, 865 | { 866 | "cell_type": "markdown", 867 | "metadata": {}, 868 | "source": [ 869 | "#### + Gender, Pre-Click" 870 | ] 871 | }, 872 | { 873 | "cell_type": "code", 874 | "execution_count": 19, 875 | "metadata": {}, 876 | "outputs": [ 877 | { 878 | "name": "stdout", 879 | "output_type": "stream", 880 | "text": [ 881 | " OLS Regression Results \n", 882 | "==============================================================================\n", 883 | "Dep. Variable: click R-squared: 0.602\n", 884 | "Model: OLS Adj. R-squared: 0.602\n", 885 | "Method: Least Squares F-statistic: 4032.\n", 886 | "Date: Thu, 16 Feb 2023 Prob (F-statistic): 0.00\n", 887 | "Time: 20:09:31 Log-Likelihood: 5996.7\n", 888 | "No. Observations: 7999 AIC: -1.199e+04\n", 889 | "Df Residuals: 7995 BIC: -1.196e+04\n", 890 | "Df Model: 3 \n", 891 | "Covariance Type: nonrobust \n", 892 | "==============================================================================\n", 893 | " coef std err t P>|t| [0.025 0.975]\n", 894 | "------------------------------------------------------------------------------\n", 895 | "Intercept 0.0180 0.004 4.063 0.000 0.009 0.027\n", 896 | "treat -0.0041 0.003 -1.585 0.113 -0.009 0.001\n", 897 | "gender -0.0101 0.003 -3.904 0.000 -0.015 -0.005\n", 898 | "pre_click 0.6183 0.006 109.697 0.000 0.607 0.629\n", 899 | "==============================================================================\n", 900 | "Omnibus: 4157.532 Durbin-Watson: 1.995\n", 901 | "Prob(Omnibus): 0.000 Jarque-Bera (JB): 141808.657\n", 902 | "Skew: -1.870 Prob(JB): 0.00\n", 903 | "Kurtosis: 23.285 Cond. No. 8.77\n", 904 | "==============================================================================\n", 905 | "\n", 906 | "Notes:\n", 907 | "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n" 908 | ] 909 | } 910 | ], 911 | "source": [ 912 | "mod = smf.ols(formula='click ~ treat + gender + pre_click', data=df)\n", 913 | "res = mod.fit()\n", 914 | "print(res.summary())" 915 | ] 916 | }, 917 | { 918 | "cell_type": "code", 919 | "execution_count": null, 920 | "metadata": {}, 921 | "outputs": [], 922 | "source": [] 923 | } 924 | ], 925 | "metadata": { 926 | "kernelspec": { 927 | "display_name": "Python 3", 928 | "language": "python", 929 | "name": "python3" 930 | }, 931 | "language_info": { 932 | "codemirror_mode": { 933 | "name": "ipython", 934 | "version": 3 935 | }, 936 | "file_extension": ".py", 937 | "mimetype": "text/x-python", 938 | "name": "python", 939 | "nbconvert_exporter": "python", 940 | "pygments_lexer": "ipython3", 941 | "version": "3.8.5" 942 | } 943 | }, 944 | "nbformat": 4, 945 | "nbformat_minor": 4 946 | } 947 | -------------------------------------------------------------------------------- /Class Exercises/Readme: -------------------------------------------------------------------------------- 1 | These are the class exercises that I have designed to enhance students' understanding of concepts and methods in A/B testing. Most of them are in Python. 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Teaching & Learning: Digital Experimentation & A/B Testing in Product Management 2 | 3 | These course materials were initially designed for teaching the Masters of Science in Business Analytics and Marketing program since 2021, at HKU Business School, focusing on the principles and practices of A/B testing. I aim to extend their usefulness to a wider audience, encompassing both learners and educators in the field of A/B testing. The materials include both lecture notes and class exercises in Python, which can serve as a foundation for developing assignments. The course has been meticulously developed from the ground up, ensuring that all content is original. Notably, the primary source of information for this course is my practical experience and research in the field of A/B testing. Additionally, I incorporate insights and some materials from 'Kohavi et al. (2020)'. My practical experience mainly stems from my consultancy and research collaborations with the WeChat A/B testing team at Tencent, China. In addition, I welcome any input related to teaching and learning A/B tests. Understanding this knowledge will benefit practitioners by enabling them to use data for making decisions more scientifically. 4 | 5 | Reference: 6 | Kohavi, Ron, Diane Tang, and Ya Xu. Trustworthy online controlled experiments: A practical guide to a/b testing. Cambridge University Press, 2020. 7 | 8 | **Course Instructor**: [Shan Huang](https://www.shanhhuang.com/), an assistant professor in Marketing at HKU Business School. Email: shanhh@hku.hk 9 | 10 | **Course TA**： Chen Wang, who is a PhD student at HKU Business School. Email: annacwang@connect.hku.hk, 11 | 12 | **Course Description**: 13 | 14 | The newly emerging capability to rapidly deploy and iterate online controlled experiments to assist decision-making in organizations is one of the most significant innovations in today’s technology industry. As more and more social interactions, decisions, opinions, and transactions are mediated by online platforms, digital experiments are becoming increasingly crucial for firms to understand their user behaviors and make product decisions. This course will cover the most cutting-edge digital experimentation methods used in the daily operations at large technology firms. We will also share the key lessons and pitfalls encountered in practice. Topics include the statistics behind experiments, experimental design, methods of analyzing experiments, A/B testing platforms and culture in organizations, recent developments in digital experimentation, and observational causal studies. Students will also learn how to conduct and analyze online experiments using programming languages, such as python, and Large Language Models. 15 | 16 | **Course Roadmap**: 17 | 18 | 1. A comprehensive overview of AB testing 19 | 20 | 2. Statistics behind A/B testing 21 | * Statistical tests (t, z, chi-square) 22 | * Confidence intervals 23 | * Type I error & Multiple Testing 24 | * Type II Error & Power Analysis 25 | * Regression 26 | 27 | 3. Internal & External Validity 28 | * SUTVA (network interferences) 29 | * Survivorship bias 30 | * Sanity Checks (SRM, Randomisation checks, A/A tests) 31 | * Heterogeneous Treatment Effects (HTE) 32 | 33 | 4. Improve Sensitivity 34 | * Ratio metrics (e.g., lift etc.) 35 | * Increase N (pooled control group, split sample) 36 | * Increase effect size (Triggering Experiments) 37 | * Reduce variance (transform metrics and interleaving design, paried design) 38 | * Stratification (post and at assignment, block design) 39 | * Regression with controls, CUPED 40 | 41 | 42 | 5. Observational Causal Studies 43 | * Interrupted time series (ITS) 44 | * Regression discontinuity design (RDD) 45 | * Difference-in-Difference (DID) 46 | * Propensity score matching (PSM) 47 | 48 | -------------------------------------------------------------------------------- /Slides/Guest_Lecture_KennyXie.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Slides/Guest_Lecture_KennyXie.pdf -------------------------------------------------------------------------------- /Slides/Guest_Lecture_Tencent.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Slides/Guest_Lecture_Tencent.pdf -------------------------------------------------------------------------------- /Slides/JasonMa_Tencent_PresentationHK.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Slides/JasonMa_Tencent_PresentationHK.pdf -------------------------------------------------------------------------------- /Slides/L1 - AB -Testing Overview_SH.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Slides/L1 - AB -Testing Overview_SH.pdf -------------------------------------------------------------------------------- /Slides/L2-Statistics1_SH.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Slides/L2-Statistics1_SH.pdf -------------------------------------------------------------------------------- /Slides/L3-Statistics2_SH.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Slides/L3-Statistics2_SH.pdf -------------------------------------------------------------------------------- /Slides/L4-InternalExternalValidity_SH.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Slides/L4-InternalExternalValidity_SH.pdf -------------------------------------------------------------------------------- /Slides/L5-Improve_SensitivityI_SH.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Slides/L5-Improve_SensitivityI_SH.pdf -------------------------------------------------------------------------------- /Slides/L6 - ImproveSensitivity II-SH.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Slides/L6 - ImproveSensitivity II-SH.pdf -------------------------------------------------------------------------------- /Slides/L7- ObservationalCausal-SH.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shanmit/Course---Digital-Experimentation-Methods-A-B-Testing/f3d4458d60a1001ee71f46fffee2963235a23d03/Slides/L7- ObservationalCausal-SH.pdf -------------------------------------------------------------------------------- /Slides/Readme.md: -------------------------------------------------------------------------------- 1 | This folder contains presentation slides for all the lectures I've created on A/B testing. 2 | --------------------------------------------------------------------------------