├── .ipynb_checkpoints └── ABn Testing Playbook - Proportions-checkpoint.ipynb ├── ABn Testing Playbook - Proportions.ipynb ├── Data └── Sample Data.csv ├── README.md └── requirements.txt /Data/Sample Data.csv: -------------------------------------------------------------------------------- 1 | user_id,date_joined,date_visit,version,type,category,visits,clicked,clicks,liked,likes 2 | 31098458,2018-04-23,2018-04-23,B,0,E,3,0,0,0,0 3 | 31098467,2018-04-23,2018-05-08,C,1,M,2,0,0,0,0 4 | 31098468,2018-04-23,2018-05-02,A1,1,,2,0,0,0,0 5 | 31098481,2018-04-23,2018-04-23,A2,1,,2,0,0,0,0 6 | 31098487,2018-04-23,2018-04-24,C,0,E,2,0,0,0,0 7 | 31098488,2018-04-23,2018-04-24,A1,1,,2,0,0,0,0 8 | 31098496,2018-04-23,2018-04-23,A1,0,H,2,0,0,0,0 9 | 31098503,2018-04-23,2018-04-24,C,1,,2,0,0,0,0 10 | 31098507,2018-04-23,2018-04-23,C,1,,2,0,0,0,0 11 | 31098509,2018-04-23,2018-04-23,A2,1,E,2,0,0,0,0 12 | 31098531,2018-04-23,2018-04-23,C,0,,2,0,0,0,0 13 | 31098534,2018-04-23,2018-04-23,B,1,E,2,0,0,0,0 14 | 31098540,2018-04-23,2018-04-23,A1,0,,1,0,0,0,0 15 | 31098543,2018-04-23,2018-04-23,C,0,E,1,0,0,0,0 16 | 31098544,2018-04-27,2018-04-23,A1,1,E,1,0,0,0,0 17 | 31098547,2018-04-27,2018-04-23,C,1,,1,0,0,0,0 18 | 31098550,2018-04-27,2018-04-23,B,0,,1,0,0,0,0 19 | 31098551,2018-04-27,2018-04-23,C,1,E,1,0,0,0,0 20 | 31098552,2018-04-27,2018-04-23,A1,1,M,1,0,0,0,0 21 | 31098557,2018-04-27,2018-04-23,A2,0,M,1,0,0,0,0 22 | 31098563,2018-04-27,2018-04-23,C,1,M,1,0,0,0,0 23 | 31098564,2018-04-27,2018-04-23,A1,0,,1,0,0,0,0 24 | 31098567,2018-04-27,2018-04-23,C,0,,1,0,0,0,0 25 | 31098568,2018-04-27,2018-04-23,A1,0,E,1,0,0,0,0 26 | 31098573,2018-04-27,2018-04-23,A2,1,,1,0,0,0,0 27 | 31098576,2018-04-27,2018-04-23,A1,0,M,1,0,0,0,0 28 | 31098578,2018-04-27,2018-04-23,B,1,,1,0,0,0,0 29 | 31098583,2018-04-27,2018-04-23,C,0,,1,0,0,0,0 30 | 31098585,2018-04-27,2018-04-23,A2,0,H,1,0,0,0,0 31 | 31098586,2018-04-27,2018-04-23,B,1,,1,0,0,0,0 32 | 31098588,2018-04-27,2018-04-23,A1,1,,1,0,0,0,0 33 | 31098595,2018-04-27,2018-04-23,C,0,,1,0,0,0,0 34 | 31098598,2018-04-27,2018-04-23,B,0,,1,0,0,0,0 35 | 31098603,2018-04-27,2018-04-23,C,1,M,1,0,0,0,0 36 | 31098609,2018-04-27,2018-04-23,A2,1,H,1,0,0,0,0 37 | 31098612,2018-04-27,2018-04-23,A1,0,,1,0,0,0,0 38 | 31098615,2018-04-27,2018-04-23,C,1,,1,0,0,0,0 39 | 31098619,2018-04-27,2018-04-23,C,1,,1,0,0,0,0 40 | 31098626,2018-04-27,2018-04-23,B,0,,1,0,0,0,0 41 | 31098627,2018-04-27,2018-04-23,C,0,,1,0,0,0,0 42 | 31098628,2018-04-27,2018-04-23,A1,1,E,1,0,0,0,0 43 | 31098629,2018-04-27,2018-04-23,A2,1,H,1,0,0,0,0 44 | 31098631,2018-04-27,2018-04-23,C,0,E,1,0,0,0,0 45 | 31098634,2018-04-27,2018-04-23,B,0,E,1,0,0,0,0 46 | 31098637,2018-04-27,2018-04-23,A2,0,,1,0,0,0,0 47 | 31098642,2018-04-27,2018-04-23,B,1,,1,0,0,0,0 48 | 31098647,2018-04-27,2018-04-23,C,0,H,1,0,0,0,0 49 | 31098648,2018-04-27,2018-04-23,A1,0,E,1,0,0,0,0 50 | 31098652,2018-04-27,2018-04-23,A1,1,H,1,0,0,0,0 51 | 31098653,2018-04-27,2018-04-23,A2,0,,1,0,0,0,0 52 | 31098658,2018-04-27,2018-04-23,B,0,,1,0,0,0,0 53 | 31098660,2018-04-27,2018-04-23,A1,0,E,1,0,0,0,0 54 | 31098661,2018-04-27,2018-04-23,A2,0,,1,0,0,0,0 55 | 31098663,2018-04-27,2018-04-23,C,0,,1,0,0,0,0 56 | 31098669,2018-04-27,2018-04-23,A2,1,,1,0,0,0,0 57 | 31098671,2018-04-27,2018-04-23,C,0,M,1,0,0,0,0 58 | 31098675,2018-04-27,2018-04-23,C,0,E,1,0,0,0,0 59 | 31098679,2018-04-27,2018-04-23,C,1,E,1,0,0,0,0 60 | 31098680,2018-04-27,2018-04-23,A1,1,E,1,0,0,0,0 61 | 31098682,2018-04-27,2018-04-23,B,1,E,1,0,0,0,0 62 | 31098687,2018-04-27,2018-04-23,C,0,E,1,0,0,0,0 63 | 31098692,2018-04-27,2018-04-23,A1,0,,1,0,0,0,0 64 | 31098693,2018-04-27,2018-04-23,A2,1,E,1,0,0,0,0 65 | 31098694,2018-04-27,2018-04-23,B,0,,1,0,0,0,0 66 | 31098698,2018-04-27,2018-04-23,B,0,E,1,0,0,0,0 67 | 31098701,2018-04-27,2018-04-23,A2,1,H,1,0,0,0,0 68 | 31098708,2018-04-27,2018-04-23,A1,1,,1,0,0,0,0 69 | 31098709,2018-04-27,2018-04-23,A2,0,M,1,0,0,0,0 70 | 31098719,2018-04-27,2018-04-23,C,1,,1,0,0,0,0 71 | 31098720,2018-04-27,2018-04-23,A1,1,E,1,0,0,0,0 72 | 31098721,2018-04-27,2018-04-23,A2,1,,1,0,0,0,0 73 | 31098731,2018-04-27,2018-04-23,C,0,,1,0,0,0,0 74 | 31098733,2018-04-27,2018-04-23,A2,0,H,1,0,0,0,0 75 | 31098740,2018-04-27,2018-04-23,A1,0,,1,0,0,0,0 76 | 31098741,2018-04-27,2018-04-23,A2,1,E,1,0,0,0,0 77 | 31098743,2018-04-27,2018-04-23,C,0,E,1,0,0,0,0 78 | 31098745,2018-04-27,2018-04-23,A2,1,,1,0,0,0,0 79 | 31098746,2018-04-27,2018-04-23,B,0,,1,0,0,0,0 80 | 31098747,2018-04-27,2018-04-23,C,1,E,1,0,0,0,0 81 | 31098748,2018-04-27,2018-04-23,A1,0,,1,0,0,0,0 82 | 31098750,2018-04-27,2018-04-23,B,0,E,1,0,0,0,0 83 | 31098752,2018-04-27,2018-04-23,A1,0,,1,0,0,0,0 84 | 31098753,2018-04-27,2018-04-23,A2,1,,1,0,0,0,0 85 | 31098757,2018-04-27,2018-04-23,A2,0,E,1,0,0,0,0 86 | 31098328,2018-04-23,2018-05-10,A1,1,M,14,1,10,1,7 87 | 31098329,2018-04-23,2018-05-10,A2,1,M,14,1,29,1,5 88 | 31098330,2018-04-23,2018-05-08,B,1,M,13,1,19,0,0 89 | 31098331,2018-04-23,2018-05-10,C,1,H,13,1,34,0,0 90 | 31098332,2018-04-23,2018-05-09,A1,0,H,13,1,21,1,3 91 | 31098333,2018-04-23,2018-04-23,A2,0,E,12,1,21,1,5 92 | 31098334,2018-04-23,2018-05-10,B,0,E,11,1,12,0,0 93 | 31098335,2018-04-23,2018-04-23,C,0,E,11,1,13,0,0 94 | 31098336,2018-04-23,2018-05-04,A1,1,E,11,1,5,0,0 95 | 31098337,2018-04-23,2018-04-30,A2,1,,10,1,46,0,0 96 | 31098338,2018-04-23,2018-05-04,B,1,H,10,1,24,1,1 97 | 31098339,2018-04-23,2018-05-08,C,1,E,10,1,13,1,7 98 | 31098340,2018-04-23,2018-05-10,A1,1,E,9,1,46,1,3 99 | 31098341,2018-04-23,2018-05-10,A2,0,H,9,1,41,1,6 100 | 31098342,2018-04-23,2018-05-09,B,0,E,9,1,12,0,0 101 | 31098343,2018-04-23,2018-05-10,C,0,M,9,1,17,0,0 102 | 31098344,2018-04-23,2018-05-09,A1,1,E,9,1,5,0,0 103 | 31098345,2018-04-23,2018-05-08,A2,1,M,9,1,11,0,0 104 | 31098346,2018-04-23,2018-05-09,B,0,E,9,1,18,1,7 105 | 31098347,2018-04-23,2018-05-09,C,1,E,9,1,11,0,0 106 | 31098348,2018-04-23,2018-05-04,A1,1,,9,1,15,1,6 107 | 31098349,2018-04-23,2018-05-10,A2,1,H,8,1,8,1,4 108 | 31098350,2018-04-23,2018-05-10,B,0,E,8,1,20,0,0 109 | 31098351,2018-04-23,2018-04-23,C,1,E,8,1,8,0,0 110 | 31098352,2018-04-23,2018-05-02,A1,1,H,8,1,8,0,0 111 | 31098353,2018-04-23,2018-05-03,A2,1,,8,1,14,1,5 112 | 31098354,2018-04-23,2018-04-24,B,1,,8,1,34,0,0 113 | 31098355,2018-04-23,2018-05-09,C,1,H,8,1,13,0,0 114 | 31098356,2018-04-23,2018-05-01,A1,1,M,7,1,11,0,0 115 | 31098357,2018-04-23,2018-05-09,A2,1,H,7,1,13,0,0 116 | 31098358,2018-04-23,2018-05-10,B,0,H,7,1,26,1,3 117 | 31098359,2018-04-23,2018-05-09,C,0,,7,1,13,0,0 118 | 31098360,2018-04-23,2018-05-03,A1,1,E,7,1,21,1,7 119 | 31098361,2018-04-23,2018-05-01,A2,1,M,7,1,36,0,0 120 | 31098362,2018-04-23,2018-04-30,B,0,H,7,1,16,0,0 121 | 31098363,2018-04-23,2018-05-10,C,0,E,7,1,26,1,4 122 | 31098364,2018-04-23,2018-05-08,A1,0,E,7,1,15,0,0 123 | 31098365,2018-04-23,2018-05-09,A2,0,,7,1,17,1,10 124 | 31098366,2018-04-23,2018-05-04,B,1,E,7,1,38,0,0 125 | 31098367,2018-04-23,2018-05-07,C,1,M,7,1,14,1,7 126 | 31098368,2018-04-23,2018-04-26,A1,0,M,6,1,46,1,9 127 | 31098369,2018-04-23,2018-05-10,A2,0,M,6,1,16,0,0 128 | 31098370,2018-04-23,2018-05-08,B,0,E,6,1,13,0,0 129 | 31098371,2018-04-23,2018-05-07,C,0,,6,1,6,1,1 130 | 31098372,2018-04-23,2018-05-10,A1,0,E,6,1,13,1,6 131 | 31098373,2018-04-23,2018-04-23,A2,1,E,6,1,9,1,6 132 | 31098374,2018-04-23,2018-05-09,B,0,M,6,1,8,0,0 133 | 31098375,2018-04-23,2018-05-03,C,1,E,6,1,5,0,0 134 | 31098376,2018-04-23,2018-04-23,A1,0,,6,1,1,1,0 135 | 31098377,2018-04-23,2018-05-09,A2,0,,6,1,15,1,2 136 | 31098378,2018-04-23,2018-04-23,B,0,E,6,1,6,1,3 137 | 31098379,2018-04-23,2018-05-09,C,1,E,6,1,8,0,0 138 | 31098380,2018-04-23,2018-05-08,A1,1,E,6,1,14,0,0 139 | 31098381,2018-04-23,2018-05-09,A2,1,H,6,1,19,0,0 140 | 31098382,2018-04-23,2018-04-24,B,1,M,6,1,6,1,4 141 | 31098383,2018-04-23,2018-05-09,C,0,,6,1,15,0,0 142 | 31098384,2018-04-23,2018-05-09,A1,1,E,6,1,14,1,5 143 | 31098385,2018-04-23,2018-05-10,A2,0,M,6,1,9,0,0 144 | 31098386,2018-04-23,2018-05-02,B,1,E,6,1,10,0,0 145 | 31098387,2018-04-23,2018-05-09,C,0,H,6,1,9,0,0 146 | 31098388,2018-04-23,2018-05-10,A1,0,E,5,1,20,1,6 147 | 31098389,2018-04-23,2018-04-24,A2,0,,5,1,4,0,0 148 | 31098390,2018-04-23,2018-05-07,B,1,E,5,1,7,1,1 149 | 31098391,2018-04-23,2018-05-02,C,0,H,5,1,5,1,5 150 | 31098392,2018-04-23,2018-04-23,A1,0,E,5,1,3,1,8 151 | 31098393,2018-04-23,2018-05-10,A2,0,E,5,1,4,1,0 152 | 31098394,2018-04-23,2018-04-24,B,1,E,5,1,7,1,1 153 | 31098395,2018-04-23,2018-04-24,C,1,M,5,1,3,1,5 154 | 31098396,2018-04-23,2018-04-25,A1,1,,5,1,7,0,0 155 | 31098397,2018-04-23,2018-05-08,A2,1,,5,1,23,0,0 156 | 31098398,2018-04-23,2018-05-04,B,1,H,5,1,10,1,9 157 | 31098399,2018-04-23,2018-05-07,C,0,,5,1,8,0,0 158 | 31098400,2018-04-23,2018-05-10,A1,0,E,5,1,5,0,0 159 | 31098401,2018-04-23,2018-04-23,A2,1,E,5,1,3,1,4 160 | 31098402,2018-04-23,2018-05-07,B,1,M,5,1,9,0,0 161 | 31098403,2018-04-23,2018-04-24,C,1,,5,1,9,1,3 162 | 31098404,2018-04-23,2018-05-02,A1,1,,4,1,5,0,0 163 | 31098405,2018-04-23,2018-04-25,A2,1,,4,1,7,0,0 164 | 31098406,2018-04-23,2018-04-24,B,0,,4,1,10,0,0 165 | 31098407,2018-04-23,2018-05-09,C,1,E,4,1,4,1,3 166 | 31098408,2018-04-23,2018-05-07,A1,1,M,4,1,16,0,0 167 | 31098409,2018-04-23,2018-04-30,A2,0,,4,1,8,0,0 168 | 31098410,2018-04-23,2018-05-06,B,0,,4,1,38,0,0 169 | 31098411,2018-04-23,2018-04-25,C,0,E,4,1,6,1,8 170 | 31098412,2018-04-23,2018-05-04,A1,0,E,4,1,3,1,7 171 | 31098413,2018-04-23,2018-04-30,A2,1,,4,1,4,0,0 172 | 31098414,2018-04-23,2018-05-01,B,0,E,4,1,17,1,5 173 | 31098415,2018-04-23,2018-04-25,C,1,E,4,1,3,1,4 174 | 31098416,2018-04-23,2018-05-07,A1,1,,4,1,14,1,7 175 | 31098417,2018-04-23,2018-04-30,A2,0,,4,1,6,0,0 176 | 31098418,2018-04-23,2018-04-24,B,1,,4,1,8,1,2 177 | 31098419,2018-04-23,2018-05-09,C,1,M,4,1,2,1,10 178 | 31098420,2018-04-23,2018-05-07,A1,1,,4,1,6,0,0 179 | 31098421,2018-04-23,2018-05-08,A2,1,M,4,1,10,1,6 180 | 31098422,2018-04-23,2018-04-23,B,0,E,4,1,7,0,0 181 | 31098423,2018-04-23,2018-05-01,C,0,E,4,1,3,0,0 182 | 31098424,2018-04-23,2018-04-23,A1,1,E,3,1,4,1,3 183 | 31098425,2018-04-23,2018-04-23,A2,1,M,3,1,1,0,0 184 | 31098426,2018-04-23,2018-05-07,B,0,,3,1,4,1,1 185 | 31098427,2018-04-23,2018-04-26,C,0,E,3,1,8,0,0 186 | 31098428,2018-04-23,2018-04-24,A1,1,,3,1,2,0,0 187 | 31098429,2018-04-23,2018-04-30,A2,0,,3,1,6,0,0 188 | 31098430,2018-04-23,2018-04-26,B,0,,3,1,6,0,0 189 | 31098431,2018-04-23,2018-04-24,C,1,E,3,1,3,1,0 190 | 31098432,2018-04-23,2018-04-27,A1,0,,3,1,20,0,0 191 | 31098433,2018-04-23,2018-04-24,A2,0,,3,1,18,1,3 192 | 31098434,2018-04-23,2018-05-09,B,1,E,3,1,4,0,0 193 | 31098435,2018-04-23,2018-04-23,C,1,E,3,1,8,1,3 194 | 31098436,2018-04-23,2018-05-02,A1,0,,3,1,3,0,0 195 | 31098437,2018-04-23,2018-04-23,A2,0,,3,1,3,1,2 196 | 31098438,2018-04-23,2018-04-30,B,1,,3,1,36,1,5 197 | 31098439,2018-04-23,2018-04-27,C,1,,3,1,5,1,5 198 | 31098440,2018-04-23,2018-05-08,A1,0,H,3,1,3,0,0 199 | 31098441,2018-04-23,2018-04-28,A2,1,E,3,1,10,0,0 200 | 31098442,2018-04-23,2018-04-23,B,1,,3,1,5,1,4 201 | 31098443,2018-04-23,2018-05-03,C,0,,3,1,3,1,6 202 | 31098444,2018-04-23,2018-04-23,A1,0,,3,1,5,0,0 203 | 31098445,2018-04-23,2018-04-24,A2,0,,3,1,3,0,0 204 | 31098446,2018-04-23,2018-04-23,B,0,,3,1,17,0,0 205 | 31098447,2018-04-23,2018-04-29,C,0,,3,1,8,0,0 206 | 31098448,2018-04-23,2018-04-30,A1,0,,3,1,15,1,9 207 | 31098449,2018-04-23,2018-04-23,A2,0,,3,1,16,1,5 208 | 31098450,2018-04-23,2018-05-01,B,1,E,3,1,8,0,0 209 | 31098451,2018-04-23,2018-04-24,C,0,,3,1,11,1,4 210 | 31098452,2018-04-23,2018-05-04,A1,1,E,3,1,4,1,4 211 | 31098453,2018-04-23,2018-04-24,A2,0,,3,1,4,1,9 212 | 31098454,2018-04-23,2018-04-25,B,1,,3,1,13,0,0 213 | 31098455,2018-04-23,2018-05-08,C,1,E,3,1,7,0,0 214 | 31098456,2018-04-23,2018-05-07,A1,1,,3,1,16,0,0 215 | 31098457,2018-04-23,2018-05-09,A2,0,E,3,1,15,1,8 216 | 31098459,2018-04-23,2018-05-03,C,1,,3,1,6,0,0 217 | 31098460,2018-04-23,2018-05-10,A1,1,E,3,1,2,0,0 218 | 31098461,2018-04-23,2018-04-30,A2,0,E,3,1,12,0,0 219 | 31098462,2018-04-23,2018-05-01,B,0,E,3,1,1,0,0 220 | 31098463,2018-04-23,2018-05-10,C,1,E,3,1,5,1,2 221 | 31098464,2018-04-23,2018-05-08,A1,0,M,3,1,2,0,0 222 | 31098465,2018-04-23,2018-05-03,A2,0,E,3,1,8,0,0 223 | 31098466,2018-04-23,2018-04-26,B,1,M,2,1,3,1,1 224 | 31098469,2018-04-23,2018-04-23,A2,0,,2,1,2,0,0 225 | 31098470,2018-04-23,2018-04-30,B,0,,2,1,3,0,0 226 | 31098471,2018-04-23,2018-04-30,C,1,E,2,1,14,1,0 227 | 31098472,2018-04-23,2018-04-27,A1,1,,2,1,2,1,8 228 | 31098473,2018-04-23,2018-04-23,A2,0,,2,1,2,1,5 229 | 31098474,2018-04-23,2018-04-24,B,1,M,2,1,4,0,0 230 | 31098475,2018-04-23,2018-04-24,C,0,,2,1,10,0,0 231 | 31098476,2018-04-23,2018-04-27,A1,1,E,2,1,4,1,9 232 | 31098477,2018-04-23,2018-04-23,A2,0,,2,1,2,0,0 233 | 31098478,2018-04-23,2018-04-23,B,1,,2,1,6,0,0 234 | 31098479,2018-04-23,2018-04-23,C,0,,2,1,6,1,8 235 | 31098480,2018-04-23,2018-04-23,A1,0,E,2,1,3,0,0 236 | 31098482,2018-04-23,2018-04-23,B,0,,2,1,2,0,0 237 | 31098483,2018-04-23,2018-04-23,C,0,E,2,1,1,1,2 238 | 31098484,2018-04-23,2018-04-23,A1,1,,2,1,4,1,2 239 | 31098485,2018-04-23,2018-05-07,A2,0,,2,1,2,1,4 240 | 31098486,2018-04-23,2018-04-29,B,0,E,2,1,11,1,3 241 | 31098489,2018-04-23,2018-04-24,A2,0,M,2,1,10,1,8 242 | 31098490,2018-04-23,2018-04-25,B,1,,2,1,5,0,0 243 | 31098491,2018-04-23,2018-04-23,C,0,H,2,1,2,1,10 244 | 31098492,2018-04-23,2018-04-25,A1,1,E,2,1,4,0,0 245 | 31098493,2018-04-23,2018-05-01,A2,0,M,2,1,3,0,0 246 | 31098494,2018-04-23,2018-05-09,B,0,E,2,1,1,1,8 247 | 31098495,2018-04-23,2018-04-30,C,1,E,2,1,3,0,0 248 | 31098497,2018-04-23,2018-04-23,A2,0,,2,1,2,1,2 249 | 31098498,2018-04-23,2018-05-09,B,1,E,2,1,2,0,0 250 | 31098499,2018-04-23,2018-04-25,C,1,H,2,1,5,0,0 251 | 31098500,2018-04-23,2018-04-23,A1,1,,2,1,1,0,0 252 | 31098501,2018-04-23,2018-05-09,A2,1,,2,1,3,1,3 253 | 31098502,2018-04-23,2018-04-25,B,0,E,2,1,2,0,0 254 | 31098504,2018-04-23,2018-05-09,A1,1,,2,1,9,0,0 255 | 31098505,2018-04-23,2018-04-23,A2,1,E,2,1,4,1,7 256 | 31098506,2018-04-23,2018-04-30,B,0,,2,1,3,1,4 257 | 31098508,2018-04-23,2018-04-23,A1,0,,2,1,2,0,0 258 | 31098510,2018-04-23,2018-05-04,B,1,E,2,1,5,0,0 259 | 31098511,2018-04-23,2018-04-26,C,0,,2,1,2,1,8 260 | 31098512,2018-04-23,2018-04-24,A1,0,,2,1,2,1,4 261 | 31098513,2018-04-23,2018-05-10,A2,1,E,2,1,4,0,0 262 | 31098514,2018-04-23,2018-04-23,B,1,,2,1,2,1,9 263 | 31098515,2018-04-23,2018-04-23,C,1,,2,1,2,1,2 264 | 31098516,2018-04-23,2018-04-24,A1,1,,2,1,5,1,3 265 | 31098517,2018-04-23,2018-04-23,A2,1,,2,1,18,1,10 266 | 31098518,2018-04-23,2018-04-23,B,1,,2,1,2,0,0 267 | 31098519,2018-04-23,2018-04-30,C,0,M,2,1,1,0,0 268 | 31098520,2018-04-23,2018-04-25,A1,1,,2,1,1,1,9 269 | 31098521,2018-04-23,2018-04-24,A2,0,E,2,1,13,1,6 270 | 31098522,2018-04-23,2018-04-23,B,1,E,2,1,1,1,5 271 | 31098523,2018-04-23,2018-05-08,C,0,,2,1,1,0,0 272 | 31098524,2018-04-23,2018-04-23,A1,0,,2,1,1,1,5 273 | 31098525,2018-04-23,2018-05-03,A2,0,M,2,1,7,0,0 274 | 31098526,2018-04-23,2018-04-24,B,1,E,2,1,4,1,10 275 | 31098527,2018-04-23,2018-04-23,C,0,,2,1,2,1,8 276 | 31098528,2018-04-23,2018-04-23,A1,0,E,2,1,1,1,3 277 | 31098529,2018-04-23,2018-04-25,A2,1,,2,1,2,0,0 278 | 31098530,2018-04-23,2018-04-25,B,1,,2,1,2,0,0 279 | 31098532,2018-04-23,2018-04-25,A1,1,,2,1,5,0,0 280 | 31098533,2018-04-23,2018-05-01,A2,0,E,2,1,12,1,2 281 | 31098535,2018-04-23,2018-04-23,C,0,H,2,1,4,0,0 282 | 31098536,2018-04-23,2018-04-23,A1,0,,2,1,9,0,0 283 | 31098537,2018-04-23,2018-05-01,A2,1,E,2,1,6,1,10 284 | 31098538,2018-04-23,2018-04-23,B,1,E,2,1,3,1,4 285 | 31098539,2018-04-23,2018-04-27,C,0,,2,1,4,1,1 286 | 31098541,2018-04-23,2018-04-23,A2,0,E,1,1,1,0,0 287 | 31098542,2018-04-23,2018-04-23,B,0,E,1,1,2,1,4 288 | 31098545,2018-04-27,2018-04-23,A2,1,,1,1,1,1,7 289 | 31098546,2018-04-27,2018-04-23,B,1,,1,1,2,0,0 290 | 31098548,2018-04-27,2018-04-23,A1,0,E,1,1,1,0,0 291 | 31098549,2018-04-27,2018-04-23,A2,0,,1,1,1,0,0 292 | 31098553,2018-04-27,2018-04-23,A2,0,M,1,1,1,0,0 293 | 31098554,2018-04-27,2018-04-23,B,0,E,1,1,1,0,0 294 | 31098555,2018-04-27,2018-04-23,C,1,E,1,1,1,0,0 295 | 31098556,2018-04-27,2018-04-23,A1,1,E,1,1,1,1,7 296 | 31098558,2018-04-27,2018-04-23,B,0,,1,1,1,1,3 297 | 31098559,2018-04-27,2018-04-23,C,1,,1,1,1,1,6 298 | 31098560,2018-04-27,2018-04-23,A1,1,E,1,1,2,1,7 299 | 31098561,2018-04-27,2018-04-23,A2,0,,1,1,1,1,7 300 | 31098562,2018-04-27,2018-04-23,B,1,E,1,1,2,1,4 301 | 31098565,2018-04-27,2018-04-23,A2,0,,1,1,3,0,0 302 | 31098566,2018-04-27,2018-04-23,B,1,,1,1,3,1,6 303 | 31098569,2018-04-27,2018-04-23,A2,1,,1,1,1,0,0 304 | 31098570,2018-04-27,2018-04-23,B,1,,1,1,1,1,7 305 | 31098571,2018-04-27,2018-04-23,C,1,H,1,1,2,0,0 306 | 31098572,2018-04-27,2018-04-23,A1,0,,1,1,1,1,4 307 | 31098574,2018-04-27,2018-04-23,B,0,,1,1,2,1,1 308 | 31098575,2018-04-27,2018-04-23,C,0,E,1,1,2,1,8 309 | 31098577,2018-04-27,2018-04-23,A2,1,,1,1,4,0,0 310 | 31098579,2018-04-27,2018-04-23,C,1,H,1,1,2,0,0 311 | 31098580,2018-04-27,2018-04-23,A1,0,,1,1,1,0,0 312 | 31098581,2018-04-27,2018-04-23,A2,0,,1,1,2,1,5 313 | 31098582,2018-04-27,2018-04-23,B,1,E,1,1,1,1,7 314 | 31098584,2018-04-27,2018-04-23,A1,1,,1,1,1,0,0 315 | 31098587,2018-04-27,2018-04-23,C,1,M,1,1,1,1,8 316 | 31098589,2018-04-27,2018-04-23,A2,1,M,1,1,5,0,0 317 | 31098590,2018-04-27,2018-04-23,B,1,M,1,1,1,1,1 318 | 31098591,2018-04-27,2018-04-23,C,1,M,1,1,1,1,4 319 | 31098592,2018-04-27,2018-04-23,A1,0,,1,1,7,1,3 320 | 31098593,2018-04-27,2018-04-23,A2,1,E,1,1,3,1,4 321 | 31098594,2018-04-27,2018-04-23,B,0,,1,1,1,0,0 322 | 31098596,2018-04-27,2018-04-23,A1,1,E,1,1,3,0,0 323 | 31098597,2018-04-27,2018-04-23,A2,0,,1,1,2,0,0 324 | 31098599,2018-04-27,2018-04-23,C,1,E,1,1,1,1,4 325 | 31098600,2018-04-27,2018-04-23,A1,1,,1,1,1,0,0 326 | 31098601,2018-04-27,2018-04-23,A2,1,,1,1,3,1,7 327 | 31098602,2018-04-27,2018-04-23,B,1,,1,1,1,1,4 328 | 31098604,2018-04-27,2018-04-23,A1,1,,1,1,1,0,0 329 | 31098605,2018-04-27,2018-04-23,A2,0,,1,1,2,0,0 330 | 31098606,2018-04-27,2018-04-23,B,0,,1,1,1,0,0 331 | 31098607,2018-04-27,2018-04-23,C,0,,1,1,5,1,4 332 | 31098608,2018-04-27,2018-04-23,A1,1,E,1,1,1,1,7 333 | 31098610,2018-04-27,2018-04-23,B,1,,1,1,2,0,0 334 | 31098611,2018-04-27,2018-04-23,C,1,H,1,1,1,0,0 335 | 31098613,2018-04-27,2018-04-23,A2,0,E,1,1,1,0,0 336 | 31098614,2018-04-27,2018-04-23,B,1,E,1,1,8,0,0 337 | 31098616,2018-04-27,2018-04-23,A1,0,,1,1,2,0,0 338 | 31098617,2018-04-27,2018-04-23,A2,0,E,1,1,2,0,0 339 | 31098618,2018-04-27,2018-04-23,B,0,E,1,1,1,1,6 340 | 31098620,2018-04-27,2018-04-23,A1,1,,1,1,1,0,0 341 | 31098621,2018-04-27,2018-04-23,A2,1,,1,1,1,1,5 342 | 31098622,2018-04-27,2018-04-23,B,0,,1,1,5,1,10 343 | 31098623,2018-04-27,2018-04-23,C,1,,1,1,1,1,5 344 | 31098624,2018-04-27,2018-04-23,A1,1,,1,1,1,0,0 345 | 31098625,2018-04-27,2018-04-23,A2,0,,1,1,1,0,0 346 | 31098630,2018-04-27,2018-04-23,B,0,,1,1,1,0,0 347 | 31098632,2018-04-27,2018-04-23,A1,0,,1,1,2,1,7 348 | 31098633,2018-04-27,2018-04-23,A2,1,,1,1,1,1,3 349 | 31098635,2018-04-27,2018-04-23,C,0,,1,1,1,0,0 350 | 31098636,2018-04-27,2018-04-23,A1,0,E,1,1,2,0,0 351 | 31098638,2018-04-27,2018-04-23,B,0,E,1,1,1,1,10 352 | 31098639,2018-04-27,2018-04-23,C,0,E,1,1,1,0,0 353 | 31098640,2018-04-27,2018-04-23,A1,1,,1,1,1,1,7 354 | 31098641,2018-04-27,2018-04-23,A2,1,M,1,1,1,0,0 355 | 31098643,2018-04-27,2018-04-23,C,1,,1,1,1,0,0 356 | 31098644,2018-04-27,2018-04-23,A1,1,M,1,1,1,0,0 357 | 31098645,2018-04-27,2018-04-23,A2,1,E,1,1,13,0,0 358 | 31098646,2018-04-27,2018-04-23,B,1,,1,1,1,1,7 359 | 31098649,2018-04-27,2018-04-23,A2,0,E,1,1,1,0,0 360 | 31098650,2018-04-27,2018-04-23,B,1,,1,1,3,1,4 361 | 31098651,2018-04-27,2018-04-23,C,1,,1,1,8,0,0 362 | 31098654,2018-04-27,2018-04-23,B,0,E,1,1,2,1,8 363 | 31098655,2018-04-27,2018-04-23,C,0,E,1,1,1,0,0 364 | 31098656,2018-04-27,2018-04-24,A1,0,E,1,1,1,1,2 365 | 31098657,2018-04-27,2018-04-23,A2,1,E,1,1,1,0,0 366 | 31098659,2018-04-27,2018-04-23,C,1,,1,1,3,0,0 367 | 31098662,2018-04-27,2018-04-23,B,0,M,1,1,2,1,7 368 | 31098664,2018-04-27,2018-04-24,A1,1,,1,1,4,1,4 369 | 31098665,2018-04-27,2018-04-23,A2,1,,1,1,3,0,0 370 | 31098666,2018-04-27,2018-04-23,B,0,,1,1,1,0,0 371 | 31098667,2018-04-27,2018-04-23,C,1,,1,1,2,1,6 372 | 31098668,2018-04-27,2018-04-23,A1,0,,1,1,2,1,8 373 | 31098670,2018-04-27,2018-04-23,B,1,,1,1,1,1,3 374 | 31098672,2018-04-27,2018-04-23,A1,1,E,1,1,2,0,0 375 | 31098673,2018-04-27,2018-04-23,A2,1,,1,1,1,0,0 376 | 31098674,2018-04-27,2018-04-23,B,0,,1,1,2,0,0 377 | 31098676,2018-04-27,2018-04-23,A1,1,,1,1,2,1,1 378 | 31098677,2018-04-27,2018-04-23,A2,1,,1,1,1,0,0 379 | 31098678,2018-04-27,2018-04-23,B,0,E,1,1,2,1,8 380 | 31098681,2018-04-27,2018-04-23,A2,1,,1,1,3,0,0 381 | 31098683,2018-04-27,2018-04-23,C,1,,1,1,1,0,0 382 | 31098684,2018-04-27,2018-04-23,A1,1,E,1,1,1,1,10 383 | 31098685,2018-04-27,2018-04-23,A2,1,,1,1,1,0,0 384 | 31098686,2018-04-27,2018-04-23,B,0,H,1,1,1,1,7 385 | 31098688,2018-04-27,2018-04-23,A1,1,E,1,1,8,0,0 386 | 31098689,2018-04-27,2018-04-23,A2,0,,1,1,3,1,3 387 | 31098690,2018-04-27,2018-04-23,B,0,,1,1,2,1,7 388 | 31098691,2018-04-27,2018-04-23,C,0,E,1,1,1,0,0 389 | 31098695,2018-04-27,2018-04-23,C,1,E,1,1,1,1,1 390 | 31098696,2018-04-27,2018-04-23,A1,1,,1,1,2,1,9 391 | 31098697,2018-04-27,2018-04-23,A2,0,H,1,1,1,0,0 392 | 31098699,2018-04-27,2018-04-23,C,0,,1,1,1,0,0 393 | 31098700,2018-04-27,2018-04-23,A1,1,,1,1,1,0,0 394 | 31098702,2018-04-27,2018-04-23,B,1,M,1,1,5,0,0 395 | 31098703,2018-04-27,2018-04-23,C,0,H,1,1,2,0,0 396 | 31098704,2018-04-27,2018-04-23,A1,1,,1,1,1,0,0 397 | 31098705,2018-04-27,2018-04-23,A2,0,,1,1,2,1,2 398 | 31098706,2018-04-27,2018-04-23,B,0,,1,1,1,0,0 399 | 31098707,2018-04-27,2018-04-23,C,1,,1,1,2,1,6 400 | 31098710,2018-04-27,2018-04-23,B,0,,1,1,1,1,6 401 | 31098711,2018-04-27,2018-04-23,C,0,,1,1,1,0,0 402 | 31098712,2018-04-27,2018-04-23,A1,0,,1,1,5,1,4 403 | 31098713,2018-04-27,2018-04-23,A2,1,E,1,1,2,0,0 404 | 31098714,2018-04-27,2018-04-23,B,0,,1,1,1,1,9 405 | 31098715,2018-04-27,2018-04-23,C,0,E,1,1,1,0,0 406 | 31098716,2018-04-27,2018-04-23,A1,0,E,1,1,2,0,0 407 | 31098717,2018-04-27,2018-04-23,A2,0,,1,1,2,0,0 408 | 31098718,2018-04-27,2018-04-23,B,1,E,1,1,6,1,7 409 | 31098722,2018-04-27,2018-04-23,B,0,,1,1,1,0,0 410 | 31098723,2018-04-27,2018-04-23,C,0,E,1,1,2,1,7 411 | 31098724,2018-04-27,2018-04-23,A1,0,E,1,1,1,1,9 412 | 31098725,2018-04-27,2018-04-23,A2,0,E,1,1,2,1,4 413 | 31098726,2018-04-27,2018-04-23,B,1,,1,1,2,1,3 414 | 31098727,2018-04-27,2018-04-23,C,0,,1,1,2,0,0 415 | 31098728,2018-04-27,2018-04-23,A1,1,E,1,1,1,0,0 416 | 31098729,2018-04-27,2018-04-23,A2,1,,1,1,3,0,0 417 | 31098730,2018-04-27,2018-04-23,B,1,,1,1,1,1,2 418 | 31098732,2018-04-27,2018-04-23,A1,1,E,1,1,12,1,5 419 | 31098734,2018-04-27,2018-04-23,B,1,E,1,1,7,1,7 420 | 31098735,2018-04-27,2018-04-23,C,0,,1,1,8,0,0 421 | 31098736,2018-04-27,2018-04-23,A1,1,,1,1,1,1,10 422 | 31098737,2018-04-27,2018-04-23,A2,0,,1,1,2,0,0 423 | 31098738,2018-04-27,2018-04-23,B,1,,1,1,1,0,0 424 | 31098739,2018-04-27,2018-04-23,C,1,,1,1,1,0,0 425 | 31098742,2018-04-27,2018-04-23,B,0,,1,1,1,0,0 426 | 31098744,2018-04-27,2018-04-23,A1,1,E,1,1,1,0,0 427 | 31098749,2018-04-27,2018-04-23,A2,0,E,1,1,9,1,8 428 | 31098751,2018-04-27,2018-04-23,C,0,,1,1,1,1,9 429 | 31098754,2018-04-27,2018-04-23,B,1,M,1,1,2,1,4 430 | 31098755,2018-04-27,2018-04-23,C,0,,1,1,1,0,0 431 | 31098756,2018-04-27,2018-04-23,A1,0,E,1,1,1,1,5 432 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AB-Testing 2 | 3 | ## Summary 4 | A Jupyter notebook template/tutorial for running and evaluating ABn tests where the KPI of interest is binomial. 5 | 6 | ## Motivation 7 | Reduce the amount of work required to set up and evaluate website experiments. 8 | 9 | ## What is included in this notebook? 10 | - Step by step guide with instructions for setting up an experiment and explanations for how to interpret results 11 | - Deconstructed built-in python functions used to calculate test statistics 12 | - Plug and play format with call-outs to customizable cell blocks to allow the experimenter to quickly load their own data and modify formula default inputs 13 | - Links to additional resources 14 | - Explanation for how to evaluate multiple variation tests 15 | 16 | ## How to navigate this notebook? 17 | - Set Up: Pre Test 18 | - Sample Size Calculator 19 | - Alpha Cutoff (Type I Error & Bonferroni Adjustment) 20 | - Determining Minimum Detectable Effect Size 21 | - Set Up: Proportional Data 22 | - General Data Set Up (should be applicable to most data for proportion evaluation) 23 | - Test Evaluation 24 | - 2 Sample Testing (z-test) 25 | - 2+ Sample Testing (chi-square) 26 | - Marascuilo Procedure (Multi pairwise z-test) 27 | 28 | ## How to use this notebook in order to get the most out of it? 29 | - Load in a csv file (or query result) that matches the data structure (see section *Data Setup: Proportion*). 30 | - Any `Cell block` that requires the experimenter to modify/input new variables is denoted with the color red and the following heading ***~ Set up Variables ~ *** (the rest of the cell blocks should not require any modifications). 31 | - The 2 and 2+ Sample Test sections do NOT need to be run sequentially (*2+ Sample Testing* section will work without first having to run the cell blocks in the *2 Sample Testing*). 32 | 33 | ## Feedback 34 | I wanted to share some of the kind words that my colleagues used to describe the notebook. 35 | - "I used a bunch of your code from the AB testing notebook for proportions in a HP testing notebook. It worked pretty smoothly!" 36 | - "This is a fantastic notebook. It is thorough and comprehensive. You've left no stone unturned." 37 | - "Isn't it great how fast I can asnwer all these questions?! (love this notebook)" 38 | - "Starting to see some positive results... Thanks again for all your help getting the analysis notebooks together" 39 | 40 | 41 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | datetime 2 | math 3 | matplotlib.pyplot 4 | numpy 5 | pandas 6 | os 7 | scip.stats 8 | statsmodels.stats.proportion 9 | seaborn 10 | time 11 | --------------------------------------------------------------------------------