├── .github └── workflows │ └── tests.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.rst ├── example_project ├── __init__.py ├── manage.py ├── settings.py ├── templates │ ├── 404.html │ ├── 500.html │ ├── goal.html │ └── test_page.html └── urls.py ├── experiments ├── __init__.py ├── admin.py ├── admin_utils.py ├── apps.py ├── conf.py ├── counters.py ├── dateutils.py ├── experiment_counters.py ├── manager.py ├── middleware.py ├── migrations │ ├── 0001_initial.py │ ├── 0002_auto_20201013_1408.py │ ├── 0003_alter_experiment_alternatives_and_more.py │ └── __init__.py ├── models.py ├── redis_client.py ├── signal_handlers.py ├── signals.py ├── significance.py ├── south_migrations │ ├── 0001_initial.py │ ├── 0002_auto__chg_field_enrollment_goals_.py │ ├── 0003_auto__del_field_enrollment_goals__add_field_enrollment_last_seen__chg_.py │ └── __init__.py ├── static │ └── experiments │ │ ├── dashboard │ │ ├── css │ │ │ └── admin.css │ │ └── js │ │ │ ├── admin.js │ │ │ └── csrf.js │ │ └── js │ │ ├── experiments.js │ │ └── jquery.cookie.js ├── stats.py ├── templates │ ├── admin │ │ └── experiments │ │ │ ├── change_form.html │ │ │ └── results_table.html │ └── experiments │ │ ├── confirm_human.html │ │ └── goal.html ├── templatetags │ ├── __init__.py │ └── experiments.py ├── tests │ ├── __init__.py │ ├── test_admin.py │ ├── test_counter.py │ ├── test_models.py │ ├── test_signals.py │ ├── test_significance.py │ ├── test_templatetags.py │ ├── test_webuser.py │ ├── test_webuser_incorporate.py │ └── urls.py ├── urls.py ├── utils.py └── views.py ├── setup.cfg ├── setup.py ├── testrunner.py └── tox.ini /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Test suite 2 | 3 | on: 4 | push: 5 | branches: [ '**' ] 6 | pull_request: 7 | branches: [ '**' ] 8 | 9 | jobs: 10 | tox: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | max-parallel: 5 14 | matrix: 15 | python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.8', 'pypy-3.9', 'pypy-3.10'] 16 | fail-fast: false 17 | 18 | steps: 19 | - uses: actions/checkout@v2 20 | - name: Setup Redis 21 | uses: zhulik/redis-action@1.1.0 22 | - name: Set up python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v2 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | - name: Install dependencies 27 | run: | 28 | python -m pip install --upgrade pip tox tox-gh-actions 29 | pip install . 30 | - name: Run test suite with tox 31 | run: tox 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/* 2 | .vscode/ 3 | *.pyc 4 | *~ 5 | *conflicted copy* 6 | *.sublime* 7 | *experiments.db 8 | .DS_Store 9 | django_experiments.egg-info 10 | *.egg 11 | dist/ 12 | .tox 13 | .eggs/ 14 | .coverage 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2012 Mixcloud 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE README.rst 2 | recursive-include experiments *.py *.html *.js *.css *.png *.jpg 3 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Django-Experiments 2 | ================== 3 | 4 | .. image:: https://github.com/mixcloud/django-experiments/workflows/Test%20suite/badge.svg 5 | :target: https://github.com/mixcloud/django-experiments/actions 6 | 7 | Django-Experiments is an AB Testing Framework for Django. 8 | 9 | It is possible to set up an experiment through template tags only. 10 | Through the Django admin you can monitor and control experiment progress. 11 | 12 | If you don't know what AB testing is, check out `wikipedia `_. 13 | 14 | 15 | Installation 16 | ------------ 17 | 18 | Django-Experiments is best installed via pip: 19 | 20 | .. code-block:: bash 21 | 22 | pip install django-experiments 23 | 24 | This should download django-experiments and any dependencies. If downloading from the repo, 25 | pip is still the recommended way to install dependencies: 26 | 27 | .. code-block:: bash 28 | 29 | pip install -e . 30 | 31 | Dependencies 32 | ------------ 33 | - `Django `_ 34 | - `Redis `_ 35 | - `django-modeldict `_ 36 | 37 | (Detailed list in setup.py) 38 | 39 | It also requires 'django.contrib.humanize' to be in INSTALLED_APPS. 40 | 41 | Usage 42 | ----- 43 | 44 | The example project is a good place to get started and have a play. 45 | Results are stored in redis and displayed in the Django admin. The key 46 | components of this framework are: the experiments, alternatives and 47 | goals. 48 | 49 | 50 | Configuration 51 | ~~~~~~~~~~~~~ 52 | 53 | Before you can start configuring django-experiments, you must ensure 54 | you have a redis server up and running. See `redis.io `_ for downloads and documentation. 55 | 56 | This is a quick guide to configuring your settings file to the bare minimum. 57 | First, add the relevant settings for your redis server (we run it as localhost): 58 | 59 | .. code-block:: python 60 | 61 | #Example Redis Settings 62 | EXPERIMENTS_REDIS_HOST = 'localhost' 63 | EXPERIMENTS_REDIS_PORT = 6379 64 | EXPERIMENTS_REDIS_DB = 0 65 | 66 | Next, activate the apps by adding them to your INSTALLED_APPS: 67 | 68 | .. code-block:: python 69 | 70 | #Installed Apps 71 | INSTALLED_APPS = [ 72 | ... 73 | 'django.contrib.admin', 74 | 'django.contrib.humanize', 75 | 'experiments', 76 | ] 77 | 78 | Include 'django.contrib.humanize' as above if not already included. 79 | 80 | Include the app URLconf in your urls.py file: 81 | 82 | .. code-block:: python 83 | 84 | url(r'experiments/', include('experiments.urls')), 85 | 86 | We haven't configured our goals yet, we'll do that in a bit. Please ensure 87 | you have correctly configured your STATIC_URL setting. 88 | 89 | 90 | Include following JS libraries to your base template: 91 | 92 | .. code-block:: html 93 | 94 | 95 | 96 | 97 | OPTIONAL: 98 | If you want to use the built in retention goals you will need to include the retention middleware: 99 | 100 | .. code-block:: python 101 | 102 | MIDDLEWARE_CLASSES [ 103 | ... 104 | 'experiments.middleware.ExperimentsRetentionMiddleware', 105 | ] 106 | 107 | *Note, more configuration options are detailed below.* 108 | 109 | 110 | Experiments and Alternatives 111 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 112 | 113 | The experiment can be manually created in your Django admin. Adding alternatives must currently be done in template tags or by calling the relevant code, as described below. 114 | 115 | An experiment allows you to test the effect of various design 116 | alternatives on user interaction. Django-Experiments is designed to work 117 | from within django templates, to make it easier for designers. We begin 118 | by loading our module: 119 | 120 | .. code-block:: html 121 | 122 | {% load experiments %} 123 | 124 | and we then define our first experiment and alternative, using the 125 | following syntax: 126 | 127 | .. code-block:: html 128 | 129 | {% experiment EXPERIMENT ALTERNATIVE %} 130 | 131 | We are going to run an experiment called “register\_text” to see what 132 | registration link text causes more users to complete the registration 133 | process. Our first alternative must always be the “control” alternative. 134 | This is our fallback if the experiment is disabled. 135 | 136 | .. code-block:: html 137 | 138 | {% experiment register_text control %} 139 | Register now. 140 | {% endexperiment %} 141 | 142 | So while the experiment is disabled, users will see a register link 143 | saying “Register now”. Let’s define another, more polite alternative: 144 | 145 | .. code-block:: html 146 | 147 | {% experiment register_text polite %} 148 | Please register! 149 | {% endexperiment %} 150 | 151 | While experiment is disabled, users will still see the “control” 152 | alternative, and their registration link will say “Register now”. When 153 | the experiment is enabled, users will be randomly assigned to each 154 | alternative. This information is stored in the enrollment, a unique 155 | combination of the user, the experiment and which alternative they are 156 | assigned to. 157 | 158 | Make sure the experiment tag has access to the request object (not an 159 | issue for regular templates but you might have to manually add it 160 | inside an inclusion tag) or it will silently fail to work. 161 | 162 | The experiment_enroll assignment tag can also be used (note that it 163 | takes strings or variables unlike the older experiment tag): 164 | 165 | .. code-block:: html 166 | 167 | {% experiment_enroll "experiment_name" "alternative1" "alternative2" as assigned_alternative %} 168 | {% if assigned_alternative == "alternative1" or assigned_alternative == "alternative2" %} 169 | Please register! 170 | {% else %} 171 | Register now. 172 | {% endif %} 173 | 174 | You can also enroll users in experiments and find out what alternative they 175 | are part of from python. To enroll a user in an experiment and show a 176 | different result based on the alternative: 177 | 178 | .. code-block:: python 179 | 180 | from experiments.utils import participant 181 | alternative = participant(request).enroll('register_text', ['polite']) 182 | if alternative == 'polite': 183 | text_to_show = get_polite_text() 184 | elif alternative == 'control': 185 | text_to_show = get_normal_text() 186 | 187 | If you wish to find out what experiment alternative a user is part of, but not 188 | enroll them if they are not yet a member, you can use get_alternative. This 189 | will return 'control' if the user is not enrolled. 'control' is also returned 190 | for users who are enrolled in the experiment but have been assigned to the 191 | control group - there is no way to differentiate between these cases. 192 | 193 | .. code-block:: python 194 | 195 | from experiments.utils import participant 196 | alternative = participant(request).get_alternative('register_text') 197 | if alternative == 'polite': 198 | header_text = get_polite_text_summary() 199 | elif alternative == 'control': 200 | header_text = get_normal_text_summary() 201 | 202 | You can also weight the experiments using the following techniques 203 | 204 | .. code-block:: python 205 | 206 | alternative = participant(request).enroll('example_test', {'control': 99, 'v2': 1}) 207 | 208 | .. code-block:: html 209 | 210 | {% experiment example_test control 99 %}v2{% endexperiment %} 211 | {% experiment example_test v2 1 %}v2{% endexperiment %} 212 | 213 | By default the participant function expects a HttpRequest object, but you can 214 | alternatively pass a user or session as a keyword argument 215 | 216 | .. code-block:: python 217 | 218 | participant(user=current_user).get_alternative('register_text') 219 | participant(session=session).get_alternative('register_text') 220 | 221 | 222 | \*\ *Experiments will be dynamically created by default if they are 223 | defined in a template but not in the admin. This can be overridden in 224 | settings.* 225 | 226 | After creating an experiment either using the Django admin, or through 227 | template tags or code, you must enable the experiment in the Django 228 | admin or manually for it to work. 229 | 230 | 231 | 232 | Goals 233 | ~~~~~ 234 | 235 | Goals allow us to acknowledge when a user hits a certain page. You 236 | specify them in the EXPERIMENTS\_GOALS tuple in your settings. Given the 237 | example above, we would want a goal to be triggered once the user has 238 | completed the registration process. 239 | 240 | Add the goal to our EXPERIMENT_GOALS tuple in settings.py: 241 | 242 | .. code-block:: python 243 | 244 | EXPERIMENTS_GOALS = ("registration",) 245 | 246 | Goals are simple strings that uniquely identify a goal. 247 | 248 | Our registration successful page will contain the goal template tag: 249 | 250 | .. code-block:: html 251 | 252 | {% experiment_goal "registration" %} 253 | 254 | This will be fired when the user loads the page. This is not the only way of firing a goal. In total, there are four ways of recording goals: 255 | 256 | 1. **Django Template Tags** (as above). 257 | 258 | .. code-block:: html 259 | 260 | {% experiment_goal "registration" %} 261 | 262 | 2. **Server side**, using a python function somewhere in your django views: 263 | 264 | .. code-block:: python 265 | 266 | from experiments.utils import participant 267 | 268 | participant(request).goal('registration') 269 | 270 | 3. **JavaScript onclick**: 271 | 272 | .. code-block:: html 273 | 274 | 275 | 276 | (Please note, this requires CSRF authentication. Please see the `Django Docs `_) 277 | The CSRF code would be something like: 278 | 279 | .. code-block:: javascript 280 | 281 | $.ajaxSetup({ 282 | headers: 283 | { 'X-CSRFToken': Cookies.get('csrftoken') } 284 | }); 285 | 286 | 4. **Cookies**: 287 | 288 | .. code-block:: html 289 | 290 | Complete Registration 291 | 292 | Multiple goals can be recorded via the cookie using space as a separator. 293 | 294 | The goal is independent from the experiment as many experiments can all 295 | have the same goal. The goals are defined in the settings.py file for 296 | your project. 297 | 298 | Retention Goals 299 | ~~~~~~~~~~~~~~~ 300 | 301 | There are two retention goals (VISIT_PRESENT_COUNT_GOAL and VISIT_NOT_PRESENT_COUNT_GOAL that 302 | default to '_retention_present_visits' and '_retention_not_present_visits' respectively). To 303 | use these install the retention middleware. A visit is defined by no page views within 304 | SESSION_LENGTH hours (defaults to 6). 305 | 306 | VISIT_PRESENT_COUNT_GOAL does not trigger until the next visit after the user is enrolled and 307 | should be used in most cases. VISIT_NOT_PRESENT_COUNT_GOAL triggers on the first visit after 308 | enrollment and should be used in situations where the user isn't present when being enrolled 309 | (for example when sending an email). Both goals are tracked for all experiments so take care 310 | to only use one when interpreting the results. 311 | 312 | Confirming Human 313 | ~~~~~~~~~~~~~~~~ 314 | 315 | The framework can distinguish between humans and bots. By including 316 | 317 | .. code-block:: html 318 | 319 | {% load experiments %} 320 | 321 | {% experiments_confirm_human %} 322 | 323 | at some point in your code (we recommend you put it in your base.html 324 | file), unregistered users will then be confirmed as human. This can be 325 | quickly overridden in settings, but be careful - bots can really mess up 326 | your results! 327 | 328 | If you want to customize the confirm human code you can change the 329 | CONFIRM_HUMAN_SESSION_KEY setting and manage setting the value yourself. 330 | Note that you need to call confirm_human on the participant when they 331 | become confirmed as well as setting session[CONFIRM_HUMAN_SESSION_KEY] 332 | equal to True. 333 | 334 | Managing Experiments 335 | -------------------- 336 | 337 | Experiments can be managed in the Django admin (/admin/experiments/experiment/ by 338 | default). 339 | 340 | The States 341 | ~~~~~~~~~~ 342 | 343 | **Control** - The experiment is essentially disabled. All users will see 344 | the control alternative, and no data will be collected. 345 | 346 | **Enabled** - The experiment is enabled globally, for all users. 347 | 348 | 349 | Settings 350 | -------- 351 | 352 | .. code-block:: python 353 | 354 | #Experiment Goals 355 | EXPERIMENTS_GOALS = () 356 | 357 | #Auto-create experiment if doesn't exist 358 | EXPERIMENTS_AUTO_CREATE = True 359 | 360 | #Toggle whether the framework should verify user is human. Be careful. 361 | EXPERIMENTS_VERIFY_HUMAN = False 362 | 363 | #Example Redis Settings 364 | EXPERIMENTS_REDIS_HOST = 'localhost' 365 | EXPERIMENTS_REDIS_PORT = 6379 366 | EXPERIMENTS_REDIS_DB = 0 367 | 368 | See conf.py for other settings 369 | 370 | 371 | Changelog 372 | --------- 373 | UNRELEASED 374 | ~~~~~~~~~~ 375 | - Conform to common expectations in `setup.py`: 376 | - Separate `install_requires` and `tests_require` (not reading from `requirements.txt`) 377 | - Add trove classifiers including Python and Django supported versions 378 | - Fix license name (from "MIT license, see LICENSE file" to "MIT") 379 | - Make `setup.py` ready for Python 3 (read `README.rst` using codecs module) 380 | - Dropped an irrelevant workaround for ancient Python bugs 381 | - Add `setup.cfg` to support building of universal wheels (preparing for Python 3) 382 | - Tox runs `python setup.py test` (honouring both `install_requires` and `tests_require`) 383 | - Prepared `tox.ini` for Python 3 and Django 1.11 compatibility 384 | - Remove dependency on jsonfield, use Django builtin JSONField 385 | 386 | 1.2.0 387 | ~~~~~ 388 | - Add support for Django 1.10 (Thanks to @Kobold) 389 | - Make requirements.txt more flexible 390 | - Tox support added for testing on multiple Django Versions (Thanks to @Kobold again!) 391 | 392 | 1.1.6 393 | ~~~~~ 394 | - Change to use django-modeldict-yplan as its maintained 395 | - Change to use pythons inbuilt unittest and not Django's as its Deprecated) 396 | 397 | 1.1.5 398 | ~~~~~ 399 | - Removing experiment_helpers template tag library since it is no longer used and breaks under Django 1.9 (thanks david12341235) 400 | 401 | 1.1.4 402 | ~~~~~ 403 | 404 | - Removing django-jsonfield from requirements.txt (thank you to bustavo) and adding jsonfield 405 | 406 | 1.1.2 407 | ~~~~~ 408 | 409 | - Updating migrations 410 | - Documentation improvements 411 | - Updating example app 412 | 413 | 1.1.1 414 | ~~~~~ 415 | 416 | - Fixing EXPERIMENTS_AUTO_CREATE flag (previously setting it to True did nothing) 417 | 418 | 1.1.0 419 | ~~~~~ 420 | 421 | - Nexus is no longer required or used - the standard Django admin for the Experiment model takes over the functionality previously provided by Nexus - NOTE this may have some backwards incompatibilities depending on how you included the media files 422 | - Promote an experiment to a particular alternative (other than Control) through the admin 423 | - New experiment_enroll assignment tag (see below) 424 | 425 | 1.0.0 426 | ~~~~~ 427 | 428 | Bumping version to 1.0.0 because django-experiments is definitely production 429 | ready but also due to backwards incompatible changes that have been merged in. 430 | 431 | - Django 1.7 and 1.8 support (including custom user models) 432 | - Fixed numerous bugs to do with retention goals - before this update they are not trustworthy. See retention section below for more information. 433 | - Fixed bug caused by the participant cache on request 434 | - Fixed bugs related to confirm human and made the functionality pluggable 435 | - Added "force_alternative" option to participant.enroll (important note: forcing the alternative in a non-random way will generate potentially invalid results) 436 | - Removal of gargoyle integration and extra "request" parameters to methods that no longer need them such as is_enrolled (BACKWARDS INCOMPATIBLE CHANGE) 437 | - ExperimentsMiddleware changed to ExperimentsRetentionMiddleware (BACKWARDS INCOMPATIBLE CHANGE) 438 | - More tests and logging added 439 | 440 | 0.3.5 441 | ~~~~~ 442 | 443 | - Add migration scripts for south 444 | - Fix rendering when probabilities close to 100% 445 | - Reduce database load when a user performs an action multiple times 446 | 447 | 0.3.4 448 | ~~~~~ 449 | 450 | - Updated JS goal to POST method. Requires csrf javascript. 451 | - Random number on template tag goal image to prevent caching 452 | 453 | 454 | 0.3.3 455 | ~~~~~ 456 | 457 | - Static media handled by nexus again 458 | 459 | 0.3.2 460 | ~~~~~ 461 | 462 | - Fixed missing edit/delete images 463 | 464 | 0.3.1 465 | ~~~~~ 466 | 467 | - Replaced django static template tags. Supports django 1.3 again! 468 | 469 | 0.3.0 470 | ~~~~~ 471 | 472 | - Added django permission support. 473 | - Started using django static instead of nexus:media. (django 1.4 only) 474 | -------------------------------------------------------------------------------- /example_project/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mixcloud/django-experiments/df6fe5d2e83a96539206d4bdcac4bf6e3d38d2ca/example_project/__init__.py -------------------------------------------------------------------------------- /example_project/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | from django.conf import settings 6 | 7 | if __name__ == "__main__": 8 | 9 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") 10 | 11 | from django.core.management import execute_from_command_line 12 | 13 | execute_from_command_line(sys.argv) 14 | -------------------------------------------------------------------------------- /example_project/settings.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import sys 3 | 4 | # Experiments Settings 5 | EXPERIMENTS_GOALS = ( 6 | 'page_goal', 7 | 'js_goal', 8 | 'cookie_goal', 9 | ) 10 | 11 | EXPERIMENTS_AUTO_CREATE = True 12 | 13 | EXPERIMENTS_VERIFY_HUMAN = True #Careful with this setting, if it is toggled then participant counters will not increment accordingly 14 | 15 | # Redis Settings 16 | EXPERIMENTS_REDIS_HOST = 'localhost' 17 | EXPERIMENTS_REDIS_PORT = 6379 18 | EXPERIMENTS_REDIS_DB = 0 19 | 20 | 21 | # Media Settings 22 | STATIC_URL = '/static/' 23 | 24 | # Other settings 25 | # Django settings for example_project project. 26 | NEXUS_MEDIA_PREFIX = '/nexus/media/' 27 | 28 | DEBUG = True 29 | TEMPLATE_DEBUG = True 30 | 31 | ADMINS = ( 32 | # ('Your Name', 'your_email@domain.com'), 33 | ) 34 | 35 | INTERNAL_IPS = ('127.0.0.1',) 36 | 37 | MANAGERS = ADMINS 38 | 39 | PROJECT_ROOT = os.path.dirname(__file__) 40 | 41 | sys.path.insert(0, os.path.abspath(os.path.join(PROJECT_ROOT, '..'))) 42 | 43 | DATABASES = { 44 | 'default': { 45 | 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 46 | 'NAME': 'experiments.db', # Or path to database file if using sqlite3. 47 | 'USER': '', # Not used with sqlite3. 48 | 'PASSWORD': '', # Not used with sqlite3. 49 | 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. 50 | 'PORT': '', # Set to empty string for default. Not used with sqlite3. 51 | } 52 | } 53 | 54 | # Local time zone for this installation. Choices can be found here: 55 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 56 | # although not all choices may be available on all operating systems. 57 | # On Unix systems, a value of None will cause Django to use the same 58 | # timezone as the operating system. 59 | # If running in a Windows environment this must be set to the same as your 60 | # system time zone. 61 | TIME_ZONE = 'Europe/London' 62 | 63 | # Language code for this installation. All choices can be found here: 64 | # http://www.i18nguy.com/unicode/language-identifiers.html 65 | LANGUAGE_CODE = 'en-us' 66 | 67 | SITE_ID = 1 68 | 69 | # If you set this to False, Django will make some optimizations so as not 70 | # to load the internationalization machinery. 71 | USE_I18N = True 72 | 73 | # If you set this to False, Django will not format dates, numbers and 74 | # calendars according to the current locale 75 | USE_L10N = True 76 | 77 | # Absolute path to the directory that holds media. 78 | # Example: "/home/media/media.lawrence.com/" 79 | MEDIA_ROOT = '' 80 | 81 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 82 | # trailing slash if there is a path component (optional in other cases). 83 | # Examples: "http://media.lawrence.com", "http://example.com/media/" 84 | MEDIA_URL = '' 85 | 86 | # URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a 87 | # trailing slash. 88 | # Examples: "http://foo.com/media/", "/media/". 89 | ADMIN_MEDIA_PREFIX = '/admin/media/' 90 | 91 | # Make this unique, and don't share it with anybody. 92 | SECRET_KEY = 'gfjo;2r3l;hjropjf30j3fl;m234nc9p;o2mnpfnpfj' 93 | 94 | # List of callables that know how to import templates from various sources. 95 | 96 | MIDDLEWARE = ( 97 | 'django.middleware.common.CommonMiddleware', 98 | 'django.contrib.sessions.middleware.SessionMiddleware', 99 | 'django.middleware.csrf.CsrfViewMiddleware', 100 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 101 | 'django.contrib.messages.middleware.MessageMiddleware', 102 | 'experiments.middleware.ExperimentsRetentionMiddleware', 103 | ) 104 | 105 | ROOT_URLCONF = 'example_project.urls' 106 | 107 | TEMPLATES = [ 108 | { 109 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 110 | 'DIRS': [ 111 | os.path.join(PROJECT_ROOT, 'templates'), 112 | ], 113 | 'APP_DIRS': False, 114 | 'OPTIONS': { 115 | 'context_processors': { 116 | "django.contrib.auth.context_processors.auth", 117 | "django.template.context_processors.debug", 118 | "django.template.context_processors.i18n", 119 | "django.template.context_processors.media", 120 | "django.template.context_processors.static", 121 | "django.template.context_processors.request", 122 | "django.contrib.messages.context_processors.messages", 123 | }, 124 | 'loaders': ( 125 | 'django.template.loaders.filesystem.Loader', 126 | 'django.template.loaders.app_directories.Loader', 127 | # 'django.template.loaders.eggs.Loader', 128 | ) 129 | }, 130 | }, 131 | ] 132 | 133 | 134 | INSTALLED_APPS = ( 135 | 'django.contrib.auth', 136 | 'django.contrib.contenttypes', 137 | 'django.contrib.sessions', 138 | 'django.contrib.humanize', 139 | 'django.contrib.messages', 140 | 'django.contrib.staticfiles', 141 | 'experiments', 142 | # Uncomment the next line to enable the admin: 143 | 'django.contrib.admin', 144 | ) 145 | -------------------------------------------------------------------------------- /example_project/templates/404.html: -------------------------------------------------------------------------------- 1 | 404 error -------------------------------------------------------------------------------- /example_project/templates/500.html: -------------------------------------------------------------------------------- 1 | 500 error -------------------------------------------------------------------------------- /example_project/templates/goal.html: -------------------------------------------------------------------------------- 1 | {% load experiments %} 2 | 3 | 4 | 5 | Goal Page 6 | 7 | 8 | {% experiment_goal "page_goal" %} 9 | Back 10 | 11 | 12 | -------------------------------------------------------------------------------- /example_project/templates/test_page.html: -------------------------------------------------------------------------------- 1 | {% load experiments %} 2 | {% load static %} 3 | 4 | 5 | 6 | Experiment Test Page 7 | 8 | 9 | 10 | 11 | 50 | 51 | 52 | 53 | {% csrf_token %} 54 | 55 | {% experiment helloworld control %} 56 | Click Me (Control) 57 | {% endexperiment %} 58 | 59 | {% experiment helloworld test %} 60 | Don't Click Me (test) 61 | {% endexperiment %} 62 | 63 | {% experiment_enroll "helloworld" "control" "test" as alternative %} 64 | {% if alternative == "test" %} 65 | You're in the "test" alternative! 66 | {% endif %} 67 | 68 | JS GOAL 69 | COOKIE GOAL 70 | 71 | {% experiments_confirm_human %} 72 | 73 | 74 | -------------------------------------------------------------------------------- /example_project/urls.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.urls import include, path 3 | from django.views.generic import TemplateView 4 | 5 | admin.autodiscover() 6 | 7 | urlpatterns = [ 8 | path('experiments/', include('experiments.urls')), 9 | path('admin/', admin.site.urls), 10 | path('', TemplateView.as_view(template_name="test_page.html"), name="test_page"), 11 | path('goal/', TemplateView.as_view(template_name="goal.html"), name="goal"), 12 | ] 13 | 14 | -------------------------------------------------------------------------------- /experiments/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mixcloud/django-experiments/df6fe5d2e83a96539206d4bdcac4bf6e3d38d2ca/experiments/__init__.py -------------------------------------------------------------------------------- /experiments/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.contrib.admin.utils import unquote 3 | from django import forms 4 | from django.http import JsonResponse, HttpResponse, HttpResponseBadRequest, HttpResponseForbidden 5 | from django.utils import timezone 6 | from experiments.admin_utils import get_result_context 7 | from experiments.models import Experiment 8 | from experiments import conf 9 | from django.urls import path 10 | from experiments.utils import participant 11 | 12 | 13 | class ExperimentAdmin(admin.ModelAdmin): 14 | list_display = ('name', 'start_date', 'end_date', 'state') 15 | list_filter = ('state', 'start_date', 'end_date') 16 | ordering = ('-start_date',) 17 | search_fields = ('=name',) 18 | actions = None 19 | readonly_fields = ['start_date', 'end_date'] 20 | 21 | def get_fieldsets(self, request, obj=None): 22 | """ 23 | Slightly different fields are shown for Add and Change: 24 | - default_alternative can only be changed 25 | - name can only be set on Add 26 | """ 27 | main_fields = ('description', 'start_date', 'end_date', 'state') 28 | 29 | if obj: 30 | main_fields += ('default_alternative',) 31 | else: 32 | main_fields = ('name',) + main_fields 33 | 34 | return ( 35 | (None, { 36 | 'fields': main_fields, 37 | }), 38 | ('Relevant Goals', { 39 | 'classes': ('collapse', 'hidden-relevant-goals'), 40 | 'fields': ('relevant_chi2_goals', 'relevant_mwu_goals'), 41 | }) 42 | ) 43 | 44 | # --------------------------------------- Default alternative 45 | 46 | def get_form(self, request, obj=None, **kwargs): 47 | """ 48 | Add the default alternative dropdown with appropriate choices 49 | """ 50 | if obj: 51 | if obj.alternatives: 52 | choices = [(alternative, alternative) for alternative in obj.alternatives.keys()] 53 | else: 54 | choices = [(conf.CONTROL_GROUP, conf.CONTROL_GROUP)] 55 | 56 | class ExperimentModelForm(forms.ModelForm): 57 | default_alternative = forms.ChoiceField(choices=choices, 58 | initial=obj.default_alternative, 59 | required=False) 60 | kwargs['form'] = ExperimentModelForm 61 | return super(ExperimentAdmin, self).get_form(request, obj=obj, **kwargs) 62 | 63 | def save_model(self, request, obj, form, change): 64 | if change: 65 | obj.set_default_alternative(form.cleaned_data['default_alternative']) 66 | obj.save() 67 | 68 | # --------------------------------------- Overriding admin views 69 | 70 | class Media: 71 | css = { 72 | "all": ( 73 | 'experiments/dashboard/css/admin.css', 74 | ), 75 | } 76 | js = ( 77 | 'https://www.google.com/jsapi', # used for charts 78 | 'experiments/dashboard/js/csrf.js', 79 | 'experiments/dashboard/js/admin.js', 80 | ) 81 | 82 | def _admin_view_context(self, extra_context=None): 83 | context = {} 84 | if extra_context: 85 | context.update(extra_context) 86 | context.update({ 87 | 'all_goals': conf.ALL_GOALS, 88 | 'control_group': conf.CONTROL_GROUP, 89 | }) 90 | return context 91 | 92 | def add_view(self, request, form_url='', extra_context=None): 93 | return super(ExperimentAdmin, self).add_view(request, 94 | form_url=form_url, 95 | extra_context=self._admin_view_context(extra_context=extra_context)) 96 | 97 | def change_view(self, request, object_id, form_url='', extra_context=None): 98 | experiment = self.get_object(request, unquote(object_id)) 99 | context = self._admin_view_context(extra_context=extra_context) 100 | context.update(get_result_context(request, experiment)) 101 | return super(ExperimentAdmin, self).change_view(request, object_id, form_url=form_url, extra_context=context) 102 | 103 | # --------------------------------------- Views for ajax functionality 104 | 105 | def get_urls(self): 106 | experiment_urls = [ 107 | path('set-alternative/', self.admin_site.admin_view(self.set_alternative_view), name='experiment_admin_set_alternative'), 108 | path('set-state/', self.admin_site.admin_view(self.set_state_view), name='experiment_admin_set_state'), 109 | ] 110 | return experiment_urls + super(ExperimentAdmin, self).get_urls() 111 | 112 | def set_alternative_view(self, request): 113 | """ 114 | Allows the admin user to change their assigned alternative 115 | """ 116 | if not request.user.has_perm('experiments.change_experiment'): 117 | return HttpResponseForbidden() 118 | 119 | experiment_name = request.POST.get("experiment") 120 | alternative_name = request.POST.get("alternative") 121 | if not (experiment_name and alternative_name): 122 | return HttpResponseBadRequest() 123 | 124 | participant(request).set_alternative(experiment_name, alternative_name) 125 | return JsonResponse({ 126 | 'success': True, 127 | 'alternative': participant(request).get_alternative(experiment_name) 128 | }) 129 | 130 | def set_state_view(self, request): 131 | """ 132 | Changes the experiment state 133 | """ 134 | if not request.user.has_perm('experiments.change_experiment'): 135 | return HttpResponseForbidden() 136 | 137 | try: 138 | state = int(request.POST.get("state", "")) 139 | except ValueError: 140 | return HttpResponseBadRequest() 141 | 142 | try: 143 | experiment = Experiment.objects.get(name=request.POST.get("experiment")) 144 | except Experiment.DoesNotExist: 145 | return HttpResponseBadRequest() 146 | 147 | experiment.state = state 148 | 149 | if state == 0: 150 | experiment.end_date = timezone.now() 151 | else: 152 | experiment.end_date = None 153 | 154 | experiment.save() 155 | 156 | return HttpResponse() 157 | 158 | admin.site.register(Experiment, ExperimentAdmin) 159 | 160 | -------------------------------------------------------------------------------- /experiments/admin_utils.py: -------------------------------------------------------------------------------- 1 | from experiments.experiment_counters import ExperimentCounter 2 | from experiments.significance import chi_square_p_value, mann_whitney 3 | from experiments.utils import participant 4 | from experiments import conf 5 | 6 | import json 7 | 8 | 9 | MIN_ACTIONS_TO_SHOW = 3 10 | 11 | 12 | def rate(a, b): 13 | if not b or a == None: 14 | return None 15 | return 100. * a / b 16 | 17 | 18 | def improvement(a, b): 19 | if not b or not a: 20 | return None 21 | return (a - b) * 100. / b 22 | 23 | 24 | def chi_squared_confidence(a_count, a_conversion, b_count, b_conversion): 25 | contingency_table = [[a_count - a_conversion, a_conversion], 26 | [b_count - b_conversion, b_conversion]] 27 | 28 | chi_square, p_value = chi_square_p_value(contingency_table) 29 | if p_value is not None: 30 | return (1 - p_value) * 100 31 | else: 32 | return None 33 | 34 | 35 | def average_actions(distribution): 36 | total_users = 0 37 | total_actions = 0 38 | for actions, frequency in distribution.items(): 39 | total_users += frequency 40 | total_actions += actions * frequency 41 | if total_users: 42 | return total_actions / float(total_users) 43 | else: 44 | return 0 45 | 46 | 47 | def fixup_distribution(distribution, count): 48 | zeros = count - sum(distribution.values()) 49 | distribution[0] = zeros + distribution.get(0, 0) 50 | return distribution 51 | 52 | 53 | def mann_whitney_confidence(a_distribution, b_distribution): 54 | p_value = mann_whitney(a_distribution, b_distribution)[1] 55 | if p_value is not None: 56 | return (1 - p_value * 2) * 100 # Two tailed probability 57 | else: 58 | return None 59 | 60 | 61 | def points_with_surrounding_gaps(points): 62 | """ 63 | This function makes sure that any gaps in the sequence provided have stopper points at their beginning 64 | and end so a graph will be drawn with correct 0 ranges. This is more efficient than filling in all points 65 | up to the maximum value. For example: 66 | 67 | input: [1,2,3,10,11,13] 68 | output [1,2,3,4,9,10,11,12,13] 69 | """ 70 | points_with_gaps = [] 71 | last_point = -1 72 | for point in points: 73 | if last_point + 1 == point: 74 | pass 75 | elif last_point + 2 == point: 76 | points_with_gaps.append(last_point + 1) 77 | else: 78 | points_with_gaps.append(last_point + 1) 79 | points_with_gaps.append(point - 1) 80 | points_with_gaps.append(point) 81 | last_point = point 82 | return points_with_gaps 83 | 84 | 85 | def conversion_distributions_to_graph_table(conversion_distributions): 86 | ordered_distributions = list(conversion_distributions.items()) 87 | total_entries = dict((name, float(sum(dist.values()) or 1)) for name, dist in ordered_distributions) 88 | graph_head = [['x'] + [name for name, dist in ordered_distributions]] 89 | 90 | points_in_any_distribution = sorted(set(k for name, dist in ordered_distributions for k in dist.keys())) 91 | points_with_gaps = points_with_surrounding_gaps(points_in_any_distribution) 92 | graph_body = [[point] + [dist.get(point, 0) / total_entries[name] for name, dist in ordered_distributions] for point in points_with_gaps] 93 | 94 | accumulator = [0] * len(ordered_distributions) 95 | for point in range(len(graph_body) - 1, -1, -1): 96 | accumulator = [graph_body[point][j + 1] + accumulator[j] for j in range(len(ordered_distributions))] 97 | graph_body[point][1:] = accumulator 98 | 99 | interesting_points = [point for point in points_in_any_distribution if max(dist.get(point, 0) for name, dist in ordered_distributions) >= MIN_ACTIONS_TO_SHOW] 100 | if len(interesting_points): 101 | highest_interesting_point = max(interesting_points) 102 | else: 103 | highest_interesting_point = 0 104 | graph_body = [g for g in graph_body if g[0] <= highest_interesting_point and g[0] != 0] 105 | 106 | graph_table = graph_head + graph_body 107 | return json.dumps(graph_table) 108 | 109 | 110 | def get_result_context(request, experiment): 111 | experiment_counter = ExperimentCounter() 112 | 113 | try: 114 | chi2_goals = experiment.relevant_chi2_goals.replace(" ", "").split(",") 115 | except AttributeError: 116 | chi2_goals = [u''] 117 | try: 118 | mwu_goals = experiment.relevant_mwu_goals.replace(" ", "").split(",") 119 | except AttributeError: 120 | mwu_goals = [u''] 121 | relevant_goals = set(chi2_goals + mwu_goals) 122 | 123 | alternatives = {} 124 | for alternative_name in experiment.alternatives.keys(): 125 | alternatives[alternative_name] = experiment_counter.participant_count(experiment, alternative_name) 126 | alternatives = sorted(alternatives.items()) 127 | 128 | control_participants = experiment_counter.participant_count(experiment, conf.CONTROL_GROUP) 129 | 130 | results = {} 131 | 132 | for goal in conf.ALL_GOALS: 133 | show_mwu = goal in mwu_goals 134 | 135 | alternatives_conversions = {} 136 | control_conversions = experiment_counter.goal_count(experiment, conf.CONTROL_GROUP, goal) 137 | control_conversion_rate = rate(control_conversions, control_participants) 138 | 139 | if show_mwu: 140 | mwu_histogram = {} 141 | control_conversion_distribution = fixup_distribution(experiment_counter.goal_distribution(experiment, conf.CONTROL_GROUP, goal), control_participants) 142 | control_average_goal_actions = average_actions(control_conversion_distribution) 143 | mwu_histogram['control'] = control_conversion_distribution 144 | else: 145 | control_average_goal_actions = None 146 | for alternative_name in experiment.alternatives.keys(): 147 | if not alternative_name == conf.CONTROL_GROUP: 148 | alternative_conversions = experiment_counter.goal_count(experiment, alternative_name, goal) 149 | alternative_participants = experiment_counter.participant_count(experiment, alternative_name) 150 | alternative_conversion_rate = rate(alternative_conversions, alternative_participants) 151 | alternative_confidence = chi_squared_confidence(alternative_participants, alternative_conversions, control_participants, control_conversions) 152 | if show_mwu: 153 | alternative_conversion_distribution = fixup_distribution(experiment_counter.goal_distribution(experiment, alternative_name, goal), alternative_participants) 154 | alternative_average_goal_actions = average_actions(alternative_conversion_distribution) 155 | alternative_distribution_confidence = mann_whitney_confidence(alternative_conversion_distribution, control_conversion_distribution) 156 | mwu_histogram[alternative_name] = alternative_conversion_distribution 157 | else: 158 | alternative_average_goal_actions = None 159 | alternative_distribution_confidence = None 160 | alternative = { 161 | 'conversions': alternative_conversions, 162 | 'conversion_rate': alternative_conversion_rate, 163 | 'improvement': improvement(alternative_conversion_rate, control_conversion_rate), 164 | 'confidence': alternative_confidence, 165 | 'average_goal_actions': alternative_average_goal_actions, 166 | 'mann_whitney_confidence': alternative_distribution_confidence, 167 | } 168 | alternatives_conversions[alternative_name] = alternative 169 | 170 | control = { 171 | 'conversions': control_conversions, 172 | 'conversion_rate': control_conversion_rate, 173 | 'average_goal_actions': control_average_goal_actions, 174 | } 175 | 176 | results[goal] = { 177 | "control": control, 178 | "alternatives": sorted(alternatives_conversions.items()), 179 | "relevant": goal in relevant_goals or relevant_goals == {u''}, 180 | "mwu": goal in mwu_goals, 181 | "mwu_histogram": conversion_distributions_to_graph_table(mwu_histogram) if show_mwu else None 182 | } 183 | 184 | return { 185 | 'experiment': experiment.to_dict(), 186 | 'alternatives': alternatives, 187 | 'control_participants': control_participants, 188 | 'results': results, 189 | 'column_count': len(alternatives_conversions) * 3 + 2, # Horrible coupling with template design 190 | 'user_alternative': participant(request).get_alternative(experiment.name), 191 | } 192 | -------------------------------------------------------------------------------- /experiments/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class ExperimentsConfig(AppConfig): 5 | name = 'experiments' 6 | label = 'experiments' 7 | 8 | def ready(self): 9 | from django.contrib.auth.signals import user_logged_in, user_logged_out 10 | from experiments.signal_handlers import transfer_enrollments_to_user, handle_user_logged_out 11 | 12 | user_logged_in.connect(transfer_enrollments_to_user, dispatch_uid="experiments_user_logged_in") 13 | user_logged_out.connect(handle_user_logged_out, dispatch_uid="experiments_user_logged_out") 14 | -------------------------------------------------------------------------------- /experiments/conf.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from itertools import chain 3 | import re 4 | 5 | CONTROL_GROUP = 'control' 6 | 7 | VISIT_PRESENT_COUNT_GOAL = '_retention_present_visits' 8 | VISIT_NOT_PRESENT_COUNT_GOAL = '_retention_not_present_visits' 9 | 10 | BUILT_IN_GOALS = ( 11 | VISIT_PRESENT_COUNT_GOAL, 12 | VISIT_NOT_PRESENT_COUNT_GOAL, 13 | ) 14 | 15 | SESSION_LENGTH = getattr(settings, 'EXPERIMENTS_SESSION_LENGTH', 6) 16 | 17 | USER_GOALS = getattr(settings, 'EXPERIMENTS_GOALS', []) 18 | ALL_GOALS = tuple(chain(USER_GOALS, BUILT_IN_GOALS)) 19 | 20 | VERIFY_HUMAN = getattr(settings, 'EXPERIMENTS_VERIFY_HUMAN', True) 21 | 22 | CONFIRM_HUMAN = getattr(settings, 'EXPERIMENTS_CONFIRM_HUMAN', True) 23 | 24 | CONFIRM_HUMAN_SESSION_KEY = getattr(settings, 'EXPERIMENTS_CONFIRM_HUMAN_SESSION_KEY', 'experiments_verified_human') 25 | 26 | REDIS_GOALS_TTL = getattr(settings, 'EXPERIMENTS_REDIS_GOALS_TTL', 300) 27 | 28 | BOT_REGEX = re.compile("(Baidu|Gigabot|Googlebot|YandexBot|AhrefsBot|TVersity|libwww-perl|Yeti|lwp-trivial|msnbot|bingbot|facebookexternalhit|Twitterbot|Twitmunin|SiteUptime|TwitterFeed|Slurp|WordPress|ZIBB|ZyBorg)", re.IGNORECASE) 29 | -------------------------------------------------------------------------------- /experiments/counters.py: -------------------------------------------------------------------------------- 1 | from django.utils.functional import cached_property 2 | 3 | from redis.exceptions import ConnectionError, ResponseError 4 | 5 | from experiments.redis_client import get_redis_client 6 | 7 | 8 | COUNTER_CACHE_KEY = 'experiments:participants:%s' 9 | COUNTER_FREQ_CACHE_KEY = 'experiments:freq:%s' 10 | 11 | 12 | class Counters(object): 13 | 14 | @cached_property 15 | def _redis(self): 16 | return get_redis_client() 17 | 18 | def increment(self, key, participant_identifier, count=1): 19 | if count == 0: 20 | return 21 | 22 | try: 23 | cache_key = COUNTER_CACHE_KEY % key 24 | freq_cache_key = COUNTER_FREQ_CACHE_KEY % key 25 | new_value = self._redis.hincrby(cache_key, participant_identifier, count) 26 | 27 | # Maintain histogram of per-user counts 28 | if new_value > count: 29 | self._redis.hincrby(freq_cache_key, new_value - count, -1) 30 | self._redis.hincrby(freq_cache_key, new_value, 1) 31 | except (ConnectionError, ResponseError): 32 | # Handle Redis failures gracefully 33 | pass 34 | 35 | def clear(self, key, participant_identifier): 36 | try: 37 | # Remove the direct entry 38 | cache_key = COUNTER_CACHE_KEY % key 39 | pipe = self._redis.pipeline() 40 | freq, _ = pipe.hget(cache_key, participant_identifier).hdel(cache_key, participant_identifier).execute() 41 | 42 | # Remove from the histogram 43 | freq_cache_key = COUNTER_FREQ_CACHE_KEY % key 44 | self._redis.hincrby(freq_cache_key, freq or 0, -1) 45 | except (ConnectionError, ResponseError): 46 | # Handle Redis failures gracefully 47 | pass 48 | 49 | def get(self, key): 50 | try: 51 | cache_key = COUNTER_CACHE_KEY % key 52 | return self._redis.hlen(cache_key) 53 | except (ConnectionError, ResponseError): 54 | # Handle Redis failures gracefully 55 | return 0 56 | 57 | def get_frequency(self, key, participant_identifier): 58 | try: 59 | cache_key = COUNTER_CACHE_KEY % key 60 | freq = self._redis.hget(cache_key, participant_identifier) 61 | return int(freq) if freq else 0 62 | except (ConnectionError, ResponseError): 63 | # Handle Redis failures gracefully 64 | return 0 65 | 66 | def get_frequencies(self, key): 67 | try: 68 | freq_cache_key = COUNTER_FREQ_CACHE_KEY % key 69 | # In some cases when there are concurrent updates going on, there can 70 | # briefly be a negative result for some frequency count. We discard these 71 | # as they shouldn't really affect the result, and they are about to become 72 | # zero anyway. 73 | return dict((int(k), int(v)) for (k, v) in self._redis.hgetall(freq_cache_key).items() if int(v) > 0) 74 | except (ConnectionError, ResponseError): 75 | # Handle Redis failures gracefully 76 | return dict() 77 | 78 | def reset(self, key): 79 | try: 80 | cache_key = COUNTER_CACHE_KEY % key 81 | self._redis.delete(cache_key) 82 | freq_cache_key = COUNTER_FREQ_CACHE_KEY % key 83 | self._redis.delete(freq_cache_key) 84 | return True 85 | except (ConnectionError, ResponseError): 86 | # Handle Redis failures gracefully 87 | return False 88 | 89 | def reset_pattern(self, pattern_key): 90 | #similar to above, but can pass pattern as arg instead 91 | try: 92 | cache_key = COUNTER_CACHE_KEY % pattern_key 93 | for key in self._redis.keys(cache_key): 94 | self._redis.delete(key) 95 | freq_cache_key = COUNTER_FREQ_CACHE_KEY % pattern_key 96 | for key in self._redis.keys(freq_cache_key): 97 | self._redis.delete(key) 98 | return True 99 | except (ConnectionError, ResponseError): 100 | # Handle Redis failures gracefully 101 | return False 102 | 103 | def reset_prefix(self, key_prefix): 104 | # Delete all data in redis for a given key prefix 105 | from experiments.utils import grouper 106 | 107 | try: 108 | for key_pattern in [COUNTER_CACHE_KEY, COUNTER_FREQ_CACHE_KEY]: 109 | match = "%s:*" % (key_pattern % key_prefix) 110 | key_iter = self._redis.scan_iter(match) 111 | 112 | # Delete keys in groups of 1000 to prevent problems with long 113 | # running experiments having many participants 114 | for keys in grouper(key_iter, 1000): 115 | # The last group will be padded with None to reach the specified batch 116 | # size, so these are filtered out here 117 | self._redis.delete(*filter(None, keys)) 118 | except (ConnectionError, ResponseError): 119 | # Handle Redis failures gracefully 120 | pass 121 | -------------------------------------------------------------------------------- /experiments/dateutils.py: -------------------------------------------------------------------------------- 1 | import calendar 2 | from datetime import datetime 3 | 4 | from django.conf import settings 5 | 6 | USE_TZ = getattr(settings, 'USE_TZ', False) 7 | if USE_TZ: 8 | from django.utils.timezone import now 9 | else: 10 | now = datetime.now 11 | 12 | 13 | def fix_awareness(value): 14 | tz_aware_value = value.tzinfo is not None and value.tzinfo.utcoffset(value) is not None 15 | if USE_TZ and not tz_aware_value: 16 | from django.utils.timezone import get_current_timezone 17 | return value.replace(tzinfo=get_current_timezone()) 18 | elif not USE_TZ and tz_aware_value: 19 | return value.replace(tzinfo=None) 20 | else: 21 | return value 22 | 23 | 24 | def timestamp_from_datetime(dt): 25 | if dt is None: 26 | return None 27 | return calendar.timegm(dt.utctimetuple()) 28 | 29 | 30 | def datetime_from_timestamp(ts): 31 | if ts is None: 32 | return None 33 | return datetime.utcfromtimestamp(ts) 34 | -------------------------------------------------------------------------------- /experiments/experiment_counters.py: -------------------------------------------------------------------------------- 1 | from experiments import counters, conf 2 | import logging 3 | import json 4 | 5 | PARTICIPANT_KEY = '%s:%s:participant' 6 | GOAL_KEY = '%s:%s:%s:goal' 7 | 8 | logger = logging.getLogger('experiments') 9 | 10 | class ExperimentCounter(object): 11 | def __init__(self): 12 | self.counters = counters.Counters() 13 | 14 | def increment_participant_count(self, experiment, alternative_name, participant_identifier): 15 | counter_key = PARTICIPANT_KEY % (experiment.name, alternative_name) 16 | self.counters.increment(counter_key, participant_identifier) 17 | logger.info(json.dumps({'type':'participant_add', 'experiment': experiment.name, 'alternative': alternative_name, 'participant': participant_identifier})) 18 | 19 | def increment_goal_count(self, experiment, alternative_name, goal_name, participant_identifier, count=1): 20 | counter_key = GOAL_KEY % (experiment.name, alternative_name, goal_name) 21 | self.counters.increment(counter_key, participant_identifier, count) 22 | logger.info(json.dumps({'type':'goal_hit', 'goal': goal_name, 'goal_count': count, 'experiment': experiment.name, 'alternative': alternative_name, 'participant': participant_identifier})) 23 | 24 | def remove_participant(self, experiment, alternative_name, participant_identifier): 25 | counter_key = PARTICIPANT_KEY % (experiment.name, alternative_name) 26 | self.counters.clear(counter_key, participant_identifier) 27 | logger.info(json.dumps({'type':'participant_remove', 'experiment': experiment.name, 'alternative': alternative_name, 'participant': participant_identifier})) 28 | 29 | # Remove goal records 30 | for goal_name in conf.ALL_GOALS: 31 | counter_key = GOAL_KEY % (experiment.name, alternative_name, goal_name) 32 | self.counters.clear(counter_key, participant_identifier) 33 | 34 | def participant_count(self, experiment, alternative): 35 | return self.counters.get(PARTICIPANT_KEY % (experiment.name, alternative)) 36 | 37 | def goal_count(self, experiment, alternative, goal): 38 | return self.counters.get(GOAL_KEY % (experiment.name, alternative, goal)) 39 | 40 | def participant_goal_frequencies(self, experiment, alternative, participant_identifier): 41 | for goal in conf.ALL_GOALS: 42 | yield goal, self.counters.get_frequency(GOAL_KEY % (experiment.name, alternative, goal), participant_identifier) 43 | 44 | def goal_distribution(self, experiment, alternative, goal): 45 | return self.counters.get_frequencies(GOAL_KEY % (experiment.name, alternative, goal)) 46 | 47 | def delete(self, experiment): 48 | self.counters.reset_pattern(experiment.name + "*") 49 | -------------------------------------------------------------------------------- /experiments/manager.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from experiments.models import Experiment 3 | from modeldict import ModelDict 4 | 5 | 6 | class LazyAutoCreate(object): 7 | """ 8 | A lazy version of the setting is used so that tests can change the setting and still work 9 | """ 10 | def __nonzero__(self): 11 | return self.__bool__() 12 | 13 | def __bool__(self): 14 | return getattr(settings, 'EXPERIMENTS_AUTO_CREATE', True) 15 | 16 | 17 | class ExperimentManager(ModelDict): 18 | def get_experiment(self, experiment_name): 19 | # Helper that uses self[...] so that the experiment is auto created where desired 20 | try: 21 | return self[experiment_name] 22 | except KeyError: 23 | return None 24 | 25 | 26 | experiment_manager = ExperimentManager(Experiment, key='name', value='value', instances=True, auto_create=LazyAutoCreate()) 27 | -------------------------------------------------------------------------------- /experiments/middleware.py: -------------------------------------------------------------------------------- 1 | from experiments.utils import participant 2 | 3 | try: 4 | # for Django >= 1.10 5 | from django.utils.deprecation import MiddlewareMixin 6 | except ImportError: 7 | # for Django < 1.10 8 | MiddlewareMixin = object 9 | 10 | 11 | def is_ajax(request): 12 | return request.META.get('HTTP_X_REQUESTED_WITH') == 'XMLHttpRequest' 13 | 14 | 15 | class ExperimentsRetentionMiddleware(MiddlewareMixin): 16 | def process_response(self, request, response): 17 | # Don't track, failed pages, ajax requests, logged out users or widget impressions. 18 | # We detect widgets by relying on the fact that they are flagged as being embedable 19 | if response.status_code != 200 or is_ajax(request) or getattr(response, 'xframe_options_exempt', False): 20 | return response 21 | 22 | experiment_user = participant(request) 23 | experiment_user.visit() 24 | 25 | # record cookie goal 26 | goal_name = request.COOKIES.get('experiments_goal') 27 | participant(request).goal(goal_name) 28 | 29 | return response 30 | -------------------------------------------------------------------------------- /experiments/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | try: 5 | from django.db.models import JSONField 6 | except ImportError: # Django < 3.1 7 | from jsonfield import JSONField 8 | 9 | from django.db import models, migrations 10 | import django.utils.timezone 11 | from django.conf import settings 12 | 13 | 14 | class Migration(migrations.Migration): 15 | 16 | dependencies = [ 17 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 18 | ] 19 | 20 | operations = [ 21 | migrations.CreateModel( 22 | name='Enrollment', 23 | fields=[ 24 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 25 | ('enrollment_date', models.DateTimeField(auto_now_add=True)), 26 | ('last_seen', models.DateTimeField(null=True)), 27 | ('alternative', models.CharField(max_length=50)), 28 | ], 29 | ), 30 | migrations.CreateModel( 31 | name='Experiment', 32 | fields=[ 33 | ('name', models.CharField(max_length=128, serialize=False, primary_key=True)), 34 | ('description', models.TextField(default=b'', null=True, blank=True)), 35 | ('alternatives', JSONField(default={}, blank=True)), 36 | ('relevant_chi2_goals', models.TextField(default=b'', null=True, blank=True)), 37 | ('relevant_mwu_goals', models.TextField(default=b'', null=True, blank=True)), 38 | ('state', models.IntegerField(default=0, choices=[(0, b'Default/Control'), (1, b'Enabled'), (3, b'Track')])), 39 | ('start_date', models.DateTimeField(default=django.utils.timezone.now, null=True, db_index=True, blank=True)), 40 | ('end_date', models.DateTimeField(null=True, blank=True)), 41 | ], 42 | ), 43 | migrations.AddField( 44 | model_name='enrollment', 45 | name='experiment', 46 | field=models.ForeignKey(to='experiments.Experiment', on_delete=django.db.models.deletion.CASCADE), 47 | ), 48 | migrations.AddField( 49 | model_name='enrollment', 50 | name='user', 51 | field=models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=django.db.models.deletion.CASCADE), 52 | ), 53 | migrations.AlterUniqueTogether( 54 | name='enrollment', 55 | unique_together=set([('user', 'experiment')]), 56 | ), 57 | ] 58 | -------------------------------------------------------------------------------- /experiments/migrations/0002_auto_20201013_1408.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11 on 2020-10-13 14:08 3 | from __future__ import unicode_literals 4 | 5 | import datetime 6 | from django.conf import settings 7 | from django.db import migrations, models 8 | import django.db.models.deletion 9 | 10 | 11 | class Migration(migrations.Migration): 12 | 13 | dependencies = [ 14 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 15 | ('experiments', '0001_initial'), 16 | ] 17 | 18 | operations = [ 19 | migrations.AddField( 20 | model_name='enrollment', 21 | name='session_key', 22 | field=models.CharField(max_length=40, null=True), 23 | ), 24 | migrations.AlterField( 25 | model_name='enrollment', 26 | name='user', 27 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), 28 | ), 29 | migrations.AlterUniqueTogether( 30 | name='enrollment', 31 | unique_together=set([('session_key', 'experiment'), ('user', 'experiment')]), 32 | ), 33 | ] 34 | -------------------------------------------------------------------------------- /experiments/migrations/0003_alter_experiment_alternatives_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.8 on 2022-10-31 10:11 2 | 3 | from experiments.dateutils import now 4 | from django.db import migrations, models 5 | 6 | try: 7 | from django.db.models import JSONField 8 | except ImportError: # Django < 3.1 9 | from jsonfield import JSONField 10 | 11 | 12 | class Migration(migrations.Migration): 13 | 14 | dependencies = [ 15 | ('experiments', '0002_auto_20201013_1408'), 16 | ] 17 | 18 | operations = [ 19 | migrations.AlterField( 20 | model_name='experiment', 21 | name='alternatives', 22 | field=JSONField(blank=True, default=dict), 23 | ), 24 | migrations.AlterField( 25 | model_name='experiment', 26 | name='description', 27 | field=models.TextField(blank=True, default='', null=True), 28 | ), 29 | migrations.AlterField( 30 | model_name='experiment', 31 | name='relevant_chi2_goals', 32 | field=models.TextField(blank=True, default='', null=True), 33 | ), 34 | migrations.AlterField( 35 | model_name='experiment', 36 | name='relevant_mwu_goals', 37 | field=models.TextField(blank=True, default='', null=True), 38 | ), 39 | migrations.AlterField( 40 | model_name='experiment', 41 | name='start_date', 42 | field=models.DateTimeField(blank=True, db_index=True, default=now, null=True), 43 | ), 44 | migrations.AlterField( 45 | model_name='experiment', 46 | name='state', 47 | field=models.IntegerField(choices=[(0, 'Default/Control'), (1, 'Enabled'), (3, 'Track')], default=0), 48 | ), 49 | ] 50 | -------------------------------------------------------------------------------- /experiments/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mixcloud/django-experiments/df6fe5d2e83a96539206d4bdcac4bf6e3d38d2ca/experiments/migrations/__init__.py -------------------------------------------------------------------------------- /experiments/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.core.exceptions import ValidationError 3 | from django.core.serializers.json import DjangoJSONEncoder 4 | from django.conf import settings 5 | 6 | import random 7 | import json 8 | try: 9 | from django.db.models import JSONField 10 | except ImportError: # Django < 3.1 11 | from jsonfield import JSONField 12 | 13 | from experiments.counters import Counters 14 | from experiments.dateutils import now 15 | from experiments import conf 16 | 17 | 18 | CONTROL_STATE = 0 19 | ENABLED_STATE = 1 20 | TRACK_STATE = 3 21 | 22 | STATES = ( 23 | (CONTROL_STATE, 'Default/Control'), 24 | (ENABLED_STATE, 'Enabled'), 25 | (TRACK_STATE, 'Track'), 26 | ) 27 | 28 | 29 | class Experiment(models.Model): 30 | name = models.CharField(primary_key=True, max_length=128) 31 | description = models.TextField(default="", blank=True, null=True) 32 | alternatives = JSONField(default=dict, blank=True) 33 | relevant_chi2_goals = models.TextField(default="", null=True, blank=True) 34 | relevant_mwu_goals = models.TextField(default="", null=True, blank=True) 35 | 36 | state = models.IntegerField(default=CONTROL_STATE, choices=STATES) 37 | 38 | start_date = models.DateTimeField(default=now, blank=True, null=True, db_index=True) 39 | end_date = models.DateTimeField(blank=True, null=True) 40 | 41 | def is_displaying_alternatives(self): 42 | if self.state == CONTROL_STATE: 43 | return False 44 | elif self.state == ENABLED_STATE: 45 | return True 46 | elif self.state == TRACK_STATE: 47 | return True 48 | else: 49 | raise Exception("Invalid experiment state %s!" % self.state) 50 | 51 | def is_accepting_new_users(self): 52 | if self.state == CONTROL_STATE: 53 | return False 54 | elif self.state == ENABLED_STATE: 55 | return True 56 | elif self.state == TRACK_STATE: 57 | return False 58 | else: 59 | raise Exception("Invalid experiment state %s!" % self.state) 60 | 61 | def ensure_alternative_exists(self, alternative, weight=None): 62 | if alternative not in self.alternatives: 63 | self.alternatives[alternative] = {} 64 | self.alternatives[alternative]['enabled'] = True 65 | self.save() 66 | if weight is not None and 'weight' not in self.alternatives[alternative]: 67 | self.alternatives[alternative]['weight'] = float(weight) 68 | self.save() 69 | 70 | @property 71 | def default_alternative(self): 72 | for alternative, alternative_conf in self.alternatives.items(): 73 | if alternative_conf.get('default'): 74 | return alternative 75 | return conf.CONTROL_GROUP 76 | 77 | def set_default_alternative(self, alternative): 78 | for alternative_name, alternative_conf in self.alternatives.items(): 79 | if alternative_name == alternative: 80 | alternative_conf['default'] = True 81 | elif 'default' in alternative_conf: 82 | del alternative_conf['default'] 83 | 84 | def random_alternative(self): 85 | if all('weight' in alt for alt in self.alternatives.values()): 86 | return weighted_choice([(name, details['weight']) for name, details in self.alternatives.items()]) 87 | else: 88 | return random.choice(list(self.alternatives)) 89 | 90 | def __unicode__(self): 91 | return self.name 92 | 93 | def to_dict(self): 94 | data = { 95 | 'name': self.name, 96 | 'start_date': self.start_date, 97 | 'end_date': self.end_date, 98 | 'state': self.state, 99 | 'description': self.description, 100 | 'relevant_chi2_goals': self.relevant_chi2_goals, 101 | 'relevant_mwu_goals': self.relevant_mwu_goals, 102 | 'default_alternative': self.default_alternative, 103 | 'alternatives': ','.join(self.alternatives.keys()), 104 | } 105 | return data 106 | 107 | def to_dict_serialized(self): 108 | return json.dumps(self.to_dict(), cls=DjangoJSONEncoder) 109 | 110 | def reset_counters(self): 111 | Counters().reset_prefix(self.name) 112 | 113 | def delete(self, reset_counters=True, *args, **kwargs): 114 | if reset_counters: 115 | self.reset_counters() 116 | return super(Experiment, self).delete(*args, **kwargs) 117 | 118 | 119 | 120 | class Enrollment(models.Model): 121 | """ A participant in a split testing experiment """ 122 | user = models.ForeignKey(getattr(settings, 'AUTH_USER_MODEL', 'auth.User'), on_delete=models.CASCADE, null=True) 123 | session_key = models.CharField(max_length=40, null=True) 124 | experiment = models.ForeignKey(Experiment, on_delete=models.CASCADE) 125 | enrollment_date = models.DateTimeField(auto_now_add=True) 126 | last_seen = models.DateTimeField(null=True) 127 | alternative = models.CharField(max_length=50) 128 | 129 | def clean(self): 130 | if self.user_id and self.session_key: 131 | raise ValidationError("Only one of user_id or session_key can be set") 132 | elif not self.user_id and not self.session_key: 133 | raise ValidationError("Must set a user_id or session_key") 134 | 135 | class Meta: 136 | unique_together = (('user', 'experiment'), ('session_key', 'experiment')) 137 | 138 | def __unicode__(self): 139 | if self.user_id: 140 | return u'%s - %s' % (self.user, self.experiment) 141 | else: 142 | return u'%s - %s' % (self.session_key, self.experiment) 143 | 144 | 145 | def weighted_choice(choices): 146 | total = sum(w for c, w in choices) 147 | r = random.uniform(0, total) 148 | upto = 0 149 | for c, w in choices: 150 | upto += w 151 | if upto >= r: 152 | return c 153 | -------------------------------------------------------------------------------- /experiments/redis_client.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | import redis 3 | from redis.sentinel import Sentinel 4 | 5 | def get_redis_client(): 6 | if getattr(settings, 'EXPERIMENTS_REDIS_SENTINELS', None): 7 | sentinel = Sentinel(settings.EXPERIMENTS_REDIS_SENTINELS, socket_timeout=settings.EXPERIMENTS_REDIS_SENTINELS_TIMEOUT) 8 | host, port = sentinel.discover_master(settings.EXPERIMENTS_REDIS_MASTER_NAME) 9 | else: 10 | host = getattr(settings, 'EXPERIMENTS_REDIS_HOST', 'localhost') 11 | port = getattr(settings, 'EXPERIMENTS_REDIS_PORT', 6379) 12 | 13 | password = getattr(settings, 'EXPERIMENTS_REDIS_PASSWORD', None) 14 | db = getattr(settings, 'EXPERIMENTS_REDIS_DB', 0) 15 | 16 | return redis.Redis(host=host, port=port, password=password, db=db, encoding="utf-8", decode_responses=True) 17 | -------------------------------------------------------------------------------- /experiments/signal_handlers.py: -------------------------------------------------------------------------------- 1 | from experiments.utils import participant, clear_participant_cache 2 | 3 | 4 | def transfer_enrollments_to_user(sender, request, user, **kwargs): 5 | anon_user = participant(session=request.session) 6 | authenticated_user = participant(user=user) 7 | authenticated_user.incorporate(anon_user) 8 | 9 | clear_participant_cache(request) 10 | 11 | 12 | def handle_user_logged_out(sender, request, user, **kwargs): 13 | clear_participant_cache(request) -------------------------------------------------------------------------------- /experiments/signals.py: -------------------------------------------------------------------------------- 1 | from django.dispatch import Signal 2 | 3 | user_enrolled = Signal() 4 | user_enrolled.__doc__ = """ 5 | sends arguments: 'experiment', 'alternative', 'user', 'session' 6 | """ 7 | -------------------------------------------------------------------------------- /experiments/significance.py: -------------------------------------------------------------------------------- 1 | from experiments.stats import zprob, chisqprob 2 | 3 | 4 | def mann_whitney(a_distribution, b_distribution, use_continuity=True): 5 | """Returns (u, p_value)""" 6 | MINIMUM_VALUES = 20 7 | 8 | all_values = sorted(set(a_distribution) | set(b_distribution)) 9 | 10 | count_so_far = 0 11 | a_rank_sum = 0 12 | b_rank_sum = 0 13 | a_count = 0 14 | b_count = 0 15 | 16 | variance_adjustment = 0 17 | 18 | for v in all_values: 19 | a_for_value = a_distribution.get(v, 0) 20 | b_for_value = b_distribution.get(v, 0) 21 | total_for_value = a_for_value + b_for_value 22 | average_rank = count_so_far + (1 + total_for_value) / 2.0 23 | 24 | a_rank_sum += average_rank * a_for_value 25 | b_rank_sum += average_rank * b_for_value 26 | a_count += a_for_value 27 | b_count += b_for_value 28 | count_so_far += total_for_value 29 | 30 | variance_adjustment += total_for_value ** 3 - total_for_value 31 | 32 | if a_count < MINIMUM_VALUES or b_count < MINIMUM_VALUES: 33 | return 0, None 34 | 35 | a_u = a_rank_sum - a_count * (a_count + 1) / 2.0 36 | b_u = b_rank_sum - b_count * (b_count + 1) / 2.0 37 | 38 | small_u = min(a_u, b_u) 39 | big_u = max(a_u, b_u) 40 | 41 | # These need adjusting for the huge number of ties we will have 42 | total_count = float(a_count + b_count) 43 | u_distribution_mean = a_count * b_count / 2.0 44 | u_distribution_sd = ( 45 | (a_count * b_count / (total_count * (total_count - 1))) ** 0.5 * 46 | ((total_count ** 3 - total_count - variance_adjustment) / 12.0) ** 0.5) 47 | 48 | if u_distribution_sd == 0: 49 | return small_u, None 50 | 51 | if use_continuity: 52 | # normal approximation for prob calc with continuity correction 53 | z_score = abs((big_u - 0.5 - u_distribution_mean) / u_distribution_sd) 54 | else: 55 | # normal approximation for prob calc 56 | z_score = abs((big_u - u_distribution_mean) / u_distribution_sd) 57 | 58 | return small_u, 1 - zprob(z_score) 59 | 60 | 61 | def chi_square_p_value(matrix): 62 | """ 63 | Accepts a matrix (an array of arrays, where each child array represents a row) 64 | 65 | Example from http://math.hws.edu/javamath/ryan/ChiSquare.html: 66 | 67 | Suppose you conducted a drug trial on a group of animals and you 68 | hypothesized that the animals receiving the drug would survive better than 69 | those that did not receive the drug. You conduct the study and collect the 70 | following data: 71 | 72 | Ho: The survival of the animals is independent of drug treatment. 73 | 74 | Ha: The survival of the animals is associated with drug treatment. 75 | 76 | In that case, your matrix should be: 77 | [ 78 | [ Survivors in Test, Dead in Test ], 79 | [ Survivors in Control, Dead in Control ] 80 | ] 81 | 82 | Code adapted from http://codecomments.wordpress.com/2008/02/13/computing-chi-squared-p-value-from-contingency-table-in-python/ 83 | """ 84 | try: 85 | num_rows = len(matrix) 86 | num_columns = len(matrix[0]) 87 | except TypeError: 88 | return None 89 | 90 | if num_rows != num_columns: 91 | return None 92 | 93 | # Sanity checking 94 | if num_rows == 0: 95 | return None 96 | for row in matrix: 97 | if len(row) != num_columns: 98 | return None 99 | 100 | row_sums = [] 101 | # for each row 102 | for row in matrix: 103 | # add up all the values in the row 104 | row_sums.append(sum(row)) 105 | 106 | column_sums = [] 107 | # for each column i 108 | for i in range(num_columns): 109 | column_sum = 0.0 110 | # get the i'th value from each row 111 | for row in matrix: 112 | column_sum += row[i] 113 | column_sums.append(column_sum) 114 | 115 | # the total sum could be calculated from either the rows or the columns 116 | # coerce to float to make subsequent division generate float results 117 | grand_total = float(sum(row_sums)) 118 | 119 | if grand_total <= 0: 120 | return None, None 121 | 122 | observed_test_statistic = 0.0 123 | for i in range(num_rows): 124 | for j in range(num_columns): 125 | expected_value = (row_sums[i] / grand_total) * (column_sums[j] / grand_total) * grand_total 126 | if expected_value <= 0: 127 | return None, None 128 | observed_value = matrix[i][j] 129 | observed_test_statistic += ((observed_value - expected_value) ** 2) / expected_value 130 | # See https://bitbucket.org/akoha/django-lean/issue/16/g_test-formula-is-incorrect 131 | #observed_test_statistic += 2 * (observed_value*log(observed_value/expected_value)) 132 | 133 | degrees_freedom = (num_columns - 1) * (num_rows - 1) 134 | 135 | p_value = chisqprob(observed_test_statistic, degrees_freedom) 136 | 137 | return observed_test_statistic, p_value 138 | -------------------------------------------------------------------------------- /experiments/south_migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | import datetime 3 | from south.db import db 4 | from south.v2 import SchemaMigration 5 | 6 | try: 7 | from django.contrib.auth import get_user_model 8 | except ImportError: # django < 1.5 9 | from django.contrib.auth.models import User 10 | else: 11 | User = get_user_model() 12 | 13 | 14 | class Migration(SchemaMigration): 15 | 16 | def forwards(self, orm): 17 | 18 | # Adding model 'Experiment' 19 | db.create_table('experiments_experiment', ( 20 | ('name', self.gf('django.db.models.fields.CharField')(max_length=128, primary_key=True)), 21 | ('description', self.gf('django.db.models.fields.TextField')(default='', null=True, blank=True)), 22 | ('alternatives', self.gf('jsonfield.fields.JSONField')(default='{}', blank=True)), 23 | ('relevant_goals', self.gf('django.db.models.fields.TextField')(default='', null=True, blank=True)), 24 | ('switch_key', self.gf('django.db.models.fields.CharField')(default='', max_length=50, null=True, blank=True)), 25 | ('state', self.gf('django.db.models.fields.IntegerField')(default=0)), 26 | ('start_date', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now, null=True, db_index=True, blank=True)), 27 | ('end_date', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), 28 | )) 29 | db.send_create_signal('experiments', ['Experiment']) 30 | 31 | # Adding model 'Enrollment' 32 | db.create_table('experiments_enrollment', ( 33 | ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), 34 | ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm["%s.%s" % (User._meta.app_label, User._meta.object_name)], null=True)), 35 | ('experiment', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['experiments.Experiment'])), 36 | ('enrollment_date', self.gf('django.db.models.fields.DateField')(auto_now_add=True, db_index=True, blank=True)), 37 | ('alternative', self.gf('django.db.models.fields.CharField')(max_length=50)), 38 | ('goals', self.gf('jsonfield.fields.JSONField')(default='[]', blank=True)), 39 | )) 40 | db.send_create_signal('experiments', ['Enrollment']) 41 | 42 | # Adding unique constraint on 'Enrollment', fields ['user', 'experiment'] 43 | db.create_unique('experiments_enrollment', ['user_id', 'experiment_id']) 44 | 45 | 46 | def backwards(self, orm): 47 | 48 | # Removing unique constraint on 'Enrollment', fields ['user', 'experiment'] 49 | db.delete_unique('experiments_enrollment', ['user_id', 'experiment_id']) 50 | 51 | # Deleting model 'Experiment' 52 | db.delete_table('experiments_experiment') 53 | 54 | # Deleting model 'Enrollment' 55 | db.delete_table('experiments_enrollment') 56 | 57 | 58 | models = { 59 | 'auth.group': { 60 | 'Meta': {'object_name': 'Group'}, 61 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 62 | 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), 63 | 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) 64 | }, 65 | 'auth.permission': { 66 | 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, 67 | 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), 68 | 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), 69 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 70 | 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) 71 | }, 72 | "%s.%s" % (User._meta.app_label, User._meta.module_name): { 73 | 'Meta': {'object_name': User.__name__ }, 74 | }, 75 | 'contenttypes.contenttype': { 76 | 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, 77 | 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), 78 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 79 | 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), 80 | 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) 81 | }, 82 | 'experiments.enrollment': { 83 | 'Meta': {'unique_together': "(('user', 'experiment'),)", 'object_name': 'Enrollment'}, 84 | 'alternative': ('django.db.models.fields.CharField', [], {'max_length': '50'}), 85 | 'enrollment_date': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}), 86 | 'experiment': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['experiments.Experiment']"}), 87 | 'goals': ('jsonfield.fields.JSONField', [], {'default': "'[]'", 'blank': 'True'}), 88 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 89 | 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['%s.%s']" % (User._meta.app_label, User._meta.object_name), 'null': 'True'}) 90 | }, 91 | 'experiments.experiment': { 92 | 'Meta': {'object_name': 'Experiment'}, 93 | 'alternatives': ('jsonfield.fields.JSONField', [], {'default': "'{}'", 'blank': 'True'}), 94 | 'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}), 95 | 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), 96 | 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'primary_key': 'True'}), 97 | 'relevant_goals': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}), 98 | 'start_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), 99 | 'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}), 100 | 'switch_key': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'null': 'True', 'blank': 'True'}) 101 | } 102 | } 103 | 104 | complete_apps = ['experiments'] 105 | -------------------------------------------------------------------------------- /experiments/south_migrations/0002_auto__chg_field_enrollment_goals_.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | import datetime 3 | from south.db import db 4 | from south.v2 import SchemaMigration 5 | from django.db import models 6 | 7 | try: 8 | from django.contrib.auth import get_user_model 9 | except ImportError: # django < 1.5 10 | from django.contrib.auth.models import User 11 | else: 12 | User = get_user_model() 13 | 14 | 15 | class Migration(SchemaMigration): 16 | 17 | def forwards(self, orm): 18 | 19 | # Changing field 'Enrollment.goals' 20 | db.alter_column('experiments_enrollment', 'goals', self.gf('jsonfield.fields.JSONField')(null=True, blank=True)) 21 | 22 | # Adding field 'Experiment.relevant_chi2_goals' 23 | db.add_column('experiments_experiment', 'relevant_chi2_goals', self.gf('django.db.models.fields.TextField')(default='', null=True, blank=True), keep_default=False) 24 | 25 | # Adding field 'Experiment.relevant_mwu_goals' 26 | db.add_column('experiments_experiment', 'relevant_mwu_goals', self.gf('django.db.models.fields.TextField')(default='', null=True, blank=True), keep_default=False) 27 | 28 | 29 | def backwards(self, orm): 30 | 31 | # Changing field 'Enrollment.goals' 32 | db.alter_column('experiments_enrollment', 'goals', self.gf('jsonfield.fields.JSONField')(blank=True)) 33 | 34 | # Deleting field 'Experiment.relevant_chi2_goals' 35 | db.delete_column('experiments_experiment', 'relevant_chi2_goals') 36 | 37 | # Deleting field 'Experiment.relevant_mwu_goals' 38 | db.delete_column('experiments_experiment', 'relevant_mwu_goals') 39 | 40 | 41 | models = { 42 | 'auth.group': { 43 | 'Meta': {'object_name': 'Group'}, 44 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 45 | 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), 46 | 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) 47 | }, 48 | 'auth.permission': { 49 | 'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, 50 | 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), 51 | 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), 52 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 53 | 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) 54 | }, 55 | "%s.%s" % (User._meta.app_label, User._meta.module_name): { 56 | 'Meta': {'object_name': User.__name__ }, 57 | }, 58 | 'contenttypes.contenttype': { 59 | 'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, 60 | 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), 61 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 62 | 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), 63 | 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) 64 | }, 65 | 'experiments.enrollment': { 66 | 'Meta': {'unique_together': "(('user', 'experiment'),)", 'object_name': 'Enrollment'}, 67 | 'alternative': ('django.db.models.fields.CharField', [], {'max_length': '50'}), 68 | 'enrollment_date': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}), 69 | 'experiment': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['experiments.Experiment']"}), 70 | 'goals': ('jsonfield.fields.JSONField', [], {'default': "'{}'", 'null': 'True', 'blank': 'True'}), 71 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 72 | 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['%s.%s']" % (User._meta.app_label, User._meta.object_name), 'null': 'True'}) 73 | }, 74 | 'experiments.experiment': { 75 | 'Meta': {'object_name': 'Experiment'}, 76 | 'alternatives': ('jsonfield.fields.JSONField', [], {'default': "'{}'", 'blank': 'True'}), 77 | 'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}), 78 | 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), 79 | 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'primary_key': 'True'}), 80 | 'relevant_chi2_goals': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}), 81 | 'relevant_mwu_goals': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}), 82 | 'start_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), 83 | 'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}), 84 | 'switch_key': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'null': 'True', 'blank': 'True'}) 85 | } 86 | } 87 | 88 | complete_apps = ['experiments'] 89 | -------------------------------------------------------------------------------- /experiments/south_migrations/0003_auto__del_field_enrollment_goals__add_field_enrollment_last_seen__chg_.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | import datetime 3 | from south.db import db 4 | from south.v2 import SchemaMigration 5 | from django.db import models 6 | 7 | try: 8 | from django.contrib.auth import get_user_model 9 | except ImportError: # django < 1.5 10 | from django.contrib.auth.models import User 11 | else: 12 | User = get_user_model() 13 | 14 | 15 | class Migration(SchemaMigration): 16 | 17 | def forwards(self, orm): 18 | 19 | # Deleting field 'Enrollment.goals' 20 | db.delete_column('experiments_enrollment', 'goals') 21 | 22 | # Adding field 'Enrollment.last_seen' 23 | db.add_column('experiments_enrollment', 'last_seen', self.gf('django.db.models.fields.DateTimeField')(null=True), keep_default=False) 24 | 25 | # Changing field 'Enrollment.enrollment_date' 26 | db.alter_column('experiments_enrollment', 'enrollment_date', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)) 27 | 28 | 29 | def backwards(self, orm): 30 | 31 | # Adding field 'Enrollment.goals' 32 | db.add_column('experiments_enrollment', 'goals', self.gf('jsonfield.fields.JSONField')(default='{}', null=True, blank=True), keep_default=False) 33 | 34 | # Deleting field 'Enrollment.last_seen' 35 | db.delete_column('experiments_enrollment', 'last_seen') 36 | 37 | # Changing field 'Enrollment.enrollment_date' 38 | db.alter_column('experiments_enrollment', 'enrollment_date', self.gf('django.db.models.fields.DateField')(auto_now_add=True, blank=True)) 39 | 40 | 41 | models = { 42 | 'auth.group': { 43 | 'Meta': {'object_name': 'Group'}, 44 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 45 | 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), 46 | 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) 47 | }, 48 | 'auth.permission': { 49 | 'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, 50 | 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), 51 | 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), 52 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 53 | 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) 54 | }, 55 | "%s.%s" % (User._meta.app_label, User._meta.module_name): { 56 | 'Meta': {'object_name': User.__name__ }, 57 | }, 58 | 'contenttypes.contenttype': { 59 | 'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, 60 | 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), 61 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 62 | 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), 63 | 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) 64 | }, 65 | 'experiments.enrollment': { 66 | 'Meta': {'unique_together': "(('user', 'experiment'),)", 'object_name': 'Enrollment'}, 67 | 'alternative': ('django.db.models.fields.CharField', [], {'max_length': '50'}), 68 | 'enrollment_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), 69 | 'experiment': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['experiments.Experiment']"}), 70 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 71 | 'last_seen': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), 72 | 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['%s.%s']" % (User._meta.app_label, User._meta.object_name), 'null': 'True'}) 73 | }, 74 | 'experiments.experiment': { 75 | 'Meta': {'object_name': 'Experiment'}, 76 | 'alternatives': ('jsonfield.fields.JSONField', [], {'default': "'{}'", 'blank': 'True'}), 77 | 'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}), 78 | 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), 79 | 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'primary_key': 'True'}), 80 | 'relevant_chi2_goals': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}), 81 | 'relevant_mwu_goals': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}), 82 | 'start_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), 83 | 'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}), 84 | 'switch_key': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'null': 'True', 'blank': 'True'}) 85 | } 86 | } 87 | 88 | complete_apps = ['experiments'] 89 | -------------------------------------------------------------------------------- /experiments/south_migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mixcloud/django-experiments/df6fe5d2e83a96539206d4bdcac4bf6e3d38d2ca/experiments/south_migrations/__init__.py -------------------------------------------------------------------------------- /experiments/static/experiments/dashboard/css/admin.css: -------------------------------------------------------------------------------- 1 | a.experiment-state-selected:link { 2 | text-decoration: underline; 3 | font-weight: bold; 4 | } 5 | 6 | h5.experiment-dates { 7 | margin-top: 0; 8 | letter-spacing: normal; 9 | } 10 | 11 | .hidden-relevant-goals { 12 | /* Hides the actual Django admin "Relevant Goals" section so it can 13 | be replaced with easier to use checkboxes */ 14 | visibility: hidden; 15 | height: 0; 16 | margin: 0; 17 | } 18 | 19 | .form-row.field-state { 20 | /* This is hidden so the controls at the top of the page can be used instead */ 21 | display: none; 22 | } 23 | 24 | .experiment-results-container { 25 | max-width: 1000px; 26 | margin: 30px auto; 27 | } 28 | 29 | .experiment-results-table { 30 | width: 100%; 31 | } 32 | 33 | .experiment-results-table th { 34 | cursor: pointer; 35 | } 36 | 37 | thead th.experiment-toggle-goals { 38 | color: #aaa; 39 | } 40 | 41 | .experiment-results-table tbody tr td:nth-child(2n) { 42 | background: #eee; 43 | } 44 | 45 | .experiment-results-table.experiment-hide-irrelevant .experiment-irrelevant-goal { 46 | display: none; 47 | } 48 | 49 | .experiment-alternative-enrolled { 50 | display: none; 51 | } 52 | 53 | .experiment-selected-alternative .experiment-alternative-join { 54 | display: none; 55 | } 56 | 57 | .experiment-selected-alternative .experiment-alternative-enrolled { 58 | display: inline; 59 | } 60 | 61 | .experiment-alternative-join { 62 | display: inline; 63 | } 64 | 65 | .experiment-high-confidence, 66 | .experiment-positive-improvement { 67 | color: green; 68 | font-weight: bold; 69 | } 70 | 71 | .experiment-negative-improvement { 72 | color: red; 73 | font-weight: bold; 74 | } 75 | 76 | .experiment-high-confidence:after { 77 | content: ' ✓'; 78 | } 79 | 80 | .experiment-low-confidence:after { 81 | content: ' ✗'; 82 | } 83 | 84 | .experiment-mwu-goal { 85 | cursor: pointer; 86 | text-decoration: underline; 87 | } -------------------------------------------------------------------------------- /experiments/static/experiments/dashboard/js/admin.js: -------------------------------------------------------------------------------- 1 | google.load('visualization', '1.0', {'packages':['corechart']}); 2 | 3 | (function($) { 4 | 5 | $(function() { 6 | var $table = $('#experiment-results-table'); 7 | 8 | $('#experiment-toggle-goals').click(function() { $table.toggleClass('experiment-hide-irrelevant'); return false; }); 9 | 10 | // ------------------------------ Changing the alternative 11 | 12 | $('[data-alternative]').click(function() { 13 | var $this = $(this); 14 | 15 | if ($this.hasClass('experiment-selected-alternative')) { 16 | return false; 17 | } 18 | 19 | var $currentSelected = $('.experiment-selected-alternative[data-alternative]'); 20 | 21 | $.ajax({ 22 | url: $table.data('set-alternative-url'), 23 | data: { 24 | experiment: $table.data('experiment-name'), 25 | alternative: $this.data('alternative') 26 | }, 27 | type: 'POST', 28 | dataType: 'json', 29 | success: function(data) { 30 | if (data && data.success) { 31 | $('[data-alternative="' + data.alternative + '"]').addClass('experiment-selected-alternative'); 32 | } else { 33 | $currentSelected.addClass('experiment-selected-alternative'); 34 | } 35 | }, 36 | error: function() { 37 | $currentSelected.addClass('experiment-selected-alternative'); 38 | } 39 | }); 40 | 41 | $('[data-alternative]').removeClass('experiment-selected-alternative'); 42 | 43 | return false; 44 | }); 45 | 46 | // ------------------------------ Changing the state 47 | 48 | $('[data-set-state]').click(function() { 49 | var $this = $(this), 50 | $stateSelect = $('#id_state'); 51 | 52 | $('[data-set-state]').removeClass('experiment-state-selected'); 53 | 54 | $.ajax({ 55 | url: $table.data('set-state-url'), 56 | data: { 57 | experiment: $table.data('experiment-name'), 58 | state: $this.data('set-state') 59 | }, 60 | type: 'POST', 61 | success: function() { 62 | $this.addClass('experiment-state-selected'); 63 | $stateSelect.val($this.data('set-state')); 64 | }, 65 | error: function() { 66 | $('[data-set-state="' + $stateSelect.val() + '"]').addClass('experiment-state-selected'); 67 | } 68 | }); 69 | 70 | return false; 71 | }); 72 | 73 | // ------------------------------ Showing MWU charts 74 | 75 | $('[data-chart-goal]').click(function() { 76 | var goal = $(this).data('chart-goal'); 77 | 78 | $('#' + goal + '_mwu_row').toggle(); 79 | 80 | var $graph = $('#' + goal + '_chart'); 81 | 82 | if (!$graph.data('rendered')) { 83 | $graph.data('rendered', true); 84 | 85 | var chartData = google.visualization.arrayToDataTable(window.Experiments.EXPERIMENT_CHART_DATA[goal]), 86 | chart = new google.visualization.LineChart($graph[0]), 87 | options = { 88 | height: 750, 89 | hAxis: { 90 | title: 'Performed action at least this many times', 91 | logScale: true 92 | }, 93 | vAxis : { 94 | title: 'Fraction of users' 95 | }, 96 | legend : { 97 | position: 'top', 98 | alignment: 'center' 99 | }, 100 | chartArea: { 101 | width: "75%", 102 | height: "75%" 103 | } 104 | }; 105 | 106 | chart.draw(chartData, options); 107 | } 108 | }); 109 | 110 | // ------------------------------ Relevant goal checkbox inputs 111 | 112 | function getGoalList(goalType) { 113 | if (goalType === 'chi2') { 114 | return $('#id_relevant_chi2_goals').val() + ','; 115 | } 116 | if (goalType === 'mwu') { 117 | return $('#id_relevant_mwu_goals').val() + ',' 118 | } 119 | } 120 | 121 | function setGoalList(goalType, goalList) { 122 | if (goalType === 'chi2') { 123 | return $('#id_relevant_chi2_goals').val(goalList.replace(/,$/, '')); 124 | } 125 | if (goalType === 'mwu') { 126 | return $('#id_relevant_mwu_goals').val(goalList.replace(/,$/, '')); 127 | } 128 | } 129 | 130 | var chi2Goals = getGoalList('chi2'), 131 | mwuGoals = getGoalList('mwu'); 132 | 133 | var $goals = $('#goal-list').children().each(function() { 134 | var $tr = $(this); 135 | if (chi2Goals.indexOf($tr.data('goal') + ',') > -1) { 136 | $tr.find('[data-goal-type="chi2"]').attr('checked', true); 137 | } 138 | if (mwuGoals.indexOf($tr.data('goal') + ',') > -1) { 139 | $tr.find('[data-goal-type="mwu"]').attr('checked', true); 140 | } 141 | }); 142 | 143 | $goals.bind('click', function(event) { 144 | var $target = $(event.target); 145 | if ($target.is(':checkbox')) { 146 | var goalType = $target.data('goal-type'), 147 | goalList = getGoalList(goalType), 148 | goal = $target.closest('tr').data('goal'); 149 | 150 | if ($target.is(':checked')) { 151 | if (goalList.indexOf(goal + ',') === -1) { 152 | goalList += goal; 153 | } 154 | } else { 155 | goalList = goalList.replace(goal + ',', ''); 156 | } 157 | setGoalList(goalType, goalList); 158 | } 159 | }); 160 | }); 161 | 162 | })(django.jQuery); -------------------------------------------------------------------------------- /experiments/static/experiments/dashboard/js/csrf.js: -------------------------------------------------------------------------------- 1 | (function($) { 2 | // Mostly copied from the Django docs 3 | 4 | $.ajaxSetup({ 5 | beforeSend: function(xhr, settings) { 6 | if (/^(GET|HEAD|OPTIONS|TRACE)$/.test(settings.type)) { 7 | return; 8 | } 9 | 10 | if (!this.crossDomain) { 11 | xhr.setRequestHeader("X-CSRFToken", getCookie('csrftoken')); 12 | } 13 | } 14 | }); 15 | 16 | function getCookie(name) { 17 | var cookieValue = null; 18 | if (document.cookie && document.cookie != '') { 19 | var cookies = document.cookie.split(';'); 20 | for (var i = 0; i < cookies.length; i++) { 21 | var cookie = $.trim(cookies[i]); 22 | // Does this cookie string begin with the name we want? 23 | if (cookie.substring(0, name.length + 1) == (name + '=')) { 24 | cookieValue = decodeURIComponent(cookie.substring(name.length + 1)); 25 | break; 26 | } 27 | } 28 | } 29 | return cookieValue; 30 | } 31 | })(django.jQuery); 32 | -------------------------------------------------------------------------------- /experiments/static/experiments/js/experiments.js: -------------------------------------------------------------------------------- 1 | experiments = function() { 2 | return { 3 | confirm_human: function() { 4 | $.post("/experiments/confirm_human/"); 5 | }, 6 | goal: function(goal_name) { 7 | $.post("/experiments/goal/" + goal_name + "/"); 8 | } 9 | }; 10 | }(); 11 | 12 | if (document.addEventListener) { 13 | // sets the cookie in the capturing phase so that in the bubbling phase we guarantee that if a request is being issued it will contain the new cookie as well 14 | document.addEventListener("click", function(event) { 15 | if ((event.target).hasAttribute('data-experiments-goal')) { 16 | $.cookie("experiments_goal", $(event.target).data('experiments-goal'), { path: '/' }); 17 | } 18 | }, true); 19 | } else { // IE 8 20 | $(document).delegate('[data-experiments-goal]', 'click', function(e) { 21 | // if a request is fired by the click event, the cookie might get set after it, thus the goal will be recorded with the next request (if there will be one) 22 | $.cookie("experiments_goal", $(this).data('experiments-goal'), { path: '/' }); 23 | }); 24 | } 25 | -------------------------------------------------------------------------------- /experiments/static/experiments/js/jquery.cookie.js: -------------------------------------------------------------------------------- 1 | /*jshint eqnull:true */ 2 | /*! 3 | * jQuery Cookie Plugin v1.1 4 | * https://github.com/carhartl/jquery-cookie 5 | * 6 | * Copyright 2011, Klaus Hartl 7 | * Dual licensed under the MIT or GPL Version 2 licenses. 8 | * http://www.opensource.org/licenses/mit-license.php 9 | * http://www.opensource.org/licenses/GPL-2.0 10 | */ 11 | (function($, document) { 12 | 13 | var pluses = /\+/g; 14 | function raw(s) { 15 | return s; 16 | } 17 | function decoded(s) { 18 | return decodeURIComponent(s.replace(pluses, ' ')); 19 | } 20 | 21 | $.cookie = function(key, value, options) { 22 | 23 | // key and at least value given, set cookie... 24 | if (arguments.length > 1 && (!/Object/.test(Object.prototype.toString.call(value)) || value == null)) { 25 | options = $.extend({}, $.cookie.defaults, options); 26 | 27 | if (value == null) { 28 | options.expires = -1; 29 | } 30 | 31 | if (typeof options.expires === 'number') { 32 | var days = options.expires, t = options.expires = new Date(); 33 | t.setDate(t.getDate() + days); 34 | } 35 | 36 | value = String(value); 37 | 38 | return (document.cookie = [ 39 | encodeURIComponent(key), '=', options.raw ? value : encodeURIComponent(value), 40 | options.expires ? '; expires=' + options.expires.toUTCString() : '', // use expires attribute, max-age is not supported by IE 41 | options.path ? '; path=' + options.path : '', 42 | options.domain ? '; domain=' + options.domain : '', 43 | options.secure ? '; secure' : '' 44 | ].join('')); 45 | } 46 | 47 | // key and possibly options given, get cookie... 48 | options = value || $.cookie.defaults || {}; 49 | var decode = options.raw ? raw : decoded; 50 | var cookies = document.cookie.split('; '); 51 | for (var i = 0, parts; (parts = cookies[i] && cookies[i].split('=')); i++) { 52 | if (decode(parts.shift()) === key) { 53 | return decode(parts.join('=')); 54 | } 55 | } 56 | return null; 57 | }; 58 | 59 | $.cookie.defaults = {}; 60 | 61 | })(jQuery, document); -------------------------------------------------------------------------------- /experiments/stats.py: -------------------------------------------------------------------------------- 1 | from math import fabs, exp, sqrt, log, pi 2 | 3 | 4 | def zprob(z): 5 | """ 6 | Returns the area under the normal curve 'to the left of' the given z value. 7 | Thus, 8 | for z<0, zprob(z) = 1-tail probability 9 | for z>0, 1.0-zprob(z) = 1-tail probability 10 | for any z, 2.0*(1.0-zprob(abs(z))) = 2-tail probability 11 | Originally adapted from Gary Perlman code by Gary Strangman. 12 | 13 | Usage: zprob(z) 14 | """ 15 | Z_MAX = 6.0 # maximum meaningful z-value 16 | if z == 0.0: 17 | x = 0.0 18 | else: 19 | y = 0.5 * fabs(z) 20 | if y >= (Z_MAX * 0.5): 21 | x = 1.0 22 | elif (y < 1.0): 23 | w = y * y 24 | x = ((((((((0.000124818987 * w 25 | - 0.001075204047) * w + 0.005198775019) * w 26 | - 0.019198292004) * w + 0.059054035642) * w 27 | - 0.151968751364) * w + 0.319152932694) * w 28 | - 0.531923007300) * w + 0.797884560593) * y * 2.0 29 | else: 30 | y = y - 2.0 31 | x = (((((((((((((-0.000045255659 * y 32 | + 0.000152529290) * y - 0.000019538132) * y 33 | - 0.000676904986) * y + 0.001390604284) * y 34 | - 0.000794620820) * y - 0.002034254874) * y 35 | + 0.006549791214) * y - 0.010557625006) * y 36 | + 0.011630447319) * y - 0.009279453341) * y 37 | + 0.005353579108) * y - 0.002141268741) * y 38 | + 0.000535310849) * y + 0.999936657524 39 | if z > 0.0: 40 | prob = ((x + 1.0) * 0.5) 41 | else: 42 | prob = ((1.0 - x) * 0.5) 43 | return prob 44 | 45 | 46 | def chisqprob(chisq, df): 47 | """ 48 | Returns the (1-tailed) probability value associated with the provided 49 | chi-square value and df. 50 | 51 | Originally adapted from Gary Perlman code by Gary Strangman. 52 | 53 | Usage: chisqprob(chisq,df) 54 | """ 55 | BIG = 20.0 56 | 57 | def ex(x): 58 | BIG = 20.0 59 | if x < -BIG: 60 | return 0.0 61 | else: 62 | return exp(x) 63 | 64 | if chisq <= 0 or df < 1: 65 | return 1.0 66 | 67 | a = 0.5 * chisq 68 | if df % 2 == 0: 69 | even = 1 70 | else: 71 | even = 0 72 | if df > 1: 73 | y = ex(-a) 74 | if even: 75 | s = y 76 | else: 77 | s = 2.0 * zprob(-sqrt(chisq)) 78 | if (df > 2): 79 | chisq = 0.5 * (df - 1.0) 80 | if even: 81 | z = 1.0 82 | else: 83 | z = 0.5 84 | if a > BIG: 85 | if even: 86 | e = 0.0 87 | else: 88 | e = log(sqrt(pi)) 89 | c = log(a) 90 | while (z <= chisq): 91 | e = log(z) + e 92 | s = s + ex(c * z - a - e) 93 | z = z + 1.0 94 | return s 95 | else: 96 | if even: 97 | e = 1.0 98 | else: 99 | e = 1.0 / sqrt(pi) / sqrt(a) 100 | c = 0.0 101 | while (z <= chisq): 102 | e = e * (a / float(z)) 103 | c = c + e 104 | z = z + 1.0 105 | return (c * y + s) 106 | else: 107 | return s 108 | -------------------------------------------------------------------------------- /experiments/templates/admin/experiments/change_form.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/change_form.html" %} 2 | 3 | {% block content_title %}{% if original %}

{{ original.name }}

{% else %}{{ block.super }}{% endif %}{% endblock %} 4 | 5 | {% block object-tools %} 6 | {{ block.super }} 7 | 8 | {% if original %} 9 |
{{ original.start_date }} ‐ {% if original.end_date %}{{ original.end_date }}{% else %}now{% endif %}
10 | 11 |
12 | {% include "admin/experiments/results_table.html" %} 13 |
14 | {% endif %} 15 | {% endblock object-tools %} 16 | 17 | {% block object-tools-items %} 18 | {% if original %} 19 | {# These have to be links to get the correct admin styling #} 20 |
  • {% if experiment.default_alternative != control_group %}Default: {% endif %}{{ experiment.default_alternative }}
    • 21 |
  • Track
    • 22 |
  • Enabled
    • 23 | {% endif %} 24 | {% endblock object-tools-items %} 25 | 26 | {% block field_sets %} 27 | 28 | {{ block.super }} 29 | 30 | {% endblock field_sets %} 31 | 32 | {% block after_field_sets %} 33 |
    34 |
    35 |

    Relevant Goals

    36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | {% for goal in all_goals %} 47 | 48 | 49 | 50 | 51 | 52 | {% endfor %} 53 | 54 |
    GoalΧ²U
    {{ goal }}
    55 |
    56 |
    57 | {% endblock after_field_sets %} 58 | -------------------------------------------------------------------------------- /experiments/templates/admin/experiments/results_table.html: -------------------------------------------------------------------------------- 1 | {% load humanize %} 2 | 3 | {# Somewhere to dump chart data as we loop through the goals #} 4 | 5 | 6 | 7 | 8 | 9 | 10 | 14 | {% for alternative, participants in alternatives %} 15 | {% if alternative != 'control' %} 16 | 20 | {% endif %} 21 | {% endfor %} 22 | 23 | 24 | 25 | 26 | {% for goal, data in results.items %} 27 | 28 | 35 | 36 | 42 | 43 | {% for alternative_name, results in data.alternatives %} 44 | {% if alternative_name != 'control' %} 45 | 52 | 63 | 77 | {% endif %} 78 | {% endfor %} 79 | 80 | {% if data.mwu %} 81 | 82 | 86 | 87 | {% endif %} 88 | {% endfor %} 89 | 90 |
    Toggle All Goals 11 | control ({{ control_participants|intcomma }}) 12 | ShownShow for me 13 | 17 | {{ alternative }} ({{ participants|intcomma }}) 18 | ShownShow for me 19 |
    29 | {% if data.mwu %} 30 | {{ goal }} 31 | {% else %} 32 | {{ goal }} 33 | {% endif %} 34 | 37 | {{ data.control.conversions|intcomma }} ({{ data.control.conversion_rate|floatformat:2 }}% 38 | {% if data.mwu %} 39 | - APU {{ data.control.average_goal_actions|floatformat:2 }} 40 | {% endif %} 41 | ) 46 | {{ results.conversions|intcomma }} ({{ results.conversion_rate|floatformat:2 }}% 47 | {% if data.mwu %} 48 | - APU {{ results.average_goal_actions|floatformat:2 }} 49 | {% endif %} 50 | ) 51 | 53 | {% with improvement=results.improvement confidence=results.confidence %} 54 | {% if improvement != None %} 55 | 56 | {{ improvement|floatformat:2 }} % 57 | 58 | {% else %} 59 | N/A 60 | {% endif %} 61 | {% endwith %} 62 | 64 | {% with confidence=results.confidence %} 65 | {% if confidence != None %} 66 | 67 | {% if confidence >= 99.995 %}~{% endif %}{{ confidence|floatformat:2 }} % 68 | 69 | {% else %} 70 | N/A 71 | {% endif %} 72 | {% endwith %} 73 | {% if data.mwu %} 74 | MWU: {{ results.mann_whitney_confidence|floatformat:2 }}% 75 | {% endif %} 76 |
    91 | -------------------------------------------------------------------------------- /experiments/templates/experiments/confirm_human.html: -------------------------------------------------------------------------------- 1 | {% if not confirmed_human %} 2 | 5 | {% endif %} 6 | -------------------------------------------------------------------------------- /experiments/templates/experiments/goal.html: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /experiments/templatetags/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mixcloud/django-experiments/df6fe5d2e83a96539206d4bdcac4bf6e3d38d2ca/experiments/templatetags/__init__.py -------------------------------------------------------------------------------- /experiments/templatetags/experiments.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from django import template 4 | from django.urls import reverse 5 | 6 | from experiments.utils import participant 7 | from experiments.manager import experiment_manager 8 | from experiments import conf 9 | 10 | from uuid import uuid4 11 | 12 | register = template.Library() 13 | 14 | 15 | @register.inclusion_tag('experiments/goal.html') 16 | def experiment_goal(goal_name): 17 | return {'url': reverse('experiment_goal', kwargs={'goal_name': goal_name, 'cache_buster': uuid4()})} 18 | 19 | 20 | @register.inclusion_tag('experiments/confirm_human.html', takes_context=True) 21 | def experiments_confirm_human(context): 22 | request = context.get('request') 23 | if request: 24 | return {'confirmed_human': request.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False)} 25 | 26 | 27 | class ExperimentNode(template.Node): 28 | def __init__(self, node_list, experiment_name, alternative, weight, user_variable): 29 | self.node_list = node_list 30 | self.experiment_name = experiment_name 31 | self.alternative = alternative 32 | self.weight = weight 33 | self.user_variable = user_variable 34 | 35 | def render(self, context): 36 | experiment = experiment_manager.get_experiment(self.experiment_name) 37 | if experiment: 38 | experiment.ensure_alternative_exists(self.alternative, self.weight) 39 | 40 | # Get User object 41 | if self.user_variable: 42 | auth_user = self.user_variable.resolve(context) 43 | user = participant(user=auth_user) 44 | else: 45 | request = context.get('request', None) 46 | user = participant(request) 47 | 48 | # Should we render? 49 | if user.is_enrolled(self.experiment_name, self.alternative): 50 | response = self.node_list.render(context) 51 | else: 52 | response = "" 53 | 54 | return response 55 | 56 | 57 | def _parse_token_contents(token_contents): 58 | (_, experiment_name, alternative), remaining_tokens = token_contents[:3], token_contents[3:] 59 | weight = None 60 | user_variable = None 61 | 62 | for offset, token in enumerate(remaining_tokens): 63 | if '=' in token: 64 | name, expression = token.split('=', 1) 65 | if name == 'weight': 66 | weight = expression 67 | elif name == 'user': 68 | user_variable = template.Variable(expression) 69 | else: 70 | raise ValueError() 71 | elif offset == 0: 72 | # Backwards compatibility, weight as positional argument 73 | weight = token 74 | else: 75 | raise ValueError() 76 | 77 | return experiment_name, alternative, weight, user_variable 78 | 79 | 80 | @register.tag('experiment') 81 | def experiment(parser, token): 82 | """ 83 | Split Testing experiment tag has the following syntax : 84 | 85 | {% experiment %} 86 | experiment content goes here 87 | {% endexperiment %} 88 | 89 | If the alternative name is neither 'test' nor 'control' an exception is raised 90 | during rendering. 91 | """ 92 | try: 93 | token_contents = token.split_contents() 94 | experiment_name, alternative, weight, user_variable = _parse_token_contents(token_contents) 95 | 96 | node_list = parser.parse(('endexperiment', )) 97 | parser.delete_first_token() 98 | except ValueError: 99 | raise template.TemplateSyntaxError("Syntax should be like :" 100 | "{% experiment experiment_name alternative [weight=val] [user=val] %}") 101 | 102 | return ExperimentNode(node_list, experiment_name, alternative, weight, user_variable) 103 | 104 | 105 | @register.simple_tag(takes_context=True) 106 | def experiment_enroll(context, experiment_name, *alternatives, **kwargs): 107 | if 'user' in kwargs: 108 | user = participant(user=kwargs['user']) 109 | else: 110 | user = participant(request=context.get('request', None)) 111 | return user.enroll(experiment_name, list(alternatives)) 112 | -------------------------------------------------------------------------------- /experiments/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mixcloud/django-experiments/df6fe5d2e83a96539206d4bdcac4bf6e3d38d2ca/experiments/tests/__init__.py -------------------------------------------------------------------------------- /experiments/tests/test_admin.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import json 3 | 4 | from django.contrib.auth.models import User, Permission 5 | from django.urls import reverse 6 | from django.test import TestCase 7 | 8 | from experiments.models import Experiment, CONTROL_STATE, ENABLED_STATE 9 | from experiments.utils import participant 10 | 11 | 12 | class AdminTestCase(TestCase): 13 | def test_set_state(self): 14 | experiment = Experiment.objects.create(name='test_experiment', state=CONTROL_STATE) 15 | User.objects.create_superuser(username='user', email='deleted@mixcloud.com', password='pass') 16 | self.client.login(username='user', password='pass') 17 | 18 | self.assertEqual(Experiment.objects.get(pk=experiment.pk).state, CONTROL_STATE) 19 | response = self.client.post(reverse('admin:experiment_admin_set_state'), { 20 | 'experiment': experiment.name, 21 | 'state': ENABLED_STATE, 22 | }) 23 | self.assertEqual(response.status_code, 200) 24 | self.assertEqual(Experiment.objects.get(pk=experiment.pk).state, ENABLED_STATE) 25 | self.assertIsNone(Experiment.objects.get(pk=experiment.pk).end_date) 26 | 27 | response = self.client.post(reverse('admin:experiment_admin_set_state'), { 28 | 'experiment': experiment.name, 29 | 'state': CONTROL_STATE, 30 | }) 31 | self.assertEqual(response.status_code, 200) 32 | self.assertEqual(Experiment.objects.get(pk=experiment.pk).state, CONTROL_STATE) 33 | self.assertIsNotNone(Experiment.objects.get(pk=experiment.pk).end_date) 34 | 35 | def test_set_alternative(self): 36 | experiment = Experiment.objects.create(name='test_experiment', state=ENABLED_STATE) 37 | user = User.objects.create_superuser(username='user', email='deleted@mixcloud.com', password='pass') 38 | self.client.login(username='user', password='pass') 39 | 40 | participant(user=user).enroll('test_experiment', alternatives=['other1', 'other2']) 41 | 42 | for alternative in ('other2', 'control', 'other1'): 43 | response = self.client.post(reverse('admin:experiment_admin_set_alternative'), { 44 | 'experiment': experiment.name, 45 | 'alternative': alternative, 46 | }) 47 | self.assertDictEqual(json.loads(response.content.decode('utf-8')), { 48 | 'success': True, 49 | 'alternative': alternative, 50 | }) 51 | self.assertEqual(participant(user=user).get_alternative('test_experiment'), alternative) 52 | 53 | def test_permissions(self): 54 | # redirect to login if not logged in 55 | self.assertEqual(302, self.client.post(reverse('admin:experiment_admin_set_state'), {}).status_code) 56 | self.assertEqual(302, self.client.post(reverse('admin:experiment_admin_set_alternative'), {}).status_code) 57 | 58 | response = self.client.post(reverse('admin:experiment_admin_set_alternative'), {}) 59 | self.assertEqual(response.status_code, 302) 60 | 61 | # non staff user 62 | user = User.objects.create_user(username='user', password='pass') 63 | user.save() 64 | self.client.login(username='user', password='pass') 65 | 66 | self.assertEqual(302, self.client.post(reverse('admin:experiment_admin_set_state'), {}).status_code) 67 | self.assertEqual(302, self.client.post(reverse('admin:experiment_admin_set_alternative'), {}).status_code) 68 | 69 | user.is_staff = True 70 | user.save() 71 | 72 | self.assertEqual(403, self.client.post(reverse('admin:experiment_admin_set_state'), {}).status_code) 73 | self.assertEqual(403, self.client.post(reverse('admin:experiment_admin_set_alternative'), {}).status_code) 74 | 75 | permission = Permission.objects.get(codename='change_experiment') 76 | user.user_permissions.add(permission) 77 | 78 | self.assertEqual(400, self.client.post(reverse('admin:experiment_admin_set_state'), {}).status_code) 79 | self.assertEqual(400, self.client.post(reverse('admin:experiment_admin_set_alternative'), {}).status_code) 80 | -------------------------------------------------------------------------------- /experiments/tests/test_counter.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from unittest import TestCase 4 | 5 | from experiments import counters 6 | from experiments.experiment_counters import ExperimentCounter 7 | from experiments.models import Experiment 8 | from mock import patch 9 | 10 | TEST_KEY = 'CounterTestCase' 11 | 12 | 13 | class CounterTestCase(TestCase): 14 | def setUp(self): 15 | self.counters = counters.Counters() 16 | self.counters.reset(TEST_KEY) 17 | self.assertEqual(self.counters.get(TEST_KEY), 0) 18 | 19 | def tearDown(self): 20 | self.counters.reset(TEST_KEY) 21 | 22 | def test_add_item(self): 23 | self.counters.increment(TEST_KEY, 'fred') 24 | self.assertEqual(self.counters.get(TEST_KEY), 1) 25 | 26 | def test_add_multiple_items(self): 27 | self.counters.increment(TEST_KEY, 'fred') 28 | self.counters.increment(TEST_KEY, 'barney') 29 | self.counters.increment(TEST_KEY, 'george') 30 | self.counters.increment(TEST_KEY, 'george') 31 | self.assertEqual(self.counters.get(TEST_KEY), 3) 32 | 33 | def test_add_duplicate_item(self): 34 | self.counters.increment(TEST_KEY, 'fred') 35 | self.counters.increment(TEST_KEY, 'fred') 36 | self.counters.increment(TEST_KEY, 'fred') 37 | self.assertEqual(self.counters.get(TEST_KEY), 1) 38 | 39 | def test_get_frequencies(self): 40 | self.counters.increment(TEST_KEY, 'fred') 41 | self.counters.increment(TEST_KEY, 'barney') 42 | self.counters.increment(TEST_KEY, 'george') 43 | self.counters.increment(TEST_KEY, 'roger') 44 | self.counters.increment(TEST_KEY, 'roger') 45 | self.counters.increment(TEST_KEY, 'roger') 46 | self.counters.increment(TEST_KEY, 'roger') 47 | self.assertEqual(self.counters.get_frequencies(TEST_KEY), {1: 3, 4: 1}) 48 | 49 | def test_delete_key(self): 50 | self.counters.increment(TEST_KEY, 'fred') 51 | self.counters.reset(TEST_KEY) 52 | self.assertEqual(self.counters.get(TEST_KEY), 0) 53 | 54 | def test_clear_value(self): 55 | self.counters.increment(TEST_KEY, 'fred') 56 | self.counters.increment(TEST_KEY, 'fred') 57 | self.counters.increment(TEST_KEY, 'fred') 58 | self.counters.increment(TEST_KEY, 'barney') 59 | self.counters.increment(TEST_KEY, 'barney') 60 | self.counters.clear(TEST_KEY, 'fred') 61 | 62 | self.assertEqual(self.counters.get(TEST_KEY), 1) 63 | self.assertEqual(self.counters.get_frequencies(TEST_KEY), {2: 1}) 64 | 65 | def test_reset_all(self): 66 | experiment = Experiment.objects.create(name='reset_test') 67 | other_experiment = Experiment.objects.create(name='reset_test_other') 68 | experiment_counter = ExperimentCounter() 69 | 70 | for exp in [experiment, other_experiment]: 71 | experiment_counter.increment_participant_count(exp, 'alt', 'fred') 72 | experiment_counter.increment_participant_count(exp, 'alt', 'fred') 73 | experiment_counter.increment_participant_count(exp, 'alt', 'fred') 74 | experiment_counter.increment_goal_count(exp, 'alt', 'goal1', 'fred') 75 | experiment_counter.increment_goal_count(exp, 'alt', 'goal2', 'fred') 76 | experiment_counter.increment_participant_count(exp, 'control', 'barney') 77 | experiment_counter.increment_participant_count(exp, 'control', 'wilma') 78 | experiment_counter.increment_participant_count(exp, 'control', 'betty') 79 | experiment_counter.increment_goal_count(exp, 'control', 'goal1', 'betty') 80 | 81 | self.counters.reset_prefix(experiment.name) 82 | self.assertEqual(experiment_counter.participant_count(experiment, 'alt'), 0) 83 | self.assertEqual(experiment_counter.participant_count(experiment, 'control'), 0) 84 | self.assertEqual(experiment_counter.goal_count(experiment, 'control', 'goal1'), 0) 85 | 86 | self.assertEqual(experiment_counter.participant_count(other_experiment, 'alt'), 1) 87 | self.assertEqual(experiment_counter.participant_count(other_experiment, 'control'), 3) 88 | self.assertEqual(experiment_counter.goal_count(other_experiment, 'control', 'goal1'), 1) 89 | 90 | 91 | @patch('experiments.counters.Counters._redis') 92 | def test_should_return_tuple_if_failing(self, patched__redis): 93 | patched__redis.side_effect = Exception 94 | 95 | self.assertEqual(self.counters.get_frequencies(TEST_KEY), dict()) 96 | -------------------------------------------------------------------------------- /experiments/tests/test_models.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from unittest import TestCase 4 | 5 | from experiments.models import Experiment, Counters 6 | from mock import patch 7 | 8 | 9 | class ExperimentModelTestCase(TestCase): 10 | @patch.object(Counters, 'reset_prefix') 11 | def test_delete_resets_counters(self, reset_prefix_mock): 12 | experiment = Experiment.objects.create(name='test_experiment') 13 | experiment.delete() 14 | reset_prefix_mock.assert_called_with('test_experiment') 15 | 16 | @patch.object(Counters, 'reset_prefix') 17 | def test_delete_does_not_reset_counters_if_flag_not_set(self, reset_prefix_mock): 18 | experiment = Experiment.objects.create(name='test_experiment') 19 | experiment.delete(reset_counters=False) 20 | reset_prefix_mock.assert_not_called() 21 | -------------------------------------------------------------------------------- /experiments/tests/test_signals.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from django.contrib.auth import get_user_model 3 | 4 | from experiments.models import Experiment, ENABLED_STATE 5 | from experiments.signals import user_enrolled 6 | from experiments.utils import participant 7 | 8 | EXPERIMENT_NAME = 'backgroundcolor' 9 | 10 | 11 | class WatchSignal(object): 12 | def __init__(self, signal): 13 | self.signal = signal 14 | self.called = False 15 | 16 | def __enter__(self): 17 | self.signal.connect(self.signal_handler) 18 | return self 19 | 20 | def __exit__(self, *args): 21 | self.signal.disconnect(self.signal_handler) 22 | 23 | def signal_handler(self, *args, **kwargs): 24 | self.called = True 25 | 26 | 27 | class SignalsTestCase(TestCase): 28 | def setUp(self): 29 | self.experiment = Experiment.objects.create(name=EXPERIMENT_NAME, state=ENABLED_STATE) 30 | User = get_user_model() 31 | self.user = User.objects.create(username='brian') 32 | 33 | def test_sends_enroll_signal(self): 34 | with WatchSignal(user_enrolled) as signal: 35 | participant(user=self.user).enroll(EXPERIMENT_NAME, ['red', 'blue']) 36 | self.assertTrue(signal.called) 37 | 38 | def test_does_not_send_enroll_signal_again(self): 39 | participant(user=self.user).enroll(EXPERIMENT_NAME, ['red', 'blue']) 40 | with WatchSignal(user_enrolled) as signal: 41 | participant(user=self.user).enroll(EXPERIMENT_NAME, ['red', 'blue']) 42 | self.assertFalse(signal.called) 43 | -------------------------------------------------------------------------------- /experiments/tests/test_significance.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | import random 3 | 4 | from experiments.significance import mann_whitney, chi_square_p_value 5 | 6 | 7 | # The hardcoded p and u values in these tests were calculated using scipy 8 | class MannWhitneyTestCase(TestCase): 9 | longMessage = True 10 | 11 | def test_empty_sets(self): 12 | mann_whitney(dict(), dict()) 13 | 14 | def test_identical_ranges(self): 15 | distribution = dict((x, 1) for x in range(50)) 16 | self.assertUandPCorrect(distribution, distribution, 1250.0, 0.49862467827855483) 17 | 18 | def test_many_repeated_values(self): 19 | self.assertUandPCorrect({0: 100, 1: 50}, {0: 110, 1: 60}, 12500.0, 0.35672951675909859) 20 | 21 | def test_large_range(self): 22 | distribution_a = dict((x, 1) for x in range(10000)) 23 | distribution_b = dict((x + 1, 1) for x in range(10000)) 24 | self.assertUandPCorrect(distribution_a, distribution_b, 49990000.5, 0.49023014794874586) 25 | 26 | def test_very_different_sizes(self): 27 | distribution_a = dict((x, 1) for x in range(10000)) 28 | distribution_b = dict((x, 1) for x in range(20)) 29 | self.assertUandPCorrect(distribution_a, distribution_b, 200.0, 0) 30 | 31 | def assertUandPCorrect(self, distribution_a, distribution_b, u, p): 32 | our_u, our_p = mann_whitney(distribution_a, distribution_b) 33 | self.assertEqual(our_u, u, "U score incorrect") 34 | self.assertAlmostEqual(our_p, p, msg="p value incorrect") 35 | 36 | class ChiSquare(TestCase): 37 | def test_equal(self): 38 | self.assertChiSquareCorrect(((100, 10), (200, 20)), 0, 1) 39 | self.assertChiSquareCorrect(((100, 100, 100), (200, 200, 200), (300, 300, 300)), 0, 1) 40 | 41 | def test_error(self): 42 | self.assertEqual(chi_square_p_value((1,)), None) 43 | self.assertEqual(chi_square_p_value(((0,2,3))), None) 44 | 45 | def test_is_none(self): 46 | self.assertEqual(chi_square_p_value(((1, 1), (1, -1))), (None, None), "Negative numbers should not be allowed") 47 | self.assertEqual(chi_square_p_value(((0, 0), (0, 0))), (None, None), "Zero sample size should not be allowed") 48 | self.assertIsNone(chi_square_p_value(((1,), (1, 2))), "Unequal matrices should not be allowed") 49 | self.assertIsNone(chi_square_p_value(((1, 2, 3), (1, 2, 3), (1, 2))), "Unequal matrices should not be allowed") 50 | self.assertIsNone(chi_square_p_value(((100, 10), (200, 20), (300, 30), (400, 40))), "Matrices have to be square") 51 | 52 | def test_stress(self): 53 | # Generate a large matrix 54 | matrix = [] 55 | for col in range(0, 100): 56 | matrix.append([]) 57 | for row in range(0, 100): 58 | matrix[col].append(random.randint(0, 10)) 59 | 60 | self.assertIsNotNone(chi_square_p_value(matrix)) 61 | 62 | def test_accept_hypothesis(self): 63 | self.assertChiSquareCorrect(((36, 14), (30, 25)), 3.418, 0.065, 3) 64 | self.assertChiSquareCorrect(((100, 50), (210, 110)), 0.04935, 0.8242, 3) 65 | self.assertChiSquareCorrect(((100, 50, 10), (110, 50, 10), (140, 55, 11)), 1.2238, 0.8741, 3) 66 | 67 | def test_reject_hypothesis(self): 68 | self.assertChiSquareCorrect(((100, 20), (200, 20)), 4.2929, 0.0383, 4) 69 | self.assertChiSquareCorrect(((100, 50, 10), (110, 70, 20), (140, 55, 6)), 13.0217, 0.0111, 3) 70 | 71 | def assertChiSquareCorrect(self, matrix, observed_test_statistic, p_value, accuracy=7): 72 | observed_test_statistic_result, p_value_result = chi_square_p_value(matrix) 73 | self.assertAlmostEqual(observed_test_statistic_result, observed_test_statistic, accuracy, 'Wrong observed result') 74 | self.assertAlmostEqual(p_value_result, p_value, accuracy, 'Wrong P Value') 75 | 76 | -------------------------------------------------------------------------------- /experiments/tests/test_templatetags.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import User 2 | from django.template import Template, Context 3 | from django.test import TestCase, override_settings, RequestFactory 4 | from experiments.models import Experiment 5 | 6 | from experiments.templatetags.experiments import _parse_token_contents 7 | from experiments.utils import participant 8 | 9 | 10 | class ExperimentTemplateTagTestCase(TestCase): 11 | """These test cases are rather nastily coupled, and are mainly intended to check the token parsing code""" 12 | 13 | def test_returns_with_standard_values(self): 14 | token_contents = ('experiment', 'backgroundcolor', 'blue') 15 | experiment_name, alternative, weight, user_resolvable = _parse_token_contents(token_contents) 16 | self.assertEqual(experiment_name, 'backgroundcolor') 17 | self.assertEqual(alternative, 'blue') 18 | 19 | def test_handles_old_style_weight(self): 20 | token_contents = ('experiment', 'backgroundcolor', 'blue', '10') 21 | experiment_name, alternative, weight, user_resolvable = _parse_token_contents(token_contents) 22 | self.assertEqual(weight, '10') 23 | 24 | def test_handles_labelled_weight(self): 25 | token_contents = ('experiment', 'backgroundcolor', 'blue', 'weight=10') 26 | experiment_name, alternative, weight, user_resolvable = _parse_token_contents(token_contents) 27 | self.assertEqual(weight, '10') 28 | 29 | def test_handles_user(self): 30 | token_contents = ('experiment', 'backgroundcolor', 'blue', 'user=commenter') 31 | experiment_name, alternative, weight, user_resolvable = _parse_token_contents(token_contents) 32 | self.assertEqual(user_resolvable.var, 'commenter') 33 | 34 | def test_handles_user_and_weight(self): 35 | token_contents = ('experiment', 'backgroundcolor', 'blue', 'user=commenter', 'weight=10') 36 | experiment_name, alternative, weight, user_resolvable = _parse_token_contents(token_contents) 37 | self.assertEqual(user_resolvable.var, 'commenter') 38 | self.assertEqual(weight, '10') 39 | 40 | def test_raises_on_insufficient_arguments(self): 41 | token_contents = ('experiment', 'backgroundcolor') 42 | self.assertRaises(ValueError, lambda: _parse_token_contents(token_contents)) 43 | 44 | 45 | class ExperimentAutoCreateTestCase(TestCase): 46 | @override_settings(EXPERIMENTS_AUTO_CREATE=False) 47 | def test_template_auto_create_off(self): 48 | request = RequestFactory().get('/') 49 | request.user = User.objects.create(username='test') 50 | Template("{% load experiments %}{% experiment test_experiment control %}{% endexperiment %}").render(Context({'request': request})) 51 | self.assertFalse(Experiment.objects.filter(name="test_experiment").exists()) 52 | 53 | def test_template_auto_create_on(self): 54 | request = RequestFactory().get('/') 55 | request.user = User.objects.create(username='test') 56 | Template("{% load experiments %}{% experiment test_experiment control %}{% endexperiment %}").render(Context({'request': request})) 57 | self.assertTrue(Experiment.objects.filter(name="test_experiment").exists()) 58 | 59 | @override_settings(EXPERIMENTS_AUTO_CREATE=False) 60 | def test_view_auto_create_off(self): 61 | user = User.objects.create(username='test') 62 | participant(user=user).enroll('test_experiment_y', alternatives=['other']) 63 | self.assertFalse(Experiment.objects.filter(name="test_experiment_y").exists()) 64 | 65 | def test_view_auto_create_on(self): 66 | user = User.objects.create(username='test') 67 | participant(user=user).enroll('test_experiment_x', alternatives=['other']) 68 | self.assertTrue(Experiment.objects.filter(name="test_experiment_x").exists()) 69 | -------------------------------------------------------------------------------- /experiments/tests/test_webuser.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from datetime import timedelta 4 | from django.http import HttpResponse 5 | 6 | from django.test import TestCase 7 | from django.test.client import RequestFactory 8 | from django.contrib.auth.models import AnonymousUser 9 | from django.contrib.auth import get_user_model 10 | from django.contrib.sessions.backends.db import SessionStore as DatabaseSession 11 | from django.utils import timezone 12 | from experiments import conf 13 | 14 | from experiments.experiment_counters import ExperimentCounter 15 | from experiments.middleware import ExperimentsRetentionMiddleware 16 | from experiments.models import Experiment, ENABLED_STATE, Enrollment 17 | from experiments.conf import CONTROL_GROUP, VISIT_PRESENT_COUNT_GOAL, VISIT_NOT_PRESENT_COUNT_GOAL 18 | from experiments.redis_client import get_redis_client 19 | from experiments.signal_handlers import transfer_enrollments_to_user 20 | from experiments.utils import participant 21 | 22 | from mock import patch 23 | 24 | import random 25 | 26 | request_factory = RequestFactory() 27 | 28 | TEST_ALTERNATIVE = 'blue' 29 | TEST_GOAL = 'buy' 30 | EXPERIMENT_NAME = 'backgroundcolor' 31 | 32 | 33 | class BaseUserTests(object): 34 | def setUp(self): 35 | self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE) 36 | self.experiment.save() 37 | self.request = request_factory.get('/') 38 | self.request.session = DatabaseSession() 39 | self.experiment_counter = ExperimentCounter() 40 | 41 | def tearDown(self): 42 | self.experiment_counter.delete(self.experiment) 43 | 44 | def test_enrollment_initially_control(self): 45 | experiment_user = participant(self.request) 46 | self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Default Enrollment wasn't control") 47 | 48 | def test_user_enrolls(self): 49 | experiment_user = participant(self.request) 50 | experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) 51 | self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE, "Wrong Alternative Set") 52 | 53 | def test_record_goal_increments_counts(self): 54 | experiment_user = participant(self.request) 55 | experiment_user.confirm_human() 56 | experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) 57 | 58 | self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0) 59 | experiment_user.goal(TEST_GOAL) 60 | self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment Goal count") 61 | 62 | def test_can_record_goal_multiple_times(self): 63 | experiment_user = participant(self.request) 64 | experiment_user.confirm_human() 65 | experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) 66 | 67 | experiment_user.goal(TEST_GOAL) 68 | experiment_user.goal(TEST_GOAL) 69 | experiment_user.goal(TEST_GOAL) 70 | self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment goal count correctly") 71 | self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), {3: 1}, "Incorrect goal count distribution") 72 | 73 | def test_counts_increment_immediately_once_confirmed_human(self): 74 | experiment_user = participant(self.request) 75 | experiment_user.confirm_human() 76 | 77 | experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) 78 | self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 1, "Did not count participant after confirm human") 79 | 80 | def test_visit_increases_goal(self): 81 | thetime = timezone.now() 82 | with patch('experiments.utils.now', return_value=thetime): 83 | experiment_user = participant(self.request) 84 | experiment_user.confirm_human() 85 | experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) 86 | 87 | experiment_user.visit() 88 | self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Not Present Visit was not correctly counted") 89 | self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted") 90 | 91 | with patch('experiments.utils.now', return_value=thetime + timedelta(hours=7)): 92 | experiment_user.visit() 93 | self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {2: 1}, "No Present Visit was not correctly counted") 94 | self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {1: 1}, "Present Visit was not correctly counted") 95 | 96 | def test_visit_twice_increases_once(self): 97 | experiment_user = participant(self.request) 98 | experiment_user.confirm_human() 99 | experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) 100 | 101 | experiment_user.visit() 102 | experiment_user.visit() 103 | 104 | self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Visit was not correctly counted") 105 | self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted") 106 | 107 | def test_user_force_enrolls(self): 108 | experiment_user = participant(self.request) 109 | experiment_user.enroll(EXPERIMENT_NAME, ['control', 'alternative1', 'alternative2'], force_alternative='alternative2') 110 | self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'alternative2') 111 | 112 | def test_user_does_not_force_enroll_to_new_alternative(self): 113 | alternatives = ['control', 'alternative1', 'alternative2'] 114 | experiment_user = participant(self.request) 115 | experiment_user.enroll(EXPERIMENT_NAME, alternatives) 116 | alternative = experiment_user.get_alternative(EXPERIMENT_NAME) 117 | self.assertIsNotNone(alternative) 118 | 119 | other_alternative = random.choice(list(set(alternatives) - set(alternative))) 120 | experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative) 121 | self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME)) 122 | 123 | def test_second_force_enroll_does_not_change_alternative(self): 124 | alternatives = ['control', 'alternative1', 'alternative2'] 125 | experiment_user = participant(self.request) 126 | experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative='alternative1') 127 | alternative = experiment_user.get_alternative(EXPERIMENT_NAME) 128 | self.assertIsNotNone(alternative) 129 | 130 | other_alternative = random.choice(list(set(alternatives) - set(alternative))) 131 | experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative) 132 | self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME)) 133 | 134 | 135 | class BaseUserAnonymousTestCase(BaseUserTests, TestCase): 136 | def setUp(self): 137 | super(BaseUserAnonymousTestCase, self).setUp() 138 | self.request.user = AnonymousUser() 139 | 140 | def test_confirm_human_increments_participant_count(self): 141 | experiment_user = participant(self.request) 142 | experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) 143 | experiment_user.goal(TEST_GOAL) 144 | 145 | self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Counted participant before confirmed human") 146 | experiment_user.confirm_human() 147 | self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 1, "Did not count participant after confirm human") 148 | 149 | def test_confirm_human_increments_goal_count(self): 150 | experiment_user = participant(self.request) 151 | experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) 152 | experiment_user.goal(TEST_GOAL) 153 | 154 | self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0, "Counted goal before confirmed human") 155 | experiment_user.confirm_human() 156 | self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not count goal after confirm human") 157 | 158 | 159 | class BaseUserAuthenticatedTestCase(BaseUserTests, TestCase): 160 | def setUp(self): 161 | super(BaseUserAuthenticatedTestCase, self).setUp() 162 | User = get_user_model() 163 | self.request.user = User(username='brian') 164 | self.request.user.save() 165 | 166 | 167 | class BotTests(object): 168 | def setUp(self): 169 | self.experiment = Experiment(name='backgroundcolor', state=ENABLED_STATE) 170 | self.experiment.save() 171 | self.experiment_counter = ExperimentCounter() 172 | 173 | def test_user_does_not_enroll(self): 174 | self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) 175 | self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results") 176 | 177 | def test_user_does_not_fire_goals(self): 178 | self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) 179 | self.experiment_user.goal(TEST_GOAL) 180 | self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results") 181 | 182 | def test_bot_in_control_group(self): 183 | self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) 184 | self.assertEqual(self.experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Bot enrolled in a group") 185 | self.assertEqual(self.experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE), False, "Bot in test alternative") 186 | self.assertEqual(self.experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP), True, "Bot not in control group") 187 | 188 | def tearDown(self): 189 | self.experiment_counter.delete(self.experiment) 190 | 191 | 192 | class LoggedOutBotTestCase(BotTests, TestCase): 193 | def setUp(self): 194 | super(LoggedOutBotTestCase, self).setUp() 195 | self.request = request_factory.get('/', HTTP_USER_AGENT='GoogleBot/2.1') 196 | self.experiment_user = participant(self.request) 197 | 198 | 199 | class LoggedInBotTestCase(BotTests, TestCase): 200 | def setUp(self): 201 | super(LoggedInBotTestCase, self).setUp() 202 | User = get_user_model() 203 | self.user = User(username='brian') 204 | self.user.is_confirmed_human = False 205 | self.user.save() 206 | 207 | self.experiment_user = participant(user=self.user) 208 | 209 | 210 | class ParticipantCacheTestCase(TestCase): 211 | def setUp(self): 212 | self.experiment = Experiment.objects.create(name='test_experiment1', state=ENABLED_STATE) 213 | self.experiment_counter = ExperimentCounter() 214 | 215 | def tearDown(self): 216 | self.experiment_counter.delete(self.experiment) 217 | 218 | def test_transfer_enrollments(self): 219 | User = get_user_model() 220 | user = User.objects.create(username='test') 221 | request = request_factory.get('/') 222 | request.session = DatabaseSession() 223 | participant(request).enroll('test_experiment1', ['alternative']) 224 | request.user = user 225 | transfer_enrollments_to_user(None, request, user) 226 | # the call to the middleware will set last_seen on the experiment 227 | # if the participant cache hasn't been wiped appropriately then the 228 | # session experiment user will be impacted instead of the authenticated 229 | # experiment user 230 | ExperimentsRetentionMiddleware(request).process_response(request, HttpResponse()) 231 | self.assertIsNotNone(Enrollment.objects.all()[0].last_seen) 232 | 233 | 234 | class ConfirmHumanTestCase(TestCase): 235 | def setUp(self): 236 | self.experiment = Experiment.objects.create(name='test_experiment1', state=ENABLED_STATE) 237 | self.experiment_counter = ExperimentCounter() 238 | self.experiment_user = participant(session=DatabaseSession()) 239 | self.alternative = self.experiment_user.enroll(self.experiment.name, ['alternative']) 240 | self.experiment_user.goal('my_goal') 241 | self.redis = get_redis_client() 242 | 243 | def tearDown(self): 244 | self.experiment_counter.delete(self.experiment) 245 | 246 | def test_confirm_human_updates_experiment(self): 247 | self.assertTrue(self.redis.exists(self.experiment_user._redis_goals_key)) 248 | self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 0) 249 | self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 0) 250 | self.experiment_user.confirm_human() 251 | self.assertFalse(self.redis.exists(self.experiment_user._redis_goals_key)) 252 | self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1) 253 | self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1) 254 | 255 | def test_confirm_human_called_twice(self): 256 | """ 257 | Ensuring that counters aren't incremented twice 258 | """ 259 | self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 0) 260 | self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 0) 261 | self.experiment_user.confirm_human() 262 | self.experiment_user.confirm_human() 263 | self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1) 264 | self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1) 265 | 266 | def test_confirm_human_sets_session(self): 267 | self.assertFalse(self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False)) 268 | self.experiment_user.confirm_human() 269 | self.assertTrue(self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False)) 270 | 271 | def test_session_already_confirmed(self): 272 | """ 273 | Testing that confirm_human works even if code outside of django-experiments updates the key 274 | """ 275 | self.experiment_user.session[conf.CONFIRM_HUMAN_SESSION_KEY] = True 276 | self.experiment_user.confirm_human() 277 | self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1) 278 | self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1) 279 | 280 | 281 | class DefaultAlternativeTestCase(TestCase): 282 | def test_default_alternative(self): 283 | experiment = Experiment.objects.create(name='test_default') 284 | self.assertEqual(experiment.default_alternative, conf.CONTROL_GROUP) 285 | experiment.ensure_alternative_exists('alt1') 286 | experiment.ensure_alternative_exists('alt2') 287 | 288 | self.assertEqual(conf.CONTROL_GROUP, participant(session=DatabaseSession()).enroll('test_default', ['alt1', 'alt2'])) 289 | experiment.set_default_alternative('alt2') 290 | experiment.save() 291 | self.assertEqual('alt2', participant(session=DatabaseSession()).enroll('test_default', ['alt1', 'alt2'])) 292 | experiment.set_default_alternative('alt1') 293 | experiment.save() 294 | self.assertEqual('alt1', participant(session=DatabaseSession()).enroll('test_default', ['alt1', 'alt2'])) 295 | -------------------------------------------------------------------------------- /experiments/tests/test_webuser_incorporate.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpResponse 2 | from django.test import TestCase, RequestFactory 3 | from django.contrib.sessions.backends.db import SessionStore as DatabaseSession 4 | 5 | from unittest import TestSuite 6 | 7 | from experiments import conf 8 | from experiments.experiment_counters import ExperimentCounter 9 | from experiments.middleware import ExperimentsRetentionMiddleware 10 | from experiments.signal_handlers import transfer_enrollments_to_user 11 | from experiments.utils import DummyUser, WebUser, participant 12 | from experiments.models import Experiment, ENABLED_STATE, Enrollment 13 | 14 | from django.contrib.auth import get_user_model 15 | 16 | TEST_ALTERNATIVE = 'blue' 17 | EXPERIMENT_NAME = 'backgroundcolor' 18 | 19 | 20 | class BaseUserIncorporateTestCase(object): 21 | def __init__(self, *args, **kwargs): 22 | super(BaseUserIncorporateTestCase, self).__init__(*args, **kwargs) 23 | self.experiment_counter = ExperimentCounter() 24 | 25 | def test_can_incorporate(self): 26 | self.incorporating.incorporate(self.incorporated) 27 | 28 | def test_incorporates_enrollment_from_other(self): 29 | if not self._has_data(): 30 | return 31 | 32 | try: 33 | experiment = Experiment.objects.create(name=EXPERIMENT_NAME, state=ENABLED_STATE) 34 | self.incorporated.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) 35 | self.incorporating.incorporate(self.incorporated) 36 | self.assertEqual(self.incorporating.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE) 37 | finally: 38 | self.experiment_counter.delete(experiment) 39 | 40 | def _has_data(self): 41 | return not isinstance(self.incorporated, DummyUser) and not isinstance(self.incorporating, DummyUser) 42 | 43 | 44 | def dummy(incorporating): 45 | return DummyUser() 46 | 47 | 48 | def anonymous(incorporating): 49 | return WebUser(session=DatabaseSession()) 50 | 51 | 52 | def authenticated(incorporating): 53 | User = get_user_model() 54 | return WebUser(user=User.objects.create(username=['incorporating_user', 'incorporated_user'][incorporating])) 55 | 56 | user_factories = (dummy, anonymous, authenticated) 57 | 58 | 59 | def load_tests(loader, standard_tests, _): 60 | suite = TestSuite() 61 | suite.addTests(standard_tests) 62 | 63 | for incorporating in user_factories: 64 | for incorporated in user_factories: 65 | test_case = build_test_case(incorporating, incorporated) 66 | tests = loader.loadTestsFromTestCase(test_case) 67 | suite.addTests(tests) 68 | return suite 69 | 70 | 71 | def build_test_case(incorporating, incorporated): 72 | class InstantiatedTestCase(BaseUserIncorporateTestCase, TestCase): 73 | 74 | def setUp(self): 75 | super(InstantiatedTestCase, self).setUp() 76 | self.incorporating = incorporating(True) 77 | self.incorporated = incorporated(False) 78 | InstantiatedTestCase.__name__ = "BaseUserIncorporateTestCase_into_%s_from_%s" % (incorporating.__name__, incorporated.__name__) 79 | return InstantiatedTestCase 80 | 81 | 82 | class IncorporateTestCase(TestCase): 83 | def setUp(self): 84 | self.experiment = Experiment.objects.create(name=EXPERIMENT_NAME, state=ENABLED_STATE) 85 | self.experiment_counter = ExperimentCounter() 86 | 87 | User = get_user_model() 88 | self.user = User.objects.create(username='incorporate_user') 89 | self.user.is_confirmed_human = True 90 | 91 | request_factory = RequestFactory() 92 | self.request = request_factory.get('/') 93 | self.request.session = DatabaseSession() 94 | participant(self.request).confirm_human() 95 | 96 | def tearDown(self): 97 | self.experiment_counter.delete(self.experiment) 98 | 99 | def _login(self): 100 | self.request.user = self.user 101 | transfer_enrollments_to_user(None, self.request, self.user) 102 | 103 | def test_visit_incorporate(self): 104 | alternative = participant(self.request).enroll(self.experiment.name, ['alternative']) 105 | 106 | ExperimentsRetentionMiddleware(self.request).process_response(self.request, HttpResponse()) 107 | 108 | self.assertEqual( 109 | dict(self.experiment_counter.participant_goal_frequencies(self.experiment, 110 | alternative, 111 | participant(self.request)._participant_identifier()))[conf.VISIT_NOT_PRESENT_COUNT_GOAL], 112 | 1 113 | ) 114 | 115 | self.assertFalse(Enrollment.objects.filter(user__isnull=False).exists()) 116 | self._login() 117 | 118 | self.assertTrue(Enrollment.objects.filter(user__isnull=False).exists()) 119 | self.assertIsNotNone(Enrollment.objects.all()[0].last_seen) 120 | self.assertEqual( 121 | dict(self.experiment_counter.participant_goal_frequencies(self.experiment, 122 | alternative, 123 | participant(self.request)._participant_identifier()))[conf.VISIT_NOT_PRESENT_COUNT_GOAL], 124 | 1 125 | ) 126 | self.assertEqual(self.experiment_counter.goal_count(self.experiment, alternative, conf.VISIT_NOT_PRESENT_COUNT_GOAL), 1) 127 | self.assertEqual(self.experiment_counter.participant_count(self.experiment, alternative), 1) 128 | -------------------------------------------------------------------------------- /experiments/tests/urls.py: -------------------------------------------------------------------------------- 1 | from experiments.urls import urlpatterns 2 | from django.urls import path 3 | from django.contrib import admin 4 | 5 | 6 | urlpatterns += [ 7 | path('admin/', admin.site.urls), 8 | ] 9 | -------------------------------------------------------------------------------- /experiments/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | from experiments import views 3 | 4 | urlpatterns = [ 5 | path('goal///', views.record_experiment_goal, name="experiment_goal"), 6 | path('confirm_human/', views.confirm_human, name="experiment_confirm_human"), 7 | path('change_alternative///', views.change_alternative, name="experiment_change_alternative"), 8 | ] 9 | -------------------------------------------------------------------------------- /experiments/utils.py: -------------------------------------------------------------------------------- 1 | from django.db import IntegrityError 2 | from django.utils.functional import cached_property 3 | 4 | from redis.exceptions import ConnectionError, ResponseError 5 | 6 | from experiments.models import Enrollment 7 | from experiments.manager import experiment_manager 8 | from experiments.dateutils import now, fix_awareness, datetime_from_timestamp, timestamp_from_datetime 9 | from experiments.signals import user_enrolled 10 | from experiments.experiment_counters import ExperimentCounter 11 | from experiments.redis_client import get_redis_client 12 | from experiments import conf 13 | 14 | from collections import namedtuple 15 | from datetime import timedelta 16 | 17 | import collections 18 | try: 19 | from collections.abc import Mapping 20 | except ImportError: # Python < 3.10 21 | from collections import Mapping 22 | import numbers 23 | import logging 24 | import json 25 | 26 | try: 27 | from itertools import zip_longest as izip_longest 28 | except ImportError: 29 | from itertools import izip_longest 30 | 31 | logger = logging.getLogger('experiments') 32 | 33 | 34 | UNCONFIRMED_HUMAN_GOALS_REDIS_KEY = "experiments:goals:%s" 35 | 36 | 37 | def participant(request=None, session=None, user=None): 38 | # This caches the experiment user on the request object because WebUser can involve database lookups that 39 | # it caches. Signals are attached to login/logout to clear the cache using clear_participant_cache 40 | if request and hasattr(request, '_experiments_user'): 41 | return request._experiments_user 42 | else: 43 | result = _get_participant(request, session, user) 44 | if request: 45 | request._experiments_user = result 46 | return result 47 | 48 | 49 | def clear_participant_cache(request): 50 | if hasattr(request, '_experiments_user'): 51 | del request._experiments_user 52 | 53 | 54 | def _get_participant(request, session, user): 55 | if request and hasattr(request, 'user') and not user: 56 | user = request.user 57 | if request and hasattr(request, 'session') and not session: 58 | session = request.session 59 | 60 | if request and conf.BOT_REGEX.search(request.META.get("HTTP_USER_AGENT", "")): 61 | return DummyUser() 62 | elif user and user.is_authenticated: 63 | if getattr(user, 'is_confirmed_human', True): 64 | return WebUser(user=user, request=request) 65 | else: 66 | return DummyUser() 67 | elif session: 68 | return WebUser(session=session, request=request) 69 | else: 70 | return DummyUser() 71 | 72 | 73 | EnrollmentData = namedtuple('EnrollmentData', ['experiment', 'alternative', 'enrollment_date', 'last_seen']) 74 | 75 | 76 | class BaseUser(object): 77 | """Represents a user (either authenticated or session based) which can take part in experiments""" 78 | 79 | def __init__(self): 80 | self.experiment_counter = ExperimentCounter() 81 | 82 | def enroll(self, experiment_name, alternatives, force_alternative=None): 83 | """ 84 | Enroll this user in the experiment if they are not already part of it. Returns the selected alternative 85 | 86 | force_alternative: Optionally force a user in an alternative at enrollment time 87 | """ 88 | chosen_alternative = conf.CONTROL_GROUP 89 | 90 | experiment = experiment_manager.get_experiment(experiment_name) 91 | 92 | if experiment: 93 | if experiment.is_displaying_alternatives(): 94 | if isinstance(alternatives, Mapping): 95 | if conf.CONTROL_GROUP not in alternatives: 96 | experiment.ensure_alternative_exists(conf.CONTROL_GROUP, 1) 97 | for alternative, weight in alternatives.items(): 98 | experiment.ensure_alternative_exists(alternative, weight) 99 | else: 100 | alternatives_including_control = alternatives + [conf.CONTROL_GROUP] 101 | for alternative in alternatives_including_control: 102 | experiment.ensure_alternative_exists(alternative) 103 | 104 | assigned_alternative = self._get_enrollment(experiment) 105 | if assigned_alternative: 106 | chosen_alternative = assigned_alternative 107 | elif experiment.is_accepting_new_users(): 108 | if force_alternative: 109 | chosen_alternative = force_alternative 110 | else: 111 | chosen_alternative = experiment.random_alternative() 112 | self._set_enrollment(experiment, chosen_alternative) 113 | else: 114 | chosen_alternative = experiment.default_alternative 115 | 116 | return chosen_alternative 117 | 118 | def get_alternative(self, experiment_name): 119 | """ 120 | Get the alternative this user is enrolled in. 121 | """ 122 | experiment = None 123 | try: 124 | # catching the KeyError instead of using .get so that the experiment is auto created if desired 125 | experiment = experiment_manager[experiment_name] 126 | except KeyError: 127 | pass 128 | if experiment: 129 | if experiment.is_displaying_alternatives(): 130 | alternative = self._get_enrollment(experiment) 131 | if alternative is not None: 132 | return alternative 133 | else: 134 | return experiment.default_alternative 135 | return conf.CONTROL_GROUP 136 | 137 | def set_alternative(self, experiment_name, alternative): 138 | """Explicitly set the alternative the user is enrolled in for the specified experiment. 139 | 140 | This allows you to change a user between alternatives. The user and goal counts for the new 141 | alternative will be increment, but those for the old one will not be decremented. The user will 142 | be enrolled in the experiment even if the experiment would not normally accept this user.""" 143 | experiment = experiment_manager.get_experiment(experiment_name) 144 | if experiment: 145 | self._set_enrollment(experiment, alternative) 146 | 147 | def goal(self, goal_name, count=1): 148 | """Record that this user has performed a particular goal 149 | 150 | This will update the goal stats for all experiments the user is enrolled in.""" 151 | for enrollment in self._get_all_enrollments(): 152 | if enrollment.experiment.is_displaying_alternatives(): 153 | self._experiment_goal(enrollment.experiment, enrollment.alternative, goal_name, count) 154 | 155 | def confirm_human(self): 156 | """Mark that this is a real human being (not a bot) and thus results should be counted""" 157 | pass 158 | 159 | def incorporate(self, other_user): 160 | """Incorporate all enrollments and goals performed by the other user 161 | 162 | If this user is not enrolled in a given experiment, the results for the 163 | other user are incorporated. For experiments this user is already 164 | enrolled in the results of the other user are discarded. 165 | 166 | This takes a relatively large amount of time for each experiment the other 167 | user is enrolled in.""" 168 | for enrollment in other_user._get_all_enrollments(): 169 | if not self._get_enrollment(enrollment.experiment): 170 | self._set_enrollment(enrollment.experiment, enrollment.alternative, enrollment.enrollment_date, enrollment.last_seen) 171 | goals = self.experiment_counter.participant_goal_frequencies(enrollment.experiment, enrollment.alternative, other_user._participant_identifier()) 172 | for goal_name, count in goals: 173 | self.experiment_counter.increment_goal_count(enrollment.experiment, enrollment.alternative, goal_name, self._participant_identifier(), count) 174 | other_user._cancel_enrollment(enrollment.experiment) 175 | 176 | def visit(self): 177 | """Record that the user has visited the site for the purposes of retention tracking""" 178 | for enrollment in self._get_all_enrollments(): 179 | if enrollment.experiment.is_displaying_alternatives(): 180 | # We have two different goals, VISIT_NOT_PRESENT_COUNT_GOAL and VISIT_PRESENT_COUNT_GOAL. 181 | # VISIT_PRESENT_COUNT_GOAL will avoid firing on the first time we set last_seen as it is assumed that the user is 182 | # on the page and therefore it would automatically trigger and be valueless. 183 | # This should be used for experiments when we enroll the user as part of the pageview, 184 | # alternatively we can use the NOT_PRESENT GOAL which will increment on the first pageview, 185 | # this is mainly useful for notification actions when the users isn't initially present. 186 | 187 | if not enrollment.last_seen: 188 | self._experiment_goal(enrollment.experiment, enrollment.alternative, conf.VISIT_NOT_PRESENT_COUNT_GOAL, 1) 189 | self._set_last_seen(enrollment.experiment, now()) 190 | elif now() - enrollment.last_seen >= timedelta(hours=conf.SESSION_LENGTH): 191 | self._experiment_goal(enrollment.experiment, enrollment.alternative, conf.VISIT_NOT_PRESENT_COUNT_GOAL, 1) 192 | self._experiment_goal(enrollment.experiment, enrollment.alternative, conf.VISIT_PRESENT_COUNT_GOAL, 1) 193 | self._set_last_seen(enrollment.experiment, now()) 194 | 195 | def _get_enrollment(self, experiment): 196 | """Get the name of the alternative this user is enrolled in for the specified experiment 197 | 198 | `experiment` is an instance of Experiment. If the user is not currently enrolled returns None.""" 199 | raise NotImplementedError 200 | 201 | def _set_enrollment(self, experiment, alternative, enrollment_date=None, last_seen=None): 202 | """Explicitly set the alternative the user is enrolled in for the specified experiment. 203 | 204 | This allows you to change a user between alternatives. The user and goal counts for the new 205 | alternative will be increment, but those for the old one will not be decremented.""" 206 | raise NotImplementedError 207 | 208 | def is_enrolled(self, experiment_name, alternative): 209 | """Enroll this user in the experiment if they are not already part of it. Returns the selected alternative""" 210 | """Test if the user is enrolled in the supplied alternative for the given experiment. 211 | 212 | The supplied alternative will be added to the list of possible alternatives for the 213 | experiment if it is not already there. If the user is not yet enrolled in the supplied 214 | experiment they will be enrolled, and an alternative chosen at random.""" 215 | chosen_alternative = self.enroll(experiment_name, [alternative]) 216 | return alternative == chosen_alternative 217 | 218 | def _participant_identifier(self): 219 | "Unique identifier for this user in the counter store" 220 | raise NotImplementedError 221 | 222 | def _get_all_enrollments(self): 223 | "Return experiment, alternative tuples for all experiments the user is enrolled in" 224 | raise NotImplementedError 225 | 226 | def _cancel_enrollment(self, experiment): 227 | "Remove the enrollment and any goals the user has against this experiment" 228 | raise NotImplementedError 229 | 230 | def _experiment_goal(self, experiment, alternative, goal_name, count): 231 | "Record a goal against a particular experiment and alternative" 232 | raise NotImplementedError 233 | 234 | def _set_last_seen(self, experiment, last_seen): 235 | "Set the last time the user was seen associated with this experiment" 236 | raise NotImplementedError 237 | 238 | 239 | class DummyUser(BaseUser): 240 | def _get_enrollment(self, experiment): 241 | return None 242 | 243 | def _set_enrollment(self, experiment, alternative, enrollment_date=None, last_seen=None): 244 | pass 245 | 246 | def is_enrolled(self, experiment_name, alternative): 247 | return alternative == conf.CONTROL_GROUP 248 | 249 | def incorporate(self, other_user): 250 | for enrollment in other_user._get_all_enrollments(): 251 | other_user._cancel_enrollment(enrollment.experiment) 252 | 253 | def _participant_identifier(self): 254 | return "" 255 | 256 | def _get_all_enrollments(self): 257 | return [] 258 | 259 | def _is_enrolled_in_experiment(self, experiment): 260 | return False 261 | 262 | def _cancel_enrollment(self, experiment): 263 | pass 264 | 265 | def _get_goal_counts(self, experiment, alternative): 266 | return {} 267 | 268 | def _experiment_goal(self, experiment, alternative, goal_name, count): 269 | pass 270 | 271 | def _set_last_seen(self, experiment, last_seen): 272 | pass 273 | 274 | 275 | class WebUser(BaseUser): 276 | def __init__(self, user=None, session=None, request=None): 277 | self._enrollment_cache = {} 278 | self.user = user 279 | self.session = session 280 | self.request = request 281 | self._redis_goals_key = UNCONFIRMED_HUMAN_GOALS_REDIS_KEY % self._participant_identifier() 282 | super(WebUser, self).__init__() 283 | 284 | @cached_property 285 | def _redis(self): 286 | return get_redis_client() 287 | 288 | @property 289 | def _qs_kwargs(self): 290 | if self.user: 291 | return {"user": self.user} 292 | else: 293 | return {"session_key": self._session_key} 294 | 295 | def _get_enrollment(self, experiment): 296 | if experiment.name not in self._enrollment_cache: 297 | try: 298 | self._enrollment_cache[experiment.name] = Enrollment.objects.get(experiment=experiment, **self._qs_kwargs).alternative 299 | except Enrollment.DoesNotExist: 300 | self._enrollment_cache[experiment.name] = None 301 | return self._enrollment_cache[experiment.name] 302 | 303 | def _set_enrollment(self, experiment, alternative, enrollment_date=None, last_seen=None): 304 | if experiment.name in self._enrollment_cache: 305 | del self._enrollment_cache[experiment.name] 306 | 307 | try: 308 | enrollment, _ = Enrollment.objects.get_or_create(experiment=experiment, defaults={'alternative': alternative}, **self._qs_kwargs) 309 | except IntegrityError: 310 | # Already registered (db race condition under high load) 311 | return 312 | # Update alternative if it doesn't match 313 | enrollment_changed = False 314 | if enrollment.alternative != alternative: 315 | enrollment.alternative = alternative 316 | enrollment_changed = True 317 | if enrollment_date: 318 | enrollment.enrollment_date = enrollment_date 319 | enrollment_changed = True 320 | if last_seen: 321 | enrollment.last_seen = last_seen 322 | enrollment_changed = True 323 | 324 | if enrollment_changed: 325 | enrollment.save() 326 | 327 | if self._is_verified_human: 328 | self.experiment_counter.increment_participant_count(experiment, alternative, self._participant_identifier()) 329 | else: 330 | logger.info(json.dumps({'type':'participant_unconfirmed', 'experiment': experiment.name, 'alternative': alternative, 'participant': self._participant_identifier()})) 331 | 332 | user_enrolled.send(self, experiment=experiment.name, alternative=alternative, user=self.user, session=self.session) 333 | 334 | def _participant_identifier(self): 335 | if self.user: 336 | return 'user:%s' % self.user.pk 337 | else: 338 | return 'session:%s' % self._session_key 339 | 340 | def _get_all_enrollments(self): 341 | enrollments = Enrollment.objects.filter(**self._qs_kwargs).select_related("experiment") 342 | if enrollments: 343 | for enrollment in enrollments: 344 | yield EnrollmentData(enrollment.experiment, enrollment.alternative, enrollment.enrollment_date, enrollment.last_seen) 345 | 346 | def _cancel_enrollment(self, experiment): 347 | try: 348 | enrollment = Enrollment.objects.get(experiment=experiment, **self._qs_kwargs) 349 | except Enrollment.DoesNotExist: 350 | pass 351 | else: 352 | self.experiment_counter.remove_participant(experiment, enrollment.alternative, self._participant_identifier()) 353 | enrollment.delete() 354 | 355 | def _experiment_goal(self, experiment, alternative, goal_name, count): 356 | if self._is_verified_human: 357 | self.experiment_counter.increment_goal_count(experiment, alternative, goal_name, self._participant_identifier(), count) 358 | else: 359 | try: 360 | self._redis.lpush(self._redis_goals_key, json.dumps((experiment.name, alternative, goal_name, count))) 361 | # Setting an expiry on this data otherwise it could linger for a while 362 | # and also fill up redis quickly if lots of bots begin to scrape the app. 363 | # Human confirmation processes are generally quick so this defaults to a 364 | # low value (but it can be configured via Django settings) 365 | self._redis.expire(self._redis_goals_key, conf.REDIS_GOALS_TTL) 366 | except (ConnectionError, ResponseError): 367 | # Handle Redis failures gracefully 368 | pass 369 | logger.info(json.dumps({'type': 'goal_hit_unconfirmed', 'goal': goal_name, 'goal_count': count, 'experiment': experiment.name, 'alternative': alternative, 'participant': self._participant_identifier()})) 370 | 371 | def confirm_human(self): 372 | if self.user: 373 | return 374 | 375 | self.session[conf.CONFIRM_HUMAN_SESSION_KEY] = True 376 | logger.info(json.dumps({'type': 'confirm_human', 'participant': self._participant_identifier()})) 377 | 378 | # Replay enrollments 379 | for enrollment in self._get_all_enrollments(): 380 | self.experiment_counter.increment_participant_count(enrollment.experiment, enrollment.alternative, self._participant_identifier()) 381 | 382 | # Replay goals 383 | try: 384 | goals = self._redis.lrange(self._redis_goals_key, 0, -1) 385 | if goals: 386 | try: 387 | for data in goals: 388 | experiment_name, alternative, goal_name, count = json.loads(data) 389 | experiment = experiment_manager.get_experiment(experiment_name) 390 | if experiment: 391 | self.experiment_counter.increment_goal_count(experiment, alternative, goal_name, self._participant_identifier(), count) 392 | except ValueError: 393 | pass # Values from older version 394 | finally: 395 | self._redis.delete(self._redis_goals_key) 396 | except (ConnectionError, ResponseError): 397 | # Handle Redis failures gracefully 398 | pass 399 | 400 | def _set_last_seen(self, experiment, last_seen): 401 | Enrollment.objects.filter(experiment=experiment, **self._qs_kwargs).update(last_seen=last_seen) 402 | 403 | @property 404 | def _is_verified_human(self): 405 | if conf.VERIFY_HUMAN and not self.user: 406 | return self.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False) 407 | else: 408 | return True 409 | 410 | @property 411 | def _session_key(self): 412 | if not self.session: 413 | return None 414 | if 'experiments_session_key' not in self.session: 415 | if not self.session.session_key: 416 | self.session.save() # Force session key 417 | self.session['experiments_session_key'] = self.session.session_key 418 | return self.session['experiments_session_key'] 419 | 420 | 421 | def grouper(iterable, n, fillvalue=None): 422 | # Taken from the recipe at 423 | # https://docs.python.org/2.7/library/itertools.html#itertools-recipes 424 | args = [iter(iterable)] * n 425 | return izip_longest(fillvalue=fillvalue, *args) 426 | 427 | 428 | 429 | __all__ = ['participant'] 430 | -------------------------------------------------------------------------------- /experiments/views.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpResponse, HttpResponseBadRequest 2 | from django.views.decorators.cache import never_cache 3 | from django.shortcuts import get_object_or_404 4 | from django.views.decorators.http import require_POST 5 | 6 | from experiments.utils import participant 7 | from experiments.models import Experiment 8 | from experiments import conf 9 | 10 | TRANSPARENT_1X1_PNG = \ 11 | ("\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52" 12 | "\x00\x00\x00\x01\x00\x00\x00\x01\x08\x03\x00\x00\x00\x28\xcb\x34" 13 | "\xbb\x00\x00\x00\x19\x74\x45\x58\x74\x53\x6f\x66\x74\x77\x61\x72" 14 | "\x65\x00\x41\x64\x6f\x62\x65\x20\x49\x6d\x61\x67\x65\x52\x65\x61" 15 | "\x64\x79\x71\xc9\x65\x3c\x00\x00\x00\x06\x50\x4c\x54\x45\x00\x00" 16 | "\x00\x00\x00\x00\xa5\x67\xb9\xcf\x00\x00\x00\x01\x74\x52\x4e\x53" 17 | "\x00\x40\xe6\xd8\x66\x00\x00\x00\x0c\x49\x44\x41\x54\x78\xda\x62" 18 | "\x60\x00\x08\x30\x00\x00\x02\x00\x01\x4f\x6d\x59\xe1\x00\x00\x00" 19 | "\x00\x49\x45\x4e\x44\xae\x42\x60\x82\x00") 20 | 21 | 22 | @never_cache 23 | @require_POST 24 | def confirm_human(request): 25 | if conf.CONFIRM_HUMAN: 26 | experiment_user = participant(request) 27 | experiment_user.confirm_human() 28 | return HttpResponse(status=204) 29 | 30 | 31 | @never_cache 32 | def record_experiment_goal(request, goal_name, cache_buster=None): 33 | participant(request).goal(goal_name) 34 | return HttpResponse(TRANSPARENT_1X1_PNG, content_type="image/png") 35 | 36 | 37 | def change_alternative(request, experiment_name, alternative_name): 38 | experiment = get_object_or_404(Experiment, name=experiment_name) 39 | if alternative_name not in experiment.alternatives.keys(): 40 | return HttpResponseBadRequest() 41 | 42 | participant(request).set_alternative(experiment_name, alternative_name) 43 | return HttpResponse('OK') 44 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | # This flag says that the code is written to work on both Python 2 and Python 3 | # 3. If at all possible, it is good practice to do this. If you cannot, you 4 | # will need to generate wheels for each Python version that you support. 5 | universal=1 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import codecs # to use a consistent encoding 2 | from os import path 3 | from setuptools import setup, find_packages # prefer setuptools over distutils 4 | 5 | 6 | # Get the long description from the README file 7 | PATH = path.abspath(path.dirname(__file__)) 8 | with codecs.open(path.join(PATH, 'README.rst'), encoding='utf-8') as f: 9 | LONG_DESCRIPTION = f.read() 10 | 11 | setup( 12 | name='django-experiments', 13 | version='1.2.1', 14 | description='Python Django AB Testing Framework', 15 | long_description=LONG_DESCRIPTION, 16 | author='Mixcloud', 17 | author_email='technical@mixcloud.com', 18 | url='https://github.com/mixcloud/django-experiments', 19 | packages=find_packages(exclude=["example_project"]), 20 | include_package_data=True, 21 | license='MIT', 22 | install_requires=[ 23 | 'django>=1.11', 24 | 'django-modeldict-yplan>=1.5.0', 25 | 'redis>=2.4.9', 26 | ], 27 | tests_require=[ 28 | 'mock>=1.0.1', 29 | 'tox>=2.3.1', 30 | ], 31 | test_suite="testrunner.runtests", 32 | classifiers=[ 33 | 'Development Status :: 5 - Production/Stable', 34 | 'Intended Audience :: Developers', 35 | 'License :: OSI Approved :: MIT License', 36 | 'Topic :: Internet :: WWW/HTTP', 37 | 'Topic :: Software Development :: Libraries', 38 | 'Programming Language :: Python :: 2', 39 | 'Programming Language :: Python :: 2.7', 40 | 'Programming Language :: Python :: 3', 41 | 'Programming Language :: Python :: 3.5', 42 | 'Programming Language :: Python :: 3.6', 43 | 'Framework :: Django', 44 | 'Framework :: Django :: 1.11', 45 | ], 46 | ) 47 | -------------------------------------------------------------------------------- /testrunner.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from django.conf import settings 5 | 6 | import django 7 | 8 | def runtests(): 9 | test_dir = os.path.dirname(os.path.abspath(__file__)) 10 | sys.path.insert(0, test_dir) 11 | 12 | settings.configure( 13 | DEBUG=True, 14 | DATABASES={ 15 | 'default': { 16 | 'ENGINE': 'django.db.backends.sqlite3', 17 | } 18 | }, 19 | INSTALLED_APPS=('django.contrib.auth', 20 | 'django.contrib.contenttypes', 21 | 'django.contrib.sessions', 22 | 'django.contrib.admin', 23 | 'django.contrib.messages', 24 | 'experiments',), 25 | ROOT_URLCONF='experiments.tests.urls', 26 | MIDDLEWARE = ( 27 | 'django.contrib.sessions.middleware.SessionMiddleware', 28 | 'django.middleware.csrf.CsrfViewMiddleware', 29 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 30 | 'django.contrib.messages.middleware.MessageMiddleware' 31 | ), 32 | SECRET_KEY="foobarbaz", 33 | TEMPLATES = [ 34 | { 35 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 36 | 'DIRS': [], 37 | 'APP_DIRS': True, 38 | 'OPTIONS': { 39 | 'context_processors': [ 40 | 'django.contrib.auth.context_processors.auth', 41 | 'django.contrib.messages.context_processors.messages', 42 | ], 43 | }, 44 | }, 45 | ], 46 | ) 47 | django.setup() 48 | 49 | 50 | from django.test.utils import get_runner 51 | TestRunner = get_runner(settings) 52 | test_runner = TestRunner(verbosity=1, failfast=False) 53 | failures = test_runner.run_tests(['experiments', ]) 54 | sys.exit(bool(failures)) 55 | 56 | 57 | if __name__ == '__main__': 58 | runtests() 59 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | 7 | [tox] 8 | minversion=2.7 9 | 10 | skip_missing_interpreters=True 11 | 12 | envlist = 13 | {py,pypy}{35,36,37,38}-django{2.2} 14 | {py,pypy}{36,37,38,39}-django{3.0,3.1} 15 | {py,pypy}{36,37,38,39,py310}-django{3.2} 16 | {py,pypy}{38,39,310}-django{4.0} 17 | {py,pypy}{38,39,310,311}-django{4.1} 18 | {py,pypy}{38,39,310,311,312}-django{4.2} 19 | {py,pypy}{310,311,312}-django{5.0} 20 | 21 | [gh-actions] 22 | python = 23 | 3.5: py35 24 | 3.6: py36 25 | 3.7: py37 26 | 3.8: py38 27 | 3.9: py39 28 | 3.10: py310 29 | 3.11: py311 30 | pypy-3.8: pypy38 31 | pypy-3.9: pypy39 32 | pypy-3.10: pypy310 33 | pypy-3.11: pypy311 34 | pypy-3.12: pypy312 35 | 36 | [testenv] 37 | commands = 38 | python example_project/manage.py makemigrations --check --dry-run experiments 39 | python -W error::DeprecationWarning testrunner.py 40 | 41 | deps = 42 | mock 43 | django2.2: Django==2.2.* 44 | django2.2: jsonfield>=1.0.3,<3 45 | django3.0: Django==3.0.* 46 | django3.0: jsonfield>=1.0.3,<3 47 | django3.1: Django==3.1.* 48 | django3.1: jsonfield>=1.0.3,<3 49 | django3.2: Django==3.2.* 50 | django4.0: Django==4.0.* 51 | django4.1: Django==4.1.* 52 | django4.2: Django==4.2.* 53 | django5.0: Django==5.0.* 54 | --------------------------------------------------------------------------------