├── basket ├── __init__.py ├── base │ ├── __init__.py │ ├── models.py │ ├── tests │ │ ├── __init__.py │ │ └── test__utils.py │ ├── templates │ │ ├── 404.html │ │ ├── 500.html │ │ └── admin │ │ │ ├── base_site.html │ │ │ └── login.html │ ├── authentication.py │ └── utils.py ├── news │ ├── __init__.py │ ├── backends │ │ ├── __init__.py │ │ └── common.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ ├── process_maintenance_queue.py │ │ │ ├── sync_newsletters.py │ │ │ ├── process_donations_queue.py │ │ │ ├── process_fxa_queue.py │ │ │ └── process_fxa_data.py │ ├── migrations │ │ ├── __init__.py │ │ ├── 0006_merge.py │ │ ├── 0002_delete_subscriber.py │ │ ├── 0015_delete_smsmessage.py │ │ ├── 0008_auto_20160605_1345.py │ │ ├── 0014_move_sms_messages.py │ │ ├── 0007_auto_20160531_1454.py │ │ ├── 0010_auto_20160607_0815.py │ │ ├── 0011_auto_20160607_1203.py │ │ ├── 0004_queuedtask.py │ │ ├── 0009_transactionalemailmessage.py │ │ ├── 0005_convert_newsletter_vendor_id.py │ │ ├── 0003_auto_20151202_0808.py │ │ ├── 0012_auto_20170713_1021.py │ │ └── 0013_auto_20170907_1216.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_middleware.py │ │ ├── test_send_welcomes.py │ │ ├── test_confirm.py │ │ ├── test_fields.py │ │ ├── test_models.py │ │ ├── test_newsletters.py │ │ ├── test_forms.py │ │ ├── test_users.py │ │ ├── test_backends_sfmc.py │ │ └── test_sfdc.py │ ├── context_processors.py │ ├── templates │ │ └── news │ │ │ ├── get_involved │ │ │ └── steward_email.txt │ │ │ ├── donation_notify_email.txt │ │ │ ├── formerror.html │ │ │ ├── thankyou.html │ │ │ └── newsletters.html │ ├── apps.py │ ├── celery.py │ ├── urls.py │ ├── fields.py │ ├── middleware.py │ ├── forms.py │ ├── admin.py │ ├── country_codes.py │ ├── fixtures │ │ └── newsletters.json │ └── newsletters.py ├── errors.py ├── wsgi.py └── urls.py ├── docs ├── _static │ └── .gitkeep ├── _templates │ └── .gitkeep ├── index.rst ├── production_environments.rst ├── install.rst ├── Makefile ├── conf.py └── newsletter_api.rst ├── .dockerignore ├── requirements.txt ├── .coveragerc ├── requirements ├── docs.txt ├── dev.txt └── prod.txt ├── bin ├── run-fxa-events-worker.sh ├── run-donate-worker.sh ├── run-fxa-activity-worker.sh ├── run-clock.sh ├── run-dev.sh ├── run-tests.sh ├── post-deploy.sh ├── run-worker.sh ├── run-prod.sh ├── jenkins │ └── buildandpush.sh ├── tag-release.sh └── irc-notify.sh ├── .demo_env ├── docker ├── bin │ ├── apt-install │ ├── run_tests.sh │ ├── check_if_tag.sh │ ├── build_images.sh │ ├── push2dockerhub.sh │ ├── set_git_env_vars.sh │ ├── docker_build.sh │ └── push2deis.sh └── envfiles │ └── test.env ├── setup.cfg ├── Procfile ├── env-dist ├── jenkins ├── branches │ ├── prod.yml │ └── master.yml ├── global.yml ├── utils.groovy └── default.groovy ├── manage.py ├── .pyup.yml ├── .gitignore ├── .editorconfig ├── README.rst ├── .circleci └── config.yml ├── Dockerfile ├── contribute.json ├── docker-compose.yml ├── Jenkinsfile └── newrelic.ini /basket/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /basket/base/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /basket/base/models.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /basket/news/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/_static/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/_templates/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /basket/base/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /basket/news/backends/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /basket/news/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /basket/news/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /basket/news/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | celerybeat-schedule 3 | *.db 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # for heroku/deis 2 | 3 | -r requirements/prod.txt 4 | 5 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = news 3 | omit = 4 | news/tests/* 5 | news/migrations/* 6 | -------------------------------------------------------------------------------- /basket/news/tests/__init__.py: -------------------------------------------------------------------------------- 1 | default_app_config = 'basket.news.apps.BasketNewsConfig' 2 | -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- 1 | # requirements for building the docs 2 | 3 | Sphinx==1.2.2 4 | 5 | -------------------------------------------------------------------------------- /bin/run-fxa-events-worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | exec python manage.py process_fxa_queue 4 | -------------------------------------------------------------------------------- /bin/run-donate-worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | exec python manage.py process_donations_queue 4 | -------------------------------------------------------------------------------- /bin/run-fxa-activity-worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | exec python manage.py process_fxa_data --cron 4 | -------------------------------------------------------------------------------- /bin/run-clock.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | exec celery -A basket.news beat -l "${CELERY_LOG_LEVEL:-warning}" 4 | -------------------------------------------------------------------------------- /.demo_env: -------------------------------------------------------------------------------- 1 | DEBUG=False 2 | DATABASE_URL=sqlite:///basket.db 3 | CELERY_ALWAYS_EAGER=True 4 | SECRET_KEY=sssssssssshhhhhhhhhhh 5 | -------------------------------------------------------------------------------- /bin/run-dev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | urlwait 6 | bin/post-deploy.sh 7 | ./manage.py runserver 0.0.0.0:8000 8 | -------------------------------------------------------------------------------- /docker/bin/apt-install: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | apt-get update 6 | apt-get install -y --no-install-recommends "$@" 7 | rm -rf /var/lib/apt/lists/* 8 | -------------------------------------------------------------------------------- /bin/run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -exo pipefail 4 | 5 | flake8 basket 6 | bin/post-deploy.sh 7 | py.test --junitxml=test-results/test-results.xml basket 8 | -------------------------------------------------------------------------------- /basket/news/context_processors.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings as django_settings 2 | 3 | 4 | def settings(request): 5 | return {'settings': django_settings} 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore=E121,E123,E124,E125,E126,E127,E128,E501 3 | exclude=migrations 4 | 5 | [pytest] 6 | DJANGO_SETTINGS_MODULE=basket.settings 7 | addopts = --ignore=vendor 8 | -------------------------------------------------------------------------------- /basket/base/templates/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 404 - Page not found - Basket 4 | 5 | 6 |

404 - Page not found - Basket

7 | 8 | 9 | -------------------------------------------------------------------------------- /basket/base/templates/500.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 500 - Internal Server Error - Basket 4 | 5 | 6 |

500 - Internal Server Error - Basket

7 | 8 | 9 | -------------------------------------------------------------------------------- /docker/envfiles/test.env: -------------------------------------------------------------------------------- 1 | DEBUG=False 2 | DEV=False 3 | ALLOWED_HOSTS=* 4 | SECRET_KEY=ssssssssshhhhhhhhhh 5 | ADMINS=["thedude@example.com"] 6 | CELERY_ALWAYS_EAGER=True 7 | DATABASE_URL=sqlite:///basket.db 8 | -------------------------------------------------------------------------------- /basket/news/templates/news/get_involved/steward_email.txt: -------------------------------------------------------------------------------- 1 | Name: {{ contributor_name }} 2 | Email: {{ contributor_email }} 3 | Area of Interest: {{ interest }} 4 | Language: {{ lang }} 5 | Comment: {{ message }} 6 | -------------------------------------------------------------------------------- /bin/post-deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | READ_ONLY_MODE=$(echo "$READ_ONLY_MODE" | tr '[:upper:]' '[:lower:]') 4 | 5 | if [[ "$READ_ONLY_MODE" != "true" ]]; then 6 | python manage.py migrate --noinput 7 | fi 8 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: bin/run-prod.sh 2 | worker: bin/run-worker.sh 3 | donateworker: bin/run-donate-worker.sh 4 | fxaworker: bin/run-fxa-activity-worker.sh 5 | fxaeventsworker: bin/run-fxa-events-worker.sh 6 | clock: bin/run-clock.sh 7 | -------------------------------------------------------------------------------- /env-dist: -------------------------------------------------------------------------------- 1 | DEBUG=1 2 | 3 | # change this to something secret when deployed 4 | SECRET_KEY=sssssssshhhhhhhhhh 5 | 6 | # this is so you don't need a real queue for development 7 | CELERY_ALWAYS_EAGER=1 8 | 9 | ALLOWED_HOSTS='*' 10 | -------------------------------------------------------------------------------- /basket/base/authentication.py: -------------------------------------------------------------------------------- 1 | from mozilla_django_oidc.auth import OIDCAuthenticationBackend 2 | from django.contrib.auth.backends import ModelBackend 3 | 4 | 5 | class OIDCModelBackend(OIDCAuthenticationBackend, ModelBackend): 6 | pass 7 | -------------------------------------------------------------------------------- /jenkins/branches/prod.yml: -------------------------------------------------------------------------------- 1 | require_tag: true 2 | regions: 3 | - oregon-a 4 | - oregon-b 5 | - frankfurt 6 | - tokyo 7 | apps: 8 | - basket-prod 9 | apps_rw: 10 | - basket-admin 11 | apps_post_deploy: 12 | - basket-admin 13 | -------------------------------------------------------------------------------- /jenkins/branches/master.yml: -------------------------------------------------------------------------------- 1 | regions: 2 | - oregon-a 3 | - oregon-b 4 | - frankfurt 5 | - tokyo 6 | apps: 7 | - basket-dev 8 | - basket-stage 9 | apps_rw: 10 | - basket-admin-stage 11 | apps_post_deploy: 12 | - basket-dev 13 | - basket-admin-stage 14 | -------------------------------------------------------------------------------- /basket/news/templates/news/donation_notify_email.txt: -------------------------------------------------------------------------------- 1 | A donation transaction update failed to apply because it could not be found in Salesforce. 2 | 3 | Transaction ID: {{ txn_id }} 4 | Type Lost: {{ type_lost }} 5 | Reason Lost: {{ reason_lost }} 6 | Server: {{ server_name }} 7 | -------------------------------------------------------------------------------- /basket/base/templates/admin/base_site.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/base.html" %} 2 | {% load i18n %} 3 | 4 | {% block title %}{{ title }} | Basket{% endblock %} 5 | 6 | {% block branding %} 7 |

Basket {{ foo }}

8 | {% endblock %} 9 | 10 | {% block nav-global %}{% endblock %} 11 | -------------------------------------------------------------------------------- /basket/news/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class BasketNewsConfig(AppConfig): 5 | def ready(self): 6 | # This will make sure the app is always imported when 7 | # Django starts so that shared_task will use this app. 8 | import basket.news.celery # noqa 9 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "basket.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /.pyup.yml: -------------------------------------------------------------------------------- 1 | # autogenerated pyup.io config file 2 | # see https://pyup.io/docs/configuration/ for all available options 3 | 4 | # Once a month, pyup will create a PR with all the updates 5 | schedule: "every month" 6 | 7 | # Check the master branch 8 | branch: master 9 | 10 | # Set it not to close stale PRs 11 | close_prs: False 12 | -------------------------------------------------------------------------------- /docker/bin/run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Runs unit_tests 4 | # 5 | set -exo pipefail 6 | 7 | BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 8 | source $BIN_DIR/set_git_env_vars.sh 9 | 10 | TEST_IMAGE_TAG="mozmeao/basket:${GIT_COMMIT}" 11 | docker run --rm --env-file docker/envfiles/test.env "$TEST_IMAGE_TAG" bin/run-tests.sh 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | settings/local.py 2 | *.py[co] 3 | *.sw[po] 4 | .env 5 | .coverage 6 | pip-log.txt 7 | docs/_build 8 | docs/_gh-pages 9 | build.py 10 | .DS_Store 11 | /media/img/uploads 12 | *-min.css 13 | *-all.css 14 | *-min.js 15 | *-all.js 16 | .#* 17 | .noseids 18 | *.db 19 | static 20 | celerybeat-schedule 21 | Dockerfile.jenkins 22 | test-results 23 | -------------------------------------------------------------------------------- /bin/run-worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | exec newrelic-admin run-program celery -A basket.news worker \ 4 | -P "${CELERY_POOL:-prefork}" \ 5 | -l "${CELERY_LOG_LEVEL:-warning}" \ 6 | -c "${CELERY_NUM_WORKERS:-4}" \ 7 | -Q celery,snitch 8 | -------------------------------------------------------------------------------- /basket/base/utils.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | 4 | def email_is_testing(email): 5 | """Return true if email address is at a known testing domain""" 6 | if not settings.USE_SANDBOX_BACKEND: 7 | for domain in settings.TESTING_EMAIL_DOMAINS: 8 | if email.endswith(u'@{}'.format(domain)): 9 | return True 10 | 11 | return False 12 | -------------------------------------------------------------------------------- /jenkins/global.yml: -------------------------------------------------------------------------------- 1 | regions: 2 | frankfurt: 3 | deis_profile: frankfurt 4 | name: frankfurt 5 | db_mode: ro 6 | tokyo: 7 | deis_profile: tokyo 8 | name: tokyo 9 | db_mode: ro 10 | oregon-a: 11 | deis_profile: oregon-a 12 | name: oregon-a 13 | db_mode: rw 14 | oregon-b: 15 | deis_profile: oregon-b 16 | name: oregon-b 17 | db_mode: rw 18 | -------------------------------------------------------------------------------- /basket/news/migrations/0006_merge.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('news', '0004_queuedtask'), 11 | ('news', '0005_convert_newsletter_vendor_id'), 12 | ] 13 | 14 | operations = [ 15 | ] 16 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Unix-style newlines with a newline ending every file 7 | [*] 8 | end_of_line = lf 9 | insert_final_newline = true 10 | trim_trailing_whitespace = true 11 | indent_style = space 12 | indent_size = 4 13 | 14 | [*.yml] 15 | indent_size = 2 16 | 17 | [Makefile] 18 | indent_style = tab 19 | -------------------------------------------------------------------------------- /basket/news/migrations/0002_delete_subscriber.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('news', '0001_initial'), 11 | ] 12 | 13 | operations = [ 14 | migrations.DeleteModel( 15 | name='Subscriber', 16 | ), 17 | ] 18 | -------------------------------------------------------------------------------- /basket/news/migrations/0015_delete_smsmessage.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('news', '0014_move_sms_messages'), 11 | ] 12 | 13 | operations = [ 14 | migrations.DeleteModel( 15 | name='SMSMessage', 16 | ), 17 | ] 18 | -------------------------------------------------------------------------------- /bin/run-prod.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | echo "$GIT_SHA" > static/revision.txt 4 | exec gunicorn basket.wsgi --bind "0.0.0.0:${PORT:-8000}" \ 5 | --workers "${WSGI_NUM_WORKERS:-8}" \ 6 | --worker-class "${WSGI_WORKER_CLASS:-meinheld.gmeinheld.MeinheldWorker}" \ 7 | --log-level "${WSGI_LOG_LEVEL:-info}" \ 8 | --error-logfile - \ 9 | --access-logfile - 10 | -------------------------------------------------------------------------------- /docker/bin/check_if_tag.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 4 | source $BIN_DIR/set_git_env_vars.sh 5 | 6 | if [[ -n "$GIT_TAG" ]]; then 7 | if [[ "$GIT_TAG_DATE_BASED" == true ]]; then 8 | echo "Build tagged as $GIT_TAG" 9 | exit 0 10 | else 11 | echo "Build tagged but in the wrong format: $GIT_TAG" 12 | exit 1 13 | fi 14 | else 15 | echo "Build not tagged" 16 | exit 1 17 | fi 18 | -------------------------------------------------------------------------------- /basket/news/templates/news/formerror.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% block title %}Error{% endblock %} 5 | {% block blockbots %} 6 | {% endblock %} 7 | 8 | 9 |

Error

10 | 11 |

We've had a problem processing your request. The following error(s) occurred:

12 | 13 | {{ form.errors }} 14 | 15 |

Use the "back" button to go back to the form and fix the problem(s).

16 | 17 | 18 | -------------------------------------------------------------------------------- /basket/news/migrations/0008_auto_20160605_1345.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('news', '0007_auto_20160531_1454'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='failedtask', 16 | name='task_id', 17 | field=models.CharField(max_length=255), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ====== 2 | Basket 3 | ====== 4 | 5 | Interact with our email marketing provider via a nice simple HTTP API. 6 | 7 | .. image:: https://circleci.com/gh/mozmeao/basket/tree/master.svg?style=svg 8 | :target: https://circleci.com/gh/mozmeao/basket/tree/master 9 | 10 | 11 | Docs 12 | ---- 13 | 14 | Documentation can be found at http://basket.readthedocs.org/ . 15 | 16 | 17 | License 18 | ------- 19 | 20 | This software is licensed under the `Mozilla Public License Version 2.0 `_. For more 21 | information, read the file ``LICENSE``. 22 | -------------------------------------------------------------------------------- /basket/base/tests/test__utils.py: -------------------------------------------------------------------------------- 1 | from django.test.utils import override_settings 2 | 3 | from basket.base.utils import email_is_testing 4 | 5 | 6 | @override_settings(TESTING_EMAIL_DOMAINS=['restmail.net'], 7 | USE_SANDBOX_BACKEND=False) 8 | def test_email_is_testing(): 9 | assert email_is_testing('dude@restmail.net') 10 | assert not email_is_testing('dude@restmail.net.com') 11 | assert not email_is_testing('dude@real.restmail.net') 12 | assert not email_is_testing('restmail.net@example.com') 13 | assert not email_is_testing('dude@example.com') 14 | -------------------------------------------------------------------------------- /basket/errors.py: -------------------------------------------------------------------------------- 1 | # error codes from basket-client 2 | 3 | BASKET_NETWORK_FAILURE = 1 4 | BASKET_INVALID_EMAIL = 2 5 | BASKET_UNKNOWN_EMAIL = 3 6 | BASKET_UNKNOWN_TOKEN = 4 7 | BASKET_USAGE_ERROR = 5 8 | BASKET_EMAIL_PROVIDER_AUTH_FAILURE = 6 9 | BASKET_AUTH_ERROR = 7 10 | BASKET_SSL_REQUIRED = 8 11 | BASKET_INVALID_NEWSLETTER = 9 12 | BASKET_INVALID_LANGUAGE = 10 13 | BASKET_EMAIL_NOT_CHANGED = 11 14 | BASKET_CHANGE_REQUEST_NOT_FOUND = 12 15 | BASKET_MOCK_FAILURE = 13 16 | 17 | # If you get this, report it as a bug so we can add a more specific 18 | # error code. 19 | BASKET_UNKNOWN_ERROR = 99 20 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ============================================== 2 | Welcome to Basket documentation! 3 | ============================================== 4 | 5 | This documentation explains how to install and use basket.mozilla.org. 6 | 7 | 8 | About Basket 9 | -------------------- 10 | 11 | A Python web service, basket, provides an API for all of our subscribing needs. 12 | Basket interfaces into whatever email provider we are using. 13 | 14 | 15 | Contents 16 | -------- 17 | 18 | .. toctree:: 19 | :maxdepth: 2 20 | 21 | install 22 | production_environments 23 | newsletter_api 24 | -------------------------------------------------------------------------------- /basket/news/templates/news/thankyou.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% block title %}Thank you for subscribing{% endblock %} 5 | {% block blockbots %}{% endblock %} 6 | 7 | 8 |

Thanks!

9 |

10 | If you haven’t previously confirmed a subscription to a Mozilla-related 11 | newsletter you may have to do so. Please check your inbox or your spam filter 12 | for an email from us. 13 |

14 | {% if source_url %} 15 |

Click here to return to the site.

16 | {% endif %} 17 | 18 | 19 | -------------------------------------------------------------------------------- /docker/bin/build_images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -exo pipefail 4 | 5 | BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | source $BIN_DIR/set_git_env_vars.sh 7 | 8 | DOCKER_REBUILD=false 9 | 10 | # parse cli args 11 | while [[ $# -gt 0 ]]; do 12 | key="$1" 13 | case $key in 14 | -r|--rebuild) 15 | DOCKER_REBUILD=true 16 | ;; 17 | esac 18 | shift # past argument or value 19 | done 20 | 21 | function imageExists() { 22 | docker history -q "${DOCKER_IMAGE_TAG}" > /dev/null 2>&1 23 | return $? 24 | } 25 | 26 | if ! imageExists; then 27 | docker/bin/docker_build.sh --pull 28 | fi 29 | -------------------------------------------------------------------------------- /docs/production_environments.rst: -------------------------------------------------------------------------------- 1 | .. This Source Code Form is subject to the terms of the Mozilla Public 2 | .. License, v. 2.0. If a copy of the MPL was not distributed with this 3 | .. file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | .. _ production-environments: 6 | 7 | ================ 8 | Production Environments 9 | ================ 10 | 11 | Production installs often have a few different requirements: 12 | 13 | * point Apache's ``WSGIScriptAlias`` at ``/path/to/basket/wsgi/basket.wsgi`` 14 | * jbalogh has a good example `WSGI config for Zamboni `_. 15 | * ``DEBUG = False`` in settings 16 | -------------------------------------------------------------------------------- /bin/jenkins/buildandpush.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -xe 3 | 4 | # Workaround to ignore mtime until we upgrade to Docker 1.8 5 | # See https://github.com/docker/docker/pull/12031 6 | find . -newerat 20140101 -exec touch -t 201401010000 {} \; 7 | 8 | DOCKER_IMAGE_TAG=$DOCKER_REPO:$GIT_COMMIT 9 | 10 | 11 | docker build -t $DOCKER_IMAGE_TAG . 12 | docker save $DOCKER_IMAGE_TAG | sudo docker-squash -t $DOCKER_IMAGE_TAG | docker load 13 | docker tag -f $DOCKER_IMAGE_TAG $PRIVATE_REGISTRY/$DOCKER_IMAGE_TAG 14 | docker push $PRIVATE_REGISTRY/$DOCKER_IMAGE_TAG 15 | 16 | deis login $DEIS_CONTROLLER --username $DEIS_USERNAME --password $DEIS_PASSWORD 17 | deis pull $DOCKER_REPO:$GIT_COMMIT -a $DOCKER_REPO 18 | -------------------------------------------------------------------------------- /docker/bin/push2dockerhub.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Needs DOCKER_USERNAME, DOCKER_PASSWORD, and DOCKER_REPOSITORY environment variables. 3 | set -ex 4 | 5 | BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | source $BIN_DIR/set_git_env_vars.sh 7 | 8 | DOCKER_USERNAME="${DOCKER_USERNAME:-mozjenkins}" 9 | 10 | docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD -e $DOCKER_USERNAME@example.com 11 | 12 | # Push to docker hub 13 | docker push $DOCKER_IMAGE_TAG 14 | 15 | if [[ "$GIT_TAG_DATE_BASED" == true ]]; then 16 | docker tag $DOCKER_IMAGE_TAG $DOCKER_REPOSITORY:$GIT_TAG 17 | docker push $DOCKER_REPOSITORY:$GIT_TAG 18 | docker tag $DOCKER_IMAGE_TAG $DOCKER_REPOSITORY:latest 19 | docker push $DOCKER_REPOSITORY:latest 20 | fi 21 | -------------------------------------------------------------------------------- /basket/news/templates/news/newsletters.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% block title %}{% endblock %} 5 | {% block blockbots %}{% endblock %} 6 | 7 | 8 |

Mozilla Newsletters

9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | {% for newsletter in newsletters %} 19 | 20 | 21 | 22 | 23 | 24 | {% endfor %} 25 | 26 |
TitleSlugDescription
{{ newsletter.title }}{{ newsletter.slug }}{{ newsletter.description }}
27 | 28 | 29 | -------------------------------------------------------------------------------- /jenkins/utils.groovy: -------------------------------------------------------------------------------- 1 | /** 2 | * Define utility functions. 3 | */ 4 | 5 | /** 6 | * Send a notice to #www on irc.mozilla.org with the build result 7 | * 8 | * @param stage step of build/deploy 9 | * @param result outcome of build (will be uppercased) 10 | */ 11 | def ircNotification(Map args) { 12 | def command = "bin/irc-notify.sh" 13 | for (arg in args) { 14 | command += " --${arg.key} '${arg.value}'" 15 | } 16 | sh command 17 | } 18 | 19 | def pushDockerhub() { 20 | withCredentials([[$class: 'StringBinding', 21 | credentialsId: 'DOCKER_PASSWORD', 22 | variable: 'DOCKER_PASSWORD']]) { 23 | retry(2) { 24 | sh 'docker/bin/push2dockerhub.sh' 25 | } 26 | } 27 | } 28 | 29 | return this; 30 | -------------------------------------------------------------------------------- /basket/news/migrations/0014_move_sms_messages.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations 5 | 6 | 7 | def move_sms_messages(apps, schema_editor): 8 | SMSMessage = apps.get_model('news', 'SMSMessage') 9 | LocalizedSMSMessage = apps.get_model('news', 'LocalizedSMSMessage') 10 | for sms in SMSMessage.objects.all(): 11 | LocalizedSMSMessage.objects.create( 12 | message_id=sms.message_id, 13 | vendor_id=sms.vendor_id, 14 | description=sms.description, 15 | language='en-US', 16 | country='us', 17 | ) 18 | 19 | 20 | class Migration(migrations.Migration): 21 | 22 | dependencies = [ 23 | ('news', '0013_auto_20170907_1216'), 24 | ] 25 | 26 | operations = [ 27 | migrations.RunPython(move_sms_messages), 28 | ] 29 | -------------------------------------------------------------------------------- /basket/news/migrations/0007_auto_20160531_1454.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('news', '0006_merge'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='newsletter', 16 | name='transactional', 17 | field=models.BooleanField(default=False, help_text=b'Whether this newsletter is purely for transactional messaging (e.g. Firefox Mobile download link emails).'), 18 | ), 19 | migrations.AlterField( 20 | model_name='newsletter', 21 | name='vendor_id', 22 | field=models.CharField(help_text=b"The backend vendor's identifier for this newsletter", max_length=128, blank=True), 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /bin/tag-release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | moz_git_remote="${MOZ_GIT_REMOTE:-origin}" 5 | do_push=false 6 | 7 | # parse cli args 8 | while [[ $# -ge 1 ]]; do 9 | key="$1" 10 | case $key in 11 | -p|--push) 12 | do_push=true 13 | ;; 14 | -r|--remote) 15 | moz_git_remote="$2" 16 | shift # past argument 17 | ;; 18 | esac 19 | shift # past argument or value 20 | done 21 | 22 | # ensure all tags synced 23 | git fetch --tags "$moz_git_remote" 24 | date_tag=$(date +"%Y-%m-%d") 25 | tag_suffix=0 26 | tag_value="$date_tag" 27 | while ! git tag -a $tag_value -m "tag release $tag_value" 2> /dev/null; do 28 | tag_suffix=$(( $tag_suffix + 1 )) 29 | tag_value="${date_tag}.${tag_suffix}" 30 | done 31 | echo "tagged $tag_value" 32 | if [[ "$do_push" == true ]]; then 33 | git push "$moz_git_remote" "$tag_value" 34 | git push "$moz_git_remote" HEAD:prod 35 | fi 36 | -------------------------------------------------------------------------------- /basket/news/migrations/0010_auto_20160607_0815.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations 5 | 6 | 7 | def convert_transactionals(apps, schema_editor): 8 | Newsletter = apps.get_model('news', 'Newsletter') 9 | TransactionalEmailMessage = apps.get_model('news', 'TransactionalEmailMessage') 10 | for nl in Newsletter.objects.filter(transactional=True): 11 | TransactionalEmailMessage.objects.create( 12 | message_id=nl.slug, 13 | vendor_id=nl.welcome, 14 | languages=nl.languages, 15 | description=nl.description, 16 | ) 17 | nl.delete() 18 | 19 | 20 | class Migration(migrations.Migration): 21 | 22 | dependencies = [ 23 | ('news', '0009_transactionalemailmessage'), 24 | ] 25 | 26 | operations = [ 27 | migrations.RunPython(convert_transactionals), 28 | ] 29 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | docker: 5 | - image: circleci/python:3.6 6 | environment: 7 | TEST_RESULTS: /tmp/test-results 8 | steps: 9 | - checkout 10 | - setup_remote_docker: 11 | docker_layer_caching: true 12 | - run: 13 | name: Setup 14 | command: | 15 | echo "export DOCKER_IMAGE_TAG=mozmeao/basket:${CIRCLE_SHA1}" >> $BASH_ENV 16 | mkdir -p $TEST_RESULTS 17 | - run: 18 | name: Build Image 19 | command: | 20 | docker build -t "$DOCKER_IMAGE_TAG" --pull=true . 21 | - run: 22 | name: Run Tests 23 | command: | 24 | docker run --env-file docker/envfiles/test.env --name test-run "$DOCKER_IMAGE_TAG" bin/run-tests.sh 25 | - run: docker cp test-run:/app/test-results $TEST_RESULTS 26 | - store_test_results: 27 | path: /tmp/test-results 28 | -------------------------------------------------------------------------------- /basket/news/management/commands/process_maintenance_queue.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.management import BaseCommand, CommandError 3 | 4 | from basket.news.models import QueuedTask 5 | 6 | 7 | class Command(BaseCommand): 8 | def add_arguments(self, parser): 9 | parser.add_argument( 10 | '-n', '--num-tasks', 11 | type=int, 12 | default=settings.QUEUE_BATCH_SIZE, 13 | help='Number of tasks to process ({})'.format(settings.QUEUE_BATCH_SIZE)) 14 | 15 | def handle(self, *args, **options): 16 | if settings.MAINTENANCE_MODE: 17 | raise CommandError('Command unavailable in maintenance mode') 18 | 19 | count = 0 20 | for task in QueuedTask.objects.all()[:options['num_tasks']]: 21 | task.retry() 22 | count += 1 23 | 24 | print '{} processed. {} remaining.'.format(count, QueuedTask.objects.count()) 25 | -------------------------------------------------------------------------------- /docker/bin/set_git_env_vars.sh: -------------------------------------------------------------------------------- 1 | # intended to be sourced into other scripts to set the git environment varaibles 2 | # GIT_COMMIT, GIT_COMMIT_SHORT, GIT_TAG, GIT_TAG_DATE_BASED, GIT_BRANCH, and BRANCH_NAME. 3 | 4 | if [[ -z "$GIT_COMMIT" ]]; then 5 | export GIT_COMMIT=$(git rev-parse HEAD) 6 | fi 7 | export GIT_COMMIT_SHORT="${GIT_COMMIT:0:9}" 8 | if [[ -z "$GIT_TAG" ]]; then 9 | export GIT_TAG=$(git describe --tags --exact-match $GIT_COMMIT 2> /dev/null) 10 | fi 11 | if [[ "$GIT_TAG" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}(\.[0-9])?$ ]]; then 12 | export GIT_TAG_DATE_BASED=true 13 | fi 14 | if [[ -z "$GIT_BRANCH" ]]; then 15 | export GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) 16 | export BRANCH_NAME="$GIT_BRANCH" 17 | fi 18 | if [[ -z "$DOCKER_REPOSITORY" ]]; then 19 | export DOCKER_REPOSITORY="mozmeao/basket" 20 | fi 21 | if [[ -z "$DOCKER_IMAGE_TAG" ]]; then 22 | export DOCKER_IMAGE_TAG="${DOCKER_REPOSITORY}:${GIT_COMMIT}" 23 | fi 24 | -------------------------------------------------------------------------------- /basket/news/migrations/0011_auto_20160607_1203.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('news', '0010_auto_20160607_0815'), 11 | ] 12 | 13 | operations = [ 14 | migrations.RemoveField( 15 | model_name='newsletter', 16 | name='confirm_message', 17 | ), 18 | migrations.RemoveField( 19 | model_name='newsletter', 20 | name='transactional', 21 | ), 22 | migrations.RemoveField( 23 | model_name='newsletter', 24 | name='welcome', 25 | ), 26 | migrations.AlterField( 27 | model_name='newsletter', 28 | name='vendor_id', 29 | field=models.CharField(help_text=b"The backend vendor's identifier for this newsletter", max_length=128), 30 | ), 31 | ] 32 | -------------------------------------------------------------------------------- /docker/bin/docker_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -exo pipefail 4 | 5 | BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | source $BIN_DIR/set_git_env_vars.sh 7 | 8 | DOCKER_NO_CACHE=false 9 | DOCKER_PULL=false 10 | DOCKER_CTX='.' 11 | DOCKERFILE_FINAL="Dockerfile.jenkins" 12 | 13 | # parse cli args 14 | while [[ $# -gt 1 ]]; do 15 | key="$1" 16 | case $key in 17 | -c|--context) 18 | DOCKER_CTX="$2" 19 | shift 20 | ;; 21 | -n|--no-cache) 22 | DOCKER_NO_CACHE=true 23 | ;; 24 | -p|--pull) 25 | DOCKER_PULL=true 26 | ;; 27 | esac 28 | shift # past argument or value 29 | done 30 | 31 | rm -f $DOCKERFILE_FINAL 32 | cp Dockerfile $DOCKERFILE_FINAL 33 | echo "ENV GIT_SHA ${GIT_COMMIT}" >> $DOCKERFILE_FINAL 34 | 35 | # build the docker image 36 | docker build -t "$DOCKER_IMAGE_TAG" --pull="$DOCKER_PULL" --no-cache="$DOCKER_NO_CACHE" -f "$DOCKERFILE_FINAL" "$DOCKER_CTX" 37 | -------------------------------------------------------------------------------- /basket/news/migrations/0004_queuedtask.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | import django.utils.timezone 6 | import jsonfield.fields 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('news', '0003_auto_20151202_0808'), 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='QueuedTask', 18 | fields=[ 19 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 20 | ('when', models.DateTimeField(default=django.utils.timezone.now, editable=False)), 21 | ('name', models.CharField(max_length=255)), 22 | ('args', jsonfield.fields.JSONField(default=[])), 23 | ('kwargs', jsonfield.fields.JSONField(default={})), 24 | ], 25 | options={ 26 | 'ordering': ['pk'], 27 | }, 28 | ), 29 | ] 30 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:2-slim 2 | 3 | # from https://github.com/mozmeao/docker-pythode/blob/master/Dockerfile.footer 4 | 5 | # Extra python env 6 | ENV PYTHONDONTWRITEBYTECODE=1 7 | ENV PYTHONUNBUFFERED=1 8 | ENV PIP_DISABLE_PIP_VERSION_CHECK=1 9 | 10 | # add non-priviledged user 11 | RUN adduser --uid 1000 --disabled-password --gecos '' --no-create-home webdev 12 | 13 | # Add apt script 14 | COPY docker/bin/apt-install /usr/local/bin/ 15 | 16 | # end from Dockerfile.footer 17 | 18 | RUN apt-install build-essential mysql-client-5.5 libmysqlclient-dev libxslt1.1 libxml2 libxml2-dev libxslt1-dev 19 | 20 | WORKDIR /app 21 | EXPOSE 8000 22 | CMD ["bin/run-prod.sh"] 23 | ENV DJANGO_SETTINGS_MODULE=basket.settings 24 | 25 | # Install app 26 | COPY requirements /app/requirements 27 | RUN pip install --require-hashes --no-cache-dir -r requirements/prod.txt 28 | 29 | COPY . /app 30 | RUN DEBUG=False SECRET_KEY=foo ALLOWED_HOSTS=localhost, DATABASE_URL=sqlite:// \ 31 | ./manage.py collectstatic --noinput 32 | 33 | # Change User 34 | RUN chown webdev.webdev -R . 35 | USER webdev 36 | -------------------------------------------------------------------------------- /basket/news/migrations/0009_transactionalemailmessage.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('news', '0008_auto_20160605_1345'), 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='TransactionalEmailMessage', 16 | fields=[ 17 | ('message_id', models.SlugField(help_text=b'The ID for the message that will be used by clients', serialize=False, primary_key=True)), 18 | ('vendor_id', models.CharField(help_text=b"The backend vendor's identifier for this message", max_length=50)), 19 | ('description', models.CharField(help_text=b'Optional short description of this message', max_length=200, blank=True)), 20 | ('languages', models.CharField(help_text=b'Comma-separated list of the language codes that this newsletter supports', max_length=200)), 21 | ], 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /basket/news/celery.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import os 4 | 5 | # set the default Django settings module for the 'celery' program. 6 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'basket.settings') 7 | 8 | from django.conf import settings # noqa 9 | 10 | import celery 11 | from raven.contrib.celery import register_signal, register_logger_signal 12 | from raven.contrib.django.raven_compat.models import client 13 | 14 | 15 | class Celery(celery.Celery): 16 | def on_configure(self): 17 | # register a custom filter to filter out duplicate logs 18 | register_logger_signal(client) 19 | 20 | # hook into the Celery error handler 21 | register_signal(client) 22 | 23 | 24 | app = Celery('basket') 25 | 26 | # Using a string here means the worker will not have to 27 | # pickle the object when using Windows. 28 | app.config_from_object('django.conf:settings') 29 | app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) 30 | 31 | 32 | @app.task(bind=True) 33 | def debug_task(self): 34 | print('Request: {0!r}'.format(self.request)) 35 | -------------------------------------------------------------------------------- /contribute.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Basket", 3 | "description": "A mailing list subscription service that abstracts away the list provider.", 4 | "repository": { 5 | "url": "https://github.com/mozmeao/basket", 6 | "license": "MPL2", 7 | "tests": "https://circleci.com/gh/mozmeao/basket/" 8 | }, 9 | "participate": { 10 | "home": "https://github.com/mozmeao/basket", 11 | "docs": "http://basket.readthedocs.org/en/latest/", 12 | "irc": "irc://irc.mozilla.org/#basket", 13 | "irc-contacts": ["pmac", "giorgos", "jgmize"] 14 | }, 15 | "bugs": { 16 | "list": "https://bugzilla.mozilla.org/buglist.cgi?product=Websites&component=Basket&resolution=---", 17 | "report": "https://bugzilla.mozilla.org/enter_bug.cgi?product=Websites&component=Basket" 18 | }, 19 | "urls": { 20 | "prod": "https://basket.mozilla.org/", 21 | "stage": "https://basket.allizom.org/", 22 | "dev": "https://basket-dev.allizom.org/" 23 | }, 24 | "keywords": [ 25 | "django", 26 | "mysql", 27 | "python" 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | services: 3 | db: 4 | image: mariadb 5 | environment: 6 | - MYSQL_ALLOW_EMPTY_PASSWORD=yes 7 | - MYSQL_DATABASE=basket 8 | 9 | redis: 10 | image: redis 11 | 12 | app: 13 | build: . 14 | volumes: 15 | - .:/app 16 | environment: 17 | - DATABASE_URL=mysql://root@db/basket 18 | - DEBUG=True 19 | - ALLOWED_HOSTS=localhost,127.0.0.1, 20 | - REDIS_URL=redis://redis:6379 21 | - CELERY_ALWAYS_EAGER=False 22 | - DJANGO_LOG_LEVEL=DEBUG 23 | env_file: .env 24 | 25 | web: 26 | extends: app 27 | ports: 28 | - "8000:8000" 29 | depends_on: 30 | - db 31 | - redis 32 | links: 33 | - db 34 | - redis 35 | command: 36 | ./bin/run-dev.sh 37 | 38 | worker: 39 | extends: app 40 | depends_on: 41 | - web 42 | links: 43 | - db 44 | - redis 45 | command: 46 | ./bin/run-worker.sh 47 | 48 | donate-worker: 49 | extends: app 50 | depends_on: 51 | - web 52 | links: 53 | - db 54 | - redis 55 | command: 56 | ./bin/run-donate-worker.sh 57 | -------------------------------------------------------------------------------- /basket/wsgi.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "basket.settings") 4 | 5 | from django.core.handlers.wsgi import WSGIRequest 6 | from django.core.wsgi import get_wsgi_application 7 | 8 | from raven.contrib.django.raven_compat.middleware.wsgi import Sentry 9 | from whitenoise.django import DjangoWhiteNoise 10 | 11 | try: 12 | import newrelic.agent 13 | except ImportError: 14 | newrelic = False 15 | 16 | 17 | if newrelic: 18 | newrelic_ini = os.getenv('NEWRELIC_INI_FILE', False) 19 | if newrelic_ini: 20 | newrelic.agent.initialize(newrelic_ini) 21 | else: 22 | newrelic = False 23 | 24 | IS_HTTPS = os.environ.get('HTTPS', '').strip() == 'on' 25 | 26 | 27 | class WSGIHTTPSRequest(WSGIRequest): 28 | def _get_scheme(self): 29 | if IS_HTTPS: 30 | return 'https' 31 | 32 | return super(WSGIHTTPSRequest, self)._get_scheme() 33 | 34 | 35 | application = get_wsgi_application() 36 | application.request_class = WSGIHTTPSRequest 37 | application = DjangoWhiteNoise(application) 38 | application = Sentry(application) 39 | 40 | if newrelic: 41 | application = newrelic.agent.WSGIApplicationWrapper(application) 42 | -------------------------------------------------------------------------------- /basket/news/management/commands/sync_newsletters.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.management import BaseCommand 3 | 4 | from synctool.functions import sync_data 5 | 6 | from basket.news.newsletters import clear_newsletter_cache, clear_sms_cache 7 | 8 | 9 | DEFAULT_SYNC_DOMAIN = 'basket.mozilla.org' 10 | 11 | 12 | class Command(BaseCommand): 13 | def add_arguments(self, parser): 14 | parser.add_argument('-d', '--domain', 15 | default=getattr(settings, 'SYNC_DOMAIN', DEFAULT_SYNC_DOMAIN), 16 | help='Domain of the Basket from which to sync') 17 | parser.add_argument('-k', '--key', 18 | default=settings.SYNC_KEY, 19 | help='Auth key for the sync') 20 | parser.add_argument('-c', '--clean', action='store_true', 21 | help='Delete all Newsletter data before sync') 22 | 23 | def handle(self, *args, **options): 24 | sync_data(url='https://{}/news/sync/'.format(options['domain']), 25 | clean=options['clean'], 26 | api_token=options['key']) 27 | clear_newsletter_cache() 28 | clear_sms_cache() 29 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | #!groovy 2 | 3 | @Library('github.com/mozmeao/jenkins-pipeline@20170607.1') 4 | 5 | def loadBranch(String branch) { 6 | // load the utility functions used below 7 | utils = load 'jenkins/utils.groovy' 8 | 9 | if ( fileExists("./jenkins/branches/${branch}.yml") ) { 10 | config = readYaml file: "./jenkins/branches/${branch}.yml" 11 | println "config ==> ${config}" 12 | } else { 13 | println "No config for ${branch}. Nothing to do. Good bye." 14 | return 15 | } 16 | 17 | // load the global config 18 | global_config = readYaml file: 'jenkins/global.yml' 19 | // defined in the Library loaded above 20 | setGitEnvironmentVariables() 21 | 22 | if ( config.pipeline && config.pipeline.script ) { 23 | println "Loading ./jenkins/${config.pipeline.script}.groovy" 24 | load "./jenkins/${config.pipeline.script}.groovy" 25 | } else { 26 | println "Loading ./jenkins/default.groovy" 27 | load "./jenkins/default.groovy" 28 | } 29 | } 30 | 31 | node { 32 | stage ('Prepare') { 33 | checkout scm 34 | } 35 | if ( skipTheBuild() ) { 36 | println 'Skipping this build. CI Skip detected in commit message.' 37 | } else { 38 | loadBranch(env.BRANCH_NAME) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /basket/news/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | 3 | from .views import (confirm, custom_unsub_reason, debug_user, 4 | fxa_activity, fxa_register, get_involved, list_newsletters, lookup_user, 5 | newsletters, send_recovery_message, subscribe, subscribe_sms, sync_route, 6 | unsubscribe, user, user_meta) 7 | 8 | 9 | def token_url(url_prefix, *args, **kwargs): 10 | """Require a properly formatted token as the last component of the URL.""" 11 | url_re = '^{0}/(?P[0-9A-Fa-f-]{{36}})/$'.format(url_prefix) 12 | return url(url_re, *args, **kwargs) 13 | 14 | 15 | urlpatterns = ( 16 | url('^get-involved/$', get_involved), 17 | url('^fxa-register/$', fxa_register), 18 | url('^fxa-activity/$', fxa_activity), 19 | url('^subscribe/$', subscribe), 20 | url('^subscribe_sms/$', subscribe_sms), 21 | token_url('unsubscribe', unsubscribe), 22 | token_url('user', user), 23 | token_url('user-meta', user_meta), 24 | token_url('confirm', confirm), 25 | url('^debug-user/$', debug_user), 26 | url('^lookup-user/$', lookup_user, name='lookup_user'), 27 | url('^recover/$', send_recovery_message, name='send_recovery_message'), 28 | url('^custom_unsub_reason/$', custom_unsub_reason), 29 | url('^newsletters/$', newsletters, name='newsletters_api'), 30 | url('^$', list_newsletters), 31 | ) + sync_route.urlpatterns 32 | -------------------------------------------------------------------------------- /basket/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.conf.urls import include, url 3 | from django.contrib import admin 4 | from django.views.generic import RedirectView 5 | 6 | from watchman import views as watchman_views 7 | 8 | from basket.news.views import subscribe_main, subscribe_json 9 | 10 | 11 | home_redirect = '/admin/' if settings.ADMIN_ONLY_MODE else 'https://www.mozilla.org/' 12 | 13 | urlpatterns = [ 14 | url(r'^$', RedirectView.as_view(url=home_redirect, permanent=True)), 15 | url(r'^watchman/$', watchman_views.dashboard, name="watchman.dashboard"), 16 | url(r'^healthz/$', watchman_views.ping, name="watchman.ping"), 17 | url(r'^readiness/$', watchman_views.status, name="watchman.status"), 18 | ] 19 | 20 | if not settings.ADMIN_ONLY_MODE: 21 | urlpatterns.append(url(r'^news/', include('basket.news.urls'))) 22 | urlpatterns.append(url(r'^subscribe/?$', subscribe_main)) 23 | urlpatterns.append(url(r'^subscribe\.json$', subscribe_json)) 24 | 25 | if settings.DISABLE_ADMIN: 26 | urlpatterns.append( 27 | url(r'^admin/', RedirectView.as_view(url=settings.ADMIN_REDIRECT_URL, permanent=True)) 28 | ) 29 | else: 30 | if settings.OIDC_ENABLE: 31 | urlpatterns.append(url(r'^oidc/', include('mozilla_django_oidc.urls'))) 32 | 33 | admin.autodiscover() 34 | urlpatterns.extend([ 35 | url(r'^admin/doc/', include('django.contrib.admindocs.urls')), 36 | url(r'^admin/', include(admin.site.urls)), 37 | ]) 38 | -------------------------------------------------------------------------------- /basket/news/tests/test_middleware.py: -------------------------------------------------------------------------------- 1 | from django.test import RequestFactory, override_settings 2 | 3 | from mock import Mock 4 | 5 | from basket.news.middleware import EnforceHostnameMiddleware, is_ip_address 6 | 7 | 8 | def test_is_ip_address(): 9 | assert is_ip_address('1.2.3.4') 10 | assert is_ip_address('192.168.1.1') 11 | assert not is_ip_address('basket.mozilla.org') 12 | assert not is_ip_address('42.basket.mozilla.org') 13 | 14 | 15 | @override_settings(DEBUG=False, ENFORCE_HOSTNAME=['basket.mozilla.org']) 16 | def test_enforce_hostname_middleware(): 17 | get_resp_mock = Mock() 18 | mw = EnforceHostnameMiddleware(get_resp_mock) 19 | req = RequestFactory().get('/', HTTP_HOST='basket.mozilla.org') 20 | resp = mw(req) 21 | get_resp_mock.assert_called_once_with(req) 22 | 23 | get_resp_mock.reset_mock() 24 | req = RequestFactory().get('/', HTTP_HOST='basket.allizom.org') 25 | resp = mw(req) 26 | get_resp_mock.assert_not_called() 27 | assert resp.status_code == 301 28 | assert resp['location'] == 'http://basket.mozilla.org/' 29 | 30 | # IP address should not redirect 31 | get_resp_mock.reset_mock() 32 | req = RequestFactory().get('/', HTTP_HOST='123.123.123.123') 33 | resp = mw(req) 34 | get_resp_mock.assert_called_once_with(req) 35 | 36 | # IP with port should also work 37 | get_resp_mock.reset_mock() 38 | req = RequestFactory().get('/', HTTP_HOST='1.2.3.4:12345') 39 | resp = mw(req) 40 | get_resp_mock.assert_called_once_with(req) 41 | -------------------------------------------------------------------------------- /docs/install.rst: -------------------------------------------------------------------------------- 1 | .. This Source Code Form is subject to the terms of the Mozilla Public 2 | .. License, v. 2.0. If a copy of the MPL was not distributed with this 3 | .. file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | .. _install: 6 | 7 | =========== 8 | Installing Basket 9 | =========== 10 | 11 | Requirements 12 | ============ 13 | 14 | * Python >= 2.7, < 3 15 | * MySQL (only for prod) 16 | 17 | Installation 18 | ============ 19 | 20 | Get the code 21 | ------------ 22 | 23 | :: 24 | 25 | git clone git@github.com:mozmeao/basket.git --recursive 26 | 27 | The `--recursive` is important! 28 | 29 | 30 | Make a virtualenv 31 | ----------------- 32 | 33 | Using virtualenvwrapper:: 34 | 35 | mkvirtualenv --python=python2.7 basket 36 | 37 | 38 | Install packages 39 | ---------------- 40 | 41 | :: 42 | 43 | pip install -r requirements/default.txt 44 | 45 | If you'll be using MySQL for the database:: 46 | 47 | pip install -r requirements/compiled.txt 48 | 49 | For developers:: 50 | 51 | pip install -r requirements/dev.txt 52 | 53 | 54 | Settings 55 | -------- 56 | 57 | Settings are discovered in the environment. You can either provide them via environment variables 58 | or by providing those variables in a ``.env`` file in the root of the project 59 | (along side of ``manage.py``). To get started you can copy ``env-dist`` to ``.env`` and that will 60 | provide the basics you need to run the site and the tests. 61 | 62 | Database schema 63 | --------------- 64 | 65 | :: 66 | 67 | ./manage.py migrate 68 | 69 | -------------------------------------------------------------------------------- /basket/news/backends/common.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from time import time 3 | 4 | from django_statsd.clients import statsd 5 | 6 | 7 | class UnauthorizedException(Exception): 8 | """Failure to log into the email server.""" 9 | pass 10 | 11 | 12 | class NewsletterException(Exception): 13 | """Error when trying to talk to the the email server.""" 14 | 15 | def __init__(self, msg=None, error_code=None, status_code=None): 16 | self.error_code = error_code 17 | self.status_code = status_code 18 | super(NewsletterException, self).__init__(msg) 19 | 20 | 21 | class NewsletterNoResultsException(NewsletterException): 22 | """ 23 | No results were returned from the mail server (but the request 24 | didn't report any errors) 25 | """ 26 | pass 27 | 28 | 29 | def get_timer_decorator(prefix): 30 | """ 31 | Decorator for timing and counting requests to the API 32 | """ 33 | def decorator(f): 34 | @wraps(f) 35 | def wrapped(*args, **kwargs): 36 | starttime = time() 37 | e = None 38 | try: 39 | resp = f(*args, **kwargs) 40 | except NewsletterException as e: 41 | pass 42 | except Exception: 43 | raise 44 | 45 | totaltime = int((time() - starttime) * 1000) 46 | statsd.timing(prefix + '.timing', totaltime) 47 | statsd.timing(prefix + '.{}.timing'.format(f.__name__), totaltime) 48 | statsd.incr(prefix + '.count') 49 | statsd.incr(prefix + '.{}.count'.format(f.__name__)) 50 | if e: 51 | raise 52 | else: 53 | return resp 54 | 55 | return wrapped 56 | 57 | return decorator 58 | -------------------------------------------------------------------------------- /docker/bin/push2deis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Needs DEIS_PROFILE, DEIS_APPLICATION, NEWRELIC_API_KEY and 3 | # NEWRELIC_APP_NAME environment variables. 4 | # 5 | # To set them go to Job -> Configure -> Build Environment -> Inject 6 | # passwords and Inject env variables 7 | # 8 | 9 | set -ex 10 | 11 | BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 12 | source $BIN_DIR/set_git_env_vars.sh 13 | 14 | DEIS_BIN="${DEIS_BIN:-deis2}" 15 | NR_APP="${DEIS_APPLICATION}-${REGION_NAME}" 16 | NR_DESC="Jenkins built $DOCKER_IMAGE_TAG from $GIT_COMMIT_SHORT and deployed it as Deis app $DEIS_APPLICATION in $REGION_NAME" 17 | 18 | $DEIS_BIN pull "$DOCKER_IMAGE_TAG" -a $DEIS_APPLICATION 19 | 20 | if [[ -n "$NEWRELIC_API_KEY" ]]; then 21 | echo "Pinging NewRelic about the deployment of $NR_APP" 22 | curl -H "x-api-key:$NEWRELIC_API_KEY" \ 23 | -d "deployment[app_name]=$NR_APP" \ 24 | -d "deployment[revision]=$GIT_COMMIT" \ 25 | -d "deployment[user]=MEAO Jenkins" \ 26 | -d "deployment[description]=$NR_DESC" \ 27 | https://api.newrelic.com/deployments.xml 28 | fi 29 | 30 | if [[ -n "$DATADOG_API_KEY" ]]; then 31 | echo "Pinging DataDog about the deployment of $NR_APP" 32 | dd_data=$(cat << EOF 33 | { 34 | "title": "Deployment of $NR_APP", 35 | "text": "$NR_DESC", 36 | "tags": ["region:$REGION_NAME", "appname:$DEIS_APPLICATION"], 37 | "aggregation_key": "$NR_APP", 38 | "source_type_name": "deployment", 39 | "alert_type": "info" 40 | } 41 | EOF 42 | ) 43 | curl -H "Content-type: application/json" -d "$dd_data" \ 44 | "https://app.datadoghq.com/api/v1/events?api_key=$DATADOG_API_KEY" > /dev/null 2>&1 45 | fi 46 | 47 | if [[ "$RUN_POST_DEPLOY" == 'true' ]]; then 48 | $DEIS_BIN run -a "$DEIS_APPLICATION" -- bin/post-deploy.sh 49 | fi 50 | -------------------------------------------------------------------------------- /basket/news/migrations/0005_convert_newsletter_vendor_id.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations 5 | 6 | 7 | FIELD_MAP = { 8 | 'ABOUT_MOBILE': 'Interest_Android__c', 9 | 'ABOUT_MOZILLA': 'Sub_About_Mozilla__c', 10 | 'APP_DEV': 'Sub_Apps_And_Hacks__c', 11 | 'CONNECTED_DEVICES': 'Sub_Connected_Devices__c', 12 | 'DEV_EVENTS': 'Sub_Dev_Events__c', 13 | 'FIREFOX_ACCOUNTS_JOURNEY': 'Sub_Firefox_Accounts_Journey__c', 14 | 'FIREFOX_DESKTOP': 'Interest_Firefox_Desktop__c', 15 | 'FIREFOX_FRIENDS': 'Sub_Firefox_Friends__c', 16 | 'FIREFOX_IOS': 'Interest_Firefox_iOS__c', 17 | 'FOUNDATION': 'Sub_Mozilla_Foundation__c', 18 | 'GAMEDEV_CONF': 'Sub_Game_Dev_Conference__c', 19 | 'GET_INVOLVED': 'Sub_Get_Involved__c', 20 | 'MAKER_PARTY': 'Sub_Maker_Party__c', 21 | 'MOZFEST': 'Sub_Mozilla_Festival__c', 22 | 'MOZILLA_AND_YOU': 'Sub_Firefox_And_You__c', 23 | 'MOZILLA_GENERAL': 'Interest_Mozilla__c', 24 | 'MOZILLA_PHONE': 'Sub_Mozillians__c', 25 | 'MOZ_LEARN': 'Sub_Mozilla_Learning_Network__c', 26 | 'SHAPE_WEB': 'Sub_Shape_Of_The_Web__c', 27 | 'STUDENT_AMBASSADORS': 'Sub_Student_Ambassador__c', 28 | 'VIEW_SOURCE_GLOBAL': 'Sub_View_Source_Global__c', 29 | 'VIEW_SOURCE_NA': 'Sub_View_Source_NAmerica__c', 30 | 'WEBMAKER': 'Sub_Webmaker__c', 31 | 'TEST_PILOT': 'Sub_Test_Pilot__c', 32 | 'IOS_TEST_FLIGHT': 'Sub_Test_Flight__c', 33 | } 34 | 35 | 36 | def convert_vendor_id(apps, schema_editor): 37 | Newsletter = apps.get_model('news', 'Newsletter') 38 | for nl in Newsletter.objects.all(): 39 | if nl.vendor_id in FIELD_MAP: 40 | nl.vendor_id = FIELD_MAP[nl.vendor_id] 41 | nl.save() 42 | 43 | 44 | class Migration(migrations.Migration): 45 | 46 | dependencies = [ 47 | ('news', '0003_auto_20151202_0808'), 48 | ] 49 | 50 | operations = [ 51 | migrations.RunPython(convert_vendor_id), 52 | ] 53 | -------------------------------------------------------------------------------- /basket/news/tests/test_send_welcomes.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from mock import patch 4 | 5 | from basket.news.backends.common import NewsletterException 6 | from basket.news.tasks import mogrify_message_id, send_message 7 | 8 | 9 | class TestSendMessage(TestCase): 10 | @patch('basket.news.tasks.sfmc') 11 | def test_caching_bad_message_ids(self, mock_sfmc): 12 | """Bad message IDs are cached so we don't try to send to them again""" 13 | exc = NewsletterException() 14 | exc.message = 'Invalid Customer Key' 15 | mock_sfmc.send_mail.side_effect = exc 16 | 17 | message_id = "MESSAGE_ID" 18 | for i in range(10): 19 | send_message(message_id, 'email', 'token', 'format') 20 | 21 | mock_sfmc.send_mail.assert_called_once_with(message_id, 'email', 'token', None) 22 | 23 | 24 | class TestMogrifyMessageID(TestCase): 25 | def test_mogrify_message_id_text(self): 26 | """Test adding lang and text format to message ID""" 27 | result = mogrify_message_id("MESSAGE", "en", "T") 28 | expect = "en_MESSAGE_T" 29 | self.assertEqual(expect, result) 30 | 31 | def test_mogrify_message_id_html(self): 32 | """Test adding lang and html format to message ID""" 33 | result = mogrify_message_id("MESSAGE", "en", "H") 34 | expect = "en_MESSAGE" 35 | self.assertEqual(expect, result) 36 | 37 | def test_mogrify_message_id_no_lang(self): 38 | """Test adding no lang and format to message ID""" 39 | result = mogrify_message_id("MESSAGE", None, "T") 40 | expect = "MESSAGE_T" 41 | self.assertEqual(expect, result) 42 | 43 | def test_mogrify_message_id_long_lang(self): 44 | """Test adding long lang and format to message ID""" 45 | result = mogrify_message_id("MESSAGE", "en-US", "T") 46 | expect = "en_MESSAGE_T" 47 | self.assertEqual(expect, result) 48 | 49 | def test_mogrify_message_id_upcase_lang(self): 50 | """Test adding uppercase lang and format to message ID""" 51 | result = mogrify_message_id("MESSAGE", "FR", "T") 52 | expect = "fr_MESSAGE_T" 53 | self.assertEqual(expect, result) 54 | -------------------------------------------------------------------------------- /requirements/dev.txt: -------------------------------------------------------------------------------- 1 | # This file pulls in everything a developer needs. If it's a basic package 2 | # needed to run the site, it belongs in requirements/base.txt. If it's a 3 | # package for developers (testing, etc.), it goes in this file. And if it's 4 | # a package only needed in production put it in requirements.prod.txt. 5 | 6 | -r base.txt 7 | 8 | flake8==2.0 \ 9 | --hash=sha256:8dce4f7e64cc202cc6da93eab84b2ce660110ff684b6738bba64a0a431b3bc69 10 | mccabe==0.3.1 \ 11 | --hash=sha256:bd6c080fb372aebcb0ce19e35ddac744f2abf5a7befa207db2d1097d48efe63a \ 12 | --hash=sha256:5f7ea6fb3aa9afe146d07fd6d5cedf788747d8b0c29e44732453c2b2db1e3d16 13 | mock==2.0.0 \ 14 | --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ 15 | --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba 16 | pep8==1.4.6 \ 17 | --hash=sha256:603a46e5c358ce20ac4807a0eeafac7505d1125a4c1bd8378757ada06f61bed8 18 | py==1.4.30 \ 19 | --hash=sha256:07e20ab90a550bd3c21891e0d887f0931b4098f148aec95e29b5188f161bb075 \ 20 | --hash=sha256:b703e57685ed7c280b1a51c496a4984d83d89def2a930b5e9e5da5a6ca151514 21 | pyflakes==0.9.2 \ 22 | --hash=sha256:05df584a29eeea9a2a2110dd362e53d04e0c4bb1754b4d71234f651917f3c2f0 \ 23 | --hash=sha256:02691c23ce699f252874b7c27f14cf26e3d4e82b58e5d584f000b7ab5be36a5f 24 | pytest==2.7.2 \ 25 | --hash=sha256:82924efb5ea783a72234682a0e8049d84f5eaebcaac3c8c0893b7eae97f28380 \ 26 | --hash=sha256:b30457f735420d0000d10a44bbd478cf03f8bf20e25bd77248f9bab40f4fd6a4 27 | pytest-django==2.8.0 \ 28 | --hash=sha256:d76f934e77fa073f48cc521945a49900a859e610fa029dd880d1d8b997b77c23 \ 29 | --hash=sha256:d145ac9dc7a557a719ab79770be0941004e1e038e137c34591919d9df2a790b1 30 | pytest-pythonpath==0.7 \ 31 | --hash=sha256:036bc2b62b5d3a991e45c389748ccfb61e88eb5b14b27cab4b5c1fbcd4ffd191 32 | funcsigs==1.0.1 \ 33 | --hash=sha256:2edd42db946babc214077be3626e1c496561daeb6e752e482d8d733a0d578f01 \ 34 | --hash=sha256:0726847f1463526794496423f6500cc2ed751361b6c025982ab18bc6c5af35b1 35 | pbr==1.9.1 \ 36 | --hash=sha256:fdfda7175428e7527635dfef42d0eda5b609b2d5b1ac707045dbaed7a7307c7f \ 37 | --hash=sha256:3997406c90894ebf3d1371811c1e099721440a901f946ca6dc4383350403ed51 38 | -------------------------------------------------------------------------------- /basket/news/tests/test_confirm.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from mock import patch, Mock 4 | 5 | from basket.news.backends.common import NewsletterException 6 | from basket.news.tasks import confirm_user 7 | 8 | 9 | @patch('basket.news.tasks.sfdc') 10 | @patch('basket.news.tasks.get_user_data') 11 | class TestConfirmTask(TestCase): 12 | def test_error(self, get_user_data, sfdc_mock): 13 | """ 14 | If user_data shows an error talking to ET, the task raises 15 | an exception so our task logic will retry 16 | """ 17 | get_user_data.side_effect = NewsletterException('Stuffs broke yo.') 18 | with self.assertRaises(NewsletterException): 19 | confirm_user('token') 20 | self.assertFalse(sfdc_mock.update.called) 21 | 22 | def test_normal(self, get_user_data, sfdc_mock): 23 | """If user_data is okay, and not yet confirmed, the task calls 24 | the right stuff""" 25 | token = "TOKEN" 26 | user_data = {'status': 'ok', 'optin': False, 'newsletters': Mock(), 'format': 'ZZ', 27 | 'email': 'dude@example.com', 'token': token} 28 | get_user_data.return_value = user_data 29 | confirm_user(token) 30 | sfdc_mock.update.assert_called_with(user_data, {'optin': True}) 31 | 32 | def test_already_confirmed(self, get_user_data, sfdc_mock): 33 | """If user_data already confirmed, task does nothing""" 34 | user_data = { 35 | 'status': 'ok', 36 | 'optin': True, 37 | 'newsletters': Mock(), 38 | 'format': 'ZZ', 39 | } 40 | get_user_data.return_value = user_data 41 | token = "TOKEN" 42 | confirm_user(token) 43 | self.assertFalse(sfdc_mock.update.called) 44 | 45 | @patch('basket.news.tasks.get_sfmc_doi_user') 46 | def test_user_not_found(self, doi_mock, get_user_data, sfdc_mock): 47 | """If we can't find the user, try SFMC""" 48 | get_user_data.return_value = None 49 | doi_mock.return_value = None 50 | token = "TOKEN" 51 | confirm_user(token) 52 | doi_mock.assert_called_with(token) 53 | self.assertFalse(sfdc_mock.add.called) 54 | self.assertFalse(sfdc_mock.update.called) 55 | -------------------------------------------------------------------------------- /basket/news/fields.py: -------------------------------------------------------------------------------- 1 | from django.core.validators import validate_email 2 | from django.db import models 3 | from django.forms import TextInput 4 | 5 | from product_details import product_details 6 | 7 | 8 | def parse_emails(emails_string): 9 | emails = [] 10 | for email in emails_string.split(','): 11 | email = email.strip() 12 | if email: 13 | validate_email(email) 14 | emails.append(email) 15 | 16 | return emails 17 | 18 | 19 | class CommaSeparatedEmailField(models.TextField): 20 | """TextField that stores a comma-separated list of emails.""" 21 | def pre_save(self, model_instance, add): 22 | """Remove whitespace and excess commas.""" 23 | emails = getattr(model_instance, self.attname) 24 | emails = ','.join(parse_emails(emails)) 25 | setattr(model_instance, self.attname, emails) 26 | return emails 27 | 28 | def formfield(self, **kwargs): 29 | kwargs['widget'] = TextInput(attrs={'style': 'width: 400px'}) 30 | return super(CommaSeparatedEmailField, self).formfield(**kwargs) 31 | 32 | 33 | ENGLISH_LANGUAGE_CHOICES = sorted( 34 | [(key, u'{0} ({1})'.format(key, value['English'])) 35 | for key, value in product_details.languages.items()] 36 | ) 37 | COUNTRY_CHOICES = sorted( 38 | [(key, u'{0} ({1})'.format(key, value)) 39 | for key, value in product_details.get_regions('en-US').items()] 40 | ) 41 | 42 | 43 | class CountryField(models.CharField): 44 | description = 'CharField for storing a country code.' 45 | 46 | def __init__(self, *args, **kwargs): 47 | defaults = { 48 | 'max_length': 3, 49 | 'choices': COUNTRY_CHOICES, 50 | } 51 | for key, value in defaults.items(): 52 | kwargs.setdefault(key, value) 53 | 54 | return super(CountryField, self).__init__(*args, **kwargs) 55 | 56 | 57 | class LocaleField(models.CharField): 58 | description = 'CharField for storing a locale code.' 59 | 60 | def __init__(self, *args, **kwargs): 61 | defaults = { 62 | 'max_length': 32, 63 | 'choices': ENGLISH_LANGUAGE_CHOICES, 64 | } 65 | for key, value in defaults.items(): 66 | kwargs.setdefault(key, value) 67 | 68 | return super(LocaleField, self).__init__(*args, **kwargs) 69 | -------------------------------------------------------------------------------- /basket/base/templates/admin/login.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/base_site.html" %} 2 | {% load i18n admin_static %} 3 | 4 | {% block extrastyle %}{{ block.super }}{% endblock %} 5 | 6 | {% block bodyclass %}{{ block.super }} login{% endblock %} 7 | 8 | {% block usertools %}{% endblock %} 9 | 10 | {% block nav-global %}{% endblock %} 11 | 12 | {% block content_title %}{% endblock %} 13 | 14 | {% block breadcrumbs %}{% endblock %} 15 | 16 | {% block content %} 17 | {% if form.errors and not form.non_field_errors %} 18 |

19 | {% if form.errors.items|length == 1 %}{% trans "Please correct the error below." %}{% else %}{% trans "Please correct the errors below." %}{% endif %} 20 |

21 | {% endif %} 22 | 23 | {% if form.non_field_errors %} 24 | {% for error in form.non_field_errors %} 25 |

26 | {{ error }} 27 |

28 | {% endfor %} 29 | {% endif %} 30 | 31 |
32 | {% if settings.OIDC_ENABLE %} 33 |

34 | Mozilla SSO 35 |

36 | {% else %} 37 |
{% csrf_token %} 38 |
39 | {{ form.username.errors }} 40 | {{ form.username.label_tag }} {{ form.username }} 41 |
42 |
43 | {{ form.password.errors }} 44 | {{ form.password.label_tag }} {{ form.password }} 45 | 46 |
47 | {% url 'admin_password_reset' as password_reset_url %} 48 | {% if password_reset_url %} 49 |
50 | {% trans 'Forgotten your password or username?' %} 51 |
52 | {% endif %} 53 |
54 | 55 |
56 |
57 | {% endif %} 58 | 61 |
62 | {% endblock %} 63 | -------------------------------------------------------------------------------- /basket/news/tests/test_fields.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import ValidationError 2 | from django.test import TestCase 3 | 4 | from mock import call, Mock, patch 5 | 6 | from basket.news.fields import CommaSeparatedEmailField 7 | 8 | 9 | class CommaSeparatedEmailFieldTests(TestCase): 10 | def setUp(self): 11 | self.field = CommaSeparatedEmailField(blank=True) 12 | 13 | def test_validate(self): 14 | """ 15 | Validate should run the email validator on all non-empty emails 16 | in the list. 17 | """ 18 | with patch('basket.news.fields.validate_email') as validate_email: 19 | instance = Mock() 20 | self.field.attname = 'blah' 21 | instance.blah = ' foo@example.com ,bar@example.com ' 22 | self.field.pre_save(instance, False) 23 | validate_email.assert_has_calls([ 24 | call('foo@example.com'), 25 | call('bar@example.com'), 26 | ]) 27 | 28 | validate_email.reset_mock() 29 | instance.blah = 'foo@example.com' 30 | self.field.pre_save(instance, False) 31 | validate_email.assert_has_calls([ 32 | call('foo@example.com'), 33 | ]) 34 | 35 | validate_email.reset_mock() 36 | instance.blah = '' 37 | self.field.pre_save(instance, False) 38 | self.assertFalse(validate_email.called) 39 | 40 | def test_invalid_email(self): 41 | instance = Mock() 42 | self.field.attname = 'blah' 43 | instance.blah = 'the.dude' 44 | with self.assertRaises(ValidationError): 45 | self.field.pre_save(instance, False) 46 | 47 | def test_pre_save(self): 48 | """pre_save should remove unnecessary whitespace and commas.""" 49 | instance = Mock() 50 | self.field.attname = 'blah' 51 | 52 | # Basic 53 | instance.blah = 'bob@example.com,larry@example.com' 54 | self.assertEqual(self.field.pre_save(instance, False), 55 | 'bob@example.com,larry@example.com') 56 | 57 | # Excess whitespace 58 | instance.blah = ' bob@example.com ,larry@example.com ' 59 | self.assertEqual(self.field.pre_save(instance, False), 60 | 'bob@example.com,larry@example.com') 61 | 62 | # Extra commas 63 | instance.blah = 'bob@example.com ,,,, larry@example.com ' 64 | self.assertEqual(self.field.pre_save(instance, False), 65 | 'bob@example.com,larry@example.com') 66 | -------------------------------------------------------------------------------- /bin/irc-notify.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eo pipefail 3 | 4 | # Required environment variables if using --stage: 5 | # BRANCH_NAME, BUILD_NUMBER 6 | 7 | # defaults and constants 8 | NICK="hms-hellina" 9 | CHANNEL="#basket" 10 | SERVER="irc.mozilla.org:6697" 11 | BLUE_BUILD_URL="https://ci.us-west.moz.works/blue/organizations/jenkins/basket_multibranch_pipeline" 12 | BLUE_BUILD_URL="${BLUE_BUILD_URL}/detail/${BRANCH_NAME/\//%2f}/${BUILD_NUMBER}/pipeline" 13 | # colors and styles: values from the following links 14 | # http://www.mirc.com/colors.html 15 | # http://stackoverflow.com/a/13382032 16 | RED=$'\x034' 17 | YELLOW=$'\x038' 18 | GREEN=$'\x039' 19 | BLUE=$'\x0311' 20 | BOLD=$'\x02' 21 | NORMAL=$'\x0F' 22 | 23 | # parse cli args 24 | while [[ $# -gt 1 ]]; do 25 | key="$1" 26 | case $key in 27 | --stage) 28 | STAGE="$2" 29 | shift # past argument 30 | ;; 31 | --status) 32 | STATUS="$2" 33 | shift # past argument 34 | ;; 35 | -m|--message) 36 | MESSAGE="$2" 37 | shift # past argument 38 | ;; 39 | --irc_nick) 40 | NICK="$2" 41 | shift # past argument 42 | ;; 43 | --irc_server) 44 | SERVER="$2" 45 | shift # past argument 46 | ;; 47 | --irc_channel) 48 | CHANNEL="$2" 49 | shift # past argument 50 | ;; 51 | esac 52 | shift # past argument or value 53 | done 54 | 55 | if [[ -n "$STATUS" ]]; then 56 | STATUS=$(echo "$STATUS" | tr '[:lower:]' '[:upper:]') 57 | case "$STATUS" in 58 | 'SUCCESS') 59 | STATUS_COLOR="🎉 ${BOLD}${GREEN}" 60 | ;; 61 | 'SHIPPED') 62 | STATUS_COLOR="🚢 ${BOLD}${GREEN}" 63 | ;; 64 | 'WARNING') 65 | STATUS_COLOR="⚠️ ${BOLD}${YELLOW}" 66 | ;; 67 | 'FAILURE') 68 | STATUS_COLOR="🚨 ${BOLD}${RED}" 69 | ;; 70 | *) 71 | STATUS_COLOR="✨ $BLUE" 72 | ;; 73 | esac 74 | STATUS="${STATUS_COLOR}${STATUS}${NORMAL}: " 75 | fi 76 | 77 | if [[ -n "$STAGE" ]]; then 78 | MESSAGE="${STATUS}${STAGE}:" 79 | MESSAGE="$MESSAGE Branch ${BOLD}${BRANCH_NAME}${NORMAL} build #${BUILD_NUMBER}: ${BLUE_BUILD_URL}" 80 | elif [[ -n "$MESSAGE" ]]; then 81 | MESSAGE="${STATUS}${MESSAGE}" 82 | else 83 | echo "Missing required arguments" 84 | echo 85 | echo "Usage: irc-notify.sh [--stage STAGE]|[-m MESSAGE]" 86 | echo "Optional args: --status, --irc_nick, --irc_server, --irc_channel" 87 | exit 1 88 | fi 89 | 90 | if [[ -n "$BUILD_NUMBER" ]]; then 91 | NICK="${NICK}-${BUILD_NUMBER}" 92 | fi 93 | 94 | ( 95 | echo "NICK ${NICK}" 96 | echo "USER ${NICK} 8 * : ${NICK}" 97 | sleep 5 98 | echo "JOIN ${CHANNEL}" 99 | echo "NOTICE ${CHANNEL} :${MESSAGE}" 100 | echo "QUIT" 101 | ) | openssl s_client -connect "$SERVER" > /dev/null 2>&1 102 | -------------------------------------------------------------------------------- /basket/news/middleware.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from django.conf import settings 4 | from django.core.exceptions import MiddlewareNotUsed 5 | from django.http import HttpResponsePermanentRedirect 6 | from django.http.request import split_domain_port 7 | 8 | from django_statsd.clients import statsd 9 | from django_statsd.middleware import GraphiteRequestTimingMiddleware 10 | 11 | 12 | IP_RE = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') 13 | 14 | 15 | class GraphiteViewHitCountMiddleware(GraphiteRequestTimingMiddleware): 16 | """add hit counting to statsd's request timer.""" 17 | 18 | def process_view(self, request, view_func, view_args, view_kwargs): 19 | super(GraphiteViewHitCountMiddleware, self).process_view( 20 | request, view_func, view_args, view_kwargs) 21 | if hasattr(request, '_view_name'): 22 | vmodule = request._view_module 23 | if vmodule.startswith('basket.'): 24 | vmodule = vmodule[7:] 25 | data = dict(module=vmodule, 26 | name=request._view_name, 27 | method=request.method) 28 | statsd.incr('view.count.{module}.{name}.{method}'.format(**data)) 29 | statsd.incr('view.count.{module}.{method}'.format(**data)) 30 | statsd.incr('view.count.{method}'.format(**data)) 31 | 32 | 33 | class HostnameMiddleware(object): 34 | def __init__(self, get_response): 35 | values = [getattr(settings, x) for x in ['HOSTNAME', 'DEIS_APP', 36 | 'DEIS_RELEASE', 'DEIS_DOMAIN']] 37 | self.backend_server = '.'.join(x for x in values if x) 38 | self.get_response = get_response 39 | 40 | def __call__(self, request): 41 | response = self.get_response(request) 42 | response['X-Backend-Server'] = self.backend_server 43 | return response 44 | 45 | 46 | def is_ip_address(hostname): 47 | return bool(IP_RE.match(hostname)) 48 | 49 | 50 | class EnforceHostnameMiddleware(object): 51 | """ 52 | Enforce the hostname per the ENFORCE_HOSTNAME setting in the project's settings 53 | 54 | The ENFORCE_HOSTNAME can either be a single host or a list of acceptable hosts 55 | 56 | via http://www.michaelvdw.nl/code/force-hostname-with-django-middleware-for-heroku/ 57 | """ 58 | def __init__(self, get_response): 59 | self.allowed_hosts = settings.ENFORCE_HOSTNAME 60 | self.get_response = get_response 61 | if settings.DEBUG or not self.allowed_hosts: 62 | raise MiddlewareNotUsed 63 | 64 | def __call__(self, request): 65 | """Enforce the host name""" 66 | host = request.get_host() 67 | domain, port = split_domain_port(host) 68 | if domain in self.allowed_hosts or is_ip_address(domain): 69 | return self.get_response(request) 70 | 71 | # redirect to the proper host name\ 72 | new_url = "%s://%s%s" % ( 73 | 'https' if request.is_secure() else 'http', 74 | self.allowed_hosts[0], request.get_full_path()) 75 | 76 | return HttpResponsePermanentRedirect(new_url) 77 | -------------------------------------------------------------------------------- /basket/news/management/commands/process_donations_queue.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, unicode_literals 2 | 3 | import json 4 | import sys 5 | from time import time 6 | 7 | from django.conf import settings 8 | from django.core.management import BaseCommand, CommandError 9 | 10 | import boto3 11 | import requests 12 | from django_statsd.clients import statsd 13 | from raven.contrib.django.raven_compat.models import client as sentry_client 14 | 15 | from basket.news.tasks import process_donation, process_donation_event 16 | 17 | 18 | class Command(BaseCommand): 19 | snitch_delay = 300 # 5 min 20 | snitch_last_timestamp = 0 21 | snitch_id = settings.DONATE_SNITCH_ID 22 | 23 | def snitch(self): 24 | if not self.snitch_id: 25 | return 26 | 27 | time_since = int(time() - self.snitch_last_timestamp) 28 | if time_since > self.snitch_delay: 29 | requests.post('https://nosnch.in/{}'.format(self.snitch_id)) 30 | self.snitch_last_timestamp = time() 31 | 32 | def handle(self, *args, **options): 33 | if not settings.DONATE_ACCESS_KEY_ID: 34 | raise CommandError('AWS SQS Credentials not configured') 35 | 36 | sqs = boto3.resource('sqs', 37 | region_name=settings.DONATE_QUEUE_REGION, 38 | aws_access_key_id=settings.DONATE_ACCESS_KEY_ID, 39 | aws_secret_access_key=settings.DONATE_SECRET_ACCESS_KEY) 40 | queue = sqs.Queue(settings.DONATE_QUEUE_URL) 41 | 42 | try: 43 | # Poll for messages indefinitely. 44 | while True: 45 | self.snitch() 46 | msgs = queue.receive_messages(WaitTimeSeconds=settings.DONATE_QUEUE_WAIT_TIME, 47 | MaxNumberOfMessages=10) 48 | for msg in msgs: 49 | if not (msg and msg.body): 50 | continue 51 | 52 | statsd.incr('mofo.donations.message.received') 53 | try: 54 | data = json.loads(msg.body) 55 | except ValueError as e: 56 | # body was not JSON 57 | statsd.incr('mofo.donations.message.json_error') 58 | sentry_client.captureException(data={'extra': {'msg.body': msg.body}}) 59 | print('ERROR:', e, '::', msg.body) 60 | msg.delete() 61 | continue 62 | 63 | try: 64 | etype = data['data'].setdefault('event_type', 'donation') 65 | if etype == 'donation': 66 | process_donation.delay(data['data']) 67 | else: 68 | process_donation_event.delay(data['data']) 69 | except Exception: 70 | # something's wrong with the queue. try again. 71 | statsd.incr('mofo.donations.message.queue_error') 72 | sentry_client.captureException(tags={'action': 'retried'}) 73 | continue 74 | 75 | statsd.incr('mofo.donations.message.success') 76 | msg.delete() 77 | except KeyboardInterrupt: 78 | sys.exit('\nBuh bye') 79 | -------------------------------------------------------------------------------- /jenkins/default.groovy: -------------------------------------------------------------------------------- 1 | milestone() 2 | stage ('Build Images') { 3 | // make sure we should continue 4 | env.DOCKER_REPOSITORY = 'mozmeao/basket' 5 | env.DOCKER_IMAGE_TAG = "${env.DOCKER_REPOSITORY}:${env.GIT_COMMIT}" 6 | if ( config.require_tag ) { 7 | try { 8 | sh 'docker/bin/check_if_tag.sh' 9 | } catch(err) { 10 | utils.ircNotification([stage: 'Git Tag Check', status: 'failure']) 11 | throw err 12 | } 13 | } 14 | utils.ircNotification([stage: 'Test & Deploy', status: 'starting']) 15 | lock ("basket-docker-${env.GIT_COMMIT}") { 16 | try { 17 | sh 'docker/bin/build_images.sh' 18 | sh 'docker/bin/run_tests.sh' 19 | } catch(err) { 20 | utils.ircNotification([stage: 'Docker Build', status: 'failure']) 21 | throw err 22 | } 23 | } 24 | } 25 | 26 | milestone() 27 | stage ('Push Public Images') { 28 | try { 29 | utils.pushDockerhub() 30 | } catch(err) { 31 | utils.ircNotification([stage: 'Dockerhub Push', status: 'failure']) 32 | throw err 33 | } 34 | } 35 | 36 | /** 37 | * Do region first because deployment and testing should work like this: 38 | * region1: 39 | * push image -> deploy app1 -> test app1 -> deploy app2 -> test app2 40 | * region2: 41 | * push image -> deploy app1 -> test app1 -> deploy app2 -> test app2 42 | * 43 | * A failure at any step of the above should fail the entire job 44 | */ 45 | if ( config.apps ) { 46 | milestone() 47 | // default to oregon-b only 48 | def regions = config.regions ?: ['oregon-b'] 49 | for (regionId in regions) { 50 | def region = global_config.regions[regionId] 51 | if ( region.db_mode == 'rw' && config.apps_rw ) { 52 | region_apps = config.apps + config.apps_rw 53 | } else { 54 | region_apps = config.apps 55 | } 56 | for (appname in region_apps) { 57 | appURL = "https://${appname}.${region.name}.moz.works" 58 | stageName = "Deploy ${appname}-${region.name}" 59 | lock (stageName) { 60 | milestone() 61 | stage (stageName) { 62 | // do post deploy if this is an RW app or if there are no RW apps configured 63 | if ( region.db_mode == 'rw' && config.apps_post_deploy && config.apps_post_deploy.contains(appname) ) { 64 | post_deploy = 'true' 65 | } else { 66 | post_deploy = 'false' 67 | } 68 | withEnv(["DEIS_PROFILE=${region.deis_profile}", 69 | "RUN_POST_DEPLOY=${post_deploy}", 70 | "REGION_NAME=${region.name}", 71 | "DEIS_APPLICATION=${appname}"]) { 72 | withCredentials([string(credentialsId: 'newrelic-api-key', variable: 'NEWRELIC_API_KEY')]) { 73 | withCredentials([string(credentialsId: 'datadog-api-key', variable: 'DATADOG_API_KEY')]) { 74 | try { 75 | retry(2) { 76 | sh 'docker/bin/push2deis.sh' 77 | } 78 | } catch(err) { 79 | utils.ircNotification([stage: stageName, status: 'failure']) 80 | throw err 81 | } 82 | } 83 | } 84 | } 85 | utils.ircNotification([message: appURL, status: 'shipped']) 86 | } 87 | } 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /basket/news/forms.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from django import forms 4 | from django.core.exceptions import ValidationError 5 | from django.core.validators import RegexValidator 6 | from product_details import product_details 7 | 8 | from basket.news.newsletters import newsletter_field_choices 9 | from basket.news.utils import parse_newsletters_csv, process_email, LANG_RE 10 | 11 | 12 | FORMATS = (('H', 'HTML'), ('T', 'Text')) 13 | SOURCE_URL_RE = re.compile(r'^https?://') 14 | 15 | 16 | class EmailForm(forms.Form): 17 | """Form to validate email addresses""" 18 | email = forms.EmailField() 19 | 20 | 21 | class EmailField(forms.CharField): 22 | """EmailField with better validation and value cleaning""" 23 | def to_python(self, value): 24 | value = super(EmailField, self).to_python(value) 25 | email = process_email(value) 26 | if not email: 27 | raise ValidationError('Enter a valid email address.', 'invalid') 28 | 29 | return email 30 | 31 | 32 | class NewslettersField(forms.MultipleChoiceField): 33 | """ 34 | Django form field that validates the newsletter IDs are valid 35 | 36 | * Accepts single newsletter IDs in multiple fields, and/or 37 | a comma separated list of newsletter IDs in a single field. 38 | * Validates each individual newsletter ID. 39 | * Includes newsletter group IDs. 40 | """ 41 | def __init__(self, required=True, widget=None, label=None, initial=None, 42 | help_text='', *args, **kwargs): 43 | super(NewslettersField, self).__init__(newsletter_field_choices, required, widget, label, 44 | initial, help_text, *args, **kwargs) 45 | 46 | def to_python(self, value): 47 | value = super(NewslettersField, self).to_python(value) 48 | full_list = [] 49 | for v in value: 50 | full_list.extend(parse_newsletters_csv(v)) 51 | 52 | return full_list 53 | 54 | 55 | def country_choices(): 56 | """Upper and Lower case country codes""" 57 | regions = product_details.get_regions('en-US') 58 | return regions.items() + [(code.upper(), name) for code, name in regions.iteritems()] 59 | 60 | 61 | class SubscribeForm(forms.Form): 62 | email = EmailField() 63 | newsletters = NewslettersField() 64 | privacy = forms.BooleanField() 65 | fmt = forms.ChoiceField(required=False, choices=FORMATS) 66 | source_url = forms.CharField(required=False) 67 | first_name = forms.CharField(required=False) 68 | last_name = forms.CharField(required=False) 69 | country = forms.ChoiceField(required=False, choices=country_choices) 70 | lang = forms.CharField(required=False, validators=[RegexValidator(regex=LANG_RE)]) 71 | 72 | def clean_source_url(self): 73 | source_url = self.cleaned_data['source_url'] 74 | if source_url: 75 | if SOURCE_URL_RE.match(source_url): 76 | return source_url 77 | 78 | return '' 79 | 80 | def clean_country(self): 81 | country = self.cleaned_data['country'] 82 | if country: 83 | return country.lower() 84 | 85 | return country 86 | 87 | 88 | class UpdateUserMeta(forms.Form): 89 | source_url = forms.CharField(required=False) 90 | first_name = forms.CharField(required=False) 91 | last_name = forms.CharField(required=False) 92 | country = forms.ChoiceField(required=False, choices=country_choices) 93 | lang = forms.CharField(required=False, validators=[RegexValidator(regex=LANG_RE)]) 94 | 95 | def clean_country(self): 96 | country = self.cleaned_data['country'] 97 | if country: 98 | return country.lower() 99 | 100 | return country 101 | -------------------------------------------------------------------------------- /basket/news/tests/test_models.py: -------------------------------------------------------------------------------- 1 | from django.core import mail 2 | from django.test import TestCase 3 | 4 | from mock import patch 5 | 6 | from basket.news import models 7 | 8 | 9 | class FailedTaskTest(TestCase): 10 | good_task_args = [{'case_type': 'ringer', 'email': 'dude@example.com'}, 'walter'] 11 | 12 | def test_retry_with_dict(self): 13 | """When given args with a simple dict, subtask should get matching arguments.""" 14 | task_name = 'make_a_caucasian' 15 | task = models.FailedTask.objects.create(task_id='el-dudarino', 16 | name=task_name, 17 | args=self.good_task_args) 18 | with patch.object(models.celery_app, 'send_task') as sub_mock: 19 | task.retry() 20 | 21 | sub_mock.assert_called_with(task_name, args=self.good_task_args, kwargs={}) 22 | 23 | def test_retry_with_querydict(self): 24 | """When given args with a QueryDict, subtask should get a dict.""" 25 | task_name = 'make_a_caucasian' 26 | task_args = [{'case_type': ['ringer'], 'email': ['dude@example.com']}, 'walter'] 27 | task = models.FailedTask.objects.create(task_id='el-dudarino', 28 | name=task_name, 29 | args=task_args) 30 | with patch.object(models.celery_app, 'send_task') as sub_mock: 31 | task.retry() 32 | 33 | sub_mock.assert_called_with(task_name, args=self.good_task_args, kwargs={}) 34 | 35 | def test_retry_with_querydict_not_first(self): 36 | """When given args with a QueryDict in any position, subtask should get a dict.""" 37 | task_name = 'make_a_caucasian' 38 | task_args = ['donny', {'case_type': ['ringer'], 'email': ['dude@example.com']}, 'walter'] 39 | task = models.FailedTask.objects.create(task_id='el-dudarino', 40 | name=task_name, 41 | args=task_args) 42 | with patch.object(models.celery_app, 'send_task') as sub_mock: 43 | task.retry() 44 | 45 | sub_mock.assert_called_with(task_name, args=['donny'] + self.good_task_args, kwargs={}) 46 | 47 | def test_retry_with_almost_querydict(self): 48 | """When given args with a dict with a list, subtask should get a same args.""" 49 | task_name = 'make_a_caucasian' 50 | task_args = [{'case_type': 'ringer', 'email': ['dude@example.com']}, 'walter'] 51 | task = models.FailedTask.objects.create(task_id='el-dudarino', 52 | name=task_name, 53 | args=task_args) 54 | with patch.object(models.celery_app, 'send_task') as sub_mock: 55 | task.retry() 56 | 57 | sub_mock.assert_called_with(task_name, args=task_args, kwargs={}) 58 | 59 | 60 | class InterestTests(TestCase): 61 | def test_notify_default_stewards(self): 62 | """ 63 | If there are no locale-specific stewards for the given language, 64 | notify the default stewards. 65 | """ 66 | interest = models.Interest(title='mytest', 67 | default_steward_emails='bob@example.com,bill@example.com') 68 | interest.notify_stewards('Steve', 'interested@example.com', 'en-US', 'BYE') 69 | 70 | self.assertEqual(len(mail.outbox), 1) 71 | email = mail.outbox[0] 72 | self.assertTrue('mytest' in email.subject) 73 | self.assertEqual(email.to, ['bob@example.com', 'bill@example.com']) 74 | 75 | def test_notify_locale_stewards(self): 76 | """ 77 | If there are locale-specific stewards for the given language, 78 | notify them instead of the default stewards. 79 | """ 80 | interest = models.Interest.objects.create( 81 | title='mytest', 82 | default_steward_emails='bob@example.com,bill@example.com') 83 | models.LocaleStewards.objects.create( 84 | interest=interest, 85 | locale='ach', 86 | emails='ach@example.com') 87 | interest.notify_stewards('Steve', 'interested@example.com', 'ach', 'BYE') 88 | 89 | self.assertEqual(len(mail.outbox), 1) 90 | email = mail.outbox[0] 91 | self.assertTrue('mytest' in email.subject) 92 | self.assertEqual(email.to, ['ach@example.com']) 93 | -------------------------------------------------------------------------------- /basket/news/tests/test_newsletters.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from django.test import TestCase 4 | 5 | from basket.news import newsletters, utils 6 | from basket.news.models import Newsletter, NewsletterGroup, LocalizedSMSMessage 7 | 8 | 9 | class TestSMSMessageCache(TestCase): 10 | def setUp(self): 11 | newsletters.clear_sms_cache() 12 | LocalizedSMSMessage.objects.create(message_id='the-dude', vendor_id='YOURE_NOT_WRONG_WALTER', 13 | country='us', language='de') 14 | LocalizedSMSMessage.objects.create(message_id='the-walrus', vendor_id='SHUTUP_DONNIE', 15 | country='gb', language='en-GB') 16 | 17 | def test_all_messages(self): 18 | """Messages returned should be all of the ones in the DB.""" 19 | 20 | self.assertEqual(newsletters.get_sms_messages(), { 21 | 'the-dude-us-de': 'YOURE_NOT_WRONG_WALTER', 22 | 'the-walrus-gb-en-gb': 'SHUTUP_DONNIE', 23 | }) 24 | 25 | 26 | class TestNewsletterUtils(TestCase): 27 | def setUp(self): 28 | self.newsies = [ 29 | Newsletter.objects.create( 30 | slug='bowling', 31 | title='Bowling, Man', 32 | vendor_id='BOWLING', 33 | languages='en'), 34 | Newsletter.objects.create( 35 | slug='surfing', 36 | title='Surfing, Man', 37 | vendor_id='SURFING', 38 | languages='en'), 39 | Newsletter.objects.create( 40 | slug='extorting', 41 | title='Beginning Nihilism', 42 | vendor_id='EXTORTING', 43 | languages='en'), 44 | Newsletter.objects.create( 45 | slug='papers', 46 | title='Just papers, personal papers', 47 | vendor_id='CREEDENCE', 48 | languages='en', 49 | private=True), 50 | ] 51 | self.groupies = [ 52 | NewsletterGroup.objects.create( 53 | slug='bowling', 54 | title='Bowling in Groups', 55 | active=True), 56 | NewsletterGroup.objects.create( 57 | slug='abiding', 58 | title='Be like The Dude', 59 | active=True), 60 | NewsletterGroup.objects.create( 61 | slug='failing', 62 | title='The Bums Lost!', 63 | active=False), 64 | ] 65 | self.groupies[0].newsletters.add(self.newsies[1], self.newsies[2]) 66 | 67 | def test_newseltter_private_slugs(self): 68 | self.assertEqual(newsletters.newsletter_private_slugs(), ['papers']) 69 | 70 | def test_newsletter_slugs(self): 71 | self.assertEqual(set(newsletters.newsletter_slugs()), 72 | {'bowling', 'surfing', 'extorting', 'papers'}) 73 | 74 | def test_newsletter_group_slugs(self): 75 | self.assertEqual(set(newsletters.newsletter_group_slugs()), 76 | {'bowling', 'abiding'}) 77 | 78 | def test_newsletter_and_group_slugs(self): 79 | self.assertEqual(set(newsletters.newsletter_and_group_slugs()), 80 | {'bowling', 'abiding', 'surfing', 'extorting', 'papers'}) 81 | 82 | def test_newsletter_group_newsletter_slugs(self): 83 | self.assertEqual(set(newsletters.newsletter_group_newsletter_slugs('bowling')), 84 | {'extorting', 'surfing'}) 85 | 86 | def test_parse_newsletters_for_groups(self): 87 | """If newsletter slug is a group for SUBSCRIBE, expand to group's newsletters.""" 88 | subs = utils.parse_newsletters(utils.SUBSCRIBE, ['bowling'], list()) 89 | self.assertTrue(subs['surfing']) 90 | self.assertTrue(subs['extorting']) 91 | 92 | def test_parse_newsletters_not_groups_set(self): 93 | """If newsletter slug is a group for SET mode, don't expand to group's newsletters.""" 94 | subs = utils.parse_newsletters(utils.SET, ['bowling'], list()) 95 | self.assertDictEqual(subs, {'bowling': True}) 96 | 97 | def test_parse_newsletters_not_groups_unsubscribe(self): 98 | """If newsletter slug is a group for SET mode, don't expand to group's newsletters.""" 99 | subs = utils.parse_newsletters(utils.UNSUBSCRIBE, ['bowling'], 100 | ['bowling', 'surfing', 'extorting']) 101 | self.assertDictEqual(subs, {'bowling': False}) 102 | -------------------------------------------------------------------------------- /basket/news/management/commands/process_fxa_queue.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, unicode_literals 2 | 3 | import json 4 | import sys 5 | from time import time 6 | 7 | from django.conf import settings 8 | from django.core.management import BaseCommand, CommandError 9 | 10 | import boto3 11 | import requests 12 | from django_statsd.clients import statsd 13 | from raven.contrib.django.raven_compat.models import client as sentry_client 14 | 15 | from basket.news.tasks import fxa_delete, fxa_email_changed, fxa_login, fxa_verified 16 | 17 | 18 | # TODO remove this after the cutover 19 | class FxATSProxyTask(object): 20 | """Fake task that will only fire the real task after timestamp""" 21 | def __init__(self, task, timestamp): 22 | self.task = task 23 | self.ts = timestamp 24 | 25 | def delay(self, data): 26 | if not self.ts or data['ts'] < self.ts: 27 | return 28 | 29 | self.task.delay(data) 30 | 31 | 32 | FXA_EVENT_TYPES = { 33 | 'delete': fxa_delete, 34 | 'verified': fxa_verified, 35 | 'primaryEmailChanged': fxa_email_changed, 36 | 'login': FxATSProxyTask(fxa_login, settings.FXA_LOGIN_CUTOVER_TIMESTAMP), 37 | } 38 | 39 | 40 | class Command(BaseCommand): 41 | snitch_delay = 300 # 5 min 42 | snitch_last_timestamp = 0 43 | snitch_id = settings.FXA_EVENTS_SNITCH_ID 44 | 45 | def snitch(self): 46 | if not self.snitch_id: 47 | return 48 | 49 | time_since = int(time() - self.snitch_last_timestamp) 50 | if time_since > self.snitch_delay: 51 | requests.post('https://nosnch.in/{}'.format(self.snitch_id)) 52 | self.snitch_last_timestamp = time() 53 | 54 | def handle(self, *args, **options): 55 | if not settings.FXA_EVENTS_ACCESS_KEY_ID: 56 | raise CommandError('AWS SQS Credentials not configured') 57 | 58 | if not settings.FXA_EVENTS_QUEUE_ENABLE: 59 | raise CommandError('FxA Events Queue is not enabled') 60 | 61 | sqs = boto3.resource('sqs', 62 | region_name=settings.FXA_EVENTS_QUEUE_REGION, 63 | aws_access_key_id=settings.FXA_EVENTS_ACCESS_KEY_ID, 64 | aws_secret_access_key=settings.FXA_EVENTS_SECRET_ACCESS_KEY) 65 | queue = sqs.Queue(settings.FXA_EVENTS_QUEUE_URL) 66 | 67 | try: 68 | # Poll for messages indefinitely. 69 | while True: 70 | self.snitch() 71 | msgs = queue.receive_messages(WaitTimeSeconds=settings.FXA_EVENTS_QUEUE_WAIT_TIME, 72 | MaxNumberOfMessages=10) 73 | for msg in msgs: 74 | if not (msg and msg.body): 75 | continue 76 | 77 | statsd.incr('fxa.events.message.received') 78 | try: 79 | data = json.loads(msg.body) 80 | event = json.loads(data['Message']) 81 | except ValueError as e: 82 | # body was not JSON 83 | statsd.incr('fxa.events.message.json_error') 84 | sentry_client.captureException(data={'extra': {'msg.body': msg.body}}) 85 | print('ERROR:', e, '::', msg.body) 86 | msg.delete() 87 | continue 88 | 89 | event_type = event.get('event', '__NONE__').replace(':', '-') 90 | statsd.incr('fxa.events.message.received.{}'.format(event_type)) 91 | if event_type not in FXA_EVENT_TYPES: 92 | statsd.incr('fxa.events.message.received.{}.IGNORED'.format(event_type)) 93 | # we can safely remove from the queue message types we don't need 94 | # this keeps the queue from filling up with old messages 95 | msg.delete() 96 | continue 97 | 98 | try: 99 | FXA_EVENT_TYPES[event_type].delay(event) 100 | except Exception: 101 | # something's wrong with the queue. try again. 102 | statsd.incr('fxa.events.message.queue_error') 103 | sentry_client.captureException(tags={'action': 'retried'}) 104 | continue 105 | 106 | statsd.incr('fxa.events.message.success') 107 | msg.delete() 108 | except KeyboardInterrupt: 109 | sys.exit('\nBuh bye') 110 | -------------------------------------------------------------------------------- /requirements/prod.txt: -------------------------------------------------------------------------------- 1 | -r dev.txt 2 | 3 | django-redis==4.2.0 \ 4 | --hash=sha256:9ad6b299458f7e6bfaefa8905f52560017369d82fb8fb0ed4b41adc048dbf11c 5 | gunicorn==19.7.1 \ 6 | --hash=sha256:75af03c99389535f218cc596c7de74df4763803f7b63eb09d77e92b3956b36c6 \ 7 | --hash=sha256:eee1169f0ca667be05db3351a0960765620dad53f53434262ff8901b68a1b622 8 | redis==2.10.3 \ 9 | --hash=sha256:a4fb37b02860f6b1617f6469487471fd086dd2d38bbce640c2055862b9c4019c 10 | hiredis==0.2.0 \ 11 | --hash=sha256:ca958e13128e49674aa4a96f02746f5de5973f39b57297b84d59fd44d314d5b5 12 | msgpack-python==0.4.6 \ 13 | --hash=sha256:bfcc581c9dbbf07cc2f951baf30c3249a57e20dcbd60f7e6ffc43ab3cc614794 14 | whitenoise==3.3.1 \ 15 | --hash=sha256:15f43b2e701821b95c9016cf469d29e2a546cb1c7dead584ba82c36f843995cf \ 16 | --hash=sha256:9d81515f2b5b27051910996e1e860b1332e354d9e7bcf30c98f21dcb6713e0dd 17 | newrelic==2.100.0.84 \ 18 | --hash=sha256:b75123173ac5e8a20aa9d8120e20a7bf45c38a5aa5a4672fac6ce4c3e0c8046e 19 | urlwait==0.4 \ 20 | --hash=sha256:fc39ff2c8abbcaad5043e1f79699dcb15a036cc4b0ff4d1aa825ea105d4889ff \ 21 | --hash=sha256:395fc0c2a7f9736858a2c2f449aa20c6e9da1f86bfc2d1fda4f2f5b78a5c115a 22 | MySQL-python==1.2.5 \ 23 | --hash=sha256:ab22d1322099098730a57fd59d610f60738f95a1cb68dacca2d1c47cb0cbe8ee \ 24 | --hash=sha256:811040b647e5d5686f84db415efd697e6250008b112b6909ba77ac059e140c74 25 | certifi==2017.11.5 \ 26 | --hash=sha256:244be0d93b71e93fc0a0a479862051414d0e00e16435707e5bf5000f92e04694 \ 27 | --hash=sha256:5ec74291ca1136b40f0379e1128ff80e866597e4e2c1e755739a913bbc3613c0 28 | chardet==3.0.4 \ 29 | --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \ 30 | --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae 31 | josepy==1.0.1 \ 32 | --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ 33 | --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc 34 | mozilla-django-oidc==0.4.2 \ 35 | --hash=sha256:77c29c47d67750d3c53fcd51f1aa496a2cdd65dd27a1f2a15e56ecc3c3714f19 \ 36 | --hash=sha256:650716143525bb4bae553dd8c740a1c5986baf6aeae115cba01f6a217ee5fa4f 37 | packaging==16.8 \ 38 | --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \ 39 | --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e 40 | pyOpenSSL==17.5.0 \ 41 | --hash=sha256:07a2de1a54de07448732a81e38a55df7da109b2f47f599f8bb35b0cbec69d4bd \ 42 | --hash=sha256:2c10cfba46a52c0b0950118981d61e72c1e5b1aac451ca1bc77de1a679456773 43 | pyparsing==2.2.0 \ 44 | --hash=sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010 \ 45 | --hash=sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04 \ 46 | --hash=sha256:9e8143a3e15c13713506886badd96ca4b579a87fbdf49e550dbfc057d6cb218e \ 47 | --hash=sha256:281683241b25fe9b80ec9d66017485f6deff1af5cde372469134b56ca8447a07 \ 48 | --hash=sha256:b8b3117ed9bdf45e14dcc89345ce638ec7e0e29b2b579fa1ecf32ce45ebac8a5 \ 49 | --hash=sha256:8f1e18d3fd36c6795bb7e02a39fd05c611ffc2596c1e0d995d34d67630426c18 \ 50 | --hash=sha256:e4d45427c6e20a59bf4f88c639dcc03ce30d193112047f94012102f235853a58 51 | urllib3==1.22 \ 52 | --hash=sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b \ 53 | --hash=sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f 54 | greenlet==0.4.13 \ 55 | --hash=sha256:50643fd6d54fd919f9a0a577c5f7b71f5d21f0959ab48767bd4bb73ae0839500 \ 56 | --hash=sha256:c7b04a6dc74087b1598de8d713198de4718fa30ec6cbb84959b26426c198e041 \ 57 | --hash=sha256:b6ef0cabaf5a6ecb5ac122e689d25ba12433a90c7b067b12e5f28bdb7fb78254 \ 58 | --hash=sha256:fcfadaf4bf68a27e5dc2f42cbb2f4b4ceea9f05d1d0b8f7787e640bed2801634 \ 59 | --hash=sha256:b417bb7ff680d43e7bd7a13e2e08956fa6acb11fd432f74c97b7664f8bdb6ec1 \ 60 | --hash=sha256:769b740aeebd584cd59232be84fdcaf6270b8adc356596cdea5b2152c82caaac \ 61 | --hash=sha256:c2de19c88bdb0366c976cc125dca1002ec1b346989d59524178adfd395e62421 \ 62 | --hash=sha256:5b49b3049697aeae17ef7bf21267e69972d9e04917658b4e788986ea5cc518e8 \ 63 | --hash=sha256:09ef2636ea35782364c830f07127d6c7a70542b178268714a9a9ba16318e7e8b \ 64 | --hash=sha256:f8f2a0ae8de0b49c7b5b2daca4f150fdd9c1173e854df2cce3b04123244f9f45 \ 65 | --hash=sha256:1b7df09c6598f5cfb40f843ade14ed1eb40596e75cd79b6fa2efc750ba01bb01 \ 66 | --hash=sha256:75c413551a436b462d5929255b6dc9c0c3c2b25cbeaee5271a56c7fda8ca49c0 \ 67 | --hash=sha256:58798b5d30054bb4f6cf0f712f08e6092df23a718b69000786634a265e8911a9 \ 68 | --hash=sha256:42118bf608e0288e35304b449a2d87e2ba77d1e373e8aa221ccdea073de026fa \ 69 | --hash=sha256:ad2383d39f13534f3ca5c48fe1fc0975676846dc39c2cece78c0f1f9891418e0 \ 70 | --hash=sha256:1fff21a2da5f9e03ddc5bd99131a6b8edf3d7f9d6bc29ba21784323d17806ed7 \ 71 | --hash=sha256:0fef83d43bf87a5196c91e73cb9772f945a4caaff91242766c5916d1dd1381e4 72 | meinheld==0.6.1 \ 73 | --hash=sha256:40d9dbce0165b2d9142f364d26fd6d59d3682f89d0dfe2117717a8ddad1f4133 \ 74 | --hash=sha256:293eff4983b7fcbd9134b47706b22189883fe354993bd10163c65869d141e565 75 | -------------------------------------------------------------------------------- /basket/news/migrations/0003_auto_20151202_0808.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | import basket.news.fields 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('news', '0002_delete_subscriber'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='newsletter', 17 | name='private', 18 | field=models.BooleanField(default=False, help_text=b'Whether this newsletter is private. Private newsletters require the subscribe requests to use an API key.'), 19 | preserve_default=True, 20 | ), 21 | migrations.AlterField( 22 | model_name='localestewards', 23 | name='locale', 24 | field=basket.news.fields.LocaleField(max_length=32, choices=[('ach', 'ach (Acholi)'), ('af', 'af (Afrikaans)'), ('ak', 'ak (Akan)'), ('am-et', 'am-et (Amharic)'), ('an', 'an (Aragonese)'), ('ar', 'ar (Arabic)'), ('as', 'as (Assamese)'), ('ast', 'ast (Asturian)'), ('az', 'az (Azerbaijani)'), ('be', 'be (Belarusian)'), ('bg', 'bg (Bulgarian)'), ('bm', 'bm (Bambara)'), ('bn-BD', 'bn-BD (Bengali (Bangladesh))'), ('bn-IN', 'bn-IN (Bengali (India))'), ('br', 'br (Breton)'), ('brx', 'brx (Bodo)'), ('bs', 'bs (Bosnian)'), ('ca', 'ca (Catalan)'), ('ca-valencia', 'ca-valencia (Catalan (Valencian))'), ('cak', 'cak (Kaqchikel)'), ('cs', 'cs (Czech)'), ('csb', 'csb (Kashubian)'), ('cy', 'cy (Welsh)'), ('da', 'da (Danish)'), ('dbg', 'dbg (Debug Robot)'), ('de', 'de (German)'), ('de-AT', 'de-AT (German (Austria))'), ('de-CH', 'de-CH (German (Switzerland))'), ('de-DE', 'de-DE (German (Germany))'), ('dsb', 'dsb (Lower Sorbian)'), ('ee', 'ee (Ewe)'), ('el', 'el (Greek)'), ('en-AU', 'en-AU (English (Australian))'), ('en-CA', 'en-CA (English (Canadian))'), ('en-GB', 'en-GB (English (British))'), ('en-NZ', 'en-NZ (English (New Zealand))'), ('en-US', 'en-US (English (US))'), ('en-ZA', 'en-ZA (English (South African))'), ('eo', 'eo (Esperanto)'), ('es', 'es (Spanish)'), ('es-AR', 'es-AR (Spanish (Argentina))'), ('es-CL', 'es-CL (Spanish (Chile))'), ('es-ES', 'es-ES (Spanish (Spain))'), ('es-MX', 'es-MX (Spanish (Mexico))'), ('et', 'et (Estonian)'), ('eu', 'eu (Basque)'), ('fa', 'fa (Persian)'), ('ff', 'ff (Fulah)'), ('fi', 'fi (Finnish)'), ('fj-FJ', 'fj-FJ (Fijian)'), ('fr', 'fr (French)'), ('fur-IT', 'fur-IT (Friulian)'), ('fy-NL', 'fy-NL (Frisian)'), ('ga', 'ga (Irish)'), ('ga-IE', 'ga-IE (Irish)'), ('gd', 'gd (Gaelic (Scotland))'), ('gl', 'gl (Galician)'), ('gu', 'gu (Gujarati)'), ('gu-IN', 'gu-IN (Gujarati (India))'), ('ha', 'ha (Hausa)'), ('he', 'he (Hebrew)'), ('hi', 'hi (Hindi)'), ('hi-IN', 'hi-IN (Hindi (India))'), ('hr', 'hr (Croatian)'), ('hsb', 'hsb (Upper Sorbian)'), ('hu', 'hu (Hungarian)'), ('hy-AM', 'hy-AM (Armenian)'), ('id', 'id (Indonesian)'), ('ig', 'ig (Igbo)'), ('is', 'is (Icelandic)'), ('it', 'it (Italian)'), ('ja', 'ja (Japanese)'), ('ja-JP-mac', 'ja-JP-mac (Japanese)'), ('ka', 'ka (Georgian)'), ('kk', 'kk (Kazakh)'), ('km', 'km (Khmer)'), ('kn', 'kn (Kannada)'), ('ko', 'ko (Korean)'), ('kok', 'kok (Konkani)'), ('ks', 'ks (Kashmiri)'), ('ku', 'ku (Kurdish)'), ('la', 'la (Latin)'), ('lg', 'lg (Luganda)'), ('lij', 'lij (Ligurian)'), ('ln', 'ln (Lingala)'), ('lo', 'lo (Lao)'), ('lt', 'lt (Lithuanian)'), ('lv', 'lv (Latvian)'), ('mai', 'mai (Maithili)'), ('mg', 'mg (Malagasy)'), ('mi', 'mi (Maori (Aotearoa))'), ('mk', 'mk (Macedonian)'), ('ml', 'ml (Malayalam)'), ('mn', 'mn (Mongolian)'), ('mr', 'mr (Marathi)'), ('ms', 'ms (Malay)'), ('my', 'my (Burmese)'), ('nb-NO', 'nb-NO (Norwegian (Bokm\xe5l))'), ('ne-NP', 'ne-NP (Nepali)'), ('nl', 'nl (Dutch)'), ('nn-NO', 'nn-NO (Norwegian (Nynorsk))'), ('nr', 'nr (Ndebele, South)'), ('nso', 'nso (Northern Sotho)'), ('oc', 'oc (Occitan (Lengadocian))'), ('or', 'or (Oriya)'), ('pa', 'pa (Punjabi)'), ('pa-IN', 'pa-IN (Punjabi (India))'), ('pl', 'pl (Polish)'), ('pt-BR', 'pt-BR (Portuguese (Brazilian))'), ('pt-PT', 'pt-PT (Portuguese (Portugal))'), ('rm', 'rm (Romansh)'), ('ro', 'ro (Romanian)'), ('ru', 'ru (Russian)'), ('rw', 'rw (Kinyarwanda)'), ('sa', 'sa (Sanskrit)'), ('sah', 'sah (Sakha)'), ('sat', 'sat (Santali)'), ('si', 'si (Sinhala)'), ('sk', 'sk (Slovak)'), ('sl', 'sl (Slovenian)'), ('son', 'son (Songhai)'), ('sq', 'sq (Albanian)'), ('sr', 'sr (Serbian)'), ('sr-Cyrl', 'sr-Cyrl (Serbian)'), ('sr-Latn', 'sr-Latn (Serbian)'), ('ss', 'ss (Siswati)'), ('st', 'st (Southern Sotho)'), ('sv-SE', 'sv-SE (Swedish)'), ('sw', 'sw (Swahili)'), ('ta', 'ta (Tamil)'), ('ta-IN', 'ta-IN (Tamil (India))'), ('ta-LK', 'ta-LK (Tamil (Sri Lanka))'), ('te', 'te (Telugu)'), ('th', 'th (Thai)'), ('tl', 'tl (Tagalog)'), ('tn', 'tn (Tswana)'), ('tr', 'tr (Turkish)'), ('ts', 'ts (Tsonga)'), ('tsz', 'tsz (Pur\xe9pecha)'), ('tt-RU', 'tt-RU (Tatar)'), ('uk', 'uk (Ukrainian)'), ('ur', 'ur (Urdu)'), ('uz', 'uz (Uzbek)'), ('ve', 've (Venda)'), ('vi', 'vi (Vietnamese)'), ('wo', 'wo (Wolof)'), ('x-testing', 'x-testing (Testing)'), ('xh', 'xh (Xhosa)'), ('yo', 'yo (Yoruba)'), ('zh-CN', 'zh-CN (Chinese (Simplified))'), ('zh-TW', 'zh-TW (Chinese (Traditional))'), ('zu', 'zu (Zulu)')]), 25 | preserve_default=True, 26 | ), 27 | ] 28 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " singlehtml to make a single large HTML file" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " devhelp to make HTML files and a Devhelp project" 27 | @echo " epub to make an epub" 28 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 29 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 30 | @echo " text to make text files" 31 | @echo " man to make manual pages" 32 | @echo " changes to make an overview of all changed/added/deprecated items" 33 | @echo " linkcheck to check all external links for integrity" 34 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 35 | 36 | clean: 37 | -rm -rf $(BUILDDIR)/* 38 | 39 | html: 40 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 41 | @echo 42 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 43 | 44 | dirhtml: 45 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 48 | 49 | singlehtml: 50 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 51 | @echo 52 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 53 | 54 | pickle: 55 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 56 | @echo 57 | @echo "Build finished; now you can process the pickle files." 58 | 59 | json: 60 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 61 | @echo 62 | @echo "Build finished; now you can process the JSON files." 63 | 64 | htmlhelp: 65 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 66 | @echo 67 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 68 | ".hhp project file in $(BUILDDIR)/htmlhelp." 69 | 70 | qthelp: 71 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 72 | @echo 73 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 74 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 75 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/playdoh.qhcp" 76 | @echo "To view the help file:" 77 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/playdoh.qhc" 78 | 79 | devhelp: 80 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 81 | @echo 82 | @echo "Build finished." 83 | @echo "To view the help file:" 84 | @echo "# mkdir -p $$HOME/.local/share/devhelp/playdoh" 85 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/playdoh" 86 | @echo "# devhelp" 87 | 88 | epub: 89 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 90 | @echo 91 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 92 | 93 | latex: 94 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 95 | @echo 96 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 97 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 98 | "(use \`make latexpdf' here to do that automatically)." 99 | 100 | latexpdf: 101 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 102 | @echo "Running LaTeX files through pdflatex..." 103 | make -C $(BUILDDIR)/latex all-pdf 104 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 105 | 106 | text: 107 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 108 | @echo 109 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 110 | 111 | man: 112 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 113 | @echo 114 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 115 | 116 | changes: 117 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 118 | @echo 119 | @echo "The overview file is in $(BUILDDIR)/changes." 120 | 121 | linkcheck: 122 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 123 | @echo 124 | @echo "Link check complete; look for any errors in the above output " \ 125 | "or in $(BUILDDIR)/linkcheck/output.txt." 126 | 127 | doctest: 128 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 129 | @echo "Testing of doctests in the sources finished, look at the " \ 130 | "results in $(BUILDDIR)/doctest/output.txt." 131 | -------------------------------------------------------------------------------- /basket/news/admin.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.contrib import admin, messages 3 | 4 | from basket.news.models import (APIUser, BlockedEmail, FailedTask, Interest, LocaleStewards, LocalizedSMSMessage, Newsletter, 5 | NewsletterGroup, QueuedTask, TransactionalEmailMessage) 6 | 7 | 8 | class TransactionalEmailAdmin(admin.ModelAdmin): 9 | fields = ('message_id', 'vendor_id', 'languages', 'description') 10 | list_display = ('message_id', 'vendor_id', 'languages', 'description') 11 | 12 | 13 | class LocalizedSMSMessageAdmin(admin.ModelAdmin): 14 | fields = ('message_id', 'vendor_id', 'language', 'country', 'description') 15 | list_display = ('message_id', 'vendor_id', 'language', 'country', 'description') 16 | list_editable = ('vendor_id',) 17 | 18 | 19 | class BlockedEmailAdmin(admin.ModelAdmin): 20 | fields = ('email_domain',) 21 | list_display = ('email_domain',) 22 | 23 | 24 | class NewsletterGroupAdmin(admin.ModelAdmin): 25 | fields = ('title', 'slug', 'description', 'show', 'active', 'newsletters') 26 | list_display = ('title', 'slug', 'show', 'active') 27 | list_display_links = ('title', 'slug') 28 | prepopulated_fields = {"slug": ("title",)} 29 | 30 | 31 | class LocaleStewardsInline(admin.TabularInline): 32 | model = LocaleStewards 33 | fields = ('locale', 'emails') 34 | 35 | 36 | class InterestAdmin(admin.ModelAdmin): 37 | fields = ('title', 'interest_id', '_welcome_id', 'default_steward_emails') 38 | list_display = ('title', 'interest_id', '_welcome_id', 'default_steward_emails') 39 | list_editable = ('interest_id', '_welcome_id', 'default_steward_emails') 40 | prepopulated_fields = {'interest_id': ('title',)} 41 | inlines = [LocaleStewardsInline] 42 | 43 | 44 | class APIUserAdmin(admin.ModelAdmin): 45 | list_display = ('name', 'enabled') 46 | 47 | 48 | class NewsletterAdmin(admin.ModelAdmin): 49 | fields = ('title', 'slug', 'vendor_id', 'description', 'languages', 50 | 'show', 'order', 'active', 'requires_double_optin', 'private') 51 | list_display = ('order', 'title', 'slug', 'vendor_id', 'languages', 'show', 'active', 52 | 'requires_double_optin', 'private') 53 | list_display_links = ('title', 'slug') 54 | list_editable = ('order', 'show', 'active', 'requires_double_optin', 'private') 55 | list_filter = ('show', 'active', 'requires_double_optin', 'private') 56 | prepopulated_fields = {"slug": ("title",)} 57 | search_fields = ('title', 'slug', 'description', 'vendor_id') 58 | 59 | 60 | class TaskNameFilter(admin.SimpleListFilter): 61 | """Filter to provide nicer names for task names.""" 62 | title = 'task name' 63 | parameter_name = 'name' 64 | 65 | def lookups(self, request, model_admin): 66 | qs = model_admin.get_queryset(request) 67 | names = qs.values_list('name', flat=True).distinct().order_by('name') 68 | return [(name, name.rsplit('.', 1)[1].replace('_', ' ')) for name in names] 69 | 70 | def queryset(self, request, queryset): 71 | if self.value(): 72 | return queryset.filter(name=self.value()) 73 | 74 | return queryset 75 | 76 | 77 | class QueuedTaskAdmin(admin.ModelAdmin): 78 | list_display = ('when', 'name') 79 | list_filter = (TaskNameFilter,) 80 | search_fields = ('name',) 81 | date_hierarchy = 'when' 82 | actions = ['retry_task_action'] 83 | 84 | def retry_task_action(self, request, queryset): 85 | """Admin action to retry some tasks that were queued for maintenance""" 86 | if settings.MAINTENANCE_MODE: 87 | messages.error(request, 'Maintenance mode enabled. Tasks not processed.') 88 | return 89 | 90 | count = 0 91 | for old_task in queryset: 92 | old_task.retry() 93 | count += 1 94 | messages.info(request, 'Queued %d task%s to process' % (count, '' if count == 1 else 's')) 95 | retry_task_action.short_description = u'Process task(s)' 96 | 97 | 98 | class FailedTaskAdmin(admin.ModelAdmin): 99 | list_display = ('when', 'name', 'formatted_call', 'exc') 100 | list_filter = (TaskNameFilter,) 101 | search_fields = ('name', 'exc') 102 | date_hierarchy = 'when' 103 | actions = ['retry_task_action'] 104 | 105 | def retry_task_action(self, request, queryset): 106 | """Admin action to retry some tasks that have failed previously""" 107 | count = 0 108 | for old_task in queryset: 109 | old_task.retry() 110 | count += 1 111 | messages.info(request, 'Queued %d task%s to try again' % (count, '' if count == 1 else 's')) 112 | retry_task_action.short_description = u'Retry task(s)' 113 | 114 | 115 | admin.site.register(TransactionalEmailMessage, TransactionalEmailAdmin) 116 | admin.site.register(LocalizedSMSMessage, LocalizedSMSMessageAdmin) 117 | admin.site.register(APIUser, APIUserAdmin) 118 | admin.site.register(BlockedEmail, BlockedEmailAdmin) 119 | admin.site.register(FailedTask, FailedTaskAdmin) 120 | admin.site.register(QueuedTask, QueuedTaskAdmin) 121 | admin.site.register(Interest, InterestAdmin) 122 | admin.site.register(Newsletter, NewsletterAdmin) 123 | admin.site.register(NewsletterGroup, NewsletterGroupAdmin) 124 | -------------------------------------------------------------------------------- /basket/news/tests/test_forms.py: -------------------------------------------------------------------------------- 1 | from mock import patch 2 | 3 | from basket.news import forms 4 | 5 | 6 | @patch.object(forms, 'process_email') 7 | def test_email_validation(email_mock): 8 | email_mock.return_value = None 9 | form = forms.SubscribeForm({ 10 | 'newsletters': ['dude'], 11 | 'email': 'dude@example.com', 12 | 'privacy': 'true', 13 | }) 14 | form.fields['newsletters'].choices = (('dude', 'dude'), ('walter', 'walter')) 15 | assert not form.is_valid() 16 | assert 'email' in form.errors 17 | 18 | email_mock.return_value = 'dude@example.com' 19 | form = forms.SubscribeForm({ 20 | 'newsletters': ['dude'], 21 | 'email': 'dude@example.com', 22 | 'privacy': 'true', 23 | }) 24 | form.fields['newsletters'].choices = (('dude', 'dude'), ('walter', 'walter')) 25 | assert form.is_valid(), form.errors 26 | 27 | # should result in whatever email process_email returns 28 | email_mock.return_value = 'walter@example.com' 29 | form = forms.SubscribeForm({ 30 | 'newsletters': ['dude'], 31 | 'email': 'dude@example.com', 32 | 'privacy': 'true', 33 | }) 34 | form.fields['newsletters'].choices = (('dude', 'dude'), ('walter', 'walter')) 35 | assert form.is_valid(), form.errors 36 | assert form.cleaned_data['email'] == 'walter@example.com' 37 | 38 | 39 | @patch.object(forms, 'process_email', return_value='dude@example.com') 40 | def test_newsletters_validation(email_mock): 41 | # comma separated in just one field 42 | form = forms.SubscribeForm({ 43 | 'newsletters': ['dude, walter'], 44 | 'email': 'dude@example.com', 45 | 'privacy': 'true', 46 | }) 47 | form.fields['newsletters'].choices = (('dude', 'dude'), ('walter', 'walter')) 48 | assert form.is_valid(), form.errors 49 | assert form.cleaned_data['newsletters'] == ['dude', 'walter'] 50 | 51 | # separate fields 52 | form = forms.SubscribeForm({ 53 | 'newsletters': ['dude', 'walter'], 54 | 'email': 'dude@example.com', 55 | 'privacy': 'true', 56 | }) 57 | form.fields['newsletters'].choices = (('dude', 'dude'), ('walter', 'walter')) 58 | assert form.is_valid(), form.errors 59 | assert form.cleaned_data['newsletters'] == ['dude', 'walter'] 60 | 61 | # combo of comma separated and non 62 | form = forms.SubscribeForm({ 63 | 'newsletters': ['dude', 'walter,donnie'], 64 | 'email': 'dude@example.com', 65 | 'privacy': 'true', 66 | }) 67 | form.fields['newsletters'].choices = (('dude', 'dude'), 68 | ('walter', 'walter'), 69 | ('donnie', 'donnie')) 70 | assert form.is_valid(), form.errors 71 | assert form.cleaned_data['newsletters'] == ['dude', 'walter', 'donnie'] 72 | 73 | # invalid newsletter 74 | form = forms.SubscribeForm({ 75 | 'newsletters': ['dude, walter'], 76 | 'email': 'dude@example.com', 77 | 'privacy': 'true', 78 | }) 79 | form.fields['newsletters'].choices = (('dude', 'dude'),) 80 | assert not form.is_valid() 81 | assert 'newsletters' in form.errors 82 | 83 | 84 | @patch.object(forms, 'process_email', return_value='dude@example.com') 85 | def test_privacy_required(email_mock): 86 | form = forms.SubscribeForm({ 87 | 'newsletters': ['dude, walter'], 88 | 'email': 'dude@example.com', 89 | }) 90 | form.fields['newsletters'].choices = (('dude', 'dude'),) 91 | assert not form.is_valid() 92 | assert 'privacy' in form.errors 93 | 94 | form = forms.SubscribeForm({ 95 | 'newsletters': ['dude, walter'], 96 | 'email': 'dude@example.com', 97 | 'privacy': 'false', 98 | }) 99 | form.fields['newsletters'].choices = (('dude', 'dude'),) 100 | assert not form.is_valid() 101 | assert 'privacy' in form.errors 102 | 103 | 104 | @patch.object(forms, 'process_email', return_value='dude@example.com') 105 | def test_source_url_clean(email_mock): 106 | # HTTP URL 107 | form = forms.SubscribeForm({ 108 | 'newsletters': ['dude'], 109 | 'email': 'dude@example.com', 110 | 'privacy': 'true', 111 | 'source_url': 'http://example.com', 112 | }) 113 | form.fields['newsletters'].choices = (('dude', 'dude'), ('walter', 'walter')) 114 | assert form.is_valid(), form.errors 115 | assert form.cleaned_data['source_url'] == 'http://example.com' 116 | 117 | # HTTPS URL 118 | form = forms.SubscribeForm({ 119 | 'newsletters': ['dude'], 120 | 'email': 'dude@example.com', 121 | 'privacy': 'true', 122 | 'source_url': 'https://example.com', 123 | }) 124 | form.fields['newsletters'].choices = (('dude', 'dude'), ('walter', 'walter')) 125 | assert form.is_valid(), form.errors 126 | assert form.cleaned_data['source_url'] == 'https://example.com' 127 | 128 | # JavaScript URL 129 | form = forms.SubscribeForm({ 130 | 'newsletters': ['dude'], 131 | 'email': 'dude@example.com', 132 | 'privacy': 'true', 133 | 'source_url': 'javascript:alert("dude!")', 134 | }) 135 | form.fields['newsletters'].choices = (('dude', 'dude'), ('walter', 'walter')) 136 | assert form.is_valid(), form.errors 137 | assert form.cleaned_data['source_url'] == '' 138 | -------------------------------------------------------------------------------- /basket/news/migrations/0012_auto_20170713_1021.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations 5 | import jsonfield.fields 6 | import basket.news.fields 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('news', '0011_auto_20160607_1203'), 13 | ] 14 | 15 | operations = [ 16 | migrations.AlterField( 17 | model_name='failedtask', 18 | name='args', 19 | field=jsonfield.fields.JSONField(default=list), 20 | ), 21 | migrations.AlterField( 22 | model_name='failedtask', 23 | name='kwargs', 24 | field=jsonfield.fields.JSONField(default=dict), 25 | ), 26 | migrations.AlterField( 27 | model_name='localestewards', 28 | name='locale', 29 | field=basket.news.fields.LocaleField(max_length=32, choices=[('ach', 'ach (Acholi)'), ('af', 'af (Afrikaans)'), ('ak', 'ak (Akan)'), ('am-et', 'am-et (Amharic)'), ('an', 'an (Aragonese)'), ('ar', 'ar (Arabic)'), ('as', 'as (Assamese)'), ('ast', 'ast (Asturian)'), ('az', 'az (Azerbaijani)'), ('be', 'be (Belarusian)'), ('bg', 'bg (Bulgarian)'), ('bm', 'bm (Bambara)'), ('bn-BD', 'bn-BD (Bengali (Bangladesh))'), ('bn-IN', 'bn-IN (Bengali (India))'), ('br', 'br (Breton)'), ('brx', 'brx (Bodo)'), ('bs', 'bs (Bosnian)'), ('ca', 'ca (Catalan)'), ('ca-valencia', 'ca-valencia (Catalan (Valencian))'), ('cak', 'cak (Kaqchikel)'), ('cs', 'cs (Czech)'), ('csb', 'csb (Kashubian)'), ('cy', 'cy (Welsh)'), ('da', 'da (Danish)'), ('dbg', 'dbg (Debug Robot)'), ('de', 'de (German)'), ('de-AT', 'de-AT (German (Austria))'), ('de-CH', 'de-CH (German (Switzerland))'), ('de-DE', 'de-DE (German (Germany))'), ('dsb', 'dsb (Lower Sorbian)'), ('ee', 'ee (Ewe)'), ('el', 'el (Greek)'), ('en-AU', 'en-AU (English (Australian))'), ('en-CA', 'en-CA (English (Canadian))'), ('en-GB', 'en-GB (English (British))'), ('en-NZ', 'en-NZ (English (New Zealand))'), ('en-US', 'en-US (English (US))'), ('en-ZA', 'en-ZA (English (South African))'), ('eo', 'eo (Esperanto)'), ('es', 'es (Spanish)'), ('es-AR', 'es-AR (Spanish (Argentina))'), ('es-CL', 'es-CL (Spanish (Chile))'), ('es-ES', 'es-ES (Spanish (Spain))'), ('es-MX', 'es-MX (Spanish (Mexico))'), ('et', 'et (Estonian)'), ('eu', 'eu (Basque)'), ('fa', 'fa (Persian)'), ('ff', 'ff (Fulah)'), ('fi', 'fi (Finnish)'), ('fj-FJ', 'fj-FJ (Fijian)'), ('fr', 'fr (French)'), ('fur-IT', 'fur-IT (Friulian)'), ('fy-NL', 'fy-NL (Frisian)'), ('ga', 'ga (Irish)'), ('ga-IE', 'ga-IE (Irish)'), ('gd', 'gd (Gaelic (Scotland))'), ('gl', 'gl (Galician)'), ('gn', 'gn (Guarani)'), ('gu', 'gu (Gujarati)'), ('gu-IN', 'gu-IN (Gujarati (India))'), ('ha', 'ha (Hausa)'), ('he', 'he (Hebrew)'), ('hi', 'hi (Hindi)'), ('hi-IN', 'hi-IN (Hindi (India))'), ('hr', 'hr (Croatian)'), ('hsb', 'hsb (Upper Sorbian)'), ('hu', 'hu (Hungarian)'), ('hy-AM', 'hy-AM (Armenian)'), ('id', 'id (Indonesian)'), ('ig', 'ig (Igbo)'), ('is', 'is (Icelandic)'), ('it', 'it (Italian)'), ('ja', 'ja (Japanese)'), ('ja-JP-mac', 'ja-JP-mac (Japanese)'), ('ka', 'ka (Georgian)'), ('kk', 'kk (Kazakh)'), ('km', 'km (Khmer)'), ('kn', 'kn (Kannada)'), ('ko', 'ko (Korean)'), ('kok', 'kok (Konkani)'), ('ks', 'ks (Kashmiri)'), ('ku', 'ku (Kurdish)'), ('la', 'la (Latin)'), ('lg', 'lg (Luganda)'), ('lij', 'lij (Ligurian)'), ('ln', 'ln (Lingala)'), ('lo', 'lo (Lao)'), ('lt', 'lt (Lithuanian)'), ('ltg', 'ltg (Latgalian)'), ('lv', 'lv (Latvian)'), ('mai', 'mai (Maithili)'), ('mg', 'mg (Malagasy)'), ('mi', 'mi (Maori (Aotearoa))'), ('mk', 'mk (Macedonian)'), ('ml', 'ml (Malayalam)'), ('mn', 'mn (Mongolian)'), ('mr', 'mr (Marathi)'), ('ms', 'ms (Malay)'), ('my', 'my (Burmese)'), ('nb-NO', 'nb-NO (Norwegian (Bokm\xe5l))'), ('ne-NP', 'ne-NP (Nepali)'), ('nl', 'nl (Dutch)'), ('nn-NO', 'nn-NO (Norwegian (Nynorsk))'), ('nr', 'nr (Ndebele, South)'), ('nso', 'nso (Northern Sotho)'), ('oc', 'oc (Occitan (Lengadocian))'), ('or', 'or (Oriya)'), ('pa', 'pa (Punjabi)'), ('pa-IN', 'pa-IN (Punjabi (India))'), ('pl', 'pl (Polish)'), ('pt-BR', 'pt-BR (Portuguese (Brazilian))'), ('pt-PT', 'pt-PT (Portuguese (Portugal))'), ('rm', 'rm (Romansh)'), ('ro', 'ro (Romanian)'), ('ru', 'ru (Russian)'), ('rw', 'rw (Kinyarwanda)'), ('sa', 'sa (Sanskrit)'), ('sah', 'sah (Sakha)'), ('sat', 'sat (Santali)'), ('si', 'si (Sinhala)'), ('sk', 'sk (Slovak)'), ('sl', 'sl (Slovenian)'), ('son', 'son (Songhai)'), ('sq', 'sq (Albanian)'), ('sr', 'sr (Serbian)'), ('sr-Cyrl', 'sr-Cyrl (Serbian)'), ('sr-Latn', 'sr-Latn (Serbian)'), ('ss', 'ss (Siswati)'), ('st', 'st (Southern Sotho)'), ('sv-SE', 'sv-SE (Swedish)'), ('sw', 'sw (Swahili)'), ('ta', 'ta (Tamil)'), ('ta-IN', 'ta-IN (Tamil (India))'), ('ta-LK', 'ta-LK (Tamil (Sri Lanka))'), ('te', 'te (Telugu)'), ('th', 'th (Thai)'), ('tl', 'tl (Tagalog)'), ('tn', 'tn (Tswana)'), ('tr', 'tr (Turkish)'), ('ts', 'ts (Tsonga)'), ('tsz', 'tsz (Pur\xe9pecha)'), ('tt-RU', 'tt-RU (Tatar)'), ('uk', 'uk (Ukrainian)'), ('ur', 'ur (Urdu)'), ('uz', 'uz (Uzbek)'), ('ve', 've (Venda)'), ('vi', 'vi (Vietnamese)'), ('wo', 'wo (Wolof)'), ('x-testing', 'x-testing (Testing)'), ('xh', 'xh (Xhosa)'), ('yo', 'yo (Yoruba)'), ('zh-CN', 'zh-CN (Chinese (Simplified))'), ('zh-TW', 'zh-TW (Chinese (Traditional))'), ('zu', 'zu (Zulu)')]), 30 | ), 31 | migrations.AlterField( 32 | model_name='queuedtask', 33 | name='args', 34 | field=jsonfield.fields.JSONField(default=list), 35 | ), 36 | migrations.AlterField( 37 | model_name='queuedtask', 38 | name='kwargs', 39 | field=jsonfield.fields.JSONField(default=dict), 40 | ), 41 | ] 42 | -------------------------------------------------------------------------------- /basket/news/country_codes.py: -------------------------------------------------------------------------------- 1 | country_codes_map = { 2 | 'afg': 'af', 3 | 'ala': 'ax', 4 | 'alb': 'al', 5 | 'dza': 'dz', 6 | 'asm': 'as', 7 | 'and': 'ad', 8 | 'ago': 'ao', 9 | 'aia': 'ai', 10 | 'ata': 'aq', 11 | 'atg': 'ag', 12 | 'arg': 'ar', 13 | 'arm': 'am', 14 | 'abw': 'aw', 15 | 'aus': 'au', 16 | 'aut': 'at', 17 | 'aze': 'az', 18 | 'bhs': 'bs', 19 | 'bhr': 'bh', 20 | 'bgd': 'bd', 21 | 'brb': 'bb', 22 | 'blr': 'by', 23 | 'bel': 'be', 24 | 'blz': 'bz', 25 | 'ben': 'bj', 26 | 'bmu': 'bm', 27 | 'btn': 'bt', 28 | 'bol': 'bo', 29 | 'bih': 'ba', 30 | 'bwa': 'bw', 31 | 'bvt': 'bv', 32 | 'bra': 'br', 33 | 'vgb': 'vg', 34 | 'iot': 'io', 35 | 'brn': 'bn', 36 | 'bgr': 'bg', 37 | 'bfa': 'bf', 38 | 'bdi': 'bi', 39 | 'khm': 'kh', 40 | 'cmr': 'cm', 41 | 'can': 'ca', 42 | 'cpv': 'cv', 43 | 'cym': 'ky', 44 | 'caf': 'cf', 45 | 'tcd': 'td', 46 | 'chl': 'cl', 47 | 'chn': 'cn', 48 | 'hkg': 'hk', 49 | 'mac': 'mo', 50 | 'cxr': 'cx', 51 | 'cck': 'cc', 52 | 'col': 'co', 53 | 'com': 'km', 54 | 'cog': 'cg', 55 | 'cod': 'cd', 56 | 'cok': 'ck', 57 | 'cri': 'cr', 58 | 'civ': 'ci', 59 | 'hrv': 'hr', 60 | 'cub': 'cu', 61 | 'cyp': 'cy', 62 | 'cze': 'cz', 63 | 'dnk': 'dk', 64 | 'dji': 'dj', 65 | 'dma': 'dm', 66 | 'dom': 'do', 67 | 'ecu': 'ec', 68 | 'egy': 'eg', 69 | 'slv': 'sv', 70 | 'gnq': 'gq', 71 | 'eri': 'er', 72 | 'est': 'ee', 73 | 'eth': 'et', 74 | 'flk': 'fk', 75 | 'fro': 'fo', 76 | 'fji': 'fj', 77 | 'fin': 'fi', 78 | 'fra': 'fr', 79 | 'guf': 'gf', 80 | 'pyf': 'pf', 81 | 'atf': 'tf', 82 | 'gab': 'ga', 83 | 'gmb': 'gm', 84 | 'geo': 'ge', 85 | 'deu': 'de', 86 | 'gha': 'gh', 87 | 'gib': 'gi', 88 | 'grc': 'gr', 89 | 'grl': 'gl', 90 | 'grd': 'gd', 91 | 'glp': 'gp', 92 | 'gum': 'gu', 93 | 'gtm': 'gt', 94 | 'ggy': 'gg', 95 | 'gin': 'gn', 96 | 'gnb': 'gw', 97 | 'guy': 'gy', 98 | 'hti': 'ht', 99 | 'hmd': 'hm', 100 | 'vat': 'va', 101 | 'hnd': 'hn', 102 | 'hun': 'hu', 103 | 'isl': 'is', 104 | 'ind': 'in', 105 | 'idn': 'id', 106 | 'irn': 'ir', 107 | 'irq': 'iq', 108 | 'irl': 'ie', 109 | 'imn': 'im', 110 | 'isr': 'il', 111 | 'ita': 'it', 112 | 'jam': 'jm', 113 | 'jpn': 'jp', 114 | 'jey': 'je', 115 | 'jor': 'jo', 116 | 'kaz': 'kz', 117 | 'ken': 'ke', 118 | 'kir': 'ki', 119 | 'prk': 'kp', 120 | 'kor': 'kr', 121 | 'kwt': 'kw', 122 | 'kgz': 'kg', 123 | 'lao': 'la', 124 | 'lva': 'lv', 125 | 'lbn': 'lb', 126 | 'lso': 'ls', 127 | 'lbr': 'lr', 128 | 'lby': 'ly', 129 | 'lie': 'li', 130 | 'ltu': 'lt', 131 | 'lux': 'lu', 132 | 'mkd': 'mk', 133 | 'mdg': 'mg', 134 | 'mwi': 'mw', 135 | 'mys': 'my', 136 | 'mdv': 'mv', 137 | 'mli': 'ml', 138 | 'mlt': 'mt', 139 | 'mhl': 'mh', 140 | 'mtq': 'mq', 141 | 'mrt': 'mr', 142 | 'mus': 'mu', 143 | 'myt': 'yt', 144 | 'mex': 'mx', 145 | 'fsm': 'fm', 146 | 'mda': 'md', 147 | 'mco': 'mc', 148 | 'mng': 'mn', 149 | 'mne': 'me', 150 | 'msr': 'ms', 151 | 'mar': 'ma', 152 | 'moz': 'mz', 153 | 'mmr': 'mm', 154 | 'nam': 'na', 155 | 'nru': 'nr', 156 | 'npl': 'np', 157 | 'nld': 'nl', 158 | 'ant': 'an', 159 | 'ncl': 'nc', 160 | 'nzl': 'nz', 161 | 'nic': 'ni', 162 | 'ner': 'ne', 163 | 'nga': 'ng', 164 | 'niu': 'nu', 165 | 'nfk': 'nf', 166 | 'mnp': 'mp', 167 | 'nor': 'no', 168 | 'omn': 'om', 169 | 'pak': 'pk', 170 | 'plw': 'pw', 171 | 'pse': 'ps', 172 | 'pan': 'pa', 173 | 'png': 'pg', 174 | 'pry': 'py', 175 | 'per': 'pe', 176 | 'phl': 'ph', 177 | 'pcn': 'pn', 178 | 'pol': 'pl', 179 | 'prt': 'pt', 180 | 'pri': 'pr', 181 | 'qat': 'qa', 182 | 'reu': 're', 183 | 'rou': 'ro', 184 | 'rus': 'ru', 185 | 'rwa': 'rw', 186 | 'blm': 'bl', 187 | 'shn': 'sh', 188 | 'kna': 'kn', 189 | 'lca': 'lc', 190 | 'maf': 'mf', 191 | 'spm': 'pm', 192 | 'vct': 'vc', 193 | 'wsm': 'ws', 194 | 'smr': 'sm', 195 | 'stp': 'st', 196 | 'sau': 'sa', 197 | 'sen': 'sn', 198 | 'srb': 'rs', 199 | 'syc': 'sc', 200 | 'sle': 'sl', 201 | 'sgp': 'sg', 202 | 'svk': 'sk', 203 | 'svn': 'si', 204 | 'slb': 'sb', 205 | 'som': 'so', 206 | 'zaf': 'za', 207 | 'sgs': 'gs', 208 | 'ssd': 'ss', 209 | 'esp': 'es', 210 | 'lka': 'lk', 211 | 'sdn': 'sd', 212 | 'sur': 'sr', 213 | 'sjm': 'sj', 214 | 'swz': 'sz', 215 | 'swe': 'se', 216 | 'che': 'ch', 217 | 'syr': 'sy', 218 | 'twn': 'tw', 219 | 'tjk': 'tj', 220 | 'tza': 'tz', 221 | 'tha': 'th', 222 | 'tls': 'tl', 223 | 'tgo': 'tg', 224 | 'tkl': 'tk', 225 | 'ton': 'to', 226 | 'tto': 'tt', 227 | 'tun': 'tn', 228 | 'tur': 'tr', 229 | 'tkm': 'tm', 230 | 'tca': 'tc', 231 | 'tuv': 'tv', 232 | 'uga': 'ug', 233 | 'ukr': 'ua', 234 | 'are': 'ae', 235 | 'gbr': 'gb', 236 | 'usa': 'us', 237 | 'umi': 'um', 238 | 'ury': 'uy', 239 | 'uzb': 'uz', 240 | 'vut': 'vu', 241 | 'ven': 've', 242 | 'vnm': 'vn', 243 | 'vir': 'vi', 244 | 'wlf': 'wf', 245 | 'esh': 'eh', 246 | 'yem': 'ye', 247 | 'zmb': 'zm', 248 | 'zwe': 'zw', 249 | } 250 | 251 | 252 | def convert_country_3_to_2(ccode): 253 | ccode = ccode.lower() 254 | return country_codes_map.get(ccode, None) 255 | -------------------------------------------------------------------------------- /basket/news/tests/test_users.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.core.urlresolvers import reverse 4 | from django.http import HttpResponse 5 | from django.test import TestCase 6 | from django.test.client import RequestFactory 7 | 8 | from mock import patch 9 | 10 | from basket import errors 11 | 12 | from basket.news import views 13 | from basket.news.backends.common import NewsletterException 14 | from basket.news.models import APIUser 15 | from basket.news.utils import SET, generate_token 16 | 17 | 18 | class UserTest(TestCase): 19 | def setUp(self): 20 | self.factory = RequestFactory() 21 | 22 | def test_user_set(self): 23 | """If the user view is sent a POST request, it should attempt to update 24 | the user's info. 25 | """ 26 | request = self.factory.post('/news/user/asdf/', {'fake': 'data'}) 27 | with patch.object(views, 'update_user_task') as update_user_task: 28 | update_user_task.return_value = HttpResponse() 29 | views.user(request, 'asdf') 30 | update_user_task.assert_called_with(request, SET, {'fake': 'data', 31 | 'token': 'asdf'}) 32 | 33 | @patch('basket.news.utils.sfdc') 34 | def test_user_not_in_sf(self, sfdc_mock): 35 | """A user not found in SFDC should produce an error response.""" 36 | sfdc_mock.get.side_effect = NewsletterException('DANGER!') 37 | token = generate_token() 38 | resp = self.client.get('/news/user/{}/'.format(token)) 39 | self.assertEqual(resp.status_code, 400) 40 | resp_data = json.loads(resp.content) 41 | self.assertDictEqual(resp_data, { 42 | 'status': 'error', 43 | 'desc': 'DANGER!', 44 | 'code': errors.BASKET_UNKNOWN_ERROR, 45 | }) 46 | 47 | 48 | class TestLookupUser(TestCase): 49 | """test for API lookup-user""" 50 | # Keep in mind that this API requires SSL. We make it look like an 51 | # SSL request by adding {'wsgi.url_scheme': 'https'} to the arguments 52 | # of the client.get 53 | 54 | def setUp(self): 55 | self.auth = APIUser.objects.create(name="test") 56 | self.user_data = {'status': 'ok'} 57 | self.url = reverse('lookup_user') 58 | 59 | def get(self, params=None, **extra): 60 | params = params or {} 61 | return self.client.get(self.url, data=params, **extra) 62 | 63 | def test_no_parms(self): 64 | """Passing no parms is a 400 error""" 65 | rsp = self.get() 66 | self.assertEqual(400, rsp.status_code, rsp.content) 67 | 68 | def test_both_parms(self): 69 | """Passing both parms is a 400 error""" 70 | params = { 71 | 'token': 'dummy', 72 | 'email': 'dummy@example.com', 73 | } 74 | rsp = self.get(params=params) 75 | self.assertEqual(400, rsp.status_code, rsp.content) 76 | 77 | @patch('basket.news.views.get_user_data') 78 | def test_with_token(self, get_user_data): 79 | """Passing a token gets back that user's data""" 80 | get_user_data.return_value = self.user_data 81 | params = { 82 | 'token': 'dummy', 83 | } 84 | rsp = self.get(params=params) 85 | self.assertEqual(200, rsp.status_code, rsp.content) 86 | self.assertEqual(self.user_data, json.loads(rsp.content)) 87 | 88 | def test_with_email_no_api_key(self): 89 | """Passing email without api key is a 401""" 90 | params = { 91 | 'email': 'mail@example.com', 92 | } 93 | rsp = self.get(params) 94 | self.assertEqual(401, rsp.status_code, rsp.content) 95 | 96 | def test_with_email_disabled_auth(self): 97 | """Passing email with a disabled api key is a 401""" 98 | self.auth.enabled = False 99 | self.auth.save() 100 | params = { 101 | 'email': 'mail@example.com', 102 | 'api-key': self.auth.api_key, 103 | } 104 | rsp = self.get(params) 105 | self.assertEqual(401, rsp.status_code, rsp.content) 106 | 107 | def test_with_email_bad_auth(self): 108 | """Passing email with bad api key is a 401""" 109 | params = { 110 | 'email': 'mail@example.com', 111 | 'api-key': 'BAD KEY', 112 | } 113 | rsp = self.get(params) 114 | self.assertEqual(401, rsp.status_code, rsp.content) 115 | 116 | @patch('basket.news.views.get_user_data') 117 | def test_with_email_and_auth_parm(self, get_user_data): 118 | """Passing email and valid api key parm gets user's data""" 119 | params = { 120 | 'email': 'mail@example.com', 121 | 'api-key': self.auth.api_key, 122 | } 123 | get_user_data.return_value = self.user_data 124 | rsp = self.get(params) 125 | self.assertEqual(200, rsp.status_code, rsp.content) 126 | self.assertEqual(self.user_data, json.loads(rsp.content)) 127 | 128 | @patch('basket.news.views.get_user_data') 129 | def test_with_email_and_auth_header(self, get_user_data): 130 | """Passing email and valid api key header gets user's data""" 131 | params = { 132 | 'email': 'mail@example.com', 133 | } 134 | get_user_data.return_value = self.user_data 135 | rsp = self.get(params, HTTP_X_API_KEY=self.auth.api_key) 136 | self.assertEqual(200, rsp.status_code, rsp.content) 137 | self.assertEqual(self.user_data, json.loads(rsp.content)) 138 | 139 | @patch('basket.news.views.get_user_data') 140 | def test_no_user(self, get_user_data): 141 | """If no such user, returns 404""" 142 | get_user_data.return_value = None 143 | params = { 144 | 'email': 'mail@example.com', 145 | 'api-key': self.auth.api_key, 146 | } 147 | rsp = self.get(params) 148 | self.assertEqual(404, rsp.status_code, rsp.content) 149 | -------------------------------------------------------------------------------- /basket/news/fixtures/newsletters.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "pk": 1, 4 | "model": "news.newsletter", 5 | "fields": { 6 | "vendor_id": "MOZILLA_AND_YOU", 7 | "description": "A monthly newsletter packed with tips to improve your browsing experience.", 8 | "show": true, 9 | "welcome": "", 10 | "languages": "de,en,es,fr,id,pt-BR,ru", 11 | "active": true, 12 | "title": "Firefox & You", 13 | "slug": "mozilla-and-you" 14 | } 15 | }, 16 | { 17 | "pk": 2, 18 | "model": "news.newsletter", 19 | "fields": { 20 | "vendor_id": "ABOUT_MOBILE", 21 | "description": "Get the power of Firefox in the palm of your hand.", 22 | "show": true, 23 | "welcome": "", 24 | "languages": "de,en,es,fr,id,pt-BR,ru", 25 | "active": true, 26 | "title": "Firefox for Android", 27 | "slug": "mobile" 28 | } 29 | }, 30 | { 31 | "pk": 3, 32 | "model": "news.newsletter", 33 | "fields": { 34 | "vendor_id": "FIREFOX_TIPS", 35 | "description": "Get a weekly tip on how to super-charge your Firefox experience.", 36 | "show": false, 37 | "welcome": "", 38 | "languages": "en", 39 | "active": false, 40 | "title": "Firefox Weekly Tips", 41 | "slug": "firefox-tips" 42 | } 43 | }, 44 | { 45 | "pk": 5, 46 | "model": "news.newsletter", 47 | "fields": { 48 | "vendor_id": "AURORA", 49 | "description": "Aurora", 50 | "show": false, 51 | "welcome": "", 52 | "languages": "en", 53 | "active": false, 54 | "title": "Aurora", 55 | "slug": "aurora" 56 | } 57 | }, 58 | { 59 | "pk": 6, 60 | "model": "news.newsletter", 61 | "fields": { 62 | "vendor_id": "ABOUT_MOZILLA", 63 | "description": "News from the Mozilla Project", 64 | "show": false, 65 | "welcome": "", 66 | "languages": "en", 67 | "active": true, 68 | "title": "About Mozilla", 69 | "slug": "about-mozilla" 70 | } 71 | }, 72 | { 73 | "pk": 19, 74 | "model": "news.newsletter", 75 | "fields": { 76 | "vendor_id": "FLICKS", 77 | "description": "", 78 | "show": true, 79 | "welcome": "Firefox_Flicks_Welcome", 80 | "languages": "en", 81 | "active": true, 82 | "title": "Firefox Flicks", 83 | "slug": "firefox-flicks" 84 | } 85 | }, 86 | { 87 | "pk": 18, 88 | "model": "news.newsletter", 89 | "fields": { 90 | "vendor_id": "AFFILIATES", 91 | "description": "", 92 | "show": false, 93 | "welcome": "", 94 | "languages": "en", 95 | "active": false, 96 | "title": "Firefox Affiliates", 97 | "slug": "affiliates" 98 | } 99 | }, 100 | { 101 | "pk": 17, 102 | "model": "news.newsletter", 103 | "fields": { 104 | "vendor_id": "APP_DEV", 105 | "description": "", 106 | "show": false, 107 | "welcome": "", 108 | "languages": "en", 109 | "active": false, 110 | "title": "Firefox Apps & Hacks", 111 | "slug": "app-dev" 112 | } 113 | }, 114 | { 115 | "pk": 16, 116 | "model": "news.newsletter", 117 | "fields": { 118 | "vendor_id": "MOZILLA_PHONE", 119 | "description": "", 120 | "show": false, 121 | "welcome": "", 122 | "languages": "en", 123 | "active": true, 124 | "title": "Mozillians", 125 | "slug": "mozilla-phone" 126 | } 127 | }, 128 | { 129 | "pk": 15, 130 | "model": "news.newsletter", 131 | "fields": { 132 | "vendor_id": "JOIN_MOZILLA", 133 | "description": "", 134 | "show": false, 135 | "welcome": "", 136 | "languages": "en", 137 | "active": false, 138 | "title": "Join Mozilla", 139 | "slug": "join-mozilla" 140 | } 141 | }, 142 | { 143 | "pk": 14, 144 | "model": "news.newsletter", 145 | "fields": { 146 | "vendor_id": "ADD_ONS", 147 | "description": "", 148 | "show": false, 149 | "welcome": "", 150 | "languages": "en", 151 | "active": false, 152 | "title": "Addon Development", 153 | "slug": "addon-dev" 154 | } 155 | }, 156 | { 157 | "pk": 7, 158 | "model": "news.newsletter", 159 | "fields": { 160 | "vendor_id": "DRUMBEAT_NEWS_GROUP", 161 | "description": "", 162 | "show": false, 163 | "welcome": "", 164 | "languages": "en", 165 | "active": false, 166 | "title": "Drumbeat Newsgroup", 167 | "slug": "drumbeat" 168 | } 169 | }, 170 | { 171 | "pk": 8, 172 | "model": "news.newsletter", 173 | "fields": { 174 | "vendor_id": "ABOUT_ADDONS", 175 | "description": "", 176 | "show": false, 177 | "welcome": "", 178 | "languages": "en", 179 | "active": false, 180 | "title": "About Addons", 181 | "slug": "addons" 182 | } 183 | }, 184 | { 185 | "pk": 9, 186 | "model": "news.newsletter", 187 | "fields": { 188 | "vendor_id": "ABOUT_LABS", 189 | "description": "", 190 | "show": false, 191 | "welcome": "", 192 | "languages": "en", 193 | "active": false, 194 | "title": "About Labs", 195 | "slug": "labs" 196 | } 197 | }, 198 | { 199 | "pk": 12, 200 | "model": "news.newsletter", 201 | "fields": { 202 | "vendor_id": "ABOUT_STANDARDS", 203 | "description": "", 204 | "show": false, 205 | "welcome": "", 206 | "languages": "en", 207 | "active": false, 208 | "title": "About Standards", 209 | "slug": "about-standards" 210 | } 211 | }, 212 | { 213 | "pk": 4, 214 | "model": "news.newsletter", 215 | "fields": { 216 | "vendor_id": "FIREFOX_BETA_NEWS", 217 | "description": "Read about the latest features for Firefox desktop and mobile before the final release.", 218 | "show": false, 219 | "welcome": "", 220 | "languages": "en", 221 | "active": true, 222 | "title": "Beta News", 223 | "slug": "beta" 224 | } 225 | }, 226 | { 227 | "pk": 20, 228 | "model": "news.newsletter", 229 | "fields": { 230 | "vendor_id": "STUDENT_AMBASSADORS", 231 | "description": "", 232 | "show": false, 233 | "welcome": "", 234 | "languages": "en", 235 | "active": true, 236 | "title": "Student Ambassadors", 237 | "slug": "student-ambassadors" 238 | } 239 | } 240 | ] 241 | -------------------------------------------------------------------------------- /basket/news/tests/test_backends_sfmc.py: -------------------------------------------------------------------------------- 1 | from django.core.cache import cache 2 | from django.test import TestCase 3 | 4 | from mock import patch, call, Mock 5 | 6 | from basket.news.backends.common import NewsletterException 7 | from basket.news.backends import sfmc 8 | 9 | 10 | @patch('basket.news.backends.sfmc.requests') 11 | @patch.object(sfmc.sfmc, '_client', Mock()) 12 | class TestSendSMS(TestCase): 13 | def test_single_phone_number(self, req_mock): 14 | req_mock.post.return_value.status_code = 200 15 | sfmc.sfmc.send_sms('+1234567890', 'dude') 16 | url = sfmc.sfmc.sms_api_url.format('dude') 17 | req_mock.post.assert_called_with(url, json={ 18 | 'mobileNumbers': ['1234567890'], 19 | 'Subscribe': True, 20 | 'Resubscribe': True, 21 | 'keyword': 'FFDROID', 22 | }, headers=sfmc.sfmc.auth_header, timeout=10) 23 | 24 | def test_multiple_phone_numbers(self, req_mock): 25 | req_mock.post.return_value.status_code = 200 26 | sfmc.sfmc.send_sms(['+1234567890', '+9876543210'], 'dude') 27 | url = sfmc.sfmc.sms_api_url.format('dude') 28 | req_mock.post.assert_called_with(url, json={ 29 | 'mobileNumbers': ['1234567890', '9876543210'], 30 | 'Subscribe': True, 31 | 'Resubscribe': True, 32 | 'keyword': 'FFDROID', 33 | }, headers=sfmc.sfmc.auth_header, timeout=10) 34 | 35 | 36 | @patch('basket.news.backends.sfmc.time', Mock(return_value=600)) 37 | @patch.object(sfmc.ETRefreshClient, 'build_soap_client', Mock()) 38 | @patch.object(sfmc.ETRefreshClient, 'load_wsdl', Mock()) 39 | @patch.object(sfmc.ETRefreshClient, 'request_token') 40 | @patch.object(sfmc.ETRefreshClient, 'refresh_auth_tokens_from_cache') 41 | @patch.object(sfmc.ETRefreshClient, 'cache_auth_tokens') 42 | @patch.object(sfmc.ETRefreshClient, 'token_is_expired') 43 | class TestRefreshToken(TestCase): 44 | def test_refresh_token(self, exp_mock, cache_mock, refresh_mock, request_mock): 45 | request_mock.return_value = { 46 | 'accessToken': 'good-token', 47 | 'expiresIn': 9000, 48 | 'legacyToken': 'internal-token', 49 | 'refreshToken': 'refresh-key', 50 | } 51 | client = sfmc.ETRefreshClient(params={'clientid': 'id', 'clientsecret': 'sssshhhh'}) 52 | self.assertTrue(refresh_mock.called) 53 | self.assertFalse(exp_mock.called) 54 | self.assertTrue(cache_mock.called) 55 | self.assertEqual(client.authToken, 'good-token') 56 | self.assertEqual(client.authTokenExpiration, 9600) # because of time mock 57 | self.assertEqual(client.internalAuthToken, 'internal-token') 58 | self.assertEqual(client.refreshKey, 'refresh-key') 59 | 60 | 61 | @patch.object(sfmc.ETRefreshClient, 'load_wsdl', Mock()) 62 | @patch.object(sfmc.ETRefreshClient, 'build_soap_client', Mock()) 63 | @patch.object(sfmc.ETRefreshClient, 'refresh_token', Mock()) 64 | @patch('basket.news.backends.sfmc.cache') 65 | class TestCacheTokens(TestCase): 66 | def test_cache_auth_tokens(self, cache_mock): 67 | client = sfmc.ETRefreshClient() 68 | client.authToken = 'good-token' 69 | client._old_authToken = 'old-token' 70 | client.authTokenExpiration = 9600 71 | client.authTokenExpiresIn = 100 72 | client.internalAuthToken = 'internal-token' 73 | client.refreshKey = 'refresh-key' 74 | client.cache_auth_tokens() 75 | cache_mock.set.assert_called_once_with(client.token_cache_key, { 76 | 'authToken': 'good-token', 77 | 'authTokenExpiration': 9600, 78 | 'internalAuthToken': 'internal-token', 79 | 'refreshKey': 'refresh-key', 80 | }, 700) 81 | 82 | def test_cache_auth_tokens_skip(self, cache_mock): 83 | """Should skip setting cache when token is good""" 84 | client = sfmc.ETRefreshClient() 85 | client.authToken = 'good-token' 86 | client._old_authToken = client.authToken 87 | self.assertFalse(cache_mock.set.called) 88 | 89 | def test_refresh_auth_tokens_from_cache(self, cache_mock): 90 | client = sfmc.ETRefreshClient() 91 | client.authToken = None 92 | cache_mock.get.return_value = { 93 | 'authToken': 'good-token', 94 | 'authTokenExpiration': 9600, 95 | 'internalAuthToken': 'internal-token', 96 | 'refreshKey': 'refresh-key', 97 | } 98 | client.refresh_auth_tokens_from_cache() 99 | self.assertEqual(client.authToken, 'good-token') 100 | self.assertEqual(client.authTokenExpiration, 9600) 101 | self.assertEqual(client.internalAuthToken, 'internal-token') 102 | self.assertEqual(client.refreshKey, 'refresh-key') 103 | 104 | 105 | @patch.object(sfmc.ETRefreshClient, 'load_wsdl', Mock()) 106 | @patch.object(sfmc.ETRefreshClient, 'refresh_token', Mock()) 107 | @patch('basket.news.backends.sfmc.time') 108 | @patch('basket.news.backends.sfmc.randint') 109 | class TestTokenIsExpired(TestCase): 110 | def test_token_is_expired(self, randint_mock, time_mock): 111 | client = sfmc.ETRefreshClient() 112 | client.authTokenExpiration = None 113 | self.assertTrue(client.token_is_expired()) 114 | 115 | client.authTokenExpiration = 1000 116 | time_mock.return_value = 900 117 | randint_mock.return_value = 100 118 | self.assertTrue(client.token_is_expired()) 119 | 120 | time_mock.return_value = 100 121 | self.assertFalse(client.token_is_expired()) 122 | 123 | 124 | @patch.object(sfmc.ETRefreshClient, 'load_wsdl', Mock()) 125 | @patch.object(sfmc.ETRefreshClient, 'refresh_token', Mock()) 126 | @patch('basket.news.backends.sfmc.requests') 127 | class TestRequestToken(TestCase): 128 | def setUp(self): 129 | cache.clear() 130 | 131 | def test_request_token_success(self, req_mock): 132 | client = sfmc.ETRefreshClient() 133 | req_mock.post.return_value.json.return_value = {'accessToken': 'good-token'} 134 | payload = {'refreshToken': 'token'} 135 | client.request_token(payload) 136 | # called once when first call is successful 137 | req_mock.post.assert_called_once_with(client.auth_url, json=payload) 138 | 139 | def test_request_token_first_fail(self, req_mock): 140 | """ 141 | If first call fails it should try again without refreshToken 142 | """ 143 | client = sfmc.ETRefreshClient() 144 | req_mock.post.return_value.json.side_effect = [{}, {'accessToken': 'good-token'}] 145 | payload = {'refreshToken': 'token'} 146 | client.request_token(payload) 147 | # payload should be modified 148 | self.assertEqual(req_mock.post.call_count, 2) 149 | req_mock.post.assert_has_calls([ 150 | call(client.auth_url, json={'refreshToken': 'token'}), 151 | call().json(), 152 | call(client.auth_url, json={}), 153 | call().json(), 154 | ]) 155 | 156 | def test_request_token_both_fail(self, req_mock): 157 | """If both calls fail it should raise an exception""" 158 | client = sfmc.ETRefreshClient() 159 | req_mock.post.return_value.json.return_value = {} 160 | payload = {'refreshToken': 'token'} 161 | with self.assertRaises(NewsletterException): 162 | client.request_token(payload) 163 | 164 | self.assertEqual(req_mock.post.call_count, 2) 165 | -------------------------------------------------------------------------------- /basket/news/newsletters.py: -------------------------------------------------------------------------------- 1 | """This data provides an official list of newsletters and tracks 2 | backend-specific data for working with them in the email provider. 3 | 4 | It's used to lookup the backend-specific newsletter name from a 5 | generic one passed by the user. This decouples the API from any 6 | specific email provider.""" 7 | from django.db.models.signals import post_save 8 | from django.db.models.signals import post_delete 9 | from django.core.cache import cache 10 | 11 | from basket.news.models import Newsletter, NewsletterGroup, LocalizedSMSMessage, TransactionalEmailMessage 12 | 13 | 14 | __all__ = ('clear_newsletter_cache', 'get_sms_messages', 'newsletter_field', 15 | 'newsletter_name', 'newsletter_fields') 16 | 17 | 18 | CACHE_KEY = 'newsletters_cache_data' 19 | SMS_CACHE_KEY = 'local_sms_messages_cache_data' 20 | TRANSACTIONAL_CACHE_KEY = 'transactional_messages_cache_data' 21 | 22 | 23 | def get_transactional_message_ids(): 24 | """ 25 | Returns a list of transactional message IDs that basket clients send. 26 | """ 27 | data = cache.get(TRANSACTIONAL_CACHE_KEY) 28 | if data is None: 29 | data = [tx.message_id for tx in TransactionalEmailMessage.objects.all()] 30 | cache.set(TRANSACTIONAL_CACHE_KEY, data) 31 | 32 | return data 33 | 34 | 35 | def get_sms_messages(): 36 | """ 37 | Returns a dict for which the keys are SMS message IDs that 38 | basket clients will send, and the values are the message IDs 39 | that our SMS vendor expects. 40 | """ 41 | data = cache.get(SMS_CACHE_KEY) 42 | if data is None: 43 | data = {} 44 | for msg in LocalizedSMSMessage.objects.all(): 45 | data[msg.slug] = msg.vendor_id 46 | 47 | cache.set(SMS_CACHE_KEY, data) 48 | 49 | return data 50 | 51 | 52 | def get_sms_vendor_id(message_id, country='us', language='en-US'): 53 | all_msgs = get_sms_messages() 54 | full_msg_id = LocalizedSMSMessage.make_slug(message_id, country, language) 55 | return all_msgs.get(full_msg_id) 56 | 57 | 58 | def _newsletters(): 59 | """Returns a data structure with the data about newsletters. 60 | It's cached until clear_newsletter_cache() is called, so we're 61 | not constantly hitting the database for data that rarely changes. 62 | 63 | The returned data structure looks like:: 64 | 65 | { 66 | 'by_name': { 67 | 'newsletter_name_1': a Newsletter object, 68 | 'newsletter_name_2': another Newsletter object, 69 | }, 70 | 'by_vendor_id': { 71 | 'NEWSLETTER_ID_1': a Newsletter object, 72 | 'NEWSLETTER_ID_2': another Newsletter object, 73 | }, 74 | 'groups': { 75 | 'group_slug': a list of newsletter slugs, 76 | ... 77 | } 78 | } 79 | """ 80 | data = cache.get(CACHE_KEY) 81 | if data is None: 82 | data = _get_newsletters_data() 83 | data['groups'] = _get_newsletter_groups_data() 84 | cache.set(CACHE_KEY, data) 85 | 86 | return data 87 | 88 | 89 | def _get_newsletter_groups_data(): 90 | groups = NewsletterGroup.objects.filter(active=True) 91 | return dict((nlg.slug, nlg.newsletter_slugs()) for nlg in groups) 92 | 93 | 94 | def _get_newsletters_data(): 95 | by_name = {} 96 | by_vendor_id = {} 97 | for nl in Newsletter.objects.all(): 98 | by_name[nl.slug] = nl 99 | by_vendor_id[nl.vendor_id] = nl 100 | 101 | return { 102 | 'by_name': by_name, 103 | 'by_vendor_id': by_vendor_id, 104 | } 105 | 106 | 107 | def newsletter_map(): 108 | by_name = _newsletters()['by_name'] 109 | return {name: nl.vendor_id for name, nl in by_name.iteritems()} 110 | 111 | 112 | def newsletter_inv_map(): 113 | return {v: k for k, v in newsletter_map().iteritems()} 114 | 115 | 116 | def newsletter_field(name): 117 | """Lookup the backend-specific field (vendor ID) for the newsletter""" 118 | try: 119 | return _newsletters()['by_name'][name].vendor_id 120 | except KeyError: 121 | return None 122 | 123 | 124 | def newsletter_name(field): 125 | """Lookup the generic name for this newsletter field""" 126 | try: 127 | return _newsletters()['by_vendor_id'][field].slug 128 | except KeyError: 129 | return None 130 | 131 | 132 | def newsletter_group_newsletter_slugs(name): 133 | """Return the newsletter slugs associated with a group.""" 134 | try: 135 | return _newsletters()['groups'][name] 136 | except KeyError: 137 | return None 138 | 139 | 140 | def newsletter_slugs(): 141 | """ 142 | Get a list of all the available newsletters. 143 | Returns a list of their slugs. 144 | """ 145 | return _newsletters()['by_name'].keys() 146 | 147 | 148 | def newsletter_group_slugs(): 149 | """ 150 | Get a list of all the available newsletter groups. 151 | Returns a list of their slugs. 152 | """ 153 | # using get() in case old format cached 154 | return _newsletters().get('groups', {}).keys() 155 | 156 | 157 | def newsletter_and_group_slugs(): 158 | """Return a list of all newsletter and group slugs.""" 159 | return list(set(newsletter_slugs()) | set(newsletter_group_slugs())) 160 | 161 | 162 | def newsletter_private_slugs(): 163 | """Return a list of private newsletter ids""" 164 | return [nl.slug for nl in _newsletters()['by_name'].values() if nl.private] 165 | 166 | 167 | def newsletter_inactive_slugs(): 168 | return [nl.slug for nl in _newsletters()['by_name'].values() if not nl.active] 169 | 170 | 171 | def slug_to_vendor_id(slug): 172 | """Given a newsletter's slug, return its vendor_id""" 173 | return _newsletters()['by_name'][slug].vendor_id 174 | 175 | 176 | def newsletter_fields(): 177 | """Get a list of all the newsletter backend-specific fields""" 178 | return _newsletters()['by_vendor_id'].keys() 179 | 180 | 181 | def newsletter_languages(): 182 | """ 183 | Return a set of the 2 or 5 char codes of all the languages 184 | supported by newsletters. 185 | """ 186 | lang_set = set() 187 | for newsletter in _newsletters()['by_name'].values(): 188 | lang_set |= set(newsletter.language_list) 189 | return lang_set 190 | 191 | 192 | def newsletter_field_choices(): 193 | """ 194 | Return a list of 2 tuples of newsletter slugs suitable for use in a Django form field. 195 | """ 196 | all_newsletters = newsletter_and_group_slugs() + get_transactional_message_ids() 197 | return [(slug, slug) for slug in all_newsletters] 198 | 199 | 200 | def is_supported_newsletter_language(code): 201 | """ 202 | Return True if the given language code is supported by any of the 203 | newsletters. (Only compares first two chars; case-insensitive.) 204 | """ 205 | return code[:2].lower() in [lang[:2].lower() for lang in newsletter_languages()] 206 | 207 | 208 | def clear_newsletter_cache(*args, **kwargs): 209 | cache.delete(CACHE_KEY) 210 | 211 | 212 | def clear_sms_cache(*args, **kwargs): 213 | cache.delete(SMS_CACHE_KEY) 214 | 215 | 216 | post_save.connect(clear_newsletter_cache, sender=Newsletter) 217 | post_delete.connect(clear_newsletter_cache, sender=Newsletter) 218 | post_save.connect(clear_newsletter_cache, sender=NewsletterGroup) 219 | post_delete.connect(clear_newsletter_cache, sender=NewsletterGroup) 220 | post_save.connect(clear_sms_cache, sender=LocalizedSMSMessage) 221 | post_delete.connect(clear_sms_cache, sender=LocalizedSMSMessage) 222 | -------------------------------------------------------------------------------- /basket/news/management/commands/process_fxa_data.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, unicode_literals 2 | 3 | from email.utils import formatdate 4 | from multiprocessing.dummy import Pool as ThreadPool 5 | from time import time 6 | 7 | from django.conf import settings 8 | from django.core.cache import cache 9 | from django.core.management import BaseCommand, CommandError 10 | 11 | import boto3 12 | import requests 13 | from apscheduler.schedulers.blocking import BlockingScheduler 14 | from django_statsd.clients import statsd 15 | from pathlib2 import Path 16 | from pytz import utc 17 | from raven.contrib.django.raven_compat.models import client as sentry_client 18 | 19 | from basket.news.backends.sfmc import sfmc 20 | 21 | 22 | TMP = Path('/tmp') 23 | BUCKET_DIR = 'fxa-last-active-timestamp/data' 24 | DATA_PATH = TMP.joinpath(BUCKET_DIR) 25 | FXA_IDS = {} 26 | FILE_DONE_KEY = 'fxa_activity:completed:%s' 27 | FILES_IN_PROCESS = [] 28 | TWO_WEEKS = 60 * 60 * 24 * 14 29 | UPDATE_COUNT = 0 30 | schedule = BlockingScheduler(timezone=utc) 31 | 32 | 33 | def log(message): 34 | print('process_fxa_data: %s' % message) 35 | 36 | 37 | def _fxa_id_key(fxa_id): 38 | return 'fxa_activity:%s' % fxa_id 39 | 40 | 41 | def get_fxa_time(fxa_id): 42 | fxatime = FXA_IDS.get(fxa_id) 43 | if fxatime is None: 44 | fxatime = cache.get(_fxa_id_key(fxa_id)) 45 | if fxatime: 46 | FXA_IDS[fxa_id] = fxatime 47 | 48 | return fxatime or 0 49 | 50 | 51 | def file_is_done(pathobj): 52 | is_done = bool(cache.get(FILE_DONE_KEY % pathobj.name)) 53 | if is_done: 54 | log('%s is already done' % pathobj.name) 55 | 56 | return is_done 57 | 58 | 59 | def set_file_done(pathobj): 60 | # cache done state for 2 weeks. files stay in s3 bucket for 1 week 61 | cache.set(FILE_DONE_KEY % pathobj.name, 1, timeout=TWO_WEEKS) 62 | log('set %s as done' % pathobj.name) 63 | 64 | 65 | def set_in_process_files_done(): 66 | for i in range(len(FILES_IN_PROCESS)): 67 | set_file_done(FILES_IN_PROCESS.pop()) 68 | 69 | 70 | def set_timestamps_done(timestamp_chunk): 71 | global UPDATE_COUNT 72 | for fxaid, timestamp in timestamp_chunk: 73 | FXA_IDS[fxaid] = timestamp 74 | cache.set(_fxa_id_key(fxaid), timestamp, timeout=TWO_WEEKS) 75 | UPDATE_COUNT += 1 76 | # print progress every 1,000,000 77 | if UPDATE_COUNT % 1000000 == 0: 78 | log('updated %s records' % UPDATE_COUNT) 79 | 80 | 81 | def update_fxa_records(timestamp_chunk): 82 | formatted_chunk = [format_data_for_sfmc(*vals) for vals in timestamp_chunk] 83 | try: 84 | sfmc.bulk_upsert_rows(settings.FXA_SFMC_DE, formatted_chunk) 85 | except Exception as e: 86 | log('error updating chunk: %r' % e) 87 | sentry_client.captureException() 88 | # try again later 89 | return 90 | 91 | set_timestamps_done(timestamp_chunk) 92 | 93 | 94 | def format_data_for_sfmc(fxaid, timestamp): 95 | return { 96 | 'keys': { 97 | 'FXA_ID': fxaid, 98 | }, 99 | 'values': { 100 | 'Timestamp': formatdate(timeval=timestamp, usegmt=True), 101 | }, 102 | } 103 | 104 | 105 | def chunk_fxa_data(current_timestamps, chunk_size=1000): 106 | count = 0 107 | chunk = [] 108 | for fxaid, timestamp in current_timestamps.iteritems(): 109 | curr_ts = get_fxa_time(fxaid) 110 | if timestamp > curr_ts: 111 | chunk.append((fxaid, timestamp)) 112 | count += 1 113 | if count == chunk_size: 114 | yield chunk 115 | chunk = [] 116 | count = 0 117 | 118 | if chunk: 119 | yield chunk 120 | 121 | 122 | def update_fxa_data(current_timestamps): 123 | """Store the updated timestamps in a local dict, the cache, and SFMC.""" 124 | global UPDATE_COUNT 125 | UPDATE_COUNT = 0 126 | total_count = len(current_timestamps) 127 | log('attempting to update %s fxa timestamps' % total_count) 128 | pool = ThreadPool(8) 129 | pool.map(update_fxa_records, chunk_fxa_data(current_timestamps)) 130 | pool.close() 131 | pool.join() 132 | log('updated %s fxa timestamps' % UPDATE_COUNT) 133 | set_in_process_files_done() 134 | statsd.gauge('process_fxa_data.updates', UPDATE_COUNT) 135 | 136 | 137 | def download_fxa_files(): 138 | s3 = boto3.resource('s3', 139 | aws_access_key_id=settings.FXA_ACCESS_KEY_ID, 140 | aws_secret_access_key=settings.FXA_SECRET_ACCESS_KEY) 141 | bucket = s3.Bucket(settings.FXA_S3_BUCKET) 142 | for obj in bucket.objects.filter(Prefix=BUCKET_DIR): 143 | log('found %s in s3 bucket' % obj.key) 144 | tmp_path = TMP.joinpath(obj.key) 145 | if not tmp_path.name.endswith('.csv'): 146 | continue 147 | 148 | if file_is_done(tmp_path): 149 | continue 150 | 151 | if not tmp_path.exists(): 152 | log('getting ' + obj.key) 153 | log('size is %s' % obj.size) 154 | tmp_path.parent.mkdir(parents=True, exist_ok=True) 155 | try: 156 | bucket.download_file(obj.key, str(tmp_path)) 157 | log('downloaded %s' % tmp_path) 158 | except Exception: 159 | # something went wrong, delete file 160 | log('bad things happened. deleting %s' % tmp_path) 161 | tmp_path.unlink() 162 | 163 | 164 | def get_fxa_data(): 165 | all_fxa_times = {} 166 | data_files = DATA_PATH.glob('*.csv') 167 | for tmp_path in sorted(data_files): 168 | if file_is_done(tmp_path): 169 | continue 170 | 171 | log('loading data from %s' % tmp_path) 172 | # collect all of the latest timestamps from all files in a dict first 173 | # to ensure that we have the minimum data set to compare against SFMC 174 | with tmp_path.open() as fxafile: 175 | file_count = 0 176 | for line in fxafile: 177 | file_count += 1 178 | fxaid, timestamp = line.strip().split(',') 179 | curr_ts = all_fxa_times.get(fxaid, 0) 180 | timestamp = int(timestamp) 181 | if timestamp > curr_ts: 182 | all_fxa_times[fxaid] = timestamp 183 | 184 | if file_count < 1000000: 185 | # if there were fewer than 1M rows we probably got a truncated file 186 | # try again later (typically they contain 20M) 187 | log('possibly truncated file: %s' % tmp_path) 188 | else: 189 | FILES_IN_PROCESS.append(tmp_path) 190 | 191 | # done with file either way 192 | tmp_path.unlink() 193 | 194 | return all_fxa_times 195 | 196 | 197 | @schedule.scheduled_job('interval', id='process_fxa_data', days=1, max_instances=1) 198 | def main(): 199 | start_time = time() 200 | download_fxa_files() 201 | update_fxa_data(get_fxa_data()) 202 | total_time = time() - start_time 203 | message = 'finished in %s seconds' % int(total_time) 204 | log(message) 205 | if settings.FXA_SNITCH_URL: 206 | try: 207 | requests.post(settings.FXA_SNITCH_URL, data={ 208 | 'm': message, 209 | }) 210 | except requests.RequestException: 211 | pass 212 | 213 | 214 | class Command(BaseCommand): 215 | def add_arguments(self, parser): 216 | parser.add_argument('--cron', action='store_true', default=False, 217 | help='Run the cron schedule instead of just once') 218 | 219 | def handle(self, *args, **options): 220 | if not all(getattr(settings, name) for name in ['FXA_ACCESS_KEY_ID', 221 | 'FXA_SECRET_ACCESS_KEY', 222 | 'FXA_S3_BUCKET']): 223 | raise CommandError('FXA S3 Bucket access not configured') 224 | 225 | main() 226 | if options['cron']: 227 | log('cron schedule starting') 228 | schedule.start() 229 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # playdoh documentation build configuration file, created by 4 | # sphinx-quickstart on Tue Jan 4 15:11:09 2011. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | #sys.path.insert(0, os.path.abspath('.')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # If your documentation needs a minimal Sphinx version, state it here. 24 | #needs_sphinx = '1.0' 25 | 26 | # Add any Sphinx extension module names here, as strings. They can be extensions 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 28 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage'] 29 | 30 | # Add any paths that contain templates here, relative to this directory. 31 | templates_path = ['_templates'] 32 | 33 | # The suffix of source filenames. 34 | source_suffix = '.rst' 35 | 36 | # The encoding of source files. 37 | #source_encoding = 'utf-8-sig' 38 | 39 | # The master toctree document. 40 | master_doc = 'index' 41 | 42 | # General information about the project. 43 | project = u'basket.mozilla.org' 44 | copyright = u'2014, Mozilla' 45 | 46 | # The version info for the project you're documenting, acts as replacement for 47 | # |version| and |release|, also used in various other places throughout the 48 | # built documents. 49 | # 50 | # The short X.Y version. 51 | version = '1.0' 52 | # The full version, including alpha/beta/rc tags. 53 | release = '1.0' 54 | 55 | # The language for content autogenerated by Sphinx. Refer to documentation 56 | # for a list of supported languages. 57 | #language = None 58 | 59 | # There are two options for replacing |today|: either, you set today to some 60 | # non-false value, then it is used: 61 | #today = '' 62 | # Else, today_fmt is used as the format for a strftime call. 63 | #today_fmt = '%B %d, %Y' 64 | 65 | # List of patterns, relative to source directory, that match files and 66 | # directories to ignore when looking for source files. 67 | exclude_patterns = ['_build'] 68 | 69 | # The reST default role (used for this markup: `text`) to use for all documents. 70 | #default_role = None 71 | 72 | # If true, '()' will be appended to :func: etc. cross-reference text. 73 | #add_function_parentheses = True 74 | 75 | # If true, the current module name will be prepended to all description 76 | # unit titles (such as .. function::). 77 | #add_module_names = True 78 | 79 | # If true, sectionauthor and moduleauthor directives will be shown in the 80 | # output. They are ignored by default. 81 | #show_authors = False 82 | 83 | # The name of the Pygments (syntax highlighting) style to use. 84 | pygments_style = 'sphinx' 85 | 86 | # A list of ignored prefixes for module index sorting. 87 | #modindex_common_prefix = [] 88 | 89 | 90 | # -- Options for HTML output --------------------------------------------------- 91 | 92 | # The theme to use for HTML and HTML Help pages. See the documentation for 93 | # a list of builtin themes. 94 | html_theme = 'default' 95 | 96 | # Theme options are theme-specific and customize the look and feel of a theme 97 | # further. For a list of options available for each theme, see the 98 | # documentation. 99 | #html_theme_options = {} 100 | 101 | # Add any paths that contain custom themes here, relative to this directory. 102 | #html_theme_path = [] 103 | 104 | # The name for this set of Sphinx documents. If None, it defaults to 105 | # " v documentation". 106 | #html_title = None 107 | 108 | # A shorter title for the navigation bar. Default is the same as html_title. 109 | #html_short_title = None 110 | 111 | # The name of an image file (relative to this directory) to place at the top 112 | # of the sidebar. 113 | #html_logo = None 114 | 115 | # The name of an image file (within the static path) to use as favicon of the 116 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 117 | # pixels large. 118 | #html_favicon = None 119 | 120 | # Add any paths that contain custom static files (such as style sheets) here, 121 | # relative to this directory. They are copied after the builtin static files, 122 | # so a file named "default.css" will overwrite the builtin "default.css". 123 | html_static_path = ['_static'] 124 | 125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 126 | # using the given strftime format. 127 | #html_last_updated_fmt = '%b %d, %Y' 128 | 129 | # If true, SmartyPants will be used to convert quotes and dashes to 130 | # typographically correct entities. 131 | #html_use_smartypants = True 132 | 133 | # Custom sidebar templates, maps document names to template names. 134 | #html_sidebars = {} 135 | 136 | # Additional templates that should be rendered to pages, maps page names to 137 | # template names. 138 | #html_additional_pages = {} 139 | 140 | # If false, no module index is generated. 141 | #html_domain_indices = True 142 | 143 | # If false, no index is generated. 144 | #html_use_index = True 145 | 146 | # If true, the index is split into individual pages for each letter. 147 | #html_split_index = False 148 | 149 | # If true, links to the reST sources are added to the pages. 150 | #html_show_sourcelink = True 151 | 152 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 153 | #html_show_sphinx = True 154 | 155 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 156 | #html_show_copyright = True 157 | 158 | # If true, an OpenSearch description file will be output, and all pages will 159 | # contain a tag referring to it. The value of this option must be the 160 | # base URL from which the finished HTML is served. 161 | #html_use_opensearch = '' 162 | 163 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 164 | #html_file_suffix = None 165 | 166 | # Output file base name for HTML help builder. 167 | htmlhelp_basename = 'playdohdoc' 168 | 169 | 170 | # -- Options for LaTeX output -------------------------------------------------- 171 | 172 | # The paper size ('letter' or 'a4'). 173 | #latex_paper_size = 'letter' 174 | 175 | # The font size ('10pt', '11pt' or '12pt'). 176 | #latex_font_size = '10pt' 177 | 178 | # Grouping the document tree into LaTeX files. List of tuples 179 | # (source start file, target name, title, author, documentclass [howto/manual]). 180 | latex_documents = [ 181 | ('index', 'basket.tex', u'Basket Documentation', 182 | u'Mozilla', 'manual'), 183 | ] 184 | 185 | # The name of an image file (relative to this directory) to place at the top of 186 | # the title page. 187 | #latex_logo = None 188 | 189 | # For "manual" documents, if this is true, then toplevel headings are parts, 190 | # not chapters. 191 | #latex_use_parts = False 192 | 193 | # If true, show page references after internal links. 194 | #latex_show_pagerefs = False 195 | 196 | # If true, show URL addresses after external links. 197 | #latex_show_urls = False 198 | 199 | # Additional stuff for the LaTeX preamble. 200 | #latex_preamble = '' 201 | 202 | # Documents to append as an appendix to all manuals. 203 | #latex_appendices = [] 204 | 205 | # If false, no module index is generated. 206 | #latex_domain_indices = True 207 | 208 | 209 | # -- Options for manual page output -------------------------------------------- 210 | 211 | # One entry per manual page. List of tuples 212 | # (source start file, name, description, authors, manual section). 213 | man_pages = [ 214 | ('index', 'basket.mozilla.org', u'Basket Documentation', 215 | [u'Mozilla'], 1) 216 | ] 217 | 218 | 219 | # Example configuration for intersphinx: refer to the Python standard library. 220 | intersphinx_mapping = {'http://docs.python.org/': None} 221 | -------------------------------------------------------------------------------- /newrelic.ini: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | 3 | # 4 | # This file configures the New Relic Python Agent. 5 | # 6 | # The path to the configuration file should be supplied to the function 7 | # newrelic.agent.initialize() when the agent is being initialized. 8 | # 9 | # The configuration file follows a structure similar to what you would 10 | # find for Microsoft Windows INI files. For further information on the 11 | # configuration file format see the Python ConfigParser documentation at: 12 | # 13 | # http://docs.python.org/library/configparser.html 14 | # 15 | # For further discussion on the behaviour of the Python agent that can 16 | # be configured via this configuration file see: 17 | # 18 | # http://newrelic.com/docs/python/python-agent-configuration 19 | # 20 | 21 | # --------------------------------------------------------------------------- 22 | 23 | # Here are the settings that are common to all environments. 24 | 25 | [newrelic] 26 | 27 | # You must specify the license key associated with your New 28 | # Relic account. This key binds the Python Agent's data to your 29 | # account in the New Relic service. 30 | # license_key = use NEW_RELIC_LICENSE_KEY environment variable 31 | 32 | # The appplication name. Set this to be the name of your 33 | # application as you would like it to show up in New Relic UI. 34 | # The UI will then auto-map instances of your application into a 35 | # entry on your home dashboard page. 36 | # app_name = use NEW_RELIC_APP_NAME environment variable 37 | 38 | # When "true", the agent collects performance data about your 39 | # application and reports this data to the New Relic UI at 40 | # newrelic.com. This global switch is normally overridden for 41 | # each environment below. 42 | monitor_mode = true 43 | 44 | # Sets the name of a file to log agent messages to. Useful for 45 | # debugging any issues with the agent. This is not set by 46 | # default as it is not known in advance what user your web 47 | # application processes will run as and where they have 48 | # permission to write to. Whatever you set this to you must 49 | # ensure that the permissions for the containing directory and 50 | # the file itself are correct, and that the user that your web 51 | # application runs as can write to the file. If not able to 52 | # write out a log file, it is also possible to say "stderr" and 53 | # output to standard error output. This would normally result in 54 | # output appearing in your web server log. 55 | #log_file = /tmp/newrelic-python-agent.log 56 | 57 | # Sets the level of detail of messages sent to the log file, if 58 | # a log file location has been provided. Possible values, in 59 | # increasing order of detail, are: "critical", "error", "warning", 60 | # "info" and "debug". When reporting any agent issues to New 61 | # Relic technical support, the most useful setting for the 62 | # support engineers is "debug". However, this can generate a lot 63 | # of information very quickly, so it is best not to keep the 64 | # agent at this level for longer than it takes to reproduce the 65 | # problem you are experiencing. 66 | log_level = info 67 | 68 | # The Python Agent communicates with the New Relic service using 69 | # SSL by default. Note that this does result in an increase in 70 | # CPU overhead, over and above what would occur for a non SSL 71 | # connection, to perform the encryption involved in the SSL 72 | # communication. This work is though done in a distinct thread 73 | # to those handling your web requests, so it should not impact 74 | # response times. You can if you wish revert to using a non SSL 75 | # connection, but this will result in information being sent 76 | # over a plain socket connection and will not be as secure. 77 | ssl = true 78 | 79 | # High Security Mode enforces certain security settings, and 80 | # prevents them from being overridden, so that no sensitive data 81 | # is sent to New Relic. Enabling High Security Mode means that 82 | # SSL is turned on, request parameters are not collected, and SQL 83 | # can not be sent to New Relic in its raw form. To activate High 84 | # Security Mode, it must be set to 'true' in this local .ini 85 | # configuration file AND be set to 'true' in the server-side 86 | # configuration in the New Relic user interface. For details, see 87 | # https://docs.newrelic.com/docs/subscriptions/high-security 88 | high_security = true 89 | 90 | # The Python Agent will attempt to connect directly to the New 91 | # Relic service. If there is an intermediate firewall between 92 | # your host and the New Relic service that requires you to use a 93 | # HTTP proxy, then you should set both the "proxy_host" and 94 | # "proxy_port" settings to the required values for the HTTP 95 | # proxy. The "proxy_user" and "proxy_pass" settings should 96 | # additionally be set if proxy authentication is implemented by 97 | # the HTTP proxy. The "proxy_scheme" setting dictates what 98 | # protocol scheme is used in talking to the HTTP protocol. This 99 | # would normally always be set as "http" which will result in the 100 | # agent then using a SSL tunnel through the HTTP proxy for end to 101 | # end encryption. 102 | # proxy_scheme = http 103 | # proxy_host = hostname 104 | # proxy_port = 8080 105 | # proxy_user = 106 | # proxy_pass = 107 | 108 | # Tells the transaction tracer and error collector (when 109 | # enabled) whether or not to capture the query string for the 110 | # URL and send it as the request parameters for display in the 111 | # UI. When "true", it is still possible to exclude specific 112 | # values from being captured using the "ignored_params" setting. 113 | capture_params = false 114 | 115 | # Space separated list of variables that should be removed from 116 | # the query string captured for display as the request 117 | # parameters in the UI. 118 | ignored_params = 119 | 120 | # The transaction tracer captures deep information about slow 121 | # transactions and sends this to the UI on a periodic basis. The 122 | # transaction tracer is enabled by default. Set this to "false" 123 | # to turn it off. 124 | transaction_tracer.enabled = true 125 | 126 | # Threshold in seconds for when to collect a transaction trace. 127 | # When the response time of a controller action exceeds this 128 | # threshold, a transaction trace will be recorded and sent to 129 | # the UI. Valid values are any positive float value, or (default) 130 | # "apdex_f", which will use the threshold for a dissatisfying 131 | # Apdex controller action - four times the Apdex T value. 132 | transaction_tracer.transaction_threshold = apdex_f 133 | 134 | # When the transaction tracer is on, SQL statements can 135 | # optionally be recorded. The recorder has three modes, "off" 136 | # which sends no SQL, "raw" which sends the SQL statement in its 137 | # original form, and "obfuscated", which strips out numeric and 138 | # string literals. 139 | transaction_tracer.record_sql = obfuscated 140 | 141 | # Threshold in seconds for when to collect stack trace for a SQL 142 | # call. In other words, when SQL statements exceed this 143 | # threshold, then capture and send to the UI the current stack 144 | # trace. This is helpful for pinpointing where long SQL calls 145 | # originate from in an application. 146 | transaction_tracer.stack_trace_threshold = 0.5 147 | 148 | # Determines whether the agent will capture query plans for slow 149 | # SQL queries. Only supported in MySQL and PostgreSQL. Set this 150 | # to "false" to turn it off. 151 | transaction_tracer.explain_enabled = true 152 | 153 | # Threshold for query execution time below which query plans 154 | # will not not be captured. Relevant only when "explain_enabled" 155 | # is true. 156 | transaction_tracer.explain_threshold = 0.5 157 | 158 | # Space separated list of function or method names in form 159 | # 'module:function' or 'module:class.function' for which 160 | # additional function timing instrumentation will be added. 161 | transaction_tracer.function_trace = 162 | 163 | # The error collector captures information about uncaught 164 | # exceptions or logged exceptions and sends them to UI for 165 | # viewing. The error collector is enabled by default. Set this 166 | # to "false" to turn it off. 167 | error_collector.enabled = true 168 | 169 | # To stop specific errors from reporting to the UI, set this to 170 | # a space separated list of the Python exception type names to 171 | # ignore. The exception name should be of the form 'module:class'. 172 | error_collector.ignore_errors = celery.exceptions:Retry ratelimit.exceptions:Ratelimited 173 | 174 | # Browser monitoring is the Real User Monitoring feature of the UI. 175 | # For those Python web frameworks that are supported, this 176 | # setting enables the auto-insertion of the browser monitoring 177 | # JavaScript fragments. 178 | browser_monitoring.auto_instrument = false 179 | 180 | # A thread profiling session can be scheduled via the UI when 181 | # this option is enabled. The thread profiler will periodically 182 | # capture a snapshot of the call stack for each active thread in 183 | # the application to construct a statistically representative 184 | # call tree. 185 | thread_profiler.enabled = true 186 | 187 | # --------------------------------------------------------------------------- 188 | 189 | # 190 | # The application environments. These are specific settings which 191 | # override the common environment settings. The settings related to a 192 | # specific environment will be used when the environment argument to the 193 | # newrelic.agent.initialize() function has been defined to be either 194 | # "development", "test", "staging" or "production". 195 | # 196 | 197 | [newrelic:development] 198 | monitor_mode = false 199 | 200 | [newrelic:test] 201 | monitor_mode = false 202 | 203 | [newrelic:staging] 204 | app_name = Python Application (Staging) 205 | monitor_mode = true 206 | 207 | [newrelic:production] 208 | monitor_mode = true 209 | 210 | # --------------------------------------------------------------------------- 211 | -------------------------------------------------------------------------------- /docs/newsletter_api.rst: -------------------------------------------------------------------------------- 1 | .. This Source Code Form is subject to the terms of the Mozilla Public 2 | .. License, v. 2.0. If a copy of the MPL was not distributed with this 3 | .. file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | .. _ newsletter-api: 6 | 7 | ============================ 8 | Newsletter API 9 | ============================ 10 | 11 | This "news" app provides a service for managing Mozilla newsletters. 12 | 13 | `fixtures/newsletters.json` is a fixture that can be used to load some initial 14 | data, but is probably out of date by the time you read this. 15 | 16 | Currently available newsletters can be found in JSON format via the 17 | `/news/newsletters/ API endpoint `_. 18 | 19 | If 'token-required' is specified, a token must be suffixed onto the API 20 | URL, such as:: 21 | 22 | /news/user// 23 | 24 | This is a user-specific token given away by the email backend or 25 | basket in some manner (i.e. emailed to the user from basket). This 26 | token allows clients to do more powerful things with the user. 27 | 28 | A client might also have an ``API key`` that it can use with some APIs to 29 | do privileged things, like looking up a user from their email. 30 | 31 | If 'SSL required', the call will fail if not called over a secure 32 | (SSL) connection. 33 | 34 | Whenever possible (even when the HTTP status is not 200), the response body 35 | will be a JSON-encoded dictionary with several guaranteed fields, along with 36 | any data being returned by the particular call: 37 | 38 | 'status': 'ok' if the call succeeded, 'error' if there was an error 39 | 40 | If there was an error, these fields will also be included: 41 | 42 | 'code': an integer error code taken from ``basket.errors`` 43 | in `basket-client `_. 44 | 'desc': brief English description of the error. 45 | 46 | The following URLs are available (assuming "/news" is app url): 47 | 48 | /news/subscribe 49 | --------------- 50 | 51 | This method subscribes the user to the newsletters defined in the 52 | "newsletters" field, which should be a comma-delimited list of 53 | newsletters. "email" and "newsletters" are required:: 54 | 55 | method: POST 56 | fields: email, format, country, lang, newsletters, optin, source_url, trigger_welcome, sync 57 | returns: { status: ok } on success 58 | { status: error, desc: , code: } on error 59 | SSL required if sync=Y 60 | token or API key required if sync=Y 61 | 62 | ``format`` can be any of the following values: H, html, T, or text 63 | 64 | ``country`` is the 2 letter country code for the subscriber. 65 | 66 | ``lang`` is the language code for the subscriber (e.g. de, pt-BR) 67 | 68 | ``first_name`` is the optional first name of the subscriber. 69 | 70 | ``last_name`` is the optional last name of the subscriber. 71 | 72 | ``optin`` should be set to "Y" if the user should not go through the 73 | double-optin process (email verification). Setting this option requires 74 | an API key and the use of SSL. Defaults to "N". 75 | 76 | ``trigger_welcome`` should be set to "N" if you do not want welcome emails 77 | to be sent once the user successfully subscribes and verifies their email. 78 | Defaults to "Y". 79 | 80 | ``sync`` is an optional field. If set to Y, basket will ensure the response 81 | includes the token for the provided email address, creating one if necessary. 82 | If you don't need the token, or don't need it immediately, leave off ``sync`` 83 | so Basket has the option to optimize by doing the entire subscribe in the 84 | background after returning from this call. Defaults to "N". 85 | 86 | Using ``sync=Y`` requires SSL and an API key. 87 | 88 | ``source_url`` is an optional place to add the URL of the site from which 89 | the request is being made. It's just there to give us a way of discovering 90 | which pages produce the most subscriptions. 91 | 92 | If the email address is invalid (due to format, or unrecognized domain), the error 93 | code will be ``BASKET_INVALID_EMAIL`` from the basket client. If it is likely just 94 | a misspelled domain, then basket may suggest a correction. If there is a suggestion, 95 | it will be in the error response in the ``suggestion`` parameter. 96 | 97 | If you've validated that the email address is indeed correct and don't want the validation, 98 | you may pass ``validated=true`` with your submission and the address will be accepted. It 99 | is suggested that you not use this unless you've specifically asked the user if it is really 100 | correct. 101 | 102 | /news/unsubscribe 103 | ----------------- 104 | 105 | This method unsubscribes the user from the newsletters defined in 106 | the "newsletters" field, which should be a comma-delimited list of 107 | newsletters. If the "optout" parameter is set to Y, the user will be 108 | opted out of all newsletters. "email" and either "newsletters" or 109 | "optout" is required:: 110 | 111 | method: POST 112 | fields: email, newsletters, optout 113 | returns: { status: ok } on success 114 | { status: error, desc: } on error 115 | token-required 116 | 117 | /news/user 118 | ---------- 119 | 120 | Returns information about the user including all the newsletters 121 | he/she is subscribed to:: 122 | 123 | method: GET 124 | fields: *none* 125 | returns: { 126 | status: ok, 127 | email: , 128 | format: , 129 | country: , 130 | lang: , 131 | newsletters: [, ...] 132 | } on success 133 | { 134 | status: error, 135 | desc: 136 | } on error 137 | token-required 138 | 139 | If POSTed, this method updates the user's data with the supplied 140 | fields. Note that the user is only subscribed to "newsletters" after 141 | this, meaning the user will be unsubscribed to all other 142 | newsletters. "optin" should be Y or N and opts in/out the user:: 143 | 144 | method: POST 145 | fields: email, format, country, lang, newsletters, optin 146 | returns: { status: ok } on success 147 | { status: error, desc: } on error 148 | token-required 149 | 150 | /news/newsletters 151 | ----------------- 152 | 153 | Returns information about all of the available newsletters:: 154 | 155 | method: GET 156 | fields: *none* 157 | returns: { 158 | status: ok, 159 | newsletters: { 160 | newsletter-slug: { 161 | vendor_id: "ID_FROM_EXACTTARGET", 162 | welcome: "WELCOME_MESSAGE_ID", 163 | description: "Short text description", 164 | show: boolean, // whether to always show this in lists 165 | title: "Short text title", 166 | languages: [ 167 | "<2 char lang>", 168 | ... 169 | ], 170 | active: boolean, // whether to show it at all (optional) 171 | order: 15, // in what order it should be displayed in lists 172 | requires_double_optin: boolean 173 | }, 174 | ... 175 | } 176 | } 177 | 178 | /news/debug-user 179 | ---------------- 180 | 181 | REMOVED. Will return a 404. Use the newer and better ``lookup-user`` method. 182 | 183 | /news/lookup-user 184 | ----------------- 185 | 186 | This allows retrieving user information given either their token or 187 | their email (but not both). To retrieve by email, an API key is 188 | required:: 189 | 190 | method: GET 191 | fields: token, or email and api-key 192 | returns: { status: ok, user data } on success 193 | { status: error, desc: } on error 194 | SSL required 195 | token or API key required 196 | 197 | Examples:: 198 | 199 | GET https://basket.example.com/news/lookup-user?token= 200 | GET https://basket.example.com/news/lookup-user?api-key=&email= 201 | 202 | The API key can be provided either as a GET query parameter ``api-key`` 203 | or as a request header ``X-api-key``. If both are provided, the query 204 | parameter is used. 205 | 206 | If user is not found, returns a 404 status and 'desc' is 'No such user'. 207 | 208 | On success, response is a bunch of data about the user:: 209 | 210 | { 211 | 'status': 'ok', # no errors talking to ET 212 | 'status': 'error', # errors talking to ET, see next field 213 | 'desc': 'error message' # details if status is error 214 | 'email': 'email@address', 215 | 'format': 'T'|'H', 216 | 'country': country code, 217 | 'lang': language code, 218 | 'token': UUID, 219 | 'created-date': date created, 220 | 'newsletters': list of slugs of newsletters subscribed to, 221 | 'confirmed': True if user has confirmed subscription (or was excepted), 222 | 'pending': True if we're waiting for user to confirm subscription 223 | 'master': True if we found them in the master subscribers table 224 | } 225 | 226 | Note: Because this method always calls Exact Target one or more times, it 227 | can be slower than some other Basket APIs, and will fail if ET is down. 228 | 229 | /news/recover/ 230 | -------------- 231 | 232 | This sends an email message to a user, containing a link they can use to 233 | manage their subscriptions:: 234 | 235 | method: POST 236 | fields: email 237 | returns: { status: ok } on success 238 | { status: error, desc: } on error 239 | 240 | The email address is passed as 'email' in the POST data. If it is missing 241 | or not syntactically correct, a 400 is returned. Otherwise, a message is 242 | sent to the email, containing a link to the existing subscriptions page 243 | with their token in it, so they can use it to manage their subscriptions. 244 | 245 | If the user is known in ET, the message will be sent in their preferred 246 | language and format. 247 | 248 | If the email provided is not known, a 404 status is returned. 249 | 250 | 251 | -------------------------------------------------------------------------------- /basket/news/tests/test_sfdc.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from django.test.utils import override_settings 3 | 4 | from mock import patch, Mock 5 | 6 | from basket.news.backends.sfdc import to_vendor, from_vendor 7 | 8 | 9 | @patch('basket.news.backends.sfdc.is_supported_newsletter_language', Mock(return_value=True)) 10 | class VendorConversionTests(TestCase): 11 | @patch('basket.news.backends.sfdc.newsletter_map') 12 | def test_to_vendor(self, nm_mock): 13 | nm_mock.return_value = { 14 | 'chillin': 'Sub_Chillin__c', 15 | 'bowlin': 'Sub_Bowlin__c', 16 | 'white-russian-recipes': 'Sub_Caucasians__c', 17 | } 18 | data = { 19 | 'email': 'dude@example.com', 20 | 'token': 'totally-token-man', 21 | 'format': 'H', 22 | 'country': 'US', 23 | 'lang': 'en', 24 | 'source_url': 'https://www.example.com', 25 | 'first_name': 'The', 26 | 'last_name': 'Dude', 27 | 'newsletters': [ 28 | 'chillin', 29 | 'bowlin', 30 | 'white-russian-recipes', 31 | ] 32 | } 33 | contact = { 34 | 'Email_Format__c': 'H', 35 | 'FirstName': 'The', 36 | 'LastName': 'Dude', 37 | 'Subscriber__c': True, 38 | 'Email_Language__c': 'en', 39 | 'Signup_Source_URL__c': 'https://www.example.com', 40 | 'Token__c': 'totally-token-man', 41 | 'Email': 'dude@example.com', 42 | 'MailingCountryCode': 'us', 43 | 'Sub_Chillin__c': True, 44 | 'Sub_Bowlin__c': True, 45 | 'Sub_Caucasians__c': True, 46 | } 47 | self.assertDictEqual(to_vendor(data), contact) 48 | 49 | @override_settings(TESTING_EMAIL_DOMAINS=['example.com'], 50 | USE_SANDBOX_BACKEND=False) 51 | @patch('basket.news.backends.sfdc.newsletter_map') 52 | def test_to_vendor_test_domain(self, nm_mock): 53 | """Same as main test but should flip UAT_Test_Data__c switch""" 54 | nm_mock.return_value = { 55 | 'chillin': 'Sub_Chillin__c', 56 | 'bowlin': 'Sub_Bowlin__c', 57 | 'white-russian-recipes': 'Sub_Caucasians__c', 58 | } 59 | data = { 60 | 'email': 'dude@example.com', 61 | 'token': 'totally-token-man', 62 | 'format': 'H', 63 | 'country': 'US', 64 | 'lang': 'en', 65 | 'source_url': 'https://www.example.com', 66 | 'first_name': 'The', 67 | 'last_name': 'Dude', 68 | 'newsletters': [ 69 | 'chillin', 70 | 'bowlin', 71 | 'white-russian-recipes', 72 | ] 73 | } 74 | contact = { 75 | 'Email_Format__c': 'H', 76 | 'FirstName': 'The', 77 | 'LastName': 'Dude', 78 | 'Subscriber__c': True, 79 | 'Email_Language__c': 'en', 80 | 'Signup_Source_URL__c': 'https://www.example.com', 81 | 'Token__c': 'totally-token-man', 82 | 'Email': 'dude@example.com', 83 | 'MailingCountryCode': 'us', 84 | 'Sub_Chillin__c': True, 85 | 'Sub_Bowlin__c': True, 86 | 'Sub_Caucasians__c': True, 87 | 'UAT_Test_Data__c': True, 88 | } 89 | self.assertDictEqual(to_vendor(data), contact) 90 | 91 | @patch('basket.news.backends.sfdc.newsletter_map') 92 | def test_to_vendor_dict_newsletters(self, nm_mock): 93 | nm_mock.return_value = { 94 | 'chillin': 'Sub_Chillin__c', 95 | 'bowlin': 'Sub_Bowlin__c', 96 | 'fightin': 'Sub_Fightin__c', 97 | } 98 | data = { 99 | 'email': 'dude@example.com', 100 | 'token': 'totally-token-man', 101 | 'format': 'T', 102 | 'country': 'mx', 103 | 'lang': 'es', 104 | 'source_url': 'https://www.example.com', 105 | 'first_name': 'Senior', 106 | 'last_name': 'Lebowski', 107 | 'newsletters': { 108 | 'chillin': True, 109 | 'bowlin': True, 110 | 'fightin': False, 111 | } 112 | } 113 | contact = { 114 | 'Email_Format__c': 'T', 115 | 'FirstName': 'Senior', 116 | 'LastName': 'Lebowski', 117 | 'Subscriber__c': True, 118 | 'Email_Language__c': 'es', 119 | 'Signup_Source_URL__c': 'https://www.example.com', 120 | 'Token__c': 'totally-token-man', 121 | 'Email': 'dude@example.com', 122 | 'MailingCountryCode': 'mx', 123 | 'Sub_Chillin__c': True, 124 | 'Sub_Bowlin__c': True, 125 | 'Sub_Fightin__c': False, 126 | } 127 | self.assertDictEqual(to_vendor(data), contact) 128 | 129 | @patch('basket.news.backends.sfdc.newsletter_inv_map') 130 | def test_from_vendor(self, nm_mock): 131 | nm_mock.return_value = { 132 | 'Sub_Bowlin__c': 'bowlin', 133 | 'Sub_Caucasians__c': 'white-russian-recipes', 134 | 'Sub_Chillin__c': 'chillin', 135 | 'Sub_Fightin__c': 'fightin' 136 | } 137 | data = { 138 | 'id': 'vendor-id', 139 | 'email': 'dude@example.com', 140 | 'token': 'totally-token-man', 141 | 'format': 'H', 142 | 'country': 'us', 143 | 'lang': 'en', 144 | 'source_url': 'https://www.example.com', 145 | 'first_name': 'The', 146 | 'last_name': 'Dude', 147 | 'newsletters': [ 148 | 'bowlin', 149 | 'white-russian-recipes', 150 | ] 151 | } 152 | contact = { 153 | 'Id': 'vendor-id', 154 | 'Email_Format__c': 'H', 155 | 'FirstName': 'The', 156 | 'LastName': 'Dude', 157 | 'Subscriber__c': True, 158 | 'Email_Language__c': 'en', 159 | 'Signup_Source_URL__c': 'https://www.example.com', 160 | 'Token__c': 'totally-token-man', 161 | 'Email': 'dude@example.com', 162 | 'MailingCountryCode': 'US', 163 | 'Sub_Chillin__c': False, 164 | 'Sub_Bowlin__c': True, 165 | 'Sub_Caucasians__c': True, 166 | 'Sub_Fightin__c': False, 167 | } 168 | self.assertDictEqual(from_vendor(contact), data) 169 | 170 | def test_to_vendor_blank_values(self): 171 | data = { 172 | 'email': 'dude@example.com', 173 | 'token': 'totally-token-man', 174 | 'format': 'H', 175 | 'country': 'US', 176 | 'lang': 'en', 177 | 'source_url': 'https://www.example.com', 178 | 'first_name': '', 179 | 'last_name': '', 180 | 'fsa_allow_share': 'y', 181 | 'optout': 'no', 182 | 'optin': 'true', 183 | } 184 | contact = { 185 | 'Email_Format__c': 'H', 186 | 'Subscriber__c': True, 187 | 'Email_Language__c': 'en', 188 | 'Signup_Source_URL__c': 'https://www.example.com', 189 | 'Token__c': 'totally-token-man', 190 | 'Email': 'dude@example.com', 191 | 'MailingCountryCode': 'us', 192 | 'FSA_Allow_Info_Shared__c': True, 193 | 'HasOptedOutOfEmail': False, 194 | 'Double_Opt_In__c': True, 195 | } 196 | self.assertDictEqual(to_vendor(data), contact) 197 | 198 | def test_to_vendor_boolean_casting(self): 199 | data = { 200 | 'email': 'dude@example.com', 201 | 'token': 'totally-token-man', 202 | 'format': 'H', 203 | 'country': 'US', 204 | 'lang': 'en', 205 | 'source_url': 'https://www.example.com', 206 | 'first_name': 'The', 207 | 'last_name': 'Dude', 208 | 'fsa_allow_share': 'y', 209 | 'optout': 'no', 210 | 'optin': 'true', 211 | } 212 | contact = { 213 | 'Email_Format__c': 'H', 214 | 'FirstName': 'The', 215 | 'LastName': 'Dude', 216 | 'Subscriber__c': True, 217 | 'Email_Language__c': 'en', 218 | 'Signup_Source_URL__c': 'https://www.example.com', 219 | 'Token__c': 'totally-token-man', 220 | 'Email': 'dude@example.com', 221 | 'MailingCountryCode': 'us', 222 | 'FSA_Allow_Info_Shared__c': True, 223 | 'HasOptedOutOfEmail': False, 224 | 'Double_Opt_In__c': True, 225 | } 226 | self.assertDictEqual(to_vendor(data), contact) 227 | 228 | def test_to_vendor_boolean_casting_with_booleans(self): 229 | data = { 230 | 'email': 'dude@example.com', 231 | 'token': 'totally-token-man', 232 | 'format': 'H', 233 | 'country': 'US', 234 | 'lang': 'en', 235 | 'source_url': 'https://www.example.com', 236 | 'first_name': 'The', 237 | 'last_name': 'Dude', 238 | 'fsa_allow_share': True, 239 | 'optout': False, 240 | 'optin': True, 241 | } 242 | contact = { 243 | 'Email_Format__c': 'H', 244 | 'FirstName': 'The', 245 | 'LastName': 'Dude', 246 | 'Subscriber__c': True, 247 | 'Email_Language__c': 'en', 248 | 'Signup_Source_URL__c': 'https://www.example.com', 249 | 'Token__c': 'totally-token-man', 250 | 'Email': 'dude@example.com', 251 | 'MailingCountryCode': 'us', 252 | 'FSA_Allow_Info_Shared__c': True, 253 | 'HasOptedOutOfEmail': False, 254 | 'Double_Opt_In__c': True, 255 | } 256 | self.assertDictEqual(to_vendor(data), contact) 257 | 258 | @override_settings(EXTRA_SUPPORTED_LANGS=['zh-tw']) 259 | def test_to_vendor_extra_langs(self): 260 | data = { 261 | 'email': 'dude@example.com', 262 | 'token': 'totally-token-man', 263 | 'format': 'H', 264 | 'country': 'US', 265 | 'lang': 'zh-TW', 266 | 'source_url': 'https://www.example.com', 267 | 'first_name': 'The', 268 | 'last_name': 'Dude', 269 | 'fsa_allow_share': 'y', 270 | 'optout': 'no', 271 | 'optin': 'true', 272 | } 273 | contact = { 274 | 'Email_Format__c': 'H', 275 | 'FirstName': 'The', 276 | 'LastName': 'Dude', 277 | 'Subscriber__c': True, 278 | 'Email_Language__c': 'zh-TW', 279 | 'Signup_Source_URL__c': 'https://www.example.com', 280 | 'Token__c': 'totally-token-man', 281 | 'Email': 'dude@example.com', 282 | 'MailingCountryCode': 'us', 283 | 'FSA_Allow_Info_Shared__c': True, 284 | 'HasOptedOutOfEmail': False, 285 | 'Double_Opt_In__c': True, 286 | } 287 | self.assertDictEqual(to_vendor(data), contact) 288 | -------------------------------------------------------------------------------- /basket/news/migrations/0013_auto_20170907_1216.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | import basket.news.fields 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('news', '0012_auto_20170713_1021'), 12 | ] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name='LocalizedSMSMessage', 17 | fields=[ 18 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 19 | ('message_id', models.SlugField(help_text=b'The ID for the message that will be used by clients')), 20 | ('vendor_id', models.CharField(help_text=b"The backend vendor's identifier for this message", max_length=50)), 21 | ('description', models.CharField(help_text=b'Optional short description of this message', max_length=200, blank=True)), 22 | ('language', basket.news.fields.LocaleField(default=b'en-US', max_length=32, choices=[('ach', 'ach (Acholi)'), ('af', 'af (Afrikaans)'), ('ak', 'ak (Akan)'), ('am-et', 'am-et (Amharic)'), ('an', 'an (Aragonese)'), ('ar', 'ar (Arabic)'), ('as', 'as (Assamese)'), ('ast', 'ast (Asturian)'), ('az', 'az (Azerbaijani)'), ('be', 'be (Belarusian)'), ('bg', 'bg (Bulgarian)'), ('bm', 'bm (Bambara)'), ('bn-BD', 'bn-BD (Bengali (Bangladesh))'), ('bn-IN', 'bn-IN (Bengali (India))'), ('br', 'br (Breton)'), ('brx', 'brx (Bodo)'), ('bs', 'bs (Bosnian)'), ('ca', 'ca (Catalan)'), ('ca-valencia', 'ca-valencia (Catalan (Valencian))'), ('cak', 'cak (Kaqchikel)'), ('cs', 'cs (Czech)'), ('csb', 'csb (Kashubian)'), ('cy', 'cy (Welsh)'), ('da', 'da (Danish)'), ('dbg', 'dbg (Debug Robot)'), ('de', 'de (German)'), ('de-AT', 'de-AT (German (Austria))'), ('de-CH', 'de-CH (German (Switzerland))'), ('de-DE', 'de-DE (German (Germany))'), ('dsb', 'dsb (Lower Sorbian)'), ('ee', 'ee (Ewe)'), ('el', 'el (Greek)'), ('en-AU', 'en-AU (English (Australian))'), ('en-CA', 'en-CA (English (Canadian))'), ('en-GB', 'en-GB (English (British))'), ('en-NZ', 'en-NZ (English (New Zealand))'), ('en-US', 'en-US (English (US))'), ('en-ZA', 'en-ZA (English (South African))'), ('eo', 'eo (Esperanto)'), ('es', 'es (Spanish)'), ('es-AR', 'es-AR (Spanish (Argentina))'), ('es-CL', 'es-CL (Spanish (Chile))'), ('es-ES', 'es-ES (Spanish (Spain))'), ('es-MX', 'es-MX (Spanish (Mexico))'), ('et', 'et (Estonian)'), ('eu', 'eu (Basque)'), ('fa', 'fa (Persian)'), ('ff', 'ff (Fulah)'), ('fi', 'fi (Finnish)'), ('fj-FJ', 'fj-FJ (Fijian)'), ('fr', 'fr (French)'), ('fur-IT', 'fur-IT (Friulian)'), ('fy-NL', 'fy-NL (Frisian)'), ('ga', 'ga (Irish)'), ('ga-IE', 'ga-IE (Irish)'), ('gd', 'gd (Gaelic (Scotland))'), ('gl', 'gl (Galician)'), ('gn', 'gn (Guarani)'), ('gu', 'gu (Gujarati)'), ('gu-IN', 'gu-IN (Gujarati (India))'), ('ha', 'ha (Hausa)'), ('he', 'he (Hebrew)'), ('hi', 'hi (Hindi)'), ('hi-IN', 'hi-IN (Hindi (India))'), ('hr', 'hr (Croatian)'), ('hsb', 'hsb (Upper Sorbian)'), ('hu', 'hu (Hungarian)'), ('hy-AM', 'hy-AM (Armenian)'), ('id', 'id (Indonesian)'), ('ig', 'ig (Igbo)'), ('is', 'is (Icelandic)'), ('it', 'it (Italian)'), ('ja', 'ja (Japanese)'), ('ja-JP-mac', 'ja-JP-mac (Japanese)'), ('ka', 'ka (Georgian)'), ('kk', 'kk (Kazakh)'), ('km', 'km (Khmer)'), ('kn', 'kn (Kannada)'), ('ko', 'ko (Korean)'), ('kok', 'kok (Konkani)'), ('ks', 'ks (Kashmiri)'), ('ku', 'ku (Kurdish)'), ('la', 'la (Latin)'), ('lg', 'lg (Luganda)'), ('lij', 'lij (Ligurian)'), ('ln', 'ln (Lingala)'), ('lo', 'lo (Lao)'), ('lt', 'lt (Lithuanian)'), ('ltg', 'ltg (Latgalian)'), ('lv', 'lv (Latvian)'), ('mai', 'mai (Maithili)'), ('mg', 'mg (Malagasy)'), ('mi', 'mi (Maori (Aotearoa))'), ('mk', 'mk (Macedonian)'), ('ml', 'ml (Malayalam)'), ('mn', 'mn (Mongolian)'), ('mr', 'mr (Marathi)'), ('ms', 'ms (Malay)'), ('my', 'my (Burmese)'), ('nb-NO', 'nb-NO (Norwegian (Bokm\xe5l))'), ('ne-NP', 'ne-NP (Nepali)'), ('nl', 'nl (Dutch)'), ('nn-NO', 'nn-NO (Norwegian (Nynorsk))'), ('nr', 'nr (Ndebele, South)'), ('nso', 'nso (Northern Sotho)'), ('oc', 'oc (Occitan (Lengadocian))'), ('or', 'or (Oriya)'), ('pa', 'pa (Punjabi)'), ('pa-IN', 'pa-IN (Punjabi (India))'), ('pl', 'pl (Polish)'), ('pt-BR', 'pt-BR (Portuguese (Brazilian))'), ('pt-PT', 'pt-PT (Portuguese (Portugal))'), ('rm', 'rm (Romansh)'), ('ro', 'ro (Romanian)'), ('ru', 'ru (Russian)'), ('rw', 'rw (Kinyarwanda)'), ('sa', 'sa (Sanskrit)'), ('sah', 'sah (Sakha)'), ('sat', 'sat (Santali)'), ('si', 'si (Sinhala)'), ('sk', 'sk (Slovak)'), ('sl', 'sl (Slovenian)'), ('son', 'son (Songhai)'), ('sq', 'sq (Albanian)'), ('sr', 'sr (Serbian)'), ('sr-Cyrl', 'sr-Cyrl (Serbian)'), ('sr-Latn', 'sr-Latn (Serbian)'), ('ss', 'ss (Siswati)'), ('st', 'st (Southern Sotho)'), ('sv-SE', 'sv-SE (Swedish)'), ('sw', 'sw (Swahili)'), ('ta', 'ta (Tamil)'), ('ta-IN', 'ta-IN (Tamil (India))'), ('ta-LK', 'ta-LK (Tamil (Sri Lanka))'), ('te', 'te (Telugu)'), ('th', 'th (Thai)'), ('tl', 'tl (Tagalog)'), ('tn', 'tn (Tswana)'), ('tr', 'tr (Turkish)'), ('ts', 'ts (Tsonga)'), ('tsz', 'tsz (Pur\xe9pecha)'), ('tt-RU', 'tt-RU (Tatar)'), ('uk', 'uk (Ukrainian)'), ('ur', 'ur (Urdu)'), ('uz', 'uz (Uzbek)'), ('ve', 've (Venda)'), ('vi', 'vi (Vietnamese)'), ('wo', 'wo (Wolof)'), ('x-testing', 'x-testing (Testing)'), ('xh', 'xh (Xhosa)'), ('yo', 'yo (Yoruba)'), ('zh-CN', 'zh-CN (Chinese (Simplified))'), ('zh-TW', 'zh-TW (Chinese (Traditional))'), ('zu', 'zu (Zulu)')])), 23 | ('country', basket.news.fields.CountryField(default=b'us', max_length=3, choices=[('ad', 'ad (Andorra)'), ('ae', 'ae (U.A.E.)'), ('af', 'af (Afghanistan)'), ('ag', 'ag (Antigua and Barbuda)'), ('ai', 'ai (Anguilla)'), ('al', 'al (Albania)'), ('am', 'am (Armenia)'), ('an', 'an (Netherlands Antilles)'), ('ao', 'ao (Angola)'), ('aq', 'aq (Antarctica)'), ('ar', 'ar (Argentina)'), ('as', 'as (American Samoa)'), ('at', 'at (Austria)'), ('au', 'au (Australia)'), ('aw', 'aw (Aruba)'), ('ax', 'ax (\xc5land Islands)'), ('az', 'az (Azerbaijan)'), ('ba', 'ba (Bosnia and Herzegovina)'), ('bb', 'bb (Barbados)'), ('bd', 'bd (Bangladesh)'), ('be', 'be (Belgium)'), ('bf', 'bf (Burkina Faso)'), ('bg', 'bg (Bulgaria)'), ('bh', 'bh (Bahrain)'), ('bi', 'bi (Burundi)'), ('bj', 'bj (Benin)'), ('bl', 'bl (Saint Barth\xe9lemy)'), ('bm', 'bm (Bermuda)'), ('bn', 'bn (Brunei Darussalam)'), ('bo', 'bo (Bolivia)'), ('br', 'br (Brazil)'), ('bs', 'bs (Bahamas)'), ('bt', 'bt (Bhutan)'), ('bv', 'bv (Bouvet Island)'), ('bw', 'bw (Botswana)'), ('by', 'by (Belarus)'), ('bz', 'bz (Belize)'), ('ca', 'ca (Canada)'), ('cc', 'cc (Cocos (Keeling) Islands)'), ('cd', 'cd (Congo-Kinshasa)'), ('cf', 'cf (Central African Republic)'), ('cg', 'cg (Congo-Brazzaville)'), ('ch', 'ch (Switzerland)'), ('ci', 'ci (Ivory Coast)'), ('ck', 'ck (Cook Islands)'), ('cl', 'cl (Chile)'), ('cm', 'cm (Cameroon)'), ('cn', 'cn (China)'), ('co', 'co (Colombia)'), ('cr', 'cr (Costa Rica)'), ('cu', 'cu (Cuba)'), ('cv', 'cv (Cape Verde)'), ('cx', 'cx (Christmas Island)'), ('cy', 'cy (Cyprus)'), ('cz', 'cz (Czech Republic)'), ('de', 'de (Germany)'), ('dj', 'dj (Djibouti)'), ('dk', 'dk (Denmark)'), ('dm', 'dm (Dominica)'), ('do', 'do (Dominican Republic)'), ('dz', 'dz (Algeria)'), ('ec', 'ec (Ecuador)'), ('ee', 'ee (Estonia)'), ('eg', 'eg (Egypt)'), ('eh', 'eh (Western Sahara)'), ('er', 'er (Eritrea)'), ('es', 'es (Spain)'), ('et', 'et (Ethiopia)'), ('fi', 'fi (Finland)'), ('fj', 'fj (Fiji)'), ('fk', 'fk (Falkland Islands (Malvinas))'), ('fm', 'fm (Micronesia)'), ('fo', 'fo (Faroe Islands)'), ('fr', 'fr (France)'), ('ga', 'ga (Gabon)'), ('gb', 'gb (United Kingdom)'), ('gd', 'gd (Grenada)'), ('ge', 'ge (Georgia)'), ('gf', 'gf (French Guiana)'), ('gg', 'gg (Guernsey)'), ('gh', 'gh (Ghana)'), ('gi', 'gi (Gibraltar)'), ('gl', 'gl (Greenland)'), ('gm', 'gm (Gambia)'), ('gn', 'gn (Guinea)'), ('gp', 'gp (Guadeloupe)'), ('gq', 'gq (Equatorial Guinea)'), ('gr', 'gr (Greece)'), ('gs', 'gs (South Georgia and the South Sandwich Islands)'), ('gt', 'gt (Guatemala)'), ('gu', 'gu (Guam)'), ('gw', 'gw (Guinea-Bissau)'), ('gy', 'gy (Guyana)'), ('hk', 'hk (Hong Kong)'), ('hm', 'hm (Heard Island and McDonald Islands)'), ('hn', 'hn (Honduras)'), ('hr', 'hr (Croatia)'), ('ht', 'ht (Haiti)'), ('hu', 'hu (Hungary)'), ('id', 'id (Indonesia)'), ('ie', 'ie (Ireland)'), ('il', 'il (Israel)'), ('im', 'im (Isle of Man)'), ('in', 'in (India)'), ('io', 'io (British Indian Ocean Territory)'), ('iq', 'iq (Iraq)'), ('ir', 'ir (Iran)'), ('is', 'is (Iceland)'), ('it', 'it (Italy)'), ('je', 'je (Jersey)'), ('jm', 'jm (Jamaica)'), ('jo', 'jo (Jordan)'), ('jp', 'jp (Japan)'), ('ke', 'ke (Kenya)'), ('kg', 'kg (Kyrgyzstan)'), ('kh', 'kh (Cambodia)'), ('ki', 'ki (Kiribati)'), ('km', 'km (Comoros)'), ('kn', 'kn (Saint Kitts and Nevis)'), ('kp', 'kp (North Korea)'), ('kr', 'kr (South Korea)'), ('kw', 'kw (Kuwait)'), ('ky', 'ky (Cayman Islands)'), ('kz', 'kz (Kazakhstan)'), ('la', 'la (Laos)'), ('lb', 'lb (Lebanon)'), ('lc', 'lc (Saint Lucia)'), ('li', 'li (Liechtenstein)'), ('lk', 'lk (Sri Lanka)'), ('lr', 'lr (Liberia)'), ('ls', 'ls (Lesotho)'), ('lt', 'lt (Lithuania)'), ('lu', 'lu (Luxembourg)'), ('lv', 'lv (Latvia)'), ('ly', 'ly (Libya)'), ('ma', 'ma (Morocco)'), ('mc', 'mc (Monaco)'), ('md', 'md (Moldova)'), ('me', 'me (Montenegro)'), ('mf', 'mf (Saint Martin)'), ('mg', 'mg (Madagascar)'), ('mh', 'mh (Marshall Islands)'), ('mk', 'mk (Macedonia, F.Y.R. of)'), ('ml', 'ml (Mali)'), ('mm', 'mm (Myanmar)'), ('mn', 'mn (Mongolia)'), ('mo', 'mo (Macao)'), ('mp', 'mp (Northern Mariana Islands)'), ('mq', 'mq (Martinique)'), ('mr', 'mr (Mauritania)'), ('ms', 'ms (Montserrat)'), ('mt', 'mt (Malta)'), ('mu', 'mu (Mauritius)'), ('mv', 'mv (Maldives)'), ('mw', 'mw (Malawi)'), ('mx', 'mx (Mexico)'), ('my', 'my (Malaysia)'), ('mz', 'mz (Mozambique)'), ('na', 'na (Namibia)'), ('nc', 'nc (New Caledonia)'), ('ne', 'ne (Niger)'), ('nf', 'nf (Norfolk Island)'), ('ng', 'ng (Nigeria)'), ('ni', 'ni (Nicaragua)'), ('nl', 'nl (Netherlands)'), ('no', 'no (Norway)'), ('np', 'np (Nepal)'), ('nr', 'nr (Nauru)'), ('nu', 'nu (Niue)'), ('nz', 'nz (New Zealand)'), ('om', 'om (Oman)'), ('pa', 'pa (Panama)'), ('pe', 'pe (Peru)'), ('pf', 'pf (French Polynesia)'), ('pg', 'pg (Papua New Guinea)'), ('ph', 'ph (Philippines)'), ('pk', 'pk (Pakistan)'), ('pl', 'pl (Poland)'), ('pm', 'pm (Saint Pierre and Miquelon)'), ('pn', 'pn (Pitcairn)'), ('pr', 'pr (Puerto Rico)'), ('ps', 'ps (Occupied Palestinian Territory)'), ('pt', 'pt (Portugal)'), ('pw', 'pw (Palau)'), ('py', 'py (Paraguay)'), ('qa', 'qa (Qatar)'), ('re', 're (Reunion)'), ('ro', 'ro (Romania)'), ('rs', 'rs (Serbia)'), ('ru', 'ru (Russian Federation)'), ('rw', 'rw (Rwanda)'), ('sa', 'sa (Saudi Arabia)'), ('sb', 'sb (Solomon Islands)'), ('sc', 'sc (Seychelles)'), ('sd', 'sd (Sudan)'), ('se', 'se (Sweden)'), ('sg', 'sg (Singapore)'), ('sh', 'sh (Saint Helena)'), ('si', 'si (Slovenia)'), ('sj', 'sj (Svalbard and Jan Mayen)'), ('sk', 'sk (Slovakia)'), ('sl', 'sl (Sierra Leone)'), ('sm', 'sm (San Marino)'), ('sn', 'sn (Senegal)'), ('so', 'so (Somalia)'), ('sr', 'sr (Suriname)'), ('st', 'st (Sao Tome and Principe)'), ('sv', 'sv (El Salvador)'), ('sy', 'sy (Syria)'), ('sz', 'sz (Swaziland)'), ('tc', 'tc (Turks and Caicos Islands)'), ('td', 'td (Chad)'), ('tf', 'tf (French Southern Territories)'), ('tg', 'tg (Togo)'), ('th', 'th (Thailand)'), ('tj', 'tj (Tajikistan)'), ('tk', 'tk (Tokelau)'), ('tl', 'tl (Timor-Leste)'), ('tm', 'tm (Turkmenistan)'), ('tn', 'tn (Tunisia)'), ('to', 'to (Tonga)'), ('tr', 'tr (Turkey)'), ('tt', 'tt (Trinidad and Tobago)'), ('tv', 'tv (Tuvalu)'), ('tw', 'tw (Taiwan)'), ('tz', 'tz (Tanzania)'), ('ua', 'ua (Ukraine)'), ('ug', 'ug (Uganda)'), ('um', 'um (United States Minor Outlying Islands)'), ('us', 'us (United States)'), ('uy', 'uy (Uruguay)'), ('uz', 'uz (Uzbekistan)'), ('va', 'va (Vatican City)'), ('vc', 'vc (Saint Vincent and the Grenadines)'), ('ve', 've (Venezuela)'), ('vg', 'vg (British Virgin Islands)'), ('vi', 'vi (U.S. Virgin Islands)'), ('vn', 'vn (Vietnam)'), ('vu', 'vu (Vanuatu)'), ('wf', 'wf (Wallis and Futuna)'), ('ws', 'ws (Samoa)'), ('ye', 'ye (Yemen)'), ('yt', 'yt (Mayotte)'), ('za', 'za (South Africa)'), ('zm', 'zm (Zambia)'), ('zw', 'zw (Zimbabwe)')])), 24 | ], 25 | options={ 26 | 'verbose_name': 'Localized SMS message', 27 | }, 28 | ), 29 | migrations.AlterUniqueTogether( 30 | name='localizedsmsmessage', 31 | unique_together=set([('message_id', 'language', 'country')]), 32 | ), 33 | ] 34 | --------------------------------------------------------------------------------