├── .adr-dir ├── .coveragerc ├── .dockerignore ├── .github ├── dependabot.yml └── workflows │ ├── gitlab-mirror.yml │ └── pull-request-test.yaml ├── .gitignore ├── .gitlab-ci.yml ├── .therapist.yml ├── CODE_OF_CONDUCT.md ├── Dockerfile ├── LICENSE ├── Makefile ├── Procfile ├── README.md ├── bin ├── acceptance-tests-redirector.sh ├── acceptance-tests.sh ├── run-common.sh ├── run-dev.sh ├── run-prod.sh └── update-config.sh ├── contribute.json ├── docker-compose.yml ├── docs ├── Makefile ├── _static │ └── .gitkeep ├── _templates │ └── .gitkeep ├── architecture │ └── decisions │ │ ├── 0001-record-architecture-decisions.md │ │ ├── 0002-export-asrsnippet-metadata-for-snippet-metrics-processing.md │ │ ├── 0003-convert-templates-to-database-models.md │ │ ├── 0004-stop-exporting-analytics-to-csv.md │ │ ├── 0005-frequency-capping.md │ │ ├── 0006-release-jobs-in-nightly-bundles.md │ │ ├── 0007-move-channel-targeting-to-browser.md │ │ └── 0008-simple-redirect-to-bundle-service.md ├── build-github.zsh ├── conf.py ├── contributing.rst ├── data_collection.rst ├── developing.rst ├── geolocation.rst ├── img │ └── snippet_example.png ├── index.rst ├── list_of_targetting_attributes.rst ├── overview.rst └── requirements.txt ├── manage.py ├── media └── .htaccess ├── newrelic.ini ├── redirector ├── .dockerignore ├── Dockerfile ├── config.py ├── main.py ├── redirect.py ├── requirements.in ├── requirements.txt ├── run-prod.sh └── test.py ├── requirements.in ├── requirements.txt ├── scripts ├── __init__.py ├── cron.py ├── f100s.py └── metrics.py ├── setup.cfg ├── setup.py ├── snippets ├── __init__.py ├── base │ ├── __init__.py │ ├── admin │ │ ├── __init__.py │ │ ├── actions.py │ │ ├── adminmodels.py │ │ ├── fields.py │ │ ├── filters.py │ │ └── widgets.py │ ├── app.py │ ├── bundles.py │ ├── context_processors.py │ ├── etl.py │ ├── feed.py │ ├── fields.py │ ├── filters.py │ ├── forms.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ ├── fetch_daily_metrics.py │ │ │ ├── fetch_metrics.py │ │ │ ├── generate_bundles.py │ │ │ └── update_jobs.py │ ├── middleware.py │ ├── migrations │ │ ├── 0001_squashed_0043_auto_20201104_0811.py │ │ ├── 0044_target_filtr_channel.py │ │ ├── 0045_auto_20201026_1255.py │ │ ├── 0046_auto_20201027_1337.py │ │ ├── 0047_auto_20201112_0655.py │ │ └── __init__.py │ ├── models.py │ ├── slack.py │ ├── static │ │ ├── as_preview │ │ │ ├── glyph-delete-16.svg │ │ │ ├── glyph-dismiss-16.svg │ │ │ ├── glyph-edit-16.svg │ │ │ ├── glyph-highlights-16.svg │ │ │ ├── glyph-historyItem-16.svg │ │ │ ├── glyph-import-16.svg │ │ │ ├── glyph-info-16.svg │ │ │ ├── glyph-info-option-12.svg │ │ │ ├── glyph-newWindow-16.svg │ │ │ ├── glyph-pin-12.svg │ │ │ ├── glyph-pin-16.svg │ │ │ ├── glyph-pocket-16.svg │ │ │ ├── glyph-search-16.svg │ │ │ ├── glyph-settings-16.svg │ │ │ ├── glyph-topsites-16.svg │ │ │ ├── glyph-trending-16.svg │ │ │ ├── glyph-unpin-16.svg │ │ │ ├── glyph-webextension-16.svg │ │ │ └── topic-show-more-12.svg │ │ ├── css │ │ │ ├── admin │ │ │ │ ├── ASRSnippetAdmin.css │ │ │ │ ├── CustomNameWithTags.css │ │ │ │ ├── IDFieldHighlight.css │ │ │ │ ├── InlineTemplates.css │ │ │ │ ├── JobAdmin.css │ │ │ │ ├── ListSnippetsJobs.css │ │ │ │ ├── SnippetAdmin.css │ │ │ │ ├── descriptionColorize.css │ │ │ │ └── sms-country-toggle.css │ │ │ ├── app.css │ │ │ └── templateDataWidget.css │ │ ├── favicon.ico │ │ ├── html │ │ │ └── close.html │ │ ├── img │ │ │ ├── .gitignore │ │ │ ├── glyphicons-halflings-white.png │ │ │ ├── glyphicons-halflings.png │ │ │ └── google.png │ │ ├── js │ │ │ ├── admin │ │ │ │ ├── alert-page-leaving.js │ │ │ │ ├── autoTranslatorWidget.js │ │ │ │ ├── inlineMover.js │ │ │ │ ├── jquery.are-you-sure.js │ │ │ │ ├── sms-country-toggle.js │ │ │ │ ├── templateChooserWidget.js │ │ │ │ └── templateDataWidget.js │ │ │ ├── clipboard.min.js │ │ │ ├── copy_preview.js │ │ │ └── lib │ │ │ │ └── nunjucks.min.js │ │ ├── templates │ │ │ ├── snippetDataWidget.html │ │ │ └── snippetPreviewForm.html │ │ └── vendor │ │ │ └── fullcalendar │ │ │ ├── core │ │ │ ├── locales-all.min.js │ │ │ ├── main.min.css │ │ │ └── main.min.js │ │ │ └── daygrid │ │ │ ├── main.min.css │ │ │ └── main.min.js │ ├── storage.py │ ├── templates │ │ ├── 404.jinja │ │ ├── 500.jinja │ │ ├── admin │ │ │ ├── base │ │ │ │ ├── icon │ │ │ │ │ └── delete_confirmation.html │ │ │ │ ├── job │ │ │ │ │ ├── change_form.html │ │ │ │ │ └── submit_line.html │ │ │ │ └── target │ │ │ │ │ └── change_list.html │ │ │ ├── login.html │ │ │ └── metrics_table.html │ │ ├── base │ │ │ ├── base.jinja │ │ │ ├── home.jinja │ │ │ ├── jobs_list_calendar.jinja │ │ │ ├── jobs_list_table.jinja │ │ │ ├── jobs_related_with_obj.jinja │ │ │ ├── preview_image.jinja │ │ │ ├── ratelimited.jinja │ │ │ ├── snippets_custom_name_with_tags.jinja │ │ │ └── snippets_related_with_obj.jinja │ │ ├── slack │ │ │ ├── asr_ready_for_review.jinja.json │ │ │ ├── job_canceled.jinja.json │ │ │ ├── job_completed.jinja.json │ │ │ ├── job_published.jinja.json │ │ │ ├── job_scheduled.jinja.json │ │ │ ├── legacy_published.jinja.json │ │ │ └── legacy_ready_for_review.jinja.json │ │ └── widgets │ │ │ └── jexlrange.html │ ├── tests │ │ ├── __init__.py │ │ ├── test_admin.py │ │ ├── test_admin_filters.py │ │ ├── test_bundles.py │ │ ├── test_commands.py │ │ ├── test_etl.py │ │ ├── test_feed.py │ │ ├── test_fields.py │ │ ├── test_filters.py │ │ ├── test_forms.py │ │ ├── test_middleware.py │ │ ├── test_models.py │ │ ├── test_slack.py │ │ ├── test_util.py │ │ ├── test_validators.py │ │ └── test_views.py │ ├── urls.py │ ├── util.py │ ├── validators.py │ └── views.py ├── settings.py ├── urls.py └── wsgi │ ├── __init__.py │ ├── app.py │ └── config.py └── test.env /.adr-dir: -------------------------------------------------------------------------------- 1 | docs/architecture/decisions 2 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = snippets 3 | 4 | [report] 5 | omit = */migrations/* 6 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .product_details 3 | database-dumps 4 | venv 5 | image-dumps 6 | static 7 | media 8 | product_details 9 | .env 10 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "/" 5 | schedule: 6 | interval: monthly 7 | open-pull-requests-limit: 10 8 | labels: 9 | - ":snake: Python / Django" 10 | ignore: 11 | - dependency-name: django 12 | versions: 13 | - ">= 3.a, < 4" 14 | - dependency-name: sphinx 15 | versions: 16 | - ">= 3.a, < 4" 17 | -------------------------------------------------------------------------------- /.github/workflows/gitlab-mirror.yml: -------------------------------------------------------------------------------- 1 | name: Mirror repo 2 | on: [push] 3 | 4 | jobs: 5 | mirror: 6 | if: github.repository == 'mozmeao/snippets-service' 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: mirror in gitlab 10 | uses: actions/checkout@v3 11 | with: 12 | fetch-depth: 0 13 | - uses: yesolutions/mirror-action@71cd8f5b5c9c4a461f477ecccace98850cb04bc1 14 | with: 15 | REMOTE: 'https://gitlab.com/mozmeao/snippets-service.git' 16 | GIT_USERNAME: ${{ secrets.GITLAB_USERNAME }} 17 | GIT_PASSWORD: ${{ secrets.GITLAB_PASSWORD }} 18 | -------------------------------------------------------------------------------- /.github/workflows/pull-request-test.yaml: -------------------------------------------------------------------------------- 1 | name: Test Pull Request 2 | 3 | on: 4 | pull_request: 5 | 6 | jobs: 7 | test: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: mozmeao/checkout@v1 11 | - name: Docker pull 12 | run: CI_COMMIT_SHORT_SHA=latest make pull 13 | - name: Docker build 14 | run: make build 15 | - name: Lint 16 | run: make lint 17 | - name: Check migrations 18 | run: make check-migrations 19 | - name: Test 20 | run: make test 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | *.pyc 3 | .DS_Store 4 | docs/_build 5 | .tox/ 6 | MANIFEST 7 | .coverage 8 | .runserver-ssl.* 9 | media/* 10 | database-dumps/* 11 | .product_details 12 | .vscode 13 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | DOCKER_REPOSITORY: "mozmeao/snippets" 3 | 4 | stages: 5 | - build 6 | - test 7 | - deploy 8 | 9 | build-image-django: 10 | stage: build 11 | tags: 12 | - mozmeao 13 | script: 14 | - CI_COMMIT_SHORT_SHA=latest make pull app=web 15 | - CI_COMMIT_SHORT_SHA=latest make pull app=builder 16 | - make build app=web 17 | - make push app=web 18 | # With latest tag for caching 19 | - CI_COMMIT_SHORT_SHA=latest make build app=web 20 | - CI_COMMIT_SHORT_SHA=latest make push app=web 21 | - CI_COMMIT_SHORT_SHA=latest make build app=builder 22 | - CI_COMMIT_SHORT_SHA=latest make push app=builder 23 | 24 | build-image-redirector: 25 | stage: build 26 | tags: 27 | - mozmeao 28 | script: 29 | - CI_COMMIT_SHORT_SHA=latest make pull app=redirector 30 | - make build app=redirector 31 | - make push app=redirector 32 | # With latest tag for caching 33 | - CI_COMMIT_SHORT_SHA=latest make build app=redirector 34 | - CI_COMMIT_SHORT_SHA=latest make push app=redirector 35 | 36 | check-migrations: 37 | stage: test 38 | tags: 39 | - mozmeao 40 | script: 41 | - make check-migrations 42 | 43 | run-flake8: 44 | stage: test 45 | tags: 46 | - mozmeao 47 | rules: 48 | - allow_failure: true 49 | script: 50 | - make lint 51 | 52 | run-unit-tests-django: 53 | stage: test 54 | tags: 55 | - mozmeao 56 | script: 57 | - make test-web 58 | 59 | run-unit-tests-redirector: 60 | tags: 61 | - mozmeao 62 | script: 63 | - make test-redirector 64 | 65 | .deploy: 66 | stage: deploy 67 | tags: 68 | - mozmeao 69 | - aws 70 | script: 71 | - bin/update-config.sh 72 | 73 | stage: 74 | extends: .deploy 75 | only: 76 | - stage 77 | variables: 78 | NAMESPACE: snippets-stage 79 | CLUSTERS: oregon-eks 80 | DOCKER_IMAGE_TAG: "${DOCKER_REPOSITORY}:${CI_COMMIT_SHORT_SHA}" 81 | 82 | admin: 83 | extends: .deploy 84 | only: 85 | - admin 86 | variables: 87 | NAMESPACE: snippets-admin 88 | CLUSTERS: oregon-eks 89 | DOCKER_IMAGE_TAG: "${DOCKER_REPOSITORY}:${CI_COMMIT_SHORT_SHA}" 90 | 91 | prod: 92 | extends: .deploy 93 | only: 94 | - prod 95 | variables: 96 | NAMESPACE: snippets-prod 97 | CLUSTERS: frankfurt-eks oregon-eks 98 | DOCKER_IMAGE_TAG: "${DOCKER_REPOSITORY}:redirector-${CI_COMMIT_SHORT_SHA}" 99 | -------------------------------------------------------------------------------- /.therapist.yml: -------------------------------------------------------------------------------- 1 | actions: 2 | flake8: 3 | run: flake8 {files} 4 | include: '*.py' 5 | exclude: 6 | - 'docs\*.py' 7 | - 'migrations\*.py' 8 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Community Participation Guidelines 2 | 3 | This repository is governed by Mozilla's code of conduct and etiquette guidelines. 4 | For more details, please read the 5 | [Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). 6 | 7 | ## How to Report 8 | For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. 9 | 10 | 16 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim-buster AS builder 2 | 3 | ENV PYTHONDONTWRITEBYTECODE=1 4 | ENV PYTHONUNBUFFERED=1 5 | ENV PIP_DISABLE_PIP_VERSION_CHECK=1 6 | ENV PATH="/venv/bin:$PATH" 7 | ENV LANG=C.UTF-8 8 | 9 | RUN python -m venv /venv/ 10 | 11 | # Debian slim images needs the man directories created 12 | # https://github.com/debuerreotype/debuerreotype/issues/10#issuecomment-438342078 13 | RUN bash -c 'for i in {1..8}; do mkdir -p "/usr/share/man/man$i"; done' && \ 14 | apt-get update && \ 15 | apt-get install -y --no-install-recommends build-essential libxslt1.1 libxml2 libxml2-dev \ 16 | libxslt1-dev libpq-dev && \ 17 | rm -rf /var/lib/apt/lists/* /user/share/man 18 | 19 | WORKDIR /app 20 | 21 | COPY requirements.txt . 22 | RUN pip install --require-hashes --no-cache-dir -r requirements.txt 23 | 24 | COPY . /app 25 | RUN DEBUG=False SECRET_KEY=foo ALLOWED_HOSTS=localhost,\ 26 | DATABASE_URL=sqlite:/// SITE_URL= \ 27 | ./manage.py collectstatic --noinput 28 | 29 | 30 | 31 | # Production image 32 | 33 | FROM python:3.8-slim-buster 34 | 35 | ENV PYTHONDONTWRITEBYTECODE=1 36 | ENV PYTHONUNBUFFERED=1 37 | ENV PIP_DISABLE_PIP_VERSION_CHECK=1 38 | ENV PATH="/venv/bin:$PATH" 39 | ENV LANG=C.UTF-8 40 | 41 | EXPOSE 8000 42 | CMD ["./bin/run-prod.sh"] 43 | 44 | RUN adduser --uid 1000 --disabled-password --gecos '' webdev 45 | 46 | WORKDIR /app 47 | 48 | RUN bash -c 'for i in {1..8}; do mkdir -p "/usr/share/man/man$i"; done' && \ 49 | apt-get update && \ 50 | apt-get install -y --no-install-recommends libpq-dev postgresql-client pngquant libxslt1.1 libxml2 && \ 51 | rm -rf /var/lib/apt/lists/* /usr/share/man 52 | 53 | COPY --from=builder /venv /venv 54 | COPY --from=builder /app /app 55 | 56 | RUN chown webdev.webdev -R . 57 | USER webdev 58 | 59 | ARG GIT_SHA=head 60 | ENV GIT_SHA=${GIT_SHA} 61 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DC = docker-compose 2 | 3 | all: help 4 | 5 | compile-requirements: requirements.txt 6 | ${DC} run web pip-compile --generate-hashes --reuse-hashes > requirements.txt 7 | ${DC} run web bash -c 'cd redirector && pip-compile --generate-hashes --reuse-hashes > requirements.txt' 8 | 9 | build: 10 | ${DC} build --pull --parallel $(app) 11 | 12 | push: 13 | ${DC} push $(app) 14 | 15 | pull: 16 | ${DC} pull $(app) 17 | 18 | test: test-web test-redirector 19 | 20 | test-web: 21 | ${DC} run test-web ./manage.py test --parallel 22 | 23 | test-redirector: 24 | ${DC} run test-redirector pytest test.py 25 | 26 | lint: 27 | ${DC} run web flake8 snippets redirector 28 | 29 | check-migrations: 30 | docker-compose run test-web bash -c './manage.py makemigrations | grep "No changes detected"' 31 | 32 | clean: 33 | # Docker compose images 34 | ${DC} rm 35 | # python related things 36 | find . -name '*.pyc' -exec rm -f {} + 37 | find . -name '*.pyo' -exec rm -f {} + 38 | find . -name '__pycache__' -exec rm -rf {} + 39 | # test related things 40 | -rm -f .coverage 41 | # docs files 42 | -rm -rf docs/_build/ 43 | # state files 44 | -rm -f .make.* 45 | 46 | djshell: 47 | ${DC} run web ./manage.py shell_plus 48 | 49 | 50 | help: 51 | @echo "Please use \`make ' where is one of" 52 | @echo " build - build docker images for dev" 53 | @echo " pull - pull the latest production images from Docker Hub" 54 | @echo " compile-requirements - compile requirements.in to requirements.txt" 55 | @echo " djshell - open a bash shell in the running app" 56 | @echo " clean - remove all build, test, coverage and Python artifacts" 57 | @echo " lint - check style with flake8, jshint, and stylelint" 58 | @echo " test - run tests against local files" 59 | 60 | .PHONY: all build push pull test test-web test-redirector lint check-migrations clean 61 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: ./bin/run-prod.sh 2 | clock: ./manage.py runscript cron 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # snippets service 2 | 3 | [![What's Deployed?](https://img.shields.io/badge/What's_Deployed-%3F-yellow.svg)](https://whatsdeployed.io/s/qA6/) [![Documentation RTFM](https://img.shields.io/badge/Documentation-RTFM-blue.svg)](https://abouthome-snippets-service.readthedocs.org/) [![Pipeline](https://img.shields.io/badge/CI-Pipeline-blueviolet.svg)](https://gitlab.com/mozmeao/snippets-service/pipelines) [![Pipeline](https://img.shields.io/badge/CD-Pipeline-ff69b4.svg)](https://gitlab.com/mozmeao/snippets-config/pipelines) 4 | 5 | The root of all messaging. 6 | 7 | ## Develop using Docker 8 | 9 | One time setup of your environment and database: 10 | 11 | 0. Make sure you have [docker](https://docker.io) and [docker-compose](https://github.com/docker/compose) 12 | 1. Setup an .env file with in the project root dir with at least: 13 | - `SECRET_KEY` set 14 | - `DATABASE_URL=postgres://snippets:snippets@db:5432/snippets` 15 | - `PROD_DETAILS_DIR` set to a writeable path, eg `PROD_DETAILS_DIR=/home/webdev/` 16 | 2. `$ docker-compose run --service-ports web bash` 17 | 3. `[docker]$ ./manage.py update_product_details` 18 | - If you get an error connecting to the database, you probably need to wait 19 | for a few seconds for PostgreSQL to initialize and then re-try the command. 20 | 4. `[docker]$ ./manage.py migrate` 21 | 5. `[docker]$ ./manage.py createsuperuser` (enter any user/email/pass you wish. Email is not required.) 22 | 6. `[docker]$ ./manage.py shell -c 'from snippets.base.util import *; create_countries();'` 23 | - Create a list of Countries from Product Details info. Note that create_locales() no longer exists. 24 | 25 | Start the development server: 26 | 27 | 1. `[docker]$ ./bin/run-dev.sh` 28 | 2. Navigate to `https://localhost:8443/admin` and log in with the admin account created in step #4. See an TLS Security Exception? Go to [TLS Certifcates](#tls-certificates) section. 29 | 30 | ### A note about using `run` instead of `up` 31 | 32 | `docker-compose run` is more suitable for development purposes since you get a 33 | shell and from there you can run the webserver command. This way you can debug 34 | using `set_trace()` or restart the server when things go bad. The trick here is 35 | to use `--service-ports` flag to make docker compose map the required ports. 36 | 37 | The project is configured for `docker-compose up` if that's your preference. 38 | 39 | ## TLS Certificates 40 | 41 | Firefox communicates with the snippets service only over secure HTTPS 42 | connections. For development, the `runserver_plus` command as executed in 43 | [`./bin/run-dev.sh`](https://github.com/mozmeao/snippets-service/blob/master/bin/run-dev.sh) 44 | generates and uses a self-signed certificate. 45 | 46 | You'll need to permanently accept the certificate, to allow Firefox to fetch 47 | Snippets from your development environment. 48 | 49 | ## Run the tests 50 | 51 | `$ ./manage.py test --parallel` 52 | 53 | ## Rebuild your Docker Compose Envinronment 54 | 55 | To rebuild your docker compose environment, first remove current images and 56 | containers and then run the `build` command. 57 | 58 | ```shell 59 | $ docker-compose kill 60 | $ docker-compose rm -f 61 | $ docker-compose build 62 | 63 | ``` 64 | 65 | ## Install Therapist 66 | 67 | [Therapist](https://github.com/rehandalal/therapist) is a smart pre-commit hook 68 | for git to ensure that committed code has been properly linted. 69 | 70 | Install the hooks by running: 71 | 72 | `$ therapist install` 73 | -------------------------------------------------------------------------------- /bin/acceptance-tests-redirector.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | EXIT=0 3 | BASE_URL=${1:-https://snippets.mozilla.com} 4 | URLS=( 5 | "/" 6 | "/healthz/" 7 | 8 | "/6/Firefox/62.0.1/20160922113459/WINNT_x86-msvc/en-US/release/Windows_NT%206.1/default/default/" 9 | ) 10 | 11 | function check_http_code { 12 | echo -n "Checking URL ${1} " 13 | curl -k -L -s -o /dev/null -I -w "%{http_code}" $1 | grep ${2:-200} > /dev/null 14 | if [ $? -eq 0 ]; 15 | then 16 | echo "OK" 17 | else 18 | echo "Failed" 19 | EXIT=1 20 | fi 21 | } 22 | 23 | function check_zero_content_length { 24 | echo -n "Checking zero content length of URL ${1} " 25 | test=$(curl -L -s ${1} | wc -c); 26 | if [[ $test -eq 0 ]]; 27 | then 28 | echo "OK" 29 | else 30 | echo "Failed" 31 | EXIT=1 32 | fi 33 | } 34 | 35 | function check_empty_json { 36 | echo -n "Checking empty json for URL ${1} " 37 | test=$(curl -L -s ${1}); 38 | if [ $test = '{}' ]; 39 | then 40 | echo "OK" 41 | else 42 | echo "Failed" 43 | EXIT=1 44 | fi 45 | } 46 | 47 | for url in ${URLS[*]} 48 | do 49 | check_http_code ${BASE_URL}${url} 50 | done 51 | 52 | # Check a page that throws 404. Not ideal but will surface 500s 53 | check_http_code ${BASE_URL}/foo 404 54 | 55 | check_empty_json ${BASE_URL}/6/Firefox/56.0.1/20160922113459/WINNT_x86-msvc/xx/release/Windows_NT%206.1/default/default/ 56 | 57 | exit ${EXIT} 58 | -------------------------------------------------------------------------------- /bin/acceptance-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | EXIT=0 3 | BASE_URL=${1:-https://snippets.mozilla.com} 4 | URLS=( 5 | "/" 6 | "/healthz/" 7 | "/readiness/" 8 | "/admin/" 9 | "/robots.txt" 10 | "/contribute.json" 11 | "/6/Firefox/62.0.1/20160922113459/WINNT_x86-msvc/en-US/release/Windows_NT%206.1/default/default/" 12 | "/feeds/snippets.ics" 13 | ) 14 | 15 | function check_http_code { 16 | echo -n "Checking URL ${1} " 17 | curl -k -L -s -o /dev/null -I -w "%{http_code}" $1 | grep ${2:-200} > /dev/null 18 | if [ $? -eq 0 ]; 19 | then 20 | echo "OK" 21 | else 22 | echo "Failed" 23 | EXIT=1 24 | fi 25 | } 26 | 27 | function check_zero_content_length { 28 | echo -n "Checking zero content length of URL ${1} " 29 | test=$(curl -L -s ${1} | wc -c); 30 | if [[ $test -eq 0 ]]; 31 | then 32 | echo "OK" 33 | else 34 | echo "Failed" 35 | EXIT=1 36 | fi 37 | } 38 | 39 | function check_empty_json { 40 | echo -n "Checking empty json for URL ${1} " 41 | test=$(curl -L -s ${1}); 42 | if [ $test = '{}' ]; 43 | then 44 | echo "OK" 45 | else 46 | echo "Failed" 47 | EXIT=1 48 | fi 49 | } 50 | 51 | for url in ${URLS[*]} 52 | do 53 | check_http_code ${BASE_URL}${url} 54 | done 55 | 56 | # Check a page that throws 404. Not ideal but will surface 500s 57 | check_http_code ${BASE_URL}/foo 404 58 | 59 | check_empty_json ${BASE_URL}/6/Firefox/56.0.1/20160922113459/WINNT_x86-msvc/xx/release/Windows_NT%206.1/default/default/ 60 | 61 | exit ${EXIT} 62 | -------------------------------------------------------------------------------- /bin/run-common.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ENABLE_ADMIN=$(echo "$ENABLE_ADMIN" | tr '[:upper:]' '[:lower:]') 4 | 5 | if [[ "$ENABLE_ADMIN" == "true" ]]; then 6 | python manage.py migrate --noinput 7 | fi 8 | -------------------------------------------------------------------------------- /bin/run-dev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | urlwait 4 | ./bin/run-common.sh 5 | ./manage.py runserver_plus --cert-file .runserver-ssl.crt --extra-file .env 0.0.0.0:8443 6 | -------------------------------------------------------------------------------- /bin/run-prod.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ./bin/run-common.sh 4 | 5 | echo "$GIT_SHA" > static/revision.txt 6 | 7 | exec gunicorn snippets.wsgi.app --config snippets/wsgi/config.py 8 | -------------------------------------------------------------------------------- /bin/update-config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | # env vars: CLUSTERS, CONFIG_BRANCH, CONFIG_REPO, NAMESPACE 4 | BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 5 | 6 | pushd $(mktemp -d) 7 | git clone --depth=1 -b ${CONFIG_BRANCH:=master} ${CONFIG_REPO:=github-mozmar-robot:mozmeao/snippets-config} snippets-config 8 | cd snippets-config 9 | 10 | set -u 11 | for CLUSTER in ${CLUSTERS}; do 12 | for DEPLOYMENT in {web,clock}.yaml; do 13 | DEPLOYMENT_FILE=${CLUSTER}/${NAMESPACE}/${DEPLOYMENT} 14 | if [[ -f ${DEPLOYMENT_FILE} ]]; then 15 | sed -i -e "s|image: .*|image: ${DOCKER_IMAGE_TAG}|" ${DEPLOYMENT_FILE} 16 | git add ${DEPLOYMENT_FILE} 17 | fi 18 | done 19 | done 20 | 21 | cp ${BIN_DIR}/acceptance-tests*.sh . 22 | git add acceptance-tests*.sh 23 | git commit -m "set image to ${DOCKER_IMAGE_TAG}" || echo "nothing new to commit" 24 | git push 25 | popd 26 | -------------------------------------------------------------------------------- /contribute.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Snippets Service", 3 | "description": "Service powering snippets on Firefox's about:home.", 4 | "repository": { 5 | "url": "https://github.com/mozmeao/snippets-service/", 6 | "license": "MPL 2.0", 7 | }, 8 | "urls": { 9 | "prod": "https://snippets.mozilla.com", 10 | "stage": "https://snippets-stage.us-west.moz.works" 11 | }, 12 | "bugs": { 13 | "list": "https://bugzilla.mozilla.org/buglist.cgi?product=Snippets&component=Service&resolution=---", 14 | "report": "https://bugzilla.mozilla.org/enter_bug.cgi?product=Snippets&component=Service" 15 | }, 16 | "participate": { 17 | "docs": "https://abouthome-snippets-service.readthedocs.org/", 18 | "irc": "irc://irc.mozilla.org/#snippets", 19 | "irc-contacts": [ 20 | "giorgos", 21 | "bensternthal" 22 | ] 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.4' 2 | services: 3 | db: 4 | image: postgres:11-alpine 5 | environment: 6 | - POSTGRES_PASSWORD=snippets 7 | - POSTGRES_USER=snippets 8 | - POSTGRES_DB=snippets 9 | 10 | builder: 11 | build: 12 | context: . 13 | target: builder 14 | cache_from: 15 | - mozmeao/snippets:builder-latest 16 | args: 17 | GIT_SHA: "${CI_COMMIT_SHA:-HEAD}" 18 | image: "mozmeao/snippets:builder-${CI_COMMIT_SHORT_SHA:-latest}" 19 | platform: linux/amd64 20 | 21 | web: 22 | build: 23 | context: . 24 | cache_from: 25 | - mozmeao/snippets:builder-latest 26 | - mozmeao/snippets:latest 27 | args: 28 | GIT_SHA: "${CI_COMMIT_SHA:-HEAD}" 29 | image: "mozmeao/snippets:${CI_COMMIT_SHORT_SHA:-latest}" 30 | platform: linux/amd64 31 | ports: 32 | - "8443:8443" 33 | - "8000:8000" 34 | volumes: 35 | - .:/app 36 | depends_on: 37 | - db 38 | command: 39 | ./bin/run-dev.sh 40 | 41 | redirector: 42 | build: 43 | context: ./redirector 44 | cache_from: 45 | - mozmeao/snippets:redirector-latest 46 | args: 47 | GIT_SHA: "${CI_COMMIT_SHA:-HEAD}" 48 | image: "mozmeao/snippets:redirector-${CI_COMMIT_SHORT_SHA:-latest}" 49 | platform: linux/amd64 50 | volumes: 51 | - ./redirector:/app 52 | 53 | test-web: 54 | image: "mozmeao/snippets:${CI_COMMIT_SHORT_SHA:-latest}" 55 | platform: linux/amd64 56 | depends_on: 57 | - db 58 | env_file: 59 | - test.env 60 | command: 61 | ./manage.py test --parallel 62 | 63 | test-redirector: 64 | image: "mozmeao/snippets:redirector-${CI_COMMIT_SHORT_SHA:-latest}" 65 | platform: linux/amd64 66 | command: 67 | pytest test.py 68 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " singlehtml to make a single large HTML file" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " devhelp to make HTML files and a Devhelp project" 27 | @echo " epub to make an epub" 28 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 29 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 30 | @echo " text to make text files" 31 | @echo " man to make manual pages" 32 | @echo " changes to make an overview of all changed/added/deprecated items" 33 | @echo " linkcheck to check all external links for integrity" 34 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 35 | 36 | clean: 37 | -rm -rf $(BUILDDIR)/* 38 | 39 | html: 40 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 41 | @echo 42 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 43 | 44 | dirhtml: 45 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 48 | 49 | singlehtml: 50 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 51 | @echo 52 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 53 | 54 | pickle: 55 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 56 | @echo 57 | @echo "Build finished; now you can process the pickle files." 58 | 59 | json: 60 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 61 | @echo 62 | @echo "Build finished; now you can process the JSON files." 63 | 64 | htmlhelp: 65 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 66 | @echo 67 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 68 | ".hhp project file in $(BUILDDIR)/htmlhelp." 69 | 70 | qthelp: 71 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 72 | @echo 73 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 74 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 75 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/playdoh.qhcp" 76 | @echo "To view the help file:" 77 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/playdoh.qhc" 78 | 79 | devhelp: 80 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 81 | @echo 82 | @echo "Build finished." 83 | @echo "To view the help file:" 84 | @echo "# mkdir -p $$HOME/.local/share/devhelp/playdoh" 85 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/playdoh" 86 | @echo "# devhelp" 87 | 88 | epub: 89 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 90 | @echo 91 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 92 | 93 | latex: 94 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 95 | @echo 96 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 97 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 98 | "(use \`make latexpdf' here to do that automatically)." 99 | 100 | latexpdf: 101 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 102 | @echo "Running LaTeX files through pdflatex..." 103 | make -C $(BUILDDIR)/latex all-pdf 104 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 105 | 106 | text: 107 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 108 | @echo 109 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 110 | 111 | man: 112 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 113 | @echo 114 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 115 | 116 | changes: 117 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 118 | @echo 119 | @echo "The overview file is in $(BUILDDIR)/changes." 120 | 121 | linkcheck: 122 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 123 | @echo 124 | @echo "Link check complete; look for any errors in the above output " \ 125 | "or in $(BUILDDIR)/linkcheck/output.txt." 126 | 127 | doctest: 128 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 129 | @echo "Testing of doctests in the sources finished, look at the " \ 130 | "results in $(BUILDDIR)/doctest/output.txt." 131 | -------------------------------------------------------------------------------- /docs/_static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozmeao/snippets-service/aefc553ba93689ca8d61fabbb8586deb9ca09f10/docs/_static/.gitkeep -------------------------------------------------------------------------------- /docs/_templates/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozmeao/snippets-service/aefc553ba93689ca8d61fabbb8586deb9ca09f10/docs/_templates/.gitkeep -------------------------------------------------------------------------------- /docs/architecture/decisions/0001-record-architecture-decisions.md: -------------------------------------------------------------------------------- 1 | # 1. Record architecture decisions 2 | 3 | Date: 2019-02-25 4 | 5 | ## Status 6 | 7 | Accepted 8 | 9 | ## Context 10 | 11 | We need to record the architectural decisions made on this project. 12 | 13 | ## Decision 14 | 15 | We will use Architecture Decision Records, as [described by Michael Nygard](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions). 16 | 17 | ## Consequences 18 | 19 | See Michael Nygard's article, linked above. For a lightweight ADR toolset, see Nat Pryce's [adr-tools](https://github.com/npryce/adr-tools). 20 | -------------------------------------------------------------------------------- /docs/architecture/decisions/0002-export-asrsnippet-metadata-for-snippet-metrics-processing.md: -------------------------------------------------------------------------------- 1 | # 2. Export ASRSnippet Metadata for Snippet Metrics Processing 2 | 3 | Date: 2019-02-25 4 | 5 | ## Status 6 | 7 | Accepted 8 | 9 | ## Context 10 | 11 | Data Engineers are building performance dashboards for Snippets using the 12 | metrics we collect from Firefox Telemetry. Telemetry pings include only basic 13 | information about the Snippet, like Snippet ID. 14 | 15 | For better to understand and more complete dashboards, we want to enhance the 16 | Telemetry received information with more Snippet metadata, like campaign 17 | information, included URL, main message used and others. 18 | 19 | To achieve this we will export the metadata from the Snippets Service in a 20 | machine readable format and make the file available in a Cloud Storage Provider. 21 | Then Data Engineers will import the metadata and combine them Telemetry data in 22 | unified dashboards. 23 | 24 | GitHub Issue: [#887](https://github.com/mozmeao/snippets-service/issues/887) 25 | 26 | ## Decision 27 | 28 | - Export in CSV format. 29 | - Create a cron job to export and upload resulting file to S3. 30 | - The job will run daily, on early morning UTC hours. 31 | - The job will be monitored using Dead Man's Snitch and report to the usual 32 | notification channels that project developers follow. 33 | 34 | 35 | ## Consequences 36 | 37 | ### Risks: 38 | 39 | - Cron job failures will lead to outdated CSV which in turn will provide a 40 | lesser experience to the dashboard users. This behavior will only cause 41 | inconvenience and not data loss. The attached monitor will notify developers 42 | that something is broken. 43 | 44 | - The implemented export mechanism may need refinement as the number of 45 | Snippets to be exported grows. Failure to do so may result in excessive 46 | resource usage in the snippet-admin.cron pods. 47 | -------------------------------------------------------------------------------- /docs/architecture/decisions/0003-convert-templates-to-database-models.md: -------------------------------------------------------------------------------- 1 | # 3. Convert Templates to Database Models 2 | 3 | Date: 2019-03-14 4 | 5 | ## Status 6 | 7 | Accepted 8 | 9 | ## Context 10 | 11 | Snippet Templates have been historically a combination of HTML, CSS and 12 | Javascript with Jinja2 variables. Template is saved in the database, through the 13 | `SnippetTemplate` Django Model. Upon save Django would extract the Jinja2 14 | variables and use this list to auto-generate the Snippet Admin UI. 15 | 16 | In the Snippet Admin UI, the users would select from a drop-down a Template and 17 | then a list of Inputs, Images, Textareas and Booleans would appear to represent the 18 | Jinja2 variables of the selected template. 19 | 20 | Upon save the values of those inputs would be combined into a JSON blob and 21 | saved in the database in the Snippets Django Model. 22 | 23 | This system allowed for fast iteration on templates without the need of 24 | redeploying the application. It served us well for all these years but imposes a 25 | large number of limitations and shortcomings like: 26 | 27 | - The template forms are dynamic and not real Django forms. Thus we cannot take 28 | benefit of the excellent Django form validation. Bogus input is a daily 29 | problem Snippet Editors fight with reviews. 30 | 31 | Converting Templates to real models would allow us to: 32 | 33 | - Validate type of input: text, link, URLs. 34 | 35 | - Enforce secure links. 36 | 37 | - For icons validate, type, size and dimensions of the image and thus 38 | increase the quality of the snippet. 39 | 40 | - Have complex validations like require an input when another input is set. 41 | 42 | - Give better error messages. 43 | 44 | - Have an overall better Admin UI. 45 | 46 | - Snippet Editors have to re-upload icons over and over again since all images 47 | are saved as part of a blob and not in a gallery. 48 | 49 | - Snippet Bundles are huge because they include the full blobs. Moving images 50 | outside of the bundle would drastically decrease the bundle size and reduce 51 | the CDN transfer costs. We also expect benefits on the CPU and Memory 52 | requirements on the server side and faster Activity Stream page on the client 53 | side due to the reduced size of IndexedDB. 54 | 55 | ### Why is this now possible? 56 | 57 | - Deploys used to be painful and now are not. 58 | - Also the template code is now part of Firefox and don't change that often. 59 | 60 | ## Decision 61 | 62 | We decide to move templates into real Django models. Each ASR Template will get 63 | it's own Django Model and entry in the Django Admin. 64 | 65 | Similarly images will be moved to a new Django model named `Icons` and templates 66 | will link to those icons. 67 | 68 | Each `ASRSnippet` will link to one `Template` obj using Django's model 69 | inheritance. 70 | 71 | This change will affect only the `ASRSnippet` models and the pre-ASR and 72 | `JSONSnippet` implementations will remain the same until decommissioned. 73 | 74 | Scripts to migrate from the current system the new system for all `ASRSnippets` 75 | will be created. 76 | 77 | 78 | ## Consequences 79 | 80 | New templates will now require more work to get integrated into the system. 81 | Since they also have to get coded into Firefox and ride the trains, this does 82 | not impose a real problem. 83 | 84 | Better experience for the Snippet Editors, more Snippets, less costs and many 85 | many happy faces. 86 | 87 | ## Related Issues and Milestones 88 | 89 | - https://github.com/mozmeao/snippets-service/milestone/9 90 | - https://github.com/mozmeao/snippets-service/issues/916 91 | - https://github.com/mozmeao/snippets-service/issues/841 92 | - https://github.com/mozmeao/snippets-service/issues/655 93 | -------------------------------------------------------------------------------- /docs/architecture/decisions/0004-stop-exporting-analytics-to-csv.md: -------------------------------------------------------------------------------- 1 | # 4. Stop Exporting Analytics to CSV 2 | 3 | Date: 2020-01-16 4 | 5 | ## Status 6 | 7 | Accepted 8 | 9 | ## Context 10 | 11 | As part of `2. Export ASRSnippet Metadata for Snippet Metrics 12 | Processing` we started exporting Snippet Metadata to CSVs stored in 13 | S3. 14 | 15 | The Dashboards consuming the CSVs have been long decomissioned and 16 | there's no other consumer of the data. 17 | 18 | 19 | ## Decision 20 | 21 | We decide to stop exporting metadata to CSVs for Metrics Dashboards 22 | and remove related code and exports. 23 | 24 | ## Consequences 25 | 26 | No consequences to how Snippets Service operates. Metrics analysis and 27 | Dashboards happens with Jupyter Notebooks currently. 28 | -------------------------------------------------------------------------------- /docs/architecture/decisions/0005-frequency-capping.md: -------------------------------------------------------------------------------- 1 | # 5. Frequency Capping 2 | 3 | Date: 2020-01-15 4 | 5 | ## Status 6 | 7 | Accepted 8 | 9 | ## Context 10 | 11 | Frequency Capping allows Content Managers to limit the number of 12 | impressions or interactions users have with content. Is a widely 13 | available tool in Publishing Platforms. 14 | 15 | It's usually developed on the server side where the system can decide 16 | how many times to serve the content to the requesting users which we 17 | call "Global Frequency Capping". Additionally the system may be able 18 | to limit the number of impressions per user which we call "Local" or 19 | "User Frequency Capping". 20 | 21 | For example a Content Piece can be set to 1,000,000 Global Impressions 22 | and 1 Impression per User, thus indirectly driving 1,000,000 different 23 | users to this Content. 24 | 25 | This functionality has been lacking from the Snippet Service due to 26 | technical limitations imposed by the way metrics were collected and 27 | content selection was handled on the client side. The latest 28 | developments in Firefox Messaging Center and the Firefox Telemetry 29 | Pipeline unblock this capability. [0] 30 | 31 | 32 | ## Decision 33 | 34 | We decide to implement the Frequency Capping functionality into our 35 | platform to allow Content Managers to limit the number of Impressions, 36 | Clicks and Blocks per Job. 37 | 38 | Local or User Frequency Capping will be handled on the Browser level 39 | by the Firefox Messaging Platform. The later supports only Impression 40 | Frequency Capping. 41 | 42 | The Snippets Service will provide an interface (UI) for the Content 43 | Managers to set upper limits on the number of Impressions a Job gets 44 | per Hour, Day, Week, Fortnight, Month or for the complete Browser 45 | Profile Lifetime. This information is included in the JSON generated 46 | for each Job. 47 | 48 | For Global Frequency Capping the Snippets Service will provide an 49 | interface (UI) for the Content Managers to set the limits on total 50 | worldwide number of Impressions, Clicks and Blocks per Job. 51 | 52 | Snippets Service will query Mozilla's Redash for Telemetry data every 53 | ten minutes and will fetch current impressions, clicks, blocks for 54 | each Job with set limits. 55 | 56 | When the reported numbers exceed the set limits then, the Job will be 57 | marked COMPLETE and will be pulled out of the Bundles on the next run 58 | of `update_jobs` cron job. 59 | 60 | The Frequency Capping functionality is additional to the Date 61 | Publishing controls, therefore a Job can end on a specific Date and 62 | Time or when its Global Frequency Capping Limits are met. 63 | 64 | 65 | ### Monitoring and Handling of Errors 66 | 67 | Since Global Frequency Capping depends on an external system for 68 | Metrics (Redash / Telemetry) it is possible that the latest numbers are 69 | not always available to the Snippets Service to make a decision. Such 70 | cases include scheduled or unplanned service interruptions or network 71 | errors. 72 | 73 | In co-ordination with Snippet Content Owner we decided that for cases 74 | where the Snippets Service cannot get the latest numbers for more than 75 | 24 hours, Jobs with Global Frequency Capping will get canceled. The 76 | cancellation reason will state that the Jobs where prematurely 77 | terminated due to missing metrics. 78 | 79 | The cron job responsible for fetching the Data from Telemetry is 80 | monitored by a Dead Man's Snitch. 81 | 82 | 83 | ## Consequences 84 | 85 | With this ADR implemented Snippets Service supports Global and Local 86 | Frequency Capping. 87 | 88 | Current implementation of Global Frequency Capping does go above any 89 | set limits due to: 90 | 91 | 1. The four hour interval that Browsers update Bundles. 92 | 2. The time needed by Browsers to report to Telemetry and the 93 | Telemetry Pipeline to make data available for consumption by the 94 | Snippets Service which is calculated to be about 30'. 95 | 96 | This was discussed with Snippet Content Managers and we agreed that a 97 | 5%, 10% or even 20% excess above the set limits is acceptable for now. 98 | This number may be re-evaluated when Content Managers switch to using 99 | more this functionality instead of setting Start and End Datetime 100 | limits. 101 | 102 | The excess percentage can be limited by bringing down the interval the 103 | Browsers update. Browsers can request Bundles every one hour without 104 | significant changes in the infrastructure supporting Snippets Service 105 | but with an additional cost for more CDN traffic and more active 106 | Kubernetes pods. Those changes will be decided in co-ordination with 107 | the Content Managers when requested. 108 | 109 | Local capping is not affected by the limitations above and should not 110 | go above set limits. 111 | 112 | 113 | [0] Related Snippets Telemetry Bug: https://bugzilla.mozilla.org/show_bug.cgi?id=1433214 114 | -------------------------------------------------------------------------------- /docs/architecture/decisions/0006-release-jobs-in-nightly-bundles.md: -------------------------------------------------------------------------------- 1 | # 6. Release Jobs in Nightly Bundles 2 | 3 | Date: 2020-03-31 4 | 5 | ## Status 6 | 7 | Replaced by ADR 0007. 8 | 9 | ## Context 10 | 11 | The JEXL evaluation engine in Firefox Message Router needs to be 12 | tested and performance measured. The best test input are Release Jobs 13 | tested against the current development version, i.e. Nightly. 14 | 15 | 16 | ## Decision 17 | 18 | Jobs published in Release Bundles for each locale will be also 19 | included in the Nightly Bundles. To avoid actually displaying those to 20 | Nightly users, their JEXL experssions will be appended with the 21 | expression ` && false` to force them to always evaluate to false. 22 | 23 | Firefox Messaging team will develop tests and automation to measure 24 | performance and catch regressions. 25 | 26 | This functionality is controlled by the `NIGHTLY_INCLUDES_RELEASE` 27 | enviroment variable of snippets-service. 28 | 29 | 30 | ## Consequences 31 | 32 | Nightly Bundles increase in size but, due to the small -relative to 33 | Release- number of Nightly users this will not have significant impact 34 | to CDN costs. 35 | 36 | 37 | ## Links 38 | 39 | - https://github.com/mozmeao/snippets-service/issues/1308 40 | -------------------------------------------------------------------------------- /docs/architecture/decisions/0007-move-channel-targeting-to-browser.md: -------------------------------------------------------------------------------- 1 | # 7. Move Channel targeting to browser. 2 | 3 | Date: 2020-11-12 4 | 5 | ## Status 6 | 7 | Accepted 8 | 9 | ## Context 10 | 11 | We want to be able to create more complex Targets, specifically targets that 12 | will evaluate to true for Profiles older than X weeks with X being different for 13 | each channel. 14 | 15 | The requirement comes from an initiative to reduce the number of active Jobs per week and thus reduce the programming and analyzing time required. With this change we will be able to schedule one Job for multiple channels while maintaining different targeting for each channel. 16 | 17 | 18 | ## Decision 19 | 20 | We decide to move Channel targeting from the server to the browser. To accomplish this we will take advantage of `browser.update.channel` JEXL attribute to target snippets based on channel and remove any server side code that does channel targeting. 21 | 22 | We will generate one bundle for each locale, instead of one bundle for each locale, channel combination. 23 | 24 | This ADR replaces 0006 since all Jobs for locale will be included in all channels. 25 | 26 | 27 | ## Consequences 28 | 29 | CDN traffic is expected to increase since bundles are going to include Job from all channels of a locale. The increase is not expected to be significant because traditionally most Jobs are on Release channel and it's the Release that generates from of the traffic. 30 | -------------------------------------------------------------------------------- /docs/architecture/decisions/0008-simple-redirect-to-bundle-service.md: -------------------------------------------------------------------------------- 1 | # 8. Simple Redirect to Bundle Service 2 | 3 | Date: 2020-11-19 4 | 5 | ## Status 6 | 7 | Accepted 8 | 9 | ## Context 10 | 11 | User facing production instances for the Snippets Service only do simple URL 12 | manipulation and redirect requests to a CloudFront backed S3 bucket which stores 13 | the Bundles. For this function to work, it's not required to run the whole 14 | Django app on production instances along with the required DB and Cache backing 15 | services. 16 | 17 | Bundles are pre-generated by the Admin instance of snippets-service. Snippets 18 | Admin is the only part of Snippets Service which depends on the Django app and 19 | thus requires database access. 20 | 21 | This is now possible because the support for pre "Activity Stream Router" (ASR) 22 | Snippets has been dropped and lots of legacy code was removed. The legacy code 23 | required Django and it's backing services (DB and Cache) to generate bundles on 24 | the fly. 25 | 26 | 27 | ## Decision 28 | 29 | Build a simple app that will redirect to the correct bundle. App should be 30 | deployable both to our k8s clusters and to AWS Lambda. Use Lambda service to 31 | create a failover for when kubernetes instances are not available. 32 | 33 | 34 | ## Consequences 35 | 36 | Only positive outcomes are expected from this change: 37 | - Less infrastructure to manage, as non-master DBs and replication will be removed. 38 | - Less infrastructure costs. 39 | - Higher availability due to the Lambda failover. 40 | -------------------------------------------------------------------------------- /docs/build-github.zsh: -------------------------------------------------------------------------------- 1 | #!/bin/zsh 2 | 3 | # A useful build script for projects hosted on github: 4 | # It can build your Sphinx docs and push them straight to your gh-pages branch. 5 | 6 | # Should be run from the docs directory: (cd docs && ./build-github.zsh) 7 | 8 | REPO=$(git config remote.origin.url) 9 | HERE=$(dirname $0) 10 | GH=$HERE/_gh-pages 11 | 12 | 13 | # Checkout the gh-pages branch, if necessary. 14 | if [[ ! -d $GH ]]; then 15 | git clone $REPO $GH 16 | pushd $GH 17 | git checkout -b gh-pages origin/gh-pages 18 | popd 19 | fi 20 | 21 | # Update and clean out the _gh-pages target dir. 22 | pushd $GH 23 | git pull && rm -rf * 24 | popd 25 | 26 | # Make a clean build. 27 | pushd $HERE 28 | make clean dirhtml 29 | 30 | # Move the fresh build over. 31 | cp -r _build/dirhtml/* $GH 32 | pushd $GH 33 | 34 | # Commit. 35 | git add . 36 | git commit -am "gh-pages build on $(date)" 37 | git push origin gh-pages 38 | 39 | popd 40 | popd 41 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | Check `GitHub repository `_ for up-to-date installation instructions. 5 | -------------------------------------------------------------------------------- /docs/geolocation.rst: -------------------------------------------------------------------------------- 1 | GeoLocation 2 | =========== 3 | 4 | Some snippets target specific countries. For example a snippet about a 5 | greek national holiday would target only browsers requesting snippets 6 | from Greece. 7 | 8 | To preserve user's privacy the geolocation happens on the browser 9 | level and not on the service level. Snippet bundles contain a list of 10 | targeted countries among the actual snippet data, the snippet weight 11 | and other info. 12 | 13 | The Browser pings `Mozilla Location Service`_ (MLS) to convert it's IP 14 | to a country code. Upon successful response the result is cached for 30 15 | days. Thus if a user travels from Greece to Italy for a week snippets 16 | targeting Greece will be shown while the user is in Italy. 17 | 18 | For current Firefox versions the Geo-Targeting code is part of `Activity 19 | Stream`_ as the rest of the decision engine. 20 | 21 | For pre-Quantum versions the Geo-Targeting code is part of the `JS included in 22 | the snippet bundle`_ and it's managed from the snippets service itself. It is 23 | not part of Firefox. 24 | 25 | 26 | 27 | .. _GeoDude: https://github.com/mozilla/geodude 28 | .. _Mozilla Location Service: https://location.services.mozilla.com/ 29 | .. _JS included in the snippet bundle: https://github.com/mozilla/snippets-service/blob/master/snippets/base/templates/base/includes/snippet_js.html 30 | .. _Activity Stream: https://github.com/mozilla/activity-stream/ 31 | -------------------------------------------------------------------------------- /docs/img/snippet_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozmeao/snippets-service/aefc553ba93689ca8d61fabbb8586deb9ca09f10/docs/img/snippet_example.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Snippets Service 2 | ================ 3 | 4 | The Snippets Service hosts small chunks of HTML, CSS, and JavaScript that are 5 | displayed on `about:home `_. in Firefox, among other places. 6 | 7 | 8 | Contents 9 | -------- 10 | 11 | .. toctree:: 12 | :maxdepth: 1 13 | 14 | overview 15 | developing 16 | contributing 17 | data_collection 18 | list_of_targetting_attributes 19 | geolocation 20 | -------------------------------------------------------------------------------- /docs/list_of_targetting_attributes.rst: -------------------------------------------------------------------------------- 1 | List of Targeting Attributes 2 | ============================ 3 | 4 | This is a list of available targeting attributes for ASRSnippets (Activity 5 | Stream Router). 6 | 7 | 8 | #. **Browser's Channel:** A choice among Release, Beta, ESR, Dev and Nightly. 9 | 10 | #. **Browser's Locale:** The Browser's locale. 11 | 12 | #. **Default Browser:** Is Firefox set as the Default Browser on the OS level? 13 | "Yes" or "No". 14 | 15 | #. **Profile Age (created):** How long ago was the Profile created? Can be 16 | anything from hours to months to years. Supports ranges, e.g. From one day 17 | to fourteen days, to target users during their first two weeks of using 18 | Firefox. 19 | 20 | #. **Previous Session End:** How long ago was this Profile last used? Can be 21 | anything from hours to months to years. Supports ranges, e.g. From 0 to 24 22 | hours, to targets users who open their Firefox daily. 23 | 24 | #. **Firefox Version:** Current Firefox Version. Supports ranges. 25 | 26 | #. **Uses Firefox Sync:** Is there a connected Firefox Account Syncing? "Yes" or 27 | "No". 28 | 29 | #. **Firefox Services Enabled:** Has the connected Firefox Account signed up for 30 | other Firefox Services? A choice of "Yes" or "No" for Lockwise, Monitor, 31 | Send, FPN, Notes and Pocket. 32 | 33 | #. **Country:** User's Country based on Geo-Location. 34 | 35 | #. **Is Developer:** Is the user a Developer? This is decided based on the 36 | number of times the user has opened the Firefox DevTools. We consider them a 37 | developer is opened count is equal or above 5. 38 | 39 | #. **Current Search Engine:** Current default Search Engine. 40 | 41 | #. **Total bookmarks count:** Total number of bookmarks set by the User. Supports 42 | ranges. 43 | 44 | #. **Desktop Devices Count:** Number of Desktop Devices connected to Firefox 45 | Sync. Supports ranges. 46 | 47 | #. **Mobile Devices Count:** Number of Mobile Devices connected to Firefox 48 | Sync. Supports ranges. 49 | 50 | #. **Total Devices Count:** Total number of Devices connected to Firefox Sync. 51 | Supports ranges. 52 | 53 | #. **Can Install Addons:** Can the user install Addons or blocked by the Admin? 54 | "Yes" or "No" 55 | 56 | #. **Total Addons:** Total number of installed Addons. 57 | 58 | #. **Browser Addon:** Is a Addon installed or not installed? Can target any 59 | available Addon. 60 | 61 | #. **Operating System:** User's operating system. A choice among "Windows", "Mac" 62 | and "Linux". 63 | 64 | #. **Updates Enabled:** Are updates enabled? "Yes" or "No" 65 | 66 | #. **Updates Autodownload Enabled:** Is updates auto-download enabled? "Yes" or 67 | "No". 68 | 69 | All targeting options except for Channel and Locale are optional. 70 | 71 | Also all targeting except for Channel and Locale happens on the Browser, in the 72 | Activity Stream component which selects the best Snippet to display. 73 | -------------------------------------------------------------------------------- /docs/overview.rst: -------------------------------------------------------------------------------- 1 | Overview 2 | ======== 3 | 4 | This document describes the snippet service and how it relates to about:home, 5 | the home page for Firefox. 6 | 7 | What is a snippet? 8 | ------------------ 9 | 10 | about:home is the default home page in Firefox. When you visit about:home, you 11 | normally see, among other things, a Firefox logo, a search bar, and a bit of 12 | text and an icon below the search bar. This text and icon are what is called a 13 | "snippet". 14 | 15 | Snippets can be more than just text and an icon, though. Sometimes a snippet 16 | can replace the Firefox logo with a larger image. A snippet could also pop up 17 | a video for you to watch. Snippets are made of HTML, CSS, and JavaScript, and 18 | can modify the homepage in interesting and fun ways. 19 | 20 | .. figure:: img/snippet_example.png 21 | :align: center 22 | 23 | A snippet that replaced the Firefox logo with a larger, more detailed image. 24 | 25 | The Snippets Service is a web service that serves the code for snippets. 26 | Firefox downloads code from the Snippet Service and injects the code into 27 | about:home. This allows administrators using the service to control which 28 | snippets are being shown by Firefox without having to update Firefox itself. 29 | 30 | How are snippets retrieved by Firefox? 31 | -------------------------------------- 32 | 33 | .. digraph:: snippet_download_flow 34 | 35 | load_abouthome[label="User loads\nabout:home"]; 36 | check_cache_timeout[label="Has it been\n4 hours since\nsnippets were fetched?" shape=diamond]; 37 | load_cached_snippets[label="Retrieve snippets from\nIndexedDB" shape=rectangle]; 38 | fetch_snippets[label="Fetch snippets from\nsnippets.mozilla.com" shape=rectangle]; 39 | store_snippets[label="Store new snippets in\nIndexedDB" shape=rectangle]; 40 | insert_snippets[label="Insert snippet HTML\ninto about:home"]; 41 | 42 | load_abouthome -> check_cache_timeout; 43 | check_cache_timeout -> load_cached_snippets[label="No"]; 44 | 45 | check_cache_timeout -> fetch_snippets[label="Yes"]; 46 | fetch_snippets -> store_snippets; 47 | store_snippets -> load_cached_snippets; 48 | 49 | load_cached_snippets -> insert_snippets; 50 | 51 | Firefox maintains a cache of snippet code downloaded from the Snippet Service 52 | for at least 4 hours since the last download. The cache (and a few other 53 | useful pieces of information) are stored in IndexedDB, and can be accessed by 54 | code on about:home using a global JavaScript object named ``gSnippetsMap``. 55 | 56 | When a user visits about:home, Firefox checks to see when it last downloaded 57 | new snippet code. If it has been at least 4 hours, Firefox requests new 58 | snippet code from the Snippet Service and stores it in the cache along with 59 | the current time. After this, or if it hasn't been 4 hours, Firefox loads the 60 | snippet code from the cache and injects it directly into about:home. 61 | 62 | .. note:: All Firefox does is download snippet code from the service and inject 63 | it into the page. The rest of the logic behind displaying snippets is 64 | determined by the snippet code itself, as explained in the next section. 65 | 66 | .. seealso:: 67 | 68 | `aboutHome.js `_ 69 | The JavaScript within Firefox that, among other things, handles 70 | downloading and injecting snippet code. 71 | 72 | How are snippets displayed? 73 | --------------------------- 74 | 75 | The snippet code downloaded from the Snippet Service consists of three parts: 76 | 77 | - A small block of CSS that is common to all snippets. Among other things, this 78 | hides all the snippets initially so that only the one chosen to be displayed 79 | is visible to the user. 80 | - The code for each individual snippet, surrounded by some minimal HTML. 81 | - A block of JavaScript that handles displaying the snippets and other tasks. 82 | 83 | .. note:: It's important to understand that all the code for every snippet a 84 | user might see is injected into the page. This means that any JavaScript or 85 | CSS that is included in a snippet might conflict with JavaScript or CSS from 86 | another snippet. 87 | 88 | For this reason it is important to ensure that snippet code is well-isolated 89 | and avoids overly-broad CSS selectors or the global namespace in JavaScript. 90 | 91 | Once the code is injected, the included JavaScript: 92 | 93 | - Identifies all elements in the snippet code with the ``snippet`` class as 94 | potential snippets to display. 95 | - Filters out snippets that don't match the user's location. See 96 | :doc:`geolocation` for information on how we retrieve and store 97 | geolocation data. 98 | - Filters out snippets that are only supposed to be shown to users without a 99 | Firefox account. 100 | - Filters out snippets that are only supposed to be shown to users with a 101 | certain search provider. 102 | - Chooses a random snippet from the set based on their "weight" (a higher 103 | weight makes a snippet show more often relative to snippets with lower 104 | weights). 105 | - Displays the snippet. 106 | - Triggers a ``show_snippet`` event on the ``.snippet`` element. 107 | - Modifies all ```` tags in the snippet to add the snippet ID as a 108 | URL parameter. 109 | - Logs an impression for the displayed snippet by sending a request to 110 | the snippets metrics server. These requests are sampled and only go 111 | out 10% of the time. See also :doc:`data_collection` chapter for more 112 | information on the data send to the metrics server. 113 | 114 | If no snippets are available, the code falls back to showing default snippets 115 | included within Firefox itself. 116 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | Sphinx==2.4.4 2 | sphinx_rtd_theme==1.0.0 3 | sphinx-autobuild==0.7.1 4 | docutils==0.15.2 5 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "snippets.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /media/.htaccess: -------------------------------------------------------------------------------- 1 | ExpiresActive on 2 | FileETag MTime 3 | ExpiresDefault "access plus 15 minutes" 4 | 5 | ExpiresDefault "access plus 1 month" 6 | 7 | Header set Access-Control-Allow-Origin "*" 8 | -------------------------------------------------------------------------------- /redirector/.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | venv 3 | -------------------------------------------------------------------------------- /redirector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim-buster AS builder 2 | 3 | EXPOSE 8000 4 | 5 | ENV PYTHONDONTWRITEBYTECODE=1 6 | ENV PYTHONUNBUFFERED=1 7 | ENV PIP_DISABLE_PIP_VERSION_CHECK=1 8 | ENV PATH="/venv/bin:$PATH" 9 | ENV LANG=C.UTF-8 10 | 11 | RUN python -m venv /venv/ 12 | 13 | 14 | RUN bash -c 'for i in {1..8}; do mkdir -p "/usr/share/man/man$i"; done' && \ 15 | apt-get update && \ 16 | apt-get install -y --no-install-recommends build-essential && \ 17 | rm -rf /var/lib/apt/lists/* 18 | 19 | 20 | WORKDIR /app 21 | 22 | COPY requirements.txt ./ 23 | RUN pip install --require-hashes --no-cache-dir -r requirements.txt 24 | COPY . /app 25 | 26 | 27 | # Production Image 28 | FROM python:3.8-slim-buster 29 | 30 | ENV PYTHONDONTWRITEBYTECODE=1 31 | ENV PYTHONUNBUFFERED=1 32 | ENV PIP_DISABLE_PIP_VERSION_CHECK=1 33 | ENV PATH="/venv/bin:$PATH" 34 | ENV LANG=C.UTF-8 35 | 36 | CMD ["python", "main.py"] 37 | 38 | WORKDIR /app 39 | 40 | RUN adduser --uid 1000 --disabled-password --gecos '' webdev 41 | 42 | COPY --from=builder /venv /venv 43 | COPY --from=builder /app /app 44 | 45 | RUN chown webdev.webdev -R . 46 | USER webdev 47 | 48 | ARG GIT_SHA=head 49 | ENV GIT_SHA=${GIT_SHA} 50 | -------------------------------------------------------------------------------- /redirector/config.py: -------------------------------------------------------------------------------- 1 | from os import getenv 2 | 3 | 4 | bind = '0.0.0.0:{}'.format(getenv('PORT', 8000)) 5 | workers = getenv('WSGI_NUM_WORKERS', 2) 6 | accesslog = '-' 7 | errorlog = '-' 8 | loglevel = getenv('WSGI_LOG_LEVEL', 'info') 9 | 10 | # Larger keep-alive values maybe needed when directly talking to ELBs 11 | # See https://github.com/benoitc/gunicorn/issues/1194 12 | keepalive = getenv('WSGI_KEEP_ALIVE', 2) 13 | worker_tmp_dir = '/dev/shm' 14 | worker_class = getenv('GUNICORN_WORKER_CLASS', 'meinheld.gmeinheld.MeinheldWorker') 15 | -------------------------------------------------------------------------------- /redirector/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | from bottle import redirect, response, route, run, default_app 4 | from decouple import config 5 | 6 | from redirect import calculate_redirect 7 | 8 | DEBUG = config('DEBUG', default=False, cast=bool) 9 | 10 | SNIPPET_BUNDLE_PREGEN_REDIRECT_TIMEOUT = config( 11 | 'SNIPPET_BUNDLE_PREGEN_REDIRECT_TIMEOUT', default=60 * 60 * 24, cast=int) # One day 12 | 13 | GIT_SHA = config('GIT_SHA', default='HEAD') 14 | 15 | CLUSTER_NAME = config('CLUSTER_NAME', default='cluster') 16 | K8S_NAMESPACE = config('K8S_NAMESPACE', default='namespace') 17 | K8S_POD_NAME = config('K8S_POD_NAME', default='pod') 18 | 19 | app = default_app() 20 | 21 | 22 | def set_xbackend_header(fn): 23 | def _inner(*args, **kwargs): 24 | response.add_header('X-Backend-Server', f'{CLUSTER_NAME}/{K8S_NAMESPACE}/{K8S_POD_NAME}') 25 | return fn(*args, **kwargs) 26 | 27 | return _inner 28 | 29 | 30 | @route('/') 31 | @set_xbackend_header 32 | def index(): 33 | return '' 34 | 35 | 36 | @route('/static/revision.txt') 37 | @set_xbackend_header 38 | def revision(): 39 | return GIT_SHA 40 | 41 | 42 | @route('/healthz') 43 | @route('/healthz/') 44 | @set_xbackend_header 45 | def healthz(): 46 | return 'OK' 47 | 48 | 49 | @route('/////' 50 | '////' 51 | '//') 52 | @set_xbackend_header 53 | def redirect_to_bundle(*args, **kwargs): 54 | locale, distribution, full_url = calculate_redirect(*args, **kwargs) 55 | 56 | response.set_header( 57 | 'Cache-Control', 58 | f'public, max-age={SNIPPET_BUNDLE_PREGEN_REDIRECT_TIMEOUT}' 59 | ) 60 | return redirect(full_url) 61 | 62 | 63 | if __name__ == '__main__': 64 | if DEBUG: 65 | run(host='localhost', port=8000) 66 | -------------------------------------------------------------------------------- /redirector/redirect.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import urljoin 2 | 3 | from decouple import config 4 | 5 | MEDIA_BUNDLES_PREGEN_ROOT = config('MEDIA_BUNDLES_PREGEN_ROOT', default='bundles-pregen/') 6 | SITE_URL = config('SITE_URL', default='') 7 | CDN_URL = config('CDN_URL', default='') 8 | 9 | 10 | def calculate_redirect(*args, **kwargs): 11 | product = 'Firefox' 12 | locale = kwargs['locale'].lower() 13 | 14 | # Distribution populated by client's distribution if it starts with 15 | # `experiment-`. Otherwise default to `default`. 16 | # 17 | # This is because non-Mozilla distributors of Firefox (e.g. Linux 18 | # Distributions) override the distribution field with their identification. 19 | # We want all Firefox clients to get the default bundle for locale, unless 20 | # they are part of an experiment. 21 | distribution = kwargs['distribution'].lower() 22 | if distribution.startswith('experiment-'): 23 | distribution = distribution[11:] 24 | else: 25 | distribution = 'default' 26 | 27 | filename = ( 28 | f'{MEDIA_BUNDLES_PREGEN_ROOT}/{product}/' 29 | f'{locale}/{distribution}.json' 30 | ) 31 | 32 | full_url = urljoin(CDN_URL or SITE_URL, filename) 33 | 34 | # Return calculated locale, distribution, full_url 35 | return locale, distribution, full_url 36 | -------------------------------------------------------------------------------- /redirector/requirements.in: -------------------------------------------------------------------------------- 1 | bottle==0.12.21 2 | python-decouple==3.5 3 | gunicorn==22.0.0 4 | pytest==6.2.5 5 | importlib-metadata==6.0.0 6 | meinheld==1.0.2 7 | -------------------------------------------------------------------------------- /redirector/requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with python 3.8 3 | # To update, run: 4 | # 5 | # pip-compile --generate-hashes 6 | # 7 | attrs==23.2.0 \ 8 | --hash=sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30 \ 9 | --hash=sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1 10 | # via pytest 11 | bottle==0.12.21 \ 12 | --hash=sha256:6e1c9817019dae3a8c20adacaf09035251798d2ae2fcc8ce43157ee72965f257 \ 13 | --hash=sha256:787c61b6cc02b9c229bf2663011fac53dd8fc197f7f8ad2eeede29d888d7887e 14 | # via -r requirements.in 15 | greenlet==0.4.17 \ 16 | --hash=sha256:1023d7b43ca11264ab7052cb09f5635d4afdb43df55e0854498fc63070a0b206 \ 17 | --hash=sha256:124a3ae41215f71dc91d1a3d45cbf2f84e46b543e5d60b99ecc20e24b4c8f272 \ 18 | --hash=sha256:13037e2d7ab2145300676852fa069235512fdeba4ed1e3bb4b0677a04223c525 \ 19 | --hash=sha256:3af587e9813f9bd8be9212722321a5e7be23b2bc37e6323a90e592ab0c2ef117 \ 20 | --hash=sha256:41d8835c69a78de718e466dd0e6bfd4b46125f21a67c3ff6d76d8d8059868d6b \ 21 | --hash=sha256:4481002118b2f1588fa3d821936ffdc03db80ef21186b62b90c18db4ba5e743b \ 22 | --hash=sha256:47825c3a109f0331b1e54c1173d4e57fa000aa6c96756b62852bfa1af91cd652 \ 23 | --hash=sha256:5494e3baeacc371d988345fbf8aa4bd15555b3077c40afcf1994776bb6d77eaf \ 24 | --hash=sha256:75e4c27188f28149b74e7685809f9227410fd15432a4438fc48627f518577fa5 \ 25 | --hash=sha256:97f2b01ab622a4aa4b3724a3e1fba66f47f054c434fbaa551833fa2b41e3db51 \ 26 | --hash=sha256:a34023b9eabb3525ee059f3bf33a417d2e437f7f17e341d334987d4091ae6072 \ 27 | --hash=sha256:ac85db59aa43d78547f95fc7b6fd2913e02b9e9b09e2490dfb7bbdf47b2a4914 \ 28 | --hash=sha256:be7a79988b8fdc5bbbeaed69e79cfb373da9759242f1565668be4fb7f3f37552 \ 29 | --hash=sha256:bee111161420f341a346731279dd976be161b465c1286f82cc0779baf7b729e8 \ 30 | --hash=sha256:ccd62f09f90b2730150d82f2f2ffc34d73c6ce7eac234aed04d15dc8a3023994 \ 31 | --hash=sha256:d3436110ca66fe3981031cc6aff8cc7a40d8411d173dde73ddaa5b8445385e2d \ 32 | --hash=sha256:e495096e3e2e8f7192afb6aaeba19babc4fb2bdf543d7b7fed59e00c1df7f170 \ 33 | --hash=sha256:e66a824f44892bc4ec66c58601a413419cafa9cec895e63d8da889c8a1a4fa4a 34 | # via meinheld 35 | gunicorn==22.0.0 \ 36 | --hash=sha256:350679f91b24062c86e386e198a15438d53a7a8207235a78ba1b53df4c4378d9 \ 37 | --hash=sha256:4a0b436239ff76fb33f11c07a16482c521a7e09c1ce3cc293c2330afe01bec63 38 | # via -r requirements.in 39 | importlib-metadata==6.0.0 \ 40 | --hash=sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad \ 41 | --hash=sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d 42 | # via -r requirements.in 43 | iniconfig==2.0.0 \ 44 | --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ 45 | --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 46 | # via pytest 47 | meinheld==1.0.2 \ 48 | --hash=sha256:008c76937ac2117cc69e032dc69cea9f85fc605de9bac1417f447c41c16a56d6 49 | # via -r requirements.in 50 | packaging==24.0 \ 51 | --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \ 52 | --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9 53 | # via 54 | # gunicorn 55 | # pytest 56 | pluggy==1.4.0 \ 57 | --hash=sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981 \ 58 | --hash=sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be 59 | # via pytest 60 | py==1.11.0 \ 61 | --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \ 62 | --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378 63 | # via pytest 64 | pytest==6.2.5 \ 65 | --hash=sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89 \ 66 | --hash=sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134 67 | # via -r requirements.in 68 | python-decouple==3.5 \ 69 | --hash=sha256:011d3f785367c54a72cf8a07d3a7a48bb8cc1a0f8e6c70353ca5767ebf7c8c9d \ 70 | --hash=sha256:68e4b3fcc97e24bc90eecc514852d0bf970f4ff031f5f7a6728ddafa9afefcaf 71 | # via -r requirements.in 72 | toml==0.10.2 \ 73 | --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ 74 | --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f 75 | # via pytest 76 | zipp==3.18.1 \ 77 | --hash=sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b \ 78 | --hash=sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715 79 | # via importlib-metadata 80 | -------------------------------------------------------------------------------- /redirector/run-prod.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | gunicorn main:app --config config.py 5 | -------------------------------------------------------------------------------- /redirector/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from unittest.mock import patch 4 | 5 | import main 6 | import redirect 7 | 8 | 9 | def test_redirect_calculate_redirect_locale_lower(): 10 | assert redirect.calculate_redirect(locale='el-GR', distribution='default')[0] == 'el-gr' 11 | 12 | 13 | @patch('redirect.CDN_URL', 'https://cdn.example.com') 14 | @patch('redirect.SITE_URL', 'https://www.example.com') 15 | def test_redirect_calculate_redirect_cdn_url(): 16 | full_url = redirect.calculate_redirect(locale='en-US', distribution='default')[2] 17 | assert full_url == 'https://cdn.example.com/bundles-pregen/Firefox/en-us/default.json' 18 | 19 | 20 | @patch('redirect.SITE_URL', 'https://www.example.com') 21 | def test_redirect_calculate_redirect_site_url(): 22 | full_url = redirect.calculate_redirect(locale='en-us', distribution='default')[2] 23 | assert full_url == 'https://www.example.com/bundles-pregen/Firefox/en-us/default.json' 24 | 25 | 26 | @patch('redirect.SITE_URL', 'https://www.example.com') 27 | def test_redirect_calculate_redirect_experiment(): 28 | full_url = redirect.calculate_redirect(locale='en-us', distribution='experiment-foo-bar')[2] 29 | assert full_url == 'https://www.example.com/bundles-pregen/Firefox/en-us/foo-bar.json' 30 | 31 | 32 | @patch('redirect.SITE_URL', 'https://www.example.com') 33 | def test_redirect_calculate_redirect_default_experiment(): 34 | _, distribution, full_url = redirect.calculate_redirect(locale='en-us', distribution='foo-bar') 35 | assert full_url == 'https://www.example.com/bundles-pregen/Firefox/en-us/default.json' 36 | assert distribution == 'default' 37 | 38 | 39 | def test_main_index(): 40 | assert main.index() == '' 41 | 42 | 43 | def test_main_healthz(): 44 | assert main.healthz() == 'OK' 45 | 46 | 47 | @patch('main.GIT_SHA', 'xxffxx') 48 | def test_main_revision(): 49 | assert main.revision() == 'xxffxx' 50 | 51 | 52 | @patch('main.SNIPPET_BUNDLE_PREGEN_REDIRECT_TIMEOUT', 90) 53 | @patch('main.response') 54 | @patch('main.redirect') 55 | @patch('main.calculate_redirect') 56 | def test_main_redirect_to_bundle(calculate_redirect, redirect_mock, response_mock): 57 | calculate_redirect.return_value = ('fr', 'default', 'https://www.example.com/bundle.json') 58 | main.redirect_to_bundle(locale='fr', distribution='default') 59 | assert redirect_mock.called_with('https://www.example.com/bundle.json') 60 | response_mock.set_header.assert_called_with('Cache-Control', 'public, max-age=90') 61 | -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | APScheduler==3.7.0 2 | asn1crypto==1.3.0 3 | babis==0.2.4 4 | bleach==4.1.0 5 | boto3==1.16.63 6 | Brotli==1.0.9 7 | certifi>=2023.7.22 8 | configparser==5.0.0 9 | contextlib2==0.6.0.post1 10 | cryptography>=41.0.0 11 | Django==2.2.28 12 | dj-database-url==0.5.0 13 | django-admin-list-filter-dropdown==1.0.3 14 | django-allow-cidr==0.3.1 15 | django-cache-url==3.0.0 16 | django-csp==3.7 17 | django-enforce-host==1.0.1 18 | django-extensions==3.1.5 19 | django-filter==2.4.0 20 | django-ical==1.8.3 21 | django-jinja==2.10.0 22 | django-modeladmin-reorder==0.3.1 23 | django-mozilla-product-details==0.14.1 24 | django-ratelimit==3.0.1 25 | django-redis==4.11.0 26 | django-reversion==4.0.1 27 | django-storages==1.11.1 28 | django-taggit==2.1.0 # Downpinned because 3+ needs Django 3.2 29 | django-taggit-helpers==0.1.4 30 | django-watchman==0.18.0 31 | factory-boy==3.2.1 32 | gunicorn==22.0.0 33 | hiredis==1.1.0 34 | meinheld==1.0.2 35 | mozilla-django-oidc==2.0.0 36 | newrelic==7.10.0.175 37 | Pillow>=10.2.0 38 | psycopg2==2.9.5 39 | pyjexl==0.3.0 40 | python-decouple==3.5 41 | redash-dynamic-query==1.0.4 42 | sentry-sdk==1.31.0 43 | taggit-selectize==2.7.1 44 | urlwait==1.0 45 | whitenoise==5.2.0 46 | typed-ast==1.4.0 47 | 48 | # dev 49 | Werkzeug==2.3.8 50 | flake8==3.8.4 51 | hashin==0.17.0 52 | pip-tools==6.6.2 53 | pylint==2.16.2 54 | -------------------------------------------------------------------------------- /scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozmeao/snippets-service/aefc553ba93689ca8d61fabbb8586deb9ca09f10/scripts/__init__.py -------------------------------------------------------------------------------- /scripts/cron.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import datetime 3 | import os 4 | import sys 5 | from subprocess import check_call 6 | 7 | from django.conf import settings 8 | from django.db import connection 9 | 10 | import babis 11 | from apscheduler.schedulers.blocking import BlockingScheduler 12 | 13 | from snippets.base.util import create_countries 14 | 15 | 16 | MANAGE = os.path.join(settings.ROOT, 'manage.py') 17 | schedule = BlockingScheduler() 18 | 19 | # Used by generate_bundles commands. This is intentionally here and not in 20 | # a persistent storage to force all bundle regeneration when we restart the 21 | # service, which is typically when we push new code. 22 | last_timestamp = 0 23 | 24 | 25 | def call_command(command): 26 | check_call('python {0} {1}'.format(MANAGE, command), shell=True) 27 | 28 | 29 | class scheduled_job(object): 30 | """Decorator for scheduled jobs. Takes same args as apscheduler.schedule_job.""" 31 | def __init__(self, *args, **kwargs): 32 | self.args = args 33 | self.kwargs = kwargs 34 | 35 | def __call__(self, fn): 36 | job_name = fn.__name__ 37 | self.name = job_name 38 | self.callback = fn 39 | schedule.add_job(self.run, id=job_name, *self.args, **self.kwargs) 40 | self.log('Registered.') 41 | return self.run 42 | 43 | def run(self): 44 | self.log('starting') 45 | try: 46 | self.callback() 47 | except Exception as e: 48 | self.log('CRASHED: {}'.format(e)) 49 | raise 50 | else: 51 | self.log('finished successfully') 52 | 53 | def log(self, message): 54 | msg = '[{}] Clock job {}@{}: {}'.format( 55 | datetime.datetime.utcnow(), self.name, 56 | os.getenv('K8S_NAMESPACE', 'default_app'), message) 57 | print(msg, file=sys.stderr) 58 | 59 | 60 | @scheduled_job('cron', month='*', day='*', hour='*/12', minute='10', max_instances=1, coalesce=True) 61 | @babis.decorator(ping_after=settings.DEAD_MANS_SNITCH_PRODUCT_DETAILS) 62 | def job_update_product_details(): 63 | call_command('update_product_details') 64 | connection.close() 65 | create_countries() 66 | 67 | 68 | @scheduled_job('cron', month='*', day='*', hour='*', minute='*', max_instances=1, coalesce=True) 69 | @babis.decorator(ping_after=settings.DEAD_MANS_SNITCH_UPDATE_JOBS) 70 | def job_update_jobs(): 71 | global last_timestamp 72 | call_command('update_jobs') 73 | utc_now = datetime.datetime.utcnow() 74 | if last_timestamp: 75 | call_command('generate_bundles --timestamp "{}"'.format(last_timestamp)) 76 | else: 77 | call_command('generate_bundles') 78 | last_timestamp = utc_now 79 | 80 | 81 | @babis.decorator(ping_after=settings.DEAD_MANS_SNITCH_FETCH_METRICS) 82 | def job_fetch_metrics(): 83 | call_command('fetch_metrics') 84 | 85 | 86 | @babis.decorator(ping_after=settings.DEAD_MANS_SNITCH_FETCH_DAILY_METRICS) 87 | def job_fetch_daily_metrics(): 88 | call_command('fetch_daily_metrics') 89 | 90 | 91 | if settings.REDASH_API_KEY: 92 | scheduled_job( 93 | 'cron', month='*', day='*', hour='*', minute='10', max_instances=1, coalesce=True 94 | )(job_fetch_metrics) 95 | scheduled_job( 96 | 'cron', month='*', day='*', hour='4', minute='0', max_instances=1, coalesce=True 97 | )(job_fetch_daily_metrics) 98 | 99 | 100 | def run(): 101 | try: 102 | schedule.start() 103 | except (KeyboardInterrupt, SystemExit): 104 | pass 105 | -------------------------------------------------------------------------------- /scripts/f100s.py: -------------------------------------------------------------------------------- 1 | ## 2 | # 3 | # Script to produce CSVs per locale with currently running F100s 4 | # 5 | # To be used upon request from the OMN team. Copy / paste resulting 6 | # CSVs to a Google Spreadsheet. Example 7 | # https://docs.google.com/spreadsheets/d/13N8oqMx1QOqe0V572j2VqX6gwYioO5o4cLQk1y_iGnk/edit?usp=sharing 8 | ## 9 | 10 | import csv 11 | import io 12 | import itertools 13 | 14 | from snippets.base.models import ASRSnippet, Job 15 | 16 | 17 | def writeRowCSV(csvwriter, snippet): 18 | csvwriter.writerow( 19 | [ 20 | snippet.id, 21 | snippet.name, 22 | snippet.template_ng.get_main_body(bleached=True), 23 | f'https://snippets-admin.mozilla.org/admin/base/asrsnippet/{snippet.id}/change/', 24 | snippet.get_preview_url() 25 | ] 26 | ) 27 | 28 | 29 | def run(): 30 | for locale in {'de', 'en', 'es', 'id', 'pl', 'pt-br', 'ru', 'zh-cn', 'zh-tw', 'fr'}: 31 | csvfile = io.StringIO() 32 | csvwriter = csv.writer(csvfile, dialect=csv.excel, quoting=csv.QUOTE_ALL) 33 | s = ASRSnippet.objects.filter(locale__code__contains=f',{locale}', 34 | name__icontains='f100', 35 | jobs__status=Job.PUBLISHED) 36 | snippets = {} 37 | count = 0 38 | for i in range(1, 18): 39 | ss = s.filter(name__icontains=f'week{i}_') 40 | snippets[i] = list(ss) 41 | count += ss.count() 42 | no_week_snippets = set(s) - set(itertools.chain(*snippets.values())) 43 | 44 | for number, weekly_snippets in snippets.items(): 45 | csvwriter.writerow([f'Week {number}']) 46 | for snippet in weekly_snippets: 47 | writeRowCSV(csvwriter, snippet) 48 | 49 | csvwriter.writerow([f'No Week in Name']) 50 | for snippet in no_week_snippets: 51 | writeRowCSV(csvwriter, snippet) 52 | 53 | with open(f'{locale}.csv', 'w') as f: 54 | f.write(csvfile.getvalue()) 55 | 56 | print(f'Done writing {locale}.csv') 57 | print('All Done') 58 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length=100 3 | exclude=snippets/*/migrations/* 4 | ignore=W504 5 | 6 | 7 | [tool:paul-mclendahand] 8 | github_user=mozmeao 9 | github_project=snippets-service 10 | git_remote=mozilla-rw 11 | main_branch=master -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from distutils.core import setup 4 | 5 | setup( 6 | name='snippets', 7 | version='0.1dev', 8 | description='This is https://github.com/mozilla/snippets', 9 | author='Mozilla Foundation', 10 | author_email='', 11 | url='https://github.com/mozilla/snippets' 12 | ) 13 | -------------------------------------------------------------------------------- /snippets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozmeao/snippets-service/aefc553ba93689ca8d61fabbb8586deb9ca09f10/snippets/__init__.py -------------------------------------------------------------------------------- /snippets/base/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozmeao/snippets-service/aefc553ba93689ca8d61fabbb8586deb9ca09f10/snippets/base/__init__.py -------------------------------------------------------------------------------- /snippets/base/admin/__init__.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from snippets.base import models 4 | 5 | import snippets.base.admin.adminmodels as adminmodels 6 | 7 | admin.site.register(models.Addon, adminmodels.AddonAdmin) 8 | admin.site.register(models.ASRSnippet, adminmodels.ASRSnippetAdmin) 9 | admin.site.register(models.Campaign, adminmodels.CampaignAdmin) 10 | admin.site.register(models.Category, adminmodels.CategoryAdmin) 11 | admin.site.register(models.Product, adminmodels.ProductAdmin) 12 | admin.site.register(admin.models.LogEntry, adminmodels.LogEntryAdmin) 13 | admin.site.register(models.Target, adminmodels.TargetAdmin) 14 | admin.site.register(models.Icon, adminmodels.IconAdmin) 15 | admin.site.register(models.Locale, adminmodels.LocaleAdmin) 16 | admin.site.register(models.Job, adminmodels.JobAdmin) 17 | admin.site.register(models.Distribution, adminmodels.DistributionAdmin) 18 | admin.site.register(models.DistributionBundle, adminmodels.DistributionBundleAdmin) 19 | admin.site.register(models.JobDailyPerformance, adminmodels.JobDailyPerformanceAdmin) 20 | admin.site.register(models.DailyImpressions, adminmodels.DailyImpressionsAdmin) 21 | -------------------------------------------------------------------------------- /snippets/base/admin/actions.py: -------------------------------------------------------------------------------- 1 | import csv 2 | from datetime import datetime 3 | 4 | from django.db import transaction 5 | from django.http import HttpResponse 6 | 7 | 8 | @transaction.atomic 9 | def duplicate_snippets_action(modeladmin, request, queryset): 10 | for snippet in queryset: 11 | snippet.duplicate(request.user) 12 | duplicate_snippets_action.short_description = 'Duplicate selected snippets' # noqa 13 | 14 | 15 | def export_as_csv(modeladmin, request, queryset): 16 | """Adapted from https://books.agiliq.com/projects/django-admin-cookbook/en/latest/export.html""" 17 | meta = modeladmin.model._meta 18 | field_names = [field.name for field in meta.fields] 19 | filename = f'{meta}-{datetime.today().strftime("%Y-%m-%d-%H-%M")}' 20 | 21 | response = HttpResponse(content_type='text/csv') 22 | response['Content-Disposition'] = 'attachment; filename={}.csv'.format(filename) 23 | writer = csv.writer(response) 24 | 25 | writer.writerow(field_names) 26 | for obj in queryset: 27 | writer.writerow([getattr(obj, field) for field in field_names]) 28 | 29 | return response 30 | export_as_csv.short_description = 'Export Selected to CSV' # noqa 31 | -------------------------------------------------------------------------------- /snippets/base/admin/filters.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | 3 | from django.apps import apps 4 | from django.contrib import admin 5 | from django.utils.encoding import force_text 6 | 7 | from snippets.base import models 8 | 9 | 10 | class ModifiedFilter(admin.SimpleListFilter): 11 | title = 'Last modified' 12 | parameter_name = 'last_modified' 13 | 14 | def lookups(self, request, model_admin): 15 | return ( 16 | ('24', '24 hours'), 17 | ('168', '7 days'), 18 | ('336', '14 days'), 19 | ('720', '30 days'), 20 | ('1440', '60 days'), 21 | ('all', 'All'), 22 | ) 23 | 24 | def queryset(self, request, queryset): 25 | value = self.value() 26 | if not value or value == 'all': 27 | return queryset 28 | 29 | when = datetime.utcnow() - timedelta(hours=int(value)) 30 | return queryset.exclude(modified__lt=when) 31 | 32 | def choices(self, cl): 33 | for lookup, title in self.lookup_choices: 34 | yield { 35 | 'selected': self.value() == force_text(lookup), 36 | 'query_string': cl.get_query_string({ 37 | self.parameter_name: lookup, 38 | }, []), 39 | 'display': title, 40 | } 41 | 42 | 43 | class ChannelFilter(admin.SimpleListFilter): 44 | title = 'Channel' 45 | parameter_name = 'channel' 46 | 47 | def lookups(self, request, model_admin): 48 | return ( 49 | ('release', 'Release'), 50 | ('esr', 'ESR'), 51 | ('beta', 'Beta'), 52 | ('aurora', 'Dev (Aurora)'), 53 | ('nightly', 'Nightly'), 54 | ) 55 | 56 | def queryset(self, request, queryset): 57 | if self.value() is None: 58 | return queryset 59 | 60 | if hasattr(queryset.model, 'jobs'): 61 | return queryset.filter(jobs__targets__filtr_channels__contains=self.value()).distinct() 62 | return queryset.filter(targets__filtr_channels__contains=self.value()).distinct() 63 | 64 | 65 | class TemplateFilter(admin.SimpleListFilter): 66 | title = 'template type' 67 | parameter_name = 'template' 68 | LOOKUPS = sorted([ 69 | (model.__name__, model.NAME) 70 | for model in apps.get_models() 71 | if issubclass(model, models.Template) and not model.__name__ == 'Template' 72 | ], key=lambda x: x[1]) 73 | 74 | def lookups(self, request, model_admin): 75 | return self.LOOKUPS 76 | 77 | def queryset(self, request, queryset): 78 | value = self.value() 79 | if not value: 80 | return queryset 81 | 82 | filters = {} 83 | for k, v in self.LOOKUPS: 84 | if value != k: 85 | filters['template_relation__{}'.format(k.lower())] = None 86 | 87 | return queryset.filter(**filters) 88 | 89 | 90 | class RelatedPublishedASRSnippetFilter(admin.SimpleListFilter): 91 | title = 'Currently Published' 92 | parameter_name = 'is_currently_published' 93 | 94 | def lookups(self, request, model_admin): 95 | return ( 96 | ('yes', 'Yes'), 97 | ('no', 'No'), 98 | ) 99 | 100 | def queryset(self, request, queryset): 101 | if self.value() is None: 102 | return queryset 103 | 104 | if hasattr(queryset.model, 'snippets'): 105 | if self.value() == 'yes': 106 | return queryset.filter(snippets__jobs__status=models.Job.PUBLISHED).distinct() 107 | elif self.value() == 'no': 108 | return queryset.exclude(snippets__jobs__status=models.Job.PUBLISHED).distinct() 109 | else: 110 | if self.value() == 'yes': 111 | return queryset.filter(jobs__status=models.Job.PUBLISHED).distinct() 112 | elif self.value() == 'no': 113 | return queryset.exclude(jobs__status=models.Job.PUBLISHED).distinct() 114 | 115 | 116 | class IconRelatedPublishedASRSnippetFilter(RelatedPublishedASRSnippetFilter): 117 | def queryset(self, request, queryset): 118 | if self.value() is None: 119 | return queryset 120 | 121 | icon_ids = [] 122 | for icon in queryset.all(): 123 | if icon.snippets.filter(jobs__status=models.Job.PUBLISHED).count(): 124 | icon_ids.append(icon.id) 125 | 126 | if self.value() == 'yes': 127 | return queryset.filter(id__in=icon_ids) 128 | elif self.value() == 'no': 129 | return queryset.exclude(id__in=icon_ids) 130 | -------------------------------------------------------------------------------- /snippets/base/admin/widgets.py: -------------------------------------------------------------------------------- 1 | from django.forms import widgets 2 | 3 | 4 | class JEXLMultiWidget(widgets.MultiWidget): 5 | 6 | def __init__(self, *args, **kwargs): 7 | if 'template_name' in kwargs: 8 | self.template_name = kwargs.pop('template_name') 9 | super().__init__(*args, **kwargs) 10 | 11 | def decompress(self, value): 12 | if value: 13 | return value.split(',') 14 | return [None, None] 15 | -------------------------------------------------------------------------------- /snippets/base/app.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | import session_csrf 4 | 5 | 6 | class BaseAppConfig(AppConfig): 7 | name = 'snippets.base' 8 | 9 | def ready(self): 10 | # The app is now ready. Include any monkey patches here. 11 | 12 | # Monkey patch CSRF to switch to session based CSRF. Session 13 | # based CSRF will prevent attacks from apps under the same 14 | # domain. If you're planning to host your app under it's own 15 | # domain you can remove session_csrf and use Django's CSRF 16 | # library. See also 17 | # https://github.com/mozilla/sugardough/issues/38 18 | session_csrf.monkeypatch() 19 | -------------------------------------------------------------------------------- /snippets/base/bundles.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import json 3 | import os 4 | from datetime import datetime 5 | from io import StringIO 6 | 7 | import brotli 8 | from django.conf import settings 9 | from django.core.files.base import ContentFile 10 | from django.core.files.storage import default_storage 11 | from django.db.models import Q 12 | from product_details import product_details 13 | 14 | from snippets.base import models 15 | 16 | 17 | def generate_bundles(timestamp=None, limit_to_locale=None, 18 | limit_to_distribution_bundle=None, save_to_disk=True, 19 | stdout=StringIO()): 20 | if not timestamp: 21 | stdout.write('Generating all bundles.') 22 | total_jobs = models.Job.objects.all() 23 | else: 24 | stdout.write( 25 | 'Generating bundles with Jobs modified on or after {}'.format(timestamp) 26 | ) 27 | total_jobs = models.Job.objects.filter( 28 | Q(snippet__modified__gte=timestamp) | 29 | Q(distribution__distributionbundle__modified__gte=timestamp) 30 | ).distinct() 31 | 32 | stdout.write('Processing bundles…') 33 | if limit_to_locale: 34 | all_locales_to_process = [ 35 | limit_to_locale, 36 | ] 37 | else: 38 | all_locales_to_process = set( 39 | itertools.chain.from_iterable( 40 | job.snippet.locale.code.strip(',').split(',') 41 | for job in total_jobs 42 | ) 43 | ) 44 | distribution_bundles_to_process = models.DistributionBundle.objects.filter( 45 | distributions__jobs__in=total_jobs 46 | ).distinct().order_by('id') 47 | 48 | if limit_to_distribution_bundle: 49 | distribution_bundles_to_process = distribution_bundles_to_process.filter( 50 | name__iexact=limit_to_distribution_bundle 51 | ) 52 | 53 | for distribution_bundle in distribution_bundles_to_process: 54 | distributions = distribution_bundle.distributions.all() 55 | 56 | for locale in all_locales_to_process: 57 | 58 | all_jobs = (models.Job.objects 59 | .filter(status=models.Job.PUBLISHED) 60 | .filter(distribution__in=distributions)) 61 | 62 | locales_to_process = [ 63 | key.lower() for key in product_details.languages.keys() 64 | if key.lower().startswith(locale) 65 | ] 66 | 67 | for locale_to_process in locales_to_process: 68 | filename = 'Firefox/{locale}/{distribution}.json'.format( 69 | locale=locale_to_process, 70 | distribution=distribution_bundle.code_name, 71 | ) 72 | filename = os.path.join(settings.MEDIA_BUNDLES_PREGEN_ROOT, filename) 73 | full_locale = ',{},'.format(locale_to_process.lower()) 74 | splitted_locale = ',{},'.format(locale_to_process.lower().split('-', 1)[0]) 75 | bundle_jobs = all_jobs.filter( 76 | Q(snippet__locale__code__contains=splitted_locale) | 77 | Q(snippet__locale__code__contains=full_locale)).distinct() 78 | 79 | # If DistributionBundle is not enabled, or if there are no 80 | # Published Jobs for the locale / distribution 81 | # combination, delete the current bundle file if it exists. 82 | if save_to_disk and not distribution_bundle.enabled or not bundle_jobs.exists(): 83 | if default_storage.exists(filename): 84 | stdout.write('Removing {}'.format(filename)) 85 | default_storage.delete(filename) 86 | continue 87 | 88 | data = [ 89 | job.render() for job in bundle_jobs 90 | ] 91 | bundle_content = json.dumps({ 92 | 'messages': data, 93 | 'metadata': { 94 | 'generated_at': datetime.utcnow().isoformat(), 95 | 'number_of_snippets': len(data), 96 | 'locale': locale_to_process, 97 | 'distribution_bundle': distribution_bundle.code_name, 98 | } 99 | }) 100 | 101 | # Convert str to bytes. 102 | if isinstance(bundle_content, str): 103 | bundle_content = bundle_content.encode('utf-8') 104 | 105 | if settings.BUNDLE_BROTLI_COMPRESS: 106 | content_file = ContentFile(brotli.compress(bundle_content)) 107 | content_file.content_encoding = 'br' 108 | else: 109 | content_file = ContentFile(bundle_content) 110 | 111 | if save_to_disk is True: 112 | default_storage.save(filename, content_file) 113 | stdout.write('Writing bundle {}'.format(filename)) 114 | else: 115 | return content_file 116 | 117 | # If save_to_disk is False and we reach this point, it means that we didn't 118 | # have any Jobs to return for the locale, channel, distribution combination. 119 | # Return an empty bundle 120 | if save_to_disk is False: 121 | return ContentFile( 122 | json.dumps({ 123 | 'messages': [], 124 | 'metadata': { 125 | 'generated_at': datetime.utcnow().isoformat(), 126 | 'number_of_snippets': 0, 127 | 'locale': limit_to_locale, 128 | 'distribution_bundle': limit_to_distribution_bundle, 129 | } 130 | }) 131 | ) 132 | -------------------------------------------------------------------------------- /snippets/base/context_processors.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings as django_settings 2 | 3 | 4 | def settings(request): 5 | """ 6 | Adds static-related context variables to the context. 7 | 8 | """ 9 | return {'settings': django_settings} 10 | -------------------------------------------------------------------------------- /snippets/base/feed.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from textwrap import dedent 3 | from urllib.parse import urlparse 4 | 5 | from django.conf import settings 6 | from django.db.models import Q 7 | 8 | from django_ical.views import ICalFeed 9 | 10 | from snippets.base import filters, models 11 | 12 | 13 | class JobsFeed(ICalFeed): 14 | timezone = 'UTC' 15 | title = 'Snippet Jobs' 16 | 17 | def __call__(self, request, *args, **kwargs): 18 | self.request = request 19 | return super().__call__(request, *args, **kwargs) 20 | 21 | @property 22 | def product_id(self): 23 | return '//{}/SnippetJobs?{}'.format(urlparse(settings.SITE_URL).netloc, 24 | self.request.GET.urlencode()) 25 | 26 | def items(self): 27 | queryset = (models.Job.objects 28 | .filter(Q(status=models.Job.PUBLISHED) | Q(status=models.Job.SCHEDULED)) 29 | .order_by('publish_start')) 30 | filtr = filters.JobFilter(self.request.GET, queryset=queryset) 31 | return filtr.qs 32 | 33 | def item_title(self, item): 34 | return item.snippet.name 35 | 36 | def item_link(self, item): 37 | return item.get_admin_url() 38 | 39 | def item_description(self, item): 40 | description = dedent('''\ 41 | Channels: {} 42 | Locale: {}' 43 | Preview Link: {} 44 | '''.format(', '.join(item.channels), 45 | item.snippet.locale, 46 | item.snippet.get_preview_url())) 47 | return description 48 | 49 | def item_start_datetime(self, item): 50 | return item.publish_start or item.created 51 | 52 | def item_end_datetime(self, item): 53 | return item.publish_end or (self.item_start_datetime(item) + timedelta(days=365)) 54 | 55 | def item_created(self, item): 56 | return item.created 57 | 58 | def item_updateddate(self, item): 59 | return item.modified 60 | -------------------------------------------------------------------------------- /snippets/base/fields.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | from snippets.base.validators import URLValidator 4 | 5 | 6 | class URLField(models.CharField): 7 | validators = [URLValidator] 8 | -------------------------------------------------------------------------------- /snippets/base/filters.py: -------------------------------------------------------------------------------- 1 | from distutils.util import strtobool 2 | 3 | from django.db.models import Q 4 | 5 | import django_filters 6 | 7 | from snippets.base import models 8 | 9 | 10 | class JobFilter(django_filters.FilterSet): 11 | name = django_filters.CharFilter( 12 | label='Snippet Name', 13 | method='filter_name', 14 | ) 15 | locale = django_filters.ModelChoiceFilter( 16 | label='Locale', 17 | empty_label='All Locales', 18 | queryset=models.Locale.objects.all(), 19 | field_name='snippet__locale', 20 | ) 21 | only_scheduled = django_filters.ChoiceFilter( 22 | label='Include', 23 | method='filter_scheduled', 24 | empty_label=None, 25 | null_label='All Snipppets', 26 | null_value='all', 27 | choices=(('true', 'Jobs with Start and End Date'), 28 | ('false', 'Jobs without Start and End Date')) 29 | ) 30 | 31 | def __init__(self, data=None, *args, **kwargs): 32 | # Set only_scheduled=true as default since this is the most common scenario 33 | if data is not None: 34 | data = data.copy() 35 | data['only_scheduled'] = data.get('only_scheduled', '') or 'true' 36 | else: 37 | data = { 38 | 'only_scheduled': 'true' 39 | } 40 | super().__init__(data, *args, **kwargs) 41 | 42 | def filter_name(self, queryset, name, value): 43 | # Filter based on Name, Snippet ID or Job ID 44 | if not value: 45 | return queryset 46 | 47 | try: 48 | value = int(value) 49 | except ValueError: 50 | # Not an ID 51 | return queryset.filter(snippet__name__icontains=f'{value}') 52 | else: 53 | return queryset.filter( 54 | Q(snippet__id=value) | 55 | Q(id=value) 56 | ) 57 | 58 | def filter_scheduled(self, queryset, name, value): 59 | if value == 'all': 60 | return queryset 61 | 62 | try: 63 | value = strtobool(value) 64 | except ValueError: 65 | value = True 66 | 67 | if value: 68 | return queryset.exclude(publish_end=None) 69 | return queryset.filter(publish_end=None) 70 | 71 | @property 72 | def qs(self): 73 | # Return only Published and Scheduled Snippets 74 | qs = super().qs 75 | return qs.filter( 76 | Q(status=models.Job.PUBLISHED) | 77 | Q(status=models.Job.SCHEDULED) 78 | ) 79 | 80 | class Meta: 81 | model = models.Job 82 | fields = [] 83 | -------------------------------------------------------------------------------- /snippets/base/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozmeao/snippets-service/aefc553ba93689ca8d61fabbb8586deb9ca09f10/snippets/base/management/__init__.py -------------------------------------------------------------------------------- /snippets/base/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozmeao/snippets-service/aefc553ba93689ca8d61fabbb8586deb9ca09f10/snippets/base/management/commands/__init__.py -------------------------------------------------------------------------------- /snippets/base/management/commands/fetch_daily_metrics.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime, timedelta 2 | 3 | from django.conf import settings 4 | from django.core.management.base import BaseCommand, CommandError 5 | 6 | from snippets.base import etl, models 7 | 8 | METRICS_START_DATE = date(2019, 10, 1) 9 | 10 | 11 | class Command(BaseCommand): 12 | args = "(no args)" 13 | help = "Fetch daily Job metrics" 14 | 15 | def add_arguments(self, parser): 16 | parser.add_argument( 17 | '--date', 18 | help='Fetch data for date. Defaults to yesterday. In YYYY-MM-DD format.', 19 | ) 20 | 21 | def handle(self, *args, **options): 22 | if not settings.REDASH_API_KEY: 23 | raise CommandError('Enviroment variable REDASH_API_KEY is required.') 24 | 25 | dates = [] 26 | 27 | if options['date']: 28 | dates.append(datetime.strptime(options['date'], '%Y-%m-%d').date()) 29 | else: 30 | today = date.today() 31 | try: 32 | fetched_dates = (models.JobDailyPerformance.objects 33 | .values_list('date', flat=True) 34 | .distinct()) 35 | except models.JobDailyPerformance.DoesNotExist: 36 | fetched_dates = [] 37 | 38 | check_date = METRICS_START_DATE 39 | while check_date < today: 40 | if check_date not in fetched_dates: 41 | dates.append(check_date) 42 | check_date += timedelta(days=1) 43 | 44 | for d in dates: 45 | successful_jobs = 0 46 | # Fetch Daily Impression data from last Monday relative to `d` date 47 | # to calculate Adjusted Impressions / Clicks / Blocks. 48 | last_monday = d - timedelta(days=d.weekday()) 49 | if not models.DailyImpressions.objects.filter(date=last_monday).exists(): 50 | self.stdout.write(f'Fetching impression data for {last_monday}.') 51 | if etl.update_impressions(last_monday): 52 | successful_jobs += 1 53 | else: 54 | successful_jobs += 1 55 | 56 | self.stdout.write(f'Fetching data for {d}.') 57 | if etl.update_job_metrics(d): 58 | successful_jobs += 1 59 | 60 | if successful_jobs != 2: 61 | # We didn't manage to fetch all data, something is wrong. 62 | raise CommandError('Cannot fetch data from Telemetry.') 63 | 64 | self.stdout.write(self.style.SUCCESS('Done')) 65 | -------------------------------------------------------------------------------- /snippets/base/management/commands/fetch_metrics.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from django.conf import settings 4 | from django.core.management.base import BaseCommand, CommandError 5 | 6 | import sentry_sdk 7 | from redash_dynamic_query import RedashDynamicQuery 8 | 9 | from snippets.base.models import Job 10 | 11 | 12 | class Command(BaseCommand): 13 | args = "(no args)" 14 | help = "Fetch metrics" 15 | 16 | def add_arguments(self, parser): 17 | parser.add_argument( 18 | '--force-update', 19 | action='store_true', 20 | help='Ignore last update timestamp and update all Jobs.', 21 | ) 22 | 23 | def handle(self, *args, **options): 24 | now = datetime.utcnow() 25 | 26 | if not settings.REDASH_API_KEY: 27 | raise CommandError('Enviroment variable REDASH_API_KEY is required.') 28 | 29 | redash = RedashDynamicQuery( 30 | endpoint=settings.REDASH_ENDPOINT, 31 | apikey=settings.REDASH_API_KEY, 32 | max_wait=settings.REDASH_MAX_WAIT, 33 | ) 34 | 35 | jobs = Job.objects.filter(status=Job.PUBLISHED).exclude( 36 | limit_impressions=0, 37 | limit_clicks=0, 38 | limit_blocks=0, 39 | ).order_by('id') 40 | 41 | self.stdout.write(f'Fetching Updates for {jobs.count()} Jobs.') 42 | 43 | data_fetched_global = False 44 | for job in jobs: 45 | impressions = 0 46 | clicks = 0 47 | blocks = 0 48 | bind_data = { 49 | 'start_date': job.publish_start.strftime('%Y-%m-%d'), 50 | 'end_date': now.strftime('%Y-%m-%d'), 51 | 'message_id': job.id, 52 | } 53 | 54 | data_fetched = False 55 | try: 56 | result = redash.query(settings.REDASH_JOB_QUERY_BIGQUERY_ID, bind_data) 57 | except Exception as exp: 58 | # Capture the exception but don't quit 59 | sentry_sdk.capture_exception(exp) 60 | continue 61 | 62 | try: 63 | for row in result['query_result']['data']['rows']: 64 | if row['event'] == 'IMPRESSION': 65 | impressions += row['counts'] 66 | elif row['event'] == 'BLOCK': 67 | blocks += row['counts'] 68 | elif row['event'] in ['CLICK', 'CLICK_BUTTON']: 69 | clicks += row['counts'] 70 | except KeyError as exp: 71 | # Capture the exception but don't quit 72 | sentry_sdk.capture_exception(exp) 73 | continue 74 | else: 75 | data_fetched = True 76 | 77 | # We didn't fetch data from both data sources for this job, don't 78 | # save it. 79 | if not data_fetched: 80 | continue 81 | 82 | # We fetched data for job, mark the ETL job `working` to update 83 | # DeadMansSnitch. 84 | data_fetched_global = True 85 | 86 | # Use update to avoid triggering Django signals and updating Job's 87 | # and ASRSnippet's modified date. 88 | Job.objects.filter(id=job.id).update( 89 | metric_impressions=impressions, 90 | metric_blocks=blocks, 91 | metric_clicks=clicks, 92 | metric_last_update=now, 93 | ) 94 | 95 | if jobs and not data_fetched_global: 96 | # We didn't manage to fetch data for any of the jobs. Something is 97 | # wrong. 98 | raise CommandError('Cannot fetch data from Telemetry.') 99 | 100 | self.stdout.write(self.style.SUCCESS('Done')) 101 | -------------------------------------------------------------------------------- /snippets/base/management/commands/generate_bundles.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | 3 | from snippets.base import bundles 4 | 5 | 6 | class Command(BaseCommand): 7 | args = '(no args)' 8 | help = 'Generate bundles' 9 | 10 | def add_arguments(self, parser): 11 | # Named (optional) arguments 12 | parser.add_argument( 13 | '--timestamp', 14 | help='Parse Jobs last modified after ', 15 | ) 16 | 17 | def handle(self, *args, **options): 18 | bundles.generate_bundles( 19 | timestamp=options.get('timestamp', None), 20 | stdout=self.stdout, 21 | ) 22 | -------------------------------------------------------------------------------- /snippets/base/management/commands/update_jobs.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | 3 | from django.conf import settings 4 | from django.contrib.auth import get_user_model 5 | from django.core.management.base import BaseCommand 6 | from django.db import transaction 7 | from django.db.models import F, Q 8 | 9 | from snippets.base.models import Job 10 | 11 | 12 | class Command(BaseCommand): 13 | args = "(no args)" 14 | help = "Update Jobs" 15 | 16 | @transaction.atomic 17 | def handle(self, *args, **options): 18 | now = datetime.utcnow() 19 | user = get_user_model().objects.get_or_create(username='snippets_bot')[0] 20 | count_total_completed = 0 21 | 22 | # Publish Scheduled Jobs with `publish_start` before now or without 23 | # publish_start. 24 | jobs = Job.objects.filter(status=Job.SCHEDULED).filter( 25 | Q(publish_start__lte=now - timedelta(minutes=settings.SNIPPETS_PUBLICATION_OFFSET)) | 26 | Q(publish_start=None) 27 | ) 28 | 29 | count_published = jobs.count() 30 | for job in jobs: 31 | job.change_status( 32 | status=Job.PUBLISHED, 33 | user=user, 34 | reason='Published start date reached.', 35 | ) 36 | 37 | # Disable Published Jobs with `publish_end` before now. 38 | jobs = Job.objects.filter(status=Job.PUBLISHED, publish_end__lte=now) 39 | count_publication_end = jobs.count() 40 | count_total_completed += count_publication_end 41 | 42 | for job in jobs: 43 | job.change_status( 44 | status=Job.COMPLETED, 45 | user=user, 46 | reason='Publication end date reached.', 47 | ) 48 | 49 | # Disable Jobs that reached Impression, Click or Block limits. 50 | count_limit = {} 51 | for limit in ['impressions', 'clicks', 'blocks']: 52 | jobs = (Job.objects 53 | .filter(status=Job.PUBLISHED) 54 | .exclude(**{f'limit_{limit}': 0}) 55 | .filter(**{f'limit_{limit}__lte': F(f'metric_{limit}')})) 56 | for job in jobs: 57 | job.change_status( 58 | status=Job.COMPLETED, 59 | user=user, 60 | reason=f'Limit reached: {limit}.', 61 | ) 62 | 63 | count_limit[limit] = jobs.count() 64 | count_total_completed += count_limit[limit] 65 | 66 | # Disable Jobs that have Impression, Click or Block limits but don't 67 | # have metrics data for at least 24h. This is to handle cases where the 68 | # Metrics Pipeline is broken. 69 | yesterday = datetime.utcnow() - timedelta(days=1) 70 | jobs = (Job.objects 71 | .filter(status=Job.PUBLISHED) 72 | .exclude(limit_impressions=0, limit_clicks=0, limit_blocks=0) 73 | # Exclude Jobs with limits which haven't been updated once yet. 74 | .exclude(metric_last_update='1970-01-01') 75 | .filter(metric_last_update__lt=yesterday)) 76 | for job in jobs: 77 | job.change_status( 78 | status=Job.COMPLETED, 79 | user=user, 80 | reason='Premature termination due to missing metrics.', 81 | ) 82 | count_premature_termination = jobs.count() 83 | count_total_completed += count_premature_termination 84 | 85 | count_running = Job.objects.filter(status=Job.PUBLISHED).count() 86 | 87 | self.stdout.write( 88 | f'Jobs Published: {count_published}\n' 89 | f'Jobs Completed: {count_total_completed}\n' 90 | f' - Reached Publication End Date: {count_publication_end}\n' 91 | f' - Reached Impressions Limit: {count_limit["impressions"]}\n' 92 | f' - Reached Clicks Limit: {count_limit["clicks"]}\n' 93 | f' - Reached Blocks Limit: {count_limit["blocks"]}\n' 94 | f' - Premature Termination due to missing metrics: {count_premature_termination}\n' 95 | f'Total Jobs Running: {count_running}\n' 96 | ) 97 | -------------------------------------------------------------------------------- /snippets/base/middleware.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.exceptions import MiddlewareNotUsed 3 | from django.core.validators import validate_ipv4_address, ValidationError 4 | from django.http.request import split_domain_port 5 | from django.urls import Resolver404, resolve 6 | 7 | from enforce_host import EnforceHostMiddleware 8 | 9 | from snippets.base.views import fetch_snippets 10 | 11 | 12 | class FetchSnippetsMiddleware(object): 13 | """ 14 | If the incoming request is for the fetch_snippets view, execute the view 15 | and return it before other middleware can run. 16 | 17 | fetch_snippets is a very very basic view that doesn't need any of the 18 | middleware that the rest of the site needs, such as the session or csrf 19 | middlewares. To avoid unintended issues (such as headers we don't want 20 | being added to the response) this middleware detects requests to that view 21 | and executes the view early, bypassing the rest of the middleware. 22 | """ 23 | def __init__(self, get_response): 24 | self.get_response = get_response 25 | 26 | def __call__(self, request): 27 | try: 28 | result = resolve(request.path) 29 | except Resolver404: 30 | # If we cannot resolve, continue with the next middleware. 31 | return self.get_response(request) 32 | 33 | if result.func in (fetch_snippets,): 34 | return result.func(request, *result.args, **result.kwargs) 35 | 36 | return self.get_response(request) 37 | 38 | 39 | class HostnameMiddleware(object): 40 | def __init__(self, get_response): 41 | if not settings.ENABLE_HOSTNAME_MIDDLEWARE: 42 | raise MiddlewareNotUsed 43 | 44 | values = [getattr(settings, x) for x in [ 45 | 'CLUSTER_NAME', 'K8S_NAMESPACE', 'K8S_POD_NAME']] 46 | self.backend_server = '/'.join(x for x in values if x) 47 | self.get_response = get_response 48 | 49 | def __call__(self, request): 50 | response = self.get_response(request) 51 | response['X-Backend-Server'] = self.backend_server 52 | return response 53 | 54 | 55 | # Direct copy from kitsune.sumo.middleware 56 | class EnforceHostIPMiddleware(EnforceHostMiddleware): 57 | """Modify the `EnforceHostMiddleware` to allow IP addresses""" 58 | def process_request(self, request): 59 | host = request.get_host() 60 | domain, port = split_domain_port(host) 61 | try: 62 | validate_ipv4_address(domain) 63 | except ValidationError: 64 | # not an IP address. Call the superclass 65 | return super(EnforceHostIPMiddleware, self).process_request(request) 66 | 67 | # it is an IP address 68 | return 69 | -------------------------------------------------------------------------------- /snippets/base/migrations/0044_target_filtr_channel.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.13 on 2020-10-26 10:28 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('base', '0043_auto_20201104_0811'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='target', 15 | name='filtr_channels', 16 | field=models.CharField(default='release;', max_length=255), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /snippets/base/migrations/0045_auto_20201026_1255.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.13 on 2020-10-26 12:55 2 | 3 | from django.db import migrations 4 | from django.db.models import Q 5 | 6 | def forwards(apps, schema_editor): 7 | Target = apps.get_model('base', 'Target') 8 | from snippets.base.forms import TargetAdminForm 9 | for target in Target.objects.filter( 10 | Q(on_release=True) | \ 11 | Q(on_esr=True) | \ 12 | Q(on_beta=True) | \ 13 | Q(on_aurora=True) | \ 14 | Q(on_nightly=True)): 15 | 16 | value = '' 17 | for key in ['release', 'esr', 'beta', 'aurora', 'nightly']: 18 | if getattr(target, f'on_{key}', False): 19 | value += f'{key};' 20 | value = value.strip(';') 21 | 22 | target.filtr_channels = value 23 | f = TargetAdminForm(instance=target) 24 | target.jexl_expr = f.generate_jexl_expr(f.initial) 25 | target.save() 26 | 27 | 28 | class Migration(migrations.Migration): 29 | 30 | dependencies = [ 31 | ('base', '0044_target_filtr_channel'), 32 | ] 33 | 34 | operations = [ 35 | migrations.RunPython(forwards), 36 | ] 37 | -------------------------------------------------------------------------------- /snippets/base/migrations/0046_auto_20201027_1337.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.13 on 2020-10-27 13:37 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('base', '0045_auto_20201026_1255'), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name='target', 15 | name='on_aurora', 16 | ), 17 | migrations.RemoveField( 18 | model_name='target', 19 | name='on_beta', 20 | ), 21 | migrations.RemoveField( 22 | model_name='target', 23 | name='on_esr', 24 | ), 25 | migrations.RemoveField( 26 | model_name='target', 27 | name='on_nightly', 28 | ), 29 | migrations.RemoveField( 30 | model_name='target', 31 | name='on_release', 32 | ), 33 | migrations.AlterField( 34 | model_name='target', 35 | name='filtr_channels', 36 | field=models.CharField(default='', max_length=255), 37 | ), 38 | ] 39 | -------------------------------------------------------------------------------- /snippets/base/migrations/0047_auto_20201112_0655.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.13 on 2020-11-12 06:55 2 | 3 | from django.db import migrations, models 4 | import snippets.base.validators 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('base', '0046_auto_20201027_1337'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='target', 16 | name='jexl_expr', 17 | field=models.TextField(blank=True, default='', validators=[snippets.base.validators.validate_jexl]), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /snippets/base/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozmeao/snippets-service/aefc553ba93689ca8d61fabbb8586deb9ca09f10/snippets/base/migrations/__init__.py -------------------------------------------------------------------------------- /snippets/base/slack.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.conf import settings 4 | from django.template.loader import render_to_string 5 | 6 | import requests 7 | import sentry_sdk 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | def send_slack(template_name, snippet): 13 | data = render_to_string('slack/{}.jinja.json'.format(template_name), 14 | context={'snippet': snippet}) 15 | _send_slack(data) 16 | 17 | 18 | def _send_slack(data): 19 | if not (settings.SLACK_ENABLE and settings.SLACK_WEBHOOK): 20 | logger.info('Slack is not enabled.') 21 | return 22 | 23 | try: 24 | response = requests.post(settings.SLACK_WEBHOOK, data=data.encode('utf-8'), 25 | headers={'Content-Type': 'application/json'}, 26 | timeout=4) 27 | response.raise_for_status() 28 | except requests.exceptions.RequestException as exp: 29 | sentry_sdk.capture_exception(exp) 30 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-delete-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-dismiss-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-edit-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-highlights-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-historyItem-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-import-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-info-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-info-option-12.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-newWindow-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-pin-12.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-pin-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-pocket-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-search-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-settings-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-topsites-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-trending-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-unpin-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/glyph-webextension-16.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/as_preview/topic-show-more-12.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snippets/base/static/css/admin/ASRSnippetAdmin.css: -------------------------------------------------------------------------------- 1 | #copyPreviewLink { 2 | color: white; 3 | border-radius: 4px; 4 | text-shadow: none; 5 | background: #2eb772; 6 | margin: 0 0 0 5px; 7 | line-height: 15px; 8 | padding: 5px 7px; 9 | cursor: pointer; 10 | border: none; 11 | 12 | } 13 | 14 | tr.variable input.smalltext { 15 | width: 500px; 16 | border: 1px solid rgb(204, 204, 204); 17 | display: block; 18 | } 19 | 20 | tr.variable img { 21 | float: left; 22 | } 23 | 24 | tr.variable small { 25 | display: block; 26 | clear: both; 27 | } 28 | 29 | td.field-description .vLargeTextField { 30 | width: 60em; 31 | height: 3em; 32 | } 33 | 34 | /* Make Text and URLFields larger. */ 35 | .vTextField, .vURLField { 36 | width: 80%; 37 | } 38 | 39 | /* Do not allow adding, editing or deleting locales from the ASRSnippet admin to */ 40 | /* avoid breaking the auto-localization feature */ 41 | 42 | #change_id_locale, #add_id_locale, #delete_id_locale { 43 | display: none; 44 | } 45 | 46 | #snippet_status.published { 47 | font-weight: bold; 48 | color: #2eb772; 49 | } 50 | 51 | #snippet_status.scheduled { 52 | font-weight: bold; 53 | color: orange; 54 | } 55 | 56 | #addJobButton { 57 | color: white; 58 | border-radius: 4px; 59 | text-shadow: none; 60 | background: #2eb772; 61 | margin: 0 0 0 5px; 62 | line-height: 15px; 63 | padding: 5px 7px; 64 | cursor: pointer; 65 | border: none; 66 | } 67 | -------------------------------------------------------------------------------- /snippets/base/static/css/admin/CustomNameWithTags.css: -------------------------------------------------------------------------------- 1 | .field-custom_name_with_tags #name { 2 | font-size: 120%; 3 | margin-bottom: 10px; 4 | } 5 | 6 | .field-custom_name_with_tags .tags { 7 | list-style: none; 8 | margin: 0; 9 | overflow: hidden; 10 | padding: 0; 11 | } 12 | 13 | .field-custom_name_with_tags li { 14 | float: left; 15 | list-style: none; 16 | font-size: 85%; 17 | } 18 | 19 | .field-custom_name_with_tags .tag { 20 | background: #eee; 21 | border-radius: 3px; 22 | color: #999; 23 | display: inline-block; 24 | height: 18px; 25 | line-height: 18px; 26 | padding: 0 10px 0 10px; 27 | position: relative; 28 | margin: 0 5px 5px 0; 29 | text-decoration: none; 30 | } 31 | -------------------------------------------------------------------------------- /snippets/base/static/css/admin/IDFieldHighlight.css: -------------------------------------------------------------------------------- 1 | .field-id .readonly { 2 | font-weight: bold; 3 | } 4 | -------------------------------------------------------------------------------- /snippets/base/static/css/admin/InlineTemplates.css: -------------------------------------------------------------------------------- 1 | .inline-template .inline-related fieldset { 2 | padding: 10px 3 | } 4 | 5 | .inline-template .inline-related fieldset h2 { 6 | background: #88c4e2; 7 | } 8 | -------------------------------------------------------------------------------- /snippets/base/static/css/admin/JobAdmin.css: -------------------------------------------------------------------------------- 1 | #change_id_snippet, #add_id_snippet, #delete_id_snippet { 2 | display: none; 3 | } 4 | 5 | 6 | input[name="_schedule"] { 7 | background: green; 8 | } 9 | 10 | input[name="_cancel"] { 11 | background: #a41515; 12 | } 13 | 14 | #job_status.published { 15 | font-weight: bold; 16 | color: #2eb772; 17 | } 18 | 19 | #job_status.scheduled { 20 | font-weight: bold; 21 | color: orange; 22 | } 23 | 24 | .ratio-green { 25 | color: green; 26 | } 27 | 28 | .ratio-red { 29 | color: #a41515; 30 | } 31 | 32 | #result_list .column-metric_impressions_humanized, 33 | #result_list .column-metric_clicks_humanized, 34 | #result_list .column-metric_blocks_humanized { 35 | text-align: right; 36 | } 37 | 38 | #result_list .field-metric_impressions_humanized, 39 | #result_list .field-metric_clicks_humanized, 40 | #result_list .field-metric_blocks_humanized { 41 | text-align: right; 42 | } 43 | 44 | #result_list .column-metric_clicks_ctr, 45 | #result_list .column-metric_blocks_ctr, 46 | #result_list .field-metric_clicks_ctr, 47 | #result_list .field-metric_blocks_ctr { 48 | text-align: left; 49 | } 50 | -------------------------------------------------------------------------------- /snippets/base/static/css/admin/ListSnippetsJobs.css: -------------------------------------------------------------------------------- 1 | #snippets-list, #jobs-list { 2 | margin-left: 0px; 3 | 4 | } 5 | 6 | #snippets-list li, #jobs-list li { 7 | list-style-type: disc; 8 | } 9 | -------------------------------------------------------------------------------- /snippets/base/static/css/admin/SnippetAdmin.css: -------------------------------------------------------------------------------- 1 | tr.variable input.smalltext { 2 | width: 500px; 3 | border: 1px solid rgb(204, 204, 204); 4 | display: block; 5 | } 6 | 7 | tr.variable img { 8 | float: left; 9 | } 10 | 11 | tr.variable small { 12 | display: block; 13 | clear: both; 14 | } 15 | 16 | td.field-description .vLargeTextField { 17 | width: 60em; 18 | height: 3em; 19 | } 20 | -------------------------------------------------------------------------------- /snippets/base/static/css/admin/descriptionColorize.css: -------------------------------------------------------------------------------- 1 | .description { 2 | padding: 5px 0px 12px 12px; 3 | background-color: #def6dee6; 4 | } 5 | 6 | .description code { 7 | padding: 2px 4px; 8 | font-size: 85%; 9 | background-color: #e4e4e4; 10 | border-radius: 3px; 11 | } 12 | -------------------------------------------------------------------------------- /snippets/base/static/css/admin/sms-country-toggle.css: -------------------------------------------------------------------------------- 1 | .form-row.field-country { 2 | border-bottom-width: 0; 3 | clip: rect(0 0 0 0); 4 | height: 1px; 5 | margin: -1px; 6 | overflow: hidden; 7 | padding: 0; 8 | position: absolute; 9 | width: 1px; 10 | } 11 | 12 | .form-row.field-country.visible { 13 | border-bottom-width: 1px; 14 | clip: unset; 15 | height: auto; 16 | margin: auto; 17 | padding: inherit; 18 | position: relative; 19 | width: auto; 20 | } 21 | -------------------------------------------------------------------------------- /snippets/base/static/css/app.css: -------------------------------------------------------------------------------- 1 | body { 2 | padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */ 3 | } 4 | 5 | .snippet-list { 6 | padding-top: 10px; 7 | } 8 | 9 | .json-snippet-table th:first-child, 10 | .json-snippet-table td:first-child, 11 | .snippet-table th:first-child, 12 | .snippet-table td:first-child { 13 | width: 20%; 14 | } 15 | 16 | .json-snippet-table td { 17 | vertical-align: middle; 18 | } 19 | 20 | select { 21 | width: 100%; 22 | } 23 | -------------------------------------------------------------------------------- /snippets/base/static/css/templateDataWidget.css: -------------------------------------------------------------------------------- 1 | .widget-container { 2 | float: left; 3 | border: 1px solid #DDD; 4 | } 5 | 6 | .template-data-widget table { 7 | border: none; 8 | width: 100%; 9 | } 10 | 11 | .template-data-widget td { 12 | border-right: 1px solid #DDD; 13 | padding: 10px; 14 | } 15 | 16 | .template-data-widget td:last-child { 17 | border-right: none; 18 | } 19 | 20 | .template-data-widget th { 21 | margin: 0; 22 | padding: 2px 5px 3px 5px; 23 | font-size: 11px; 24 | font-weight: bold; 25 | background: #79aec8; 26 | color: white; 27 | } 28 | 29 | .template-data-widget .variable-name { 30 | font-size: 16px; 31 | } 32 | 33 | .template-data-widget textarea { 34 | display: block; 35 | height: 200px; 36 | font-size: 12px; 37 | width: 500px; 38 | } 39 | 40 | .template-data-widget .image-input { 41 | display: block; 42 | margin-bottom: 5px; 43 | } 44 | 45 | .template-data-widget img { 46 | display: inline-block; 47 | border: 1px solid #DDD; 48 | height: 50px; 49 | } 50 | 51 | .snippet-preview-container { 52 | border-top: 1px solid #DDD; 53 | width: 100%; 54 | } 55 | 56 | .snippet-preview-form button { 57 | margin: 5px auto; 58 | display: block; 59 | width: 200px; 60 | } 61 | -------------------------------------------------------------------------------- /snippets/base/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozmeao/snippets-service/aefc553ba93689ca8d61fabbb8586deb9ca09f10/snippets/base/static/favicon.ico -------------------------------------------------------------------------------- /snippets/base/static/html/close.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Snippets Service 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /snippets/base/static/img/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozmeao/snippets-service/aefc553ba93689ca8d61fabbb8586deb9ca09f10/snippets/base/static/img/.gitignore -------------------------------------------------------------------------------- /snippets/base/static/img/glyphicons-halflings-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozmeao/snippets-service/aefc553ba93689ca8d61fabbb8586deb9ca09f10/snippets/base/static/img/glyphicons-halflings-white.png -------------------------------------------------------------------------------- /snippets/base/static/img/glyphicons-halflings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozmeao/snippets-service/aefc553ba93689ca8d61fabbb8586deb9ca09f10/snippets/base/static/img/glyphicons-halflings.png -------------------------------------------------------------------------------- /snippets/base/static/img/google.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozmeao/snippets-service/aefc553ba93689ca8d61fabbb8586deb9ca09f10/snippets/base/static/img/google.png -------------------------------------------------------------------------------- /snippets/base/static/js/admin/alert-page-leaving.js: -------------------------------------------------------------------------------- 1 | ; 2 | // Pure JS implementation of $(document).ready() 3 | // 4 | // Deals with DJango Admin loading first this file and then initializing jQuery. 5 | document.addEventListener( 6 | 'DOMContentLoaded', 7 | function() { 8 | // Translate content on locale select or template type select 9 | django.jQuery('form').areYouSure(); 10 | }, 11 | false 12 | ); 13 | -------------------------------------------------------------------------------- /snippets/base/static/js/admin/autoTranslatorWidget.js: -------------------------------------------------------------------------------- 1 | /// 2 | // 3 | // Finds all template fields that match `translation-` attributes of the 4 | // selected #id_locale option and sets their value. 5 | // 6 | // All template fields start with `id_template_relation-` because Templates are 7 | // StackedInline objects. 8 | // 9 | // Gets triggered every time the user changes locale. 10 | /// 11 | ; 12 | function autoTranslate() { 13 | let selected_locale = django.jQuery('option:selected', '#id_locale')[0]; 14 | if (! selected_locale.value) { 15 | return; 16 | } 17 | let attributes = selected_locale.attributes; 18 | let translations = JSON.parse(attributes.translations.nodeValue); 19 | Object.keys(translations).forEach(key => { 20 | django.jQuery("[id^='id_template_relation-']").filter(":visible").each(function(i, obj) { 21 | if (obj.name.endsWith('-' + key)) { 22 | django.jQuery(obj).val(translations[key]); 23 | } 24 | }); 25 | }); 26 | } 27 | 28 | // Pure JS implementation of $(document).ready() 29 | // 30 | // Deals with DJango Admin loading first this file and then initializing jQuery. 31 | document.addEventListener( 32 | 'DOMContentLoaded', 33 | function() { 34 | // Translate content on locale select or template type select 35 | django.jQuery('#id_locale').change(function() { 36 | autoTranslate(); 37 | }); 38 | }, 39 | false 40 | ); 41 | -------------------------------------------------------------------------------- /snippets/base/static/js/admin/inlineMover.js: -------------------------------------------------------------------------------- 1 | //// 2 | // Move template inlines at a higher place in the page. 3 | //// 4 | ; 5 | // Pure JS implementation of $(document).ready() 6 | // 7 | // Deals with DJango Admin loading first this file and then initializing jQuery. 8 | document.addEventListener( 9 | 'DOMContentLoaded', 10 | function() { 11 | django.jQuery('.inline-template').each(function(index) { 12 | django.jQuery(this).insertAfter(django.jQuery('.template-fieldset')); 13 | }); 14 | }, 15 | false 16 | ); 17 | -------------------------------------------------------------------------------- /snippets/base/static/js/admin/jquery.are-you-sure.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * jQuery Plugin: Are-You-Sure (Dirty Form Detection) 3 | * https://github.com/codedance/jquery.AreYouSure/ 4 | * 5 | * Copyright (c) 2012-2014, Chris Dance and PaperCut Software http://www.papercut.com/ 6 | * Dual licensed under the MIT or GPL Version 2 licenses. 7 | * http://jquery.org/license 8 | * 9 | * Author: chris.dance@papercut.com 10 | * Version: 1.9.0 11 | * Date: 13th August 2014 12 | */ 13 | (function($) { 14 | 15 | $.fn.areYouSure = function(options) { 16 | 17 | var settings = $.extend( 18 | { 19 | 'message' : 'You have unsaved changes!', 20 | 'dirtyClass' : 'dirty', 21 | 'change' : null, 22 | 'silent' : false, 23 | 'addRemoveFieldsMarksDirty' : false, 24 | 'fieldEvents' : 'change keyup propertychange input', 25 | 'fieldSelector': ":input:not(input[type=submit]):not(input[type=button])" 26 | }, options); 27 | 28 | var getValue = function($field) { 29 | if ($field.hasClass('ays-ignore') 30 | || $field.hasClass('aysIgnore') 31 | || $field.attr('data-ays-ignore') 32 | || $field.attr('name') === undefined) { 33 | return null; 34 | } 35 | 36 | if ($field.is(':disabled')) { 37 | return 'ays-disabled'; 38 | } 39 | 40 | var val; 41 | var type = $field.attr('type'); 42 | if ($field.is('select')) { 43 | type = 'select'; 44 | } 45 | 46 | switch (type) { 47 | case 'checkbox': 48 | case 'radio': 49 | val = $field.is(':checked'); 50 | break; 51 | case 'select': 52 | val = ''; 53 | $field.find('option').each(function(o) { 54 | var $option = $(this); 55 | if ($option.is(':selected')) { 56 | val += $option.val(); 57 | } 58 | }); 59 | break; 60 | default: 61 | val = $field.val(); 62 | } 63 | 64 | return val; 65 | }; 66 | 67 | var storeOrigValue = function($field) { 68 | $field.data('ays-orig', getValue($field)); 69 | }; 70 | 71 | var checkForm = function(evt) { 72 | 73 | var isFieldDirty = function($field) { 74 | var origValue = $field.data('ays-orig'); 75 | if (undefined === origValue) { 76 | return false; 77 | } 78 | return (getValue($field) != origValue); 79 | }; 80 | 81 | var $form = ($(this).is('form')) 82 | ? $(this) 83 | : $(this).parents('form'); 84 | 85 | // Test on the target first as it's the most likely to be dirty 86 | if (isFieldDirty($(evt.target))) { 87 | setDirtyStatus($form, true); 88 | return; 89 | } 90 | 91 | $fields = $form.find(settings.fieldSelector); 92 | 93 | if (settings.addRemoveFieldsMarksDirty) { 94 | // Check if field count has changed 95 | var origCount = $form.data("ays-orig-field-count"); 96 | if (origCount != $fields.length) { 97 | setDirtyStatus($form, true); 98 | return; 99 | } 100 | } 101 | 102 | // Brute force - check each field 103 | var isDirty = false; 104 | $fields.each(function() { 105 | var $field = $(this); 106 | if (isFieldDirty($field)) { 107 | isDirty = true; 108 | return false; // break 109 | } 110 | }); 111 | 112 | setDirtyStatus($form, isDirty); 113 | }; 114 | 115 | var initForm = function($form) { 116 | var fields = $form.find(settings.fieldSelector); 117 | $(fields).each(function() { storeOrigValue($(this)); }); 118 | $(fields).unbind(settings.fieldEvents, checkForm); 119 | $(fields).bind(settings.fieldEvents, checkForm); 120 | $form.data("ays-orig-field-count", $(fields).length); 121 | setDirtyStatus($form, false); 122 | }; 123 | 124 | var setDirtyStatus = function($form, isDirty) { 125 | var changed = isDirty != $form.hasClass(settings.dirtyClass); 126 | $form.toggleClass(settings.dirtyClass, isDirty); 127 | 128 | // Fire change event if required 129 | if (changed) { 130 | if (settings.change) settings.change.call($form, $form); 131 | 132 | if (isDirty) $form.trigger('dirty.areYouSure', [$form]); 133 | if (!isDirty) $form.trigger('clean.areYouSure', [$form]); 134 | $form.trigger('change.areYouSure', [$form]); 135 | } 136 | }; 137 | 138 | var rescan = function() { 139 | var $form = $(this); 140 | var fields = $form.find(settings.fieldSelector); 141 | $(fields).each(function() { 142 | var $field = $(this); 143 | if (!$field.data('ays-orig')) { 144 | storeOrigValue($field); 145 | $field.bind(settings.fieldEvents, checkForm); 146 | } 147 | }); 148 | // Check for changes while we're here 149 | $form.trigger('checkform.areYouSure'); 150 | }; 151 | 152 | var reinitialize = function() { 153 | initForm($(this)); 154 | } 155 | 156 | if (!settings.silent && !window.aysUnloadSet) { 157 | window.aysUnloadSet = true; 158 | $(window).bind('beforeunload', function() { 159 | $dirtyForms = $("form").filter('.' + settings.dirtyClass); 160 | if ($dirtyForms.length == 0) { 161 | return; 162 | } 163 | // Prevent multiple prompts - seen on Chrome and IE 164 | if (navigator.userAgent.toLowerCase().match(/msie|chrome/)) { 165 | if (window.aysHasPrompted) { 166 | return; 167 | } 168 | window.aysHasPrompted = true; 169 | window.setTimeout(function() {window.aysHasPrompted = false;}, 900); 170 | } 171 | return settings.message; 172 | }); 173 | } 174 | 175 | return this.each(function(elem) { 176 | if (!$(this).is('form')) { 177 | return; 178 | } 179 | var $form = $(this); 180 | 181 | $form.submit(function() { 182 | $form.removeClass(settings.dirtyClass); 183 | }); 184 | $form.bind('reset', function() { setDirtyStatus($form, false); }); 185 | // Add a custom events 186 | $form.bind('rescan.areYouSure', rescan); 187 | $form.bind('reinitialize.areYouSure', reinitialize); 188 | $form.bind('checkform.areYouSure', checkForm); 189 | initForm($form); 190 | }); 191 | }; 192 | })(jQuery); 193 | -------------------------------------------------------------------------------- /snippets/base/static/js/admin/sms-country-toggle.js: -------------------------------------------------------------------------------- 1 | (function () { 2 | function bindSMSCountry() { 3 | const chk_sms = document.querySelector('.form-row.field-include_sms input'); 4 | const country_row = document.querySelector('.form-row.field-country'); 5 | const country_input = document.querySelector('.form-row.field-country input'); 6 | 7 | // make sure required fields exist 8 | if (chk_sms && country_input) { 9 | // get default value from input, falling back to 'us' if empty 10 | const country_default = country_input.value || 'us'; 11 | 12 | function displayCountryField() { 13 | country_row.classList.add('visible'); 14 | } 15 | 16 | function hideCountryField() { 17 | country_row.classList.remove('visible'); 18 | 19 | // set the value back to default if user left it empty 20 | if (country_input.value === '') { 21 | country_input.value = country_default; 22 | } 23 | } 24 | 25 | // on page load, determine if we should show the country field 26 | if (chk_sms.checked) { 27 | displayCountryField(); 28 | } 29 | 30 | // toggle display of country field when SMS checkbox changes 31 | chk_sms.addEventListener('change', e => { 32 | if (chk_sms.checked) { 33 | displayCountryField(); 34 | } else { 35 | hideCountryField(); 36 | } 37 | }); 38 | } 39 | } 40 | 41 | document.addEventListener('DOMContentLoaded', bindSMSCountry); 42 | })(); 43 | -------------------------------------------------------------------------------- /snippets/base/static/js/admin/templateChooserWidget.js: -------------------------------------------------------------------------------- 1 | ; 2 | // Pure JS implementation of $(document).ready() 3 | // 4 | // Deals with DJango Admin loading first this file and then initializing jQuery. 5 | document.addEventListener( 6 | 'DOMContentLoaded', 7 | function() { 8 | function showTemplate() { 9 | django.jQuery('.inline-template').hide(); 10 | 11 | let value = django.jQuery('#id_template_chooser').val(); 12 | if (value) { 13 | django.jQuery('.' + value).show(); 14 | autoTranslate(); 15 | } 16 | // Template Chooser value is empty in two cases: a. When no template 17 | // has been selected yet and b. when user has view-only permissions. 18 | // In the later case there's a populated inline template which can 19 | // be matched with the query bellow. 20 | else { 21 | django.jQuery('.inline-related.has_original').parent('.inline-template').show(); 22 | } 23 | } 24 | 25 | // Show correct template on load 26 | showTemplate(); 27 | 28 | // Show correct template on change 29 | django.jQuery('#id_template_chooser').change(function() { 30 | showTemplate(); 31 | }); 32 | }, 33 | false 34 | ); 35 | -------------------------------------------------------------------------------- /snippets/base/static/js/copy_preview.js: -------------------------------------------------------------------------------- 1 | ; 2 | // Pure JS implementation of $(document).ready() 3 | // 4 | // Deals with DJango Admin loading first this file and then initializing jQuery. 5 | document.addEventListener( 6 | 'DOMContentLoaded', 7 | function() { 8 | var clipboard = new ClipboardJS('.btn'); 9 | clipboard.on('success', function(e) { 10 | e.trigger.innerText='Copied!'; 11 | setTimeout(function() { e.trigger.innerText=e.trigger.attributes.originalText.nodeValue; }, 1000); 12 | e.clearSelection(); 13 | }); 14 | }, 15 | false 16 | ); 17 | -------------------------------------------------------------------------------- /snippets/base/static/templates/snippetDataWidget.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | {% for variable in variables %} 7 | 8 | 9 | 27 | 28 | {% endfor %} 29 |
NameValue
{{ variable.name }} 10 | {% if variable.type == types.text or variable.type == types.body or variable.type == types.rich_text %} 11 | 12 | {% elif variable.type == types.image %} 13 | 14 | 15 | 16 | Remove image 17 |
18 | {% elif variable.type == types.smalltext %} 19 | 20 | {% elif variable.type == types.checkbox %} 21 | 22 | {% endif %} 23 | {% if variable.description %} 24 | {{ variable.description }} 25 | {% endif %} 26 |
30 | -------------------------------------------------------------------------------- /snippets/base/static/templates/snippetPreviewForm.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | 5 | 6 |
7 | -------------------------------------------------------------------------------- /snippets/base/static/vendor/fullcalendar/daygrid/main.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | FullCalendar Day Grid Plugin v4.2.0 3 | Docs & License: https://fullcalendar.io/ 4 | (c) 2019 Adam Shaw 5 | */.fc-dayGridDay-view .fc-content-skeleton,.fc-dayGridWeek-view .fc-content-skeleton{padding-bottom:1em}.fc-dayGrid-view .fc-body .fc-row{min-height:4em}.fc-row.fc-rigid{overflow:hidden}.fc-row.fc-rigid .fc-content-skeleton{position:absolute;top:0;left:0;right:0}.fc-day-top.fc-other-month{opacity:.3}.fc-dayGrid-view .fc-day-number,.fc-dayGrid-view .fc-week-number{padding:2px}.fc-dayGrid-view th.fc-day-number,.fc-dayGrid-view th.fc-week-number{padding:0 2px}.fc-ltr .fc-dayGrid-view .fc-day-top .fc-day-number{float:right}.fc-rtl .fc-dayGrid-view .fc-day-top .fc-day-number{float:left}.fc-ltr .fc-dayGrid-view .fc-day-top .fc-week-number{float:left;border-radius:0 0 3px}.fc-rtl .fc-dayGrid-view .fc-day-top .fc-week-number{float:right;border-radius:0 0 0 3px}.fc-dayGrid-view .fc-day-top .fc-week-number{min-width:1.5em;text-align:center;background-color:#f2f2f2;color:grey}.fc-dayGrid-view td.fc-week-number{text-align:center}.fc-dayGrid-view td.fc-week-number>*{display:inline-block;min-width:1.25em} -------------------------------------------------------------------------------- /snippets/base/storage.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.files.storage import FileSystemStorage 3 | from django.utils.deconstruct import deconstructible 4 | 5 | from storages.backends.s3boto3 import S3Boto3Storage 6 | 7 | 8 | # TODO 9 | class OverwriteStorage(FileSystemStorage): 10 | """ 11 | Comes from http://www.djangosnippets.org/snippets/976/ 12 | See also Django #4339, which might add this functionality to core. 13 | """ 14 | 15 | def get_available_name(self, name, max_length=None): 16 | """ 17 | Returns a filename that's free on the target storage system, and 18 | available for new content to be written to. 19 | """ 20 | if self.exists(name): 21 | self.delete(name) 22 | return name 23 | 24 | 25 | @deconstructible 26 | class S3Storage(S3Boto3Storage): 27 | cache_control_headers = getattr(settings, 'AWS_CACHE_CONTROL_HEADERS', {}) 28 | 29 | def _get_write_parameters(self, name, content): 30 | params = super()._get_write_parameters(name, content) 31 | encoding = getattr(content, 'content_encoding', params.get('ContentEncoding', None)) 32 | if encoding: 33 | params['ContentEncoding'] = encoding 34 | 35 | for filename_start, value in self.cache_control_headers.items(): 36 | if name.startswith(filename_start): 37 | params['CacheControl'] = value 38 | 39 | return params 40 | -------------------------------------------------------------------------------- /snippets/base/templates/404.jinja: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Page not found 6 | 7 | 8 |

Page not found

9 |

10 | Sorry, but we couldn't find the page you're looking for. 11 |

12 | 13 | 14 | -------------------------------------------------------------------------------- /snippets/base/templates/500.jinja: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Something went wrong! 6 | 7 | 8 |

Page not found

9 |

10 | Sorry, but something went wrong. 11 |

12 | 13 | 14 | -------------------------------------------------------------------------------- /snippets/base/templates/admin/base/icon/delete_confirmation.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/delete_confirmation.html" %} 2 | {% load i18n admin_urls static %} 3 | 4 | 5 | {% block content %} 6 | {% if perms_lacking %} 7 |

{% blocktrans with escaped_object=object %}Deleting the {{ object_name }} '{{ escaped_object }}' would result in deleting related objects, but your account doesn't have permission to delete the following types of objects:{% endblocktrans %}

8 |
    9 | {% for obj in perms_lacking %} 10 |
  • {{ obj }}
    • 11 | {% endfor %} 12 |
13 | {% elif protected %} 14 |

Cannot delete this icon because it's in use by the following ASRSnippets with Draft, Scheduled or Published Jobs.

15 |
    16 | {% for obj in protected %} 17 |
  • {{ obj }}
    • 18 | {% endfor %} 19 |
20 | {% else %} 21 |

{% blocktrans with escaped_object=object %}Are you sure you want to delete the {{ object_name }} "{{ escaped_object }}"? All of the following related items will be deleted:{% endblocktrans %}

22 | {% include "admin/includes/object_delete_summary.html" %} 23 |

{% trans "Objects" %}

24 |
    {{ deleted_objects|unordered_list }}
25 |
{% csrf_token %} 26 | 33 |
34 | {% endif %} 35 | {% endblock %} 36 | -------------------------------------------------------------------------------- /snippets/base/templates/admin/base/job/change_form.html: -------------------------------------------------------------------------------- 1 | {% extends 'admin/change_form.html' %} 2 | 3 | {% block after_field_sets %} 4 | {{ super }} 5 |
6 |

Metrics

7 | {% if not_published_yet %} 8 | Metrics will appear after Job gets published. 9 | {% else %} 10 | {% include "admin/metrics_table.html" with metrics=metrics %} 11 |

Adjusted Metrics

12 |
13 | Adjusted Metrics take into account the time a user spends in 14 | the pages that display Snippets (about:newtab and 15 | `about:home`). Only when a user spends at least {{ impression_threshold_seconds }} seconds in those 16 | pages the Snippet impression is considered valid. 17 | Otherwise it's disregarded on the basis that the user didn't spend 18 | enough time to read and comprehend the snippet. Adjusted 19 | Impressions are lower than unadjusted Impressions and as a 20 | consequence click through rates are higher. 21 |
22 | {% include "admin/metrics_table.html" with metrics=adj_metrics %} 23 |
24 | Last Update: {{ metrics_last_update|timesince }} ago. 25 |
26 | {% endif %} 27 |
28 | {% endblock %} 29 | -------------------------------------------------------------------------------- /snippets/base/templates/admin/base/job/submit_line.html: -------------------------------------------------------------------------------- 1 | {% load i18n admin_urls %} 2 |
3 | {% block submit-row %} 4 | {% if show_save %}{% endif %} 5 | {% if show_delete_link and original %} 6 | {% url opts|admin_urlname:'delete' original.pk|admin_urlquote as delete_url %} 7 |

{% trans "Delete" %}

8 | {% endif %} 9 | {% if show_duplicate %} {% endif %} 10 | {% if show_schedule %} {% endif %} 11 | {% if show_cancel %} {% endif %} 12 | {% if show_save_as_new %}{% endif %} 13 | {% if show_save_and_add_another %}{% endif %} 14 | {% if show_save_and_continue %}{% endif %} 15 | {% if show_close %}{% trans 'Close' %}{% endif %} 16 | {% endblock %} 17 |
18 | -------------------------------------------------------------------------------- /snippets/base/templates/admin/base/target/change_list.html: -------------------------------------------------------------------------------- 1 | {% extends 'admin/change_list.html' %} 2 | {% load i18n admin_urls %} 3 | 4 | 5 | {% block object-tools-items %} 6 | {% if has_add_permission %} 7 |
  • 8 | {% url cl.opts|admin_urlname:'add' as add_url %} 9 | 10 | {% blocktrans with cl.opts.verbose_name as name %}Add Custom {{ name }}{% endblocktrans %} 11 | 12 |
    • 13 | {% endif %} 14 | {{ block.super }} 15 | {% endblock %} 16 | -------------------------------------------------------------------------------- /snippets/base/templates/admin/login.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/base_site.html" %} 2 | {% load i18n admin_static %} 3 | 4 | {% block extrastyle %}{{ block.super }}{% endblock %} 5 | 6 | {% block bodyclass %}{{ block.super }} login{% endblock %} 7 | 8 | {% block usertools %}{% endblock %} 9 | 10 | {% block nav-global %}{% endblock %} 11 | 12 | {% block content_title %}{% endblock %} 13 | 14 | {% block breadcrumbs %}{% endblock %} 15 | 16 | {% block content %} 17 | {% if form.errors and not form.non_field_errors %} 18 |

    19 | {% if form.errors.items|length == 1 %}{% trans "Please correct the error below." %}{% else %}{% trans "Please correct the errors below." %}{% endif %} 20 |

    21 | {% endif %} 22 | 23 | {% if form.non_field_errors %} 24 | {% for error in form.non_field_errors %} 25 |

    26 | {{ error }} 27 |

    28 | {% endfor %} 29 | {% endif %} 30 | 31 |
    32 | {% if settings.OIDC_ENABLE %} 33 |

    34 | Mozilla SSO 35 |

    36 | {% else %} 37 |
    {% csrf_token %} 38 |
    39 | {{ form.username.errors }} 40 | {{ form.username.label_tag }} {{ form.username }} 41 |
    42 |
    43 | {{ form.password.errors }} 44 | {{ form.password.label_tag }} {{ form.password }} 45 | 46 |
    47 | {% url 'admin_password_reset' as password_reset_url %} 48 | {% if password_reset_url %} 49 |
    50 | {% trans 'Forgotten your password or username?' %} 51 |
    52 | {% endif %} 53 |
    54 | 55 |
    56 |
    57 | {% endif %} 58 | 61 |
    62 | {% endblock %} 63 | -------------------------------------------------------------------------------- /snippets/base/templates/admin/metrics_table.html: -------------------------------------------------------------------------------- 1 | {% load humanize %} 2 | 19 | 20 | {% for line in metrics %} 21 | 22 | {% for item in line %} 23 | {% if forloop.first or forloop.parentloop.first %} 24 | 27 | {% else %} 28 | 29 | {% endif %} 30 | {% endfor %} 31 | 32 | {% endfor %} 33 |
    25 | {{item|capfirst}} 26 | {{ item|intcomma }}
    34 | -------------------------------------------------------------------------------- /snippets/base/templates/base/base.jinja: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Mozilla Snippets Service 10 | 26 | {% block headextras %} 27 | {% endblock %} 28 | 29 | 30 |
    31 |

    Mozilla Snippets

    32 | {% block content %} 33 | {% endblock %} 34 |
    35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /snippets/base/templates/base/home.jinja: -------------------------------------------------------------------------------- 1 | {% extends 'base/base.jinja' %} 2 | {% block headextras %} 3 | 8 | {% endblock %} 9 | {% block content %} 10 |

    11 | made with ❤ by Mozilla Marketing and Operations Team
    12 | run by Mozilla Own Media Network Team. 13 |

    14 |

    15 | Looking for a list of currently published Jobs? Or maybe the Admin Panel? 16 |

    17 | {% endblock %} 18 | -------------------------------------------------------------------------------- /snippets/base/templates/base/jobs_list_calendar.jinja: -------------------------------------------------------------------------------- 1 | {% extends 'base/base.jinja' %} 2 | 3 | {% block headextras %} 4 | 5 | 6 | 7 | 8 | 9 | 10 | 39 | 80 | {% endblock %} 81 | 82 | {% block content %} 83 |
    84 | {{ filter.form.as_table() }} 85 | View: 86 | 90 | 93 |
    94 |
    95 |
    96 |
    97 | {% endblock %} 98 | -------------------------------------------------------------------------------- /snippets/base/templates/base/jobs_list_table.jinja: -------------------------------------------------------------------------------- 1 | {% extends 'base/base.jinja' %} 2 | 3 | {% block headextras %} 4 | 5 | 72 | 73 | 74 | {% endblock %} 75 | 76 | {% block content %} 77 |
    78 | {{ filter.form.as_table() }} 79 | View: 80 | 84 | 87 |
    88 |
    89 |
    90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | {% for obj in filter.qs %} 104 | 105 | 106 | 107 | 108 | 109 | 110 | 115 | 122 | 123 | {% endfor %} 124 | 125 |
    #SnippetLocalePublication StartPublication EndTargetsPreview Link
    {{ obj.id }}{{ obj.snippet.name }}{{ obj.snippet.locale }}{{ obj.publish_start }}{{ obj.publish_end }} 111 | {% for target in obj.targets.all() %} 112 |
  • {{ target.name }}
    • 113 | {% endfor %} 114 |     		116 | 121 |
    126 |
    127 |
    128 | {% endblock %} 129 | -------------------------------------------------------------------------------- /snippets/base/templates/base/jobs_related_with_obj.jinja: -------------------------------------------------------------------------------- 1 | 12 | -------------------------------------------------------------------------------- /snippets/base/templates/base/preview_image.jinja: -------------------------------------------------------------------------------- 1 | 18 |
    19 |
    20 | Light Theme Background 21 | 22 |
    23 |
    24 | Dark Theme Background 25 | 26 |
    27 |
    28 | -------------------------------------------------------------------------------- /snippets/base/templates/base/ratelimited.jinja: -------------------------------------------------------------------------------- 1 | {% extends 'base/base.jinja' %} 2 | {% block headextras %} 3 | 8 | {% endblock %} 9 | {% block content %} 10 |

    11 | Slow down, you 're browsing this site too fast! Try again in a few minutes. 12 |

    13 | {% endblock %} 14 | -------------------------------------------------------------------------------- /snippets/base/templates/base/snippets_custom_name_with_tags.jinja: -------------------------------------------------------------------------------- 1 | 6 |
      7 | {% for tag in obj.tags.all().order_by('name') %} 8 |
    • 9 | {{ tag.name }} 10 |
      • 11 | {% endfor %} 12 |
    13 | 14 | -------------------------------------------------------------------------------- /snippets/base/templates/base/snippets_related_with_obj.jinja: -------------------------------------------------------------------------------- 1 | 12 | -------------------------------------------------------------------------------- /snippets/base/templates/slack/asr_ready_for_review.jinja.json: -------------------------------------------------------------------------------- 1 | { 2 | "attachments": [{ 3 | "pretext": "Update on Snippet #{{ snippet.id }}", 4 | "title": "Snippet #{{ snippet.id }} is ready for review!", 5 | "title_link": "{{ snippet.get_admin_url() }}", 6 | "text": "Snippet *#{{ snippet.id }}*: *{{ snippet.name }}* by {{ snippet.creator.get_full_name() }} was just marked ready for review.", 7 | "color": "#FF7F50" 8 | }] 9 | } 10 | -------------------------------------------------------------------------------- /snippets/base/templates/slack/job_canceled.jinja.json: -------------------------------------------------------------------------------- 1 | { 2 | "attachments": [{ 3 | "pretext": "Update on Job #{{ job.id }}", 4 | "title": "Job #{{ job.id }} was canceled.", 5 | "title_link": "{{ job.get_admin_url() }}", 6 | "text": "Job #{{ job.id }} for Snippet *{{ job.snippet.name }}* (#{{ job.snippet.id }}) was just canceled.", 7 | "color": "#a41515" 8 | }] 9 | } 10 | -------------------------------------------------------------------------------- /snippets/base/templates/slack/job_completed.jinja.json: -------------------------------------------------------------------------------- 1 | { 2 | "attachments": [{ 3 | "pretext": "Update on Job #{{ job.id }}", 4 | "title": "Job #{{ job.id }} completed.", 5 | "title_link": "{{ job.get_admin_url() }}", 6 | "text": "Job #{{ job.id }} for Snippet *{{ job.snippet.name }}* (#{{ job.snippet.id }}) completed publication.{% if reason %} Reason: {{ reason }}{% endif %}", 7 | "color": "#36a64f" 8 | }] 9 | } 10 | -------------------------------------------------------------------------------- /snippets/base/templates/slack/job_published.jinja.json: -------------------------------------------------------------------------------- 1 | { 2 | "attachments": [{ 3 | "pretext": "Update on Job #{{ job.id }}", 4 | "title": "Job #{{ job.id }} was just published! :tada:", 5 | "title_link": "{{ job.get_admin_url() }}", 6 | "text": "Job #{{ job.id }} for Snippet *{{ job.snippet.name }}* (#{{ job.snippet.id }}) was just published on {{ job.channels|join(', ')}} channel(s)!", 7 | "color": "#36a64f" 8 | }] 9 | } 10 | -------------------------------------------------------------------------------- /snippets/base/templates/slack/job_scheduled.jinja.json: -------------------------------------------------------------------------------- 1 | { 2 | "attachments": [{ 3 | "pretext": "Update on Job #{{ job.id }}", 4 | "title": "Job #{{ job.id }} was scheduled for publication!", 5 | "title_link": "{{ job.get_admin_url() }}", 6 | "text": "Job #{{ job.id }} for Snippet *{{ job.snippet.name }}* (#{{ job.snippet.id }}) was just scheduled for publication.", 7 | "color": "#36a64f" 8 | }] 9 | } 10 | -------------------------------------------------------------------------------- /snippets/base/templates/slack/legacy_published.jinja.json: -------------------------------------------------------------------------------- 1 | { 2 | "attachments": [{ 3 | "pretext": "Update on Snippet #{{ snippet.id }}", 4 | "title": "Snippet #{{ snippet.id }} was just published! :tada:", 5 | "title_link": "{{ snippet.get_admin_url() }}", 6 | "text": "Snippet *#{{ snippet.id }}*: *{{ snippet.name }}* was just published on {{ snippet.channels|join(', ')}} channel(s)!", 7 | "color": "#36a64f" 8 | }] 9 | } 10 | -------------------------------------------------------------------------------- /snippets/base/templates/slack/legacy_ready_for_review.jinja.json: -------------------------------------------------------------------------------- 1 | { 2 | "attachments": [{ 3 | "pretext": "Update on Snippet #{{ snippet.id }}", 4 | "title": "Snippet #{{ snippet.id }} is ready for review!", 5 | "title_link": "{{ snippet.get_admin_url() }}", 6 | "text": "Snippet *#{{ snippet.id }}*: *{{ snippet.name }}* by {{ snippet.creator }} was just marked ready for review.", 7 | "color": "#FF7F50" 8 | }] 9 | } 10 | -------------------------------------------------------------------------------- /snippets/base/templates/widgets/jexlrange.html: -------------------------------------------------------------------------------- 1 | {% spaceless %} 2 | From and including 3 | {% with widget.subwidgets.0 as widget %} 4 | {% include widget.template_name %} 5 | {% endwith %} 6 | up to (excluded) 7 | {% with widget.subwidgets.1 as widget %} 8 | {% include widget.template_name %} 9 | {% endwith %} 10 | {% endspaceless %} 11 | -------------------------------------------------------------------------------- /snippets/base/tests/__init__.py: -------------------------------------------------------------------------------- 1 | import random 2 | import string 3 | 4 | from django.test import TransactionTestCase 5 | from django.test.utils import override_settings 6 | 7 | import factory 8 | 9 | from snippets.base import models 10 | 11 | 12 | @override_settings(SECURE_SSL_REDIRECT=False) 13 | class TestCase(TransactionTestCase): 14 | pass 15 | 16 | 17 | class UserFactory(factory.django.DjangoModelFactory): 18 | username = factory.Sequence(lambda n: 'User {0}'.format(n)) 19 | 20 | class Meta: 21 | model = 'auth.User' 22 | django_get_or_create = ('username',) 23 | 24 | 25 | class TargetFactory(factory.django.DjangoModelFactory): 26 | name = factory.Sequence(lambda n: 'Target {0}'.format(n)) 27 | creator = factory.SubFactory(UserFactory) 28 | 29 | class Meta: 30 | model = models.Target 31 | 32 | @factory.post_generation 33 | def channels(obj, create, extracted, **kwargs): 34 | if extracted is None: 35 | obj.filtr_channels = 'release' 36 | else: 37 | obj.filtr_channels = extracted 38 | 39 | 40 | class CampaignFactory(factory.django.DjangoModelFactory): 41 | name = factory.Sequence(lambda n: 'Campaign {0}'.format(n)) 42 | slug = factory.Sequence(lambda n: 'campaign_{0}'.format(n)) 43 | creator = factory.SubFactory(UserFactory) 44 | 45 | class Meta: 46 | model = models.Campaign 47 | 48 | 49 | class CategoryFactory(factory.django.DjangoModelFactory): 50 | name = factory.Sequence(lambda n: 'Campaign {0}'.format(n)) 51 | creator = factory.SubFactory(UserFactory) 52 | 53 | class Meta: 54 | model = models.Category 55 | 56 | 57 | class ProductFactory(factory.django.DjangoModelFactory): 58 | name = factory.Sequence(lambda n: 'Campaign {0}'.format(n)) 59 | creator = factory.SubFactory(UserFactory) 60 | 61 | class Meta: 62 | model = models.Product 63 | 64 | 65 | class IconFactory(factory.django.DjangoModelFactory): 66 | name = factory.Sequence(lambda n: 'Icon {0}'.format(n)) 67 | creator = factory.SubFactory(UserFactory) 68 | image = factory.django.ImageField(width=192, height=192, 69 | format='PNG', filename='example.png') 70 | 71 | class Meta: 72 | model = models.Icon 73 | 74 | 75 | class SimpleTemplateFactory(factory.django.DjangoModelFactory): 76 | text = 'This is the main text with a link.' 77 | icon = factory.SubFactory(IconFactory) 78 | 79 | class Meta: 80 | model = models.SimpleTemplate 81 | 82 | 83 | class ASRSnippetFactory(factory.django.DjangoModelFactory): 84 | creator = factory.SubFactory(UserFactory) 85 | name = factory.Sequence(lambda n: 'ASRSnippet {0}'.format(n)) 86 | category = factory.SubFactory(CategoryFactory, creator=factory.SelfAttribute('..creator')) 87 | product = factory.SubFactory(ProductFactory, creator=factory.SelfAttribute('..creator')) 88 | template_relation = factory.RelatedFactory(SimpleTemplateFactory, 'snippet') 89 | status = models.STATUS_CHOICES['Approved'] 90 | 91 | class Meta: 92 | model = models.ASRSnippet 93 | 94 | @factory.post_generation 95 | def set_template_relation_to_template_model(self, *args, **kwargs): 96 | """By default this Factory creates and relates a SimpleTemplate to this 97 | new ASRSnippet object. The Django Admin on the other hand, relates a 98 | Template obj when saving a ASRSnippet object. We use this method to 99 | make the Factory behave like Django Admin. 100 | """ 101 | self.template_relation = self.template_relation.template_ptr 102 | 103 | @factory.post_generation 104 | def locale(self, create, extracted, **kwargs): 105 | if not create: 106 | return 107 | 108 | code = extracted or 'en-us' 109 | if code[0] != ',': 110 | code = ',' + code 111 | if code[-1] != ',': 112 | code = code + ',' 113 | locale = models.Locale.objects.get_or_create(code=code, name=code)[0] 114 | self.locale = locale 115 | 116 | @factory.post_generation 117 | def add_tags(self, create, extracted, **kwargs): 118 | if not create: 119 | return 120 | 121 | if extracted: 122 | self.tags.add(*extracted) 123 | 124 | 125 | class AddonFactory(factory.django.DjangoModelFactory): 126 | name = factory.Sequence(lambda n: 'Addon {}'.format(n)) 127 | guid = factory.Sequence(lambda n: 'addon_{}'.format(n)) 128 | url = factory.Sequence(lambda n: 'https://example.com/{}'.format(n)) 129 | 130 | class Meta: 131 | model = models.Addon 132 | 133 | 134 | class LocaleFactory(factory.django.DjangoModelFactory): 135 | name = factory.Sequence(lambda n: 'Locale {}'.format(n)) 136 | code = factory.LazyAttribute(lambda o: ''.join(random.choices(string.ascii_lowercase, k=4))) 137 | 138 | class Meta: 139 | model = models.Locale 140 | 141 | 142 | class DistributionFactory(factory.django.DjangoModelFactory): 143 | name = factory.Sequence(lambda n: 'Distribution {}'.format(n)) 144 | 145 | class Meta: 146 | model = models.Distribution 147 | django_get_or_create = ('name',) 148 | 149 | 150 | class DistributionBundleFactory(factory.django.DjangoModelFactory): 151 | name = factory.Sequence(lambda n: 'Distribution Bundle {}'.format(n)) 152 | code_name = factory.Sequence(lambda n: 'distribution_bundle_{}'.format(n)) 153 | 154 | class Meta: 155 | model = models.DistributionBundle 156 | django_get_or_create = ('name', 'code_name') 157 | 158 | 159 | class JobFactory(factory.django.DjangoModelFactory): 160 | creator = factory.SubFactory(UserFactory) 161 | campaign = factory.SubFactory(CampaignFactory, creator=factory.SelfAttribute('..creator')) 162 | status = models.Job.PUBLISHED 163 | snippet = factory.SubFactory(ASRSnippetFactory, creator=factory.SelfAttribute('..creator')) 164 | distribution = factory.SubFactory(DistributionFactory, name='Default') 165 | 166 | class Meta: 167 | model = models.Job 168 | 169 | @factory.post_generation 170 | def targets(self, create, extracted, **kwargs): 171 | if not create: 172 | return 173 | 174 | if extracted is None: 175 | extracted = [TargetFactory(creator=self.creator)] 176 | 177 | for target in extracted: 178 | self.targets.add(target) 179 | -------------------------------------------------------------------------------- /snippets/base/tests/test_admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib.admin.sites import AdminSite 2 | from django.test.client import RequestFactory 3 | 4 | from unittest.mock import DEFAULT as DEFAULT_MOCK, Mock, patch 5 | 6 | from snippets.base.admin.adminmodels import ASRSnippetAdmin, JobAdmin 7 | from snippets.base.models import STATUS_CHOICES, ASRSnippet, Job 8 | from snippets.base.tests import (ASRSnippetFactory, JobFactory, 9 | TestCase, UserFactory) 10 | 11 | 12 | class ASRSnippetAdminTests(TestCase): 13 | def setUp(self): 14 | self.factory = RequestFactory() 15 | self.model_admin = ASRSnippetAdmin(ASRSnippet, None) 16 | self.model_admin.admin_site = Mock() 17 | self.user = UserFactory() 18 | 19 | def test_save_as_published(self): 20 | request = self.factory.post('/', data={ 21 | 'name': 'test', 22 | 'template': 'foo', 23 | 'status': STATUS_CHOICES['Published'], 24 | '_saveasnew': True 25 | }) 26 | request.user = self.user 27 | 28 | with patch('snippets.base.admin.admin.ModelAdmin.change_view') as change_view_mock: 29 | self.model_admin.change_view(request, 999) 30 | change_view_mock.assert_called_with(request, 999) 31 | request = change_view_mock.call_args[0][0] 32 | self.assertEqual(request.POST['status'], STATUS_CHOICES['Draft']) 33 | 34 | def test_normal_save_published(self): 35 | """Test that normal save doesn't alter `status` attribute.""" 36 | request = self.factory.post('/', data={ 37 | 'name': 'test', 38 | 'template': 'foo', 39 | 'status': STATUS_CHOICES['Published'], 40 | }) 41 | request.user = self.user 42 | 43 | with patch('snippets.base.admin.admin.ModelAdmin.change_view') as change_view_mock: 44 | self.model_admin.change_view(request, 999) 45 | change_view_mock.assert_called_with(request, 999) 46 | request = change_view_mock.call_args[0][0] 47 | self.assertEqual(request.POST['status'], str(STATUS_CHOICES['Published'])) 48 | 49 | def test_get_readonly_fields(self): 50 | asrsnippet = ASRSnippetFactory() 51 | request = self.factory.get('/') 52 | admin = ASRSnippetAdmin(ASRSnippet, AdminSite()) 53 | request.user = UserFactory 54 | 55 | # No obj 56 | readonly_fields = admin.get_readonly_fields(request, None) 57 | self.assertTrue('status' in readonly_fields) 58 | 59 | # With obj 60 | readonly_fields = admin.get_readonly_fields(request, asrsnippet) 61 | self.assertTrue('status' not in readonly_fields) 62 | 63 | 64 | class JobAdminTests(TestCase): 65 | def test_action_schedule_job(self): 66 | to_get_scheduled = JobFactory.create_batch(2, status=Job.DRAFT) 67 | already_scheduled = JobFactory(status=Job.SCHEDULED) 68 | already_published = JobFactory(status=Job.PUBLISHED) 69 | cancelled = JobFactory(status=Job.CANCELED) 70 | completed = JobFactory(status=Job.COMPLETED) 71 | JobFactory.create_batch(2, status=Job.DRAFT) 72 | 73 | queryset = Job.objects.filter(id__in=[ 74 | to_get_scheduled[0].id, 75 | to_get_scheduled[1].id, 76 | already_scheduled.id, 77 | already_published.id, 78 | cancelled.id, 79 | completed.id, 80 | ]) 81 | 82 | request = Mock() 83 | request.user = UserFactory.create() 84 | with patch.multiple('snippets.base.admin.adminmodels.messages', 85 | warning=DEFAULT_MOCK, 86 | success=DEFAULT_MOCK) as message_mocks: 87 | JobAdmin(Job, None).action_schedule_job(request, queryset) 88 | 89 | self.assertEqual( 90 | set(Job.objects.filter(status=Job.SCHEDULED)), 91 | set(to_get_scheduled + [already_scheduled]) 92 | ) 93 | self.assertTrue(message_mocks['warning'].called) 94 | self.assertTrue(message_mocks['success'].called) 95 | 96 | def test_action_cancel_job(self): 97 | to_get_canceled = [ 98 | JobFactory.create(status=Job.PUBLISHED), 99 | JobFactory.create(status=Job.SCHEDULED), 100 | ] 101 | already_cancelled = JobFactory(status=Job.CANCELED) 102 | completed = JobFactory(status=Job.COMPLETED) 103 | JobFactory.create_batch(2, status=Job.DRAFT) 104 | 105 | queryset = Job.objects.filter(id__in=[ 106 | to_get_canceled[0].id, 107 | to_get_canceled[1].id, 108 | already_cancelled.id, 109 | completed.id, 110 | ]) 111 | 112 | request = Mock() 113 | request.user = UserFactory.create() 114 | with patch.multiple('snippets.base.admin.adminmodels.messages', 115 | warning=DEFAULT_MOCK, 116 | success=DEFAULT_MOCK) as message_mocks: 117 | JobAdmin(Job, None).action_cancel_job(request, queryset) 118 | 119 | self.assertEqual( 120 | set(Job.objects.filter(status=Job.CANCELED)), 121 | set(to_get_canceled + [already_cancelled]) 122 | ) 123 | self.assertTrue(message_mocks['warning'].called) 124 | self.assertTrue(message_mocks['success'].called) 125 | 126 | def test_action_delete_job(self): 127 | to_get_deleted = [ 128 | JobFactory.create(status=Job.DRAFT), 129 | JobFactory.create(status=Job.DRAFT), 130 | ] 131 | to_remain = [ 132 | JobFactory(status=Job.CANCELED), 133 | JobFactory(status=Job.COMPLETED), 134 | JobFactory(status=Job.PUBLISHED), 135 | ] 136 | queryset = Job.objects.filter(id__in=[x.id for x in to_get_deleted + to_remain]) 137 | 138 | request = Mock() 139 | request.user = UserFactory.create() 140 | with patch.multiple('snippets.base.admin.adminmodels.messages', 141 | warning=DEFAULT_MOCK, 142 | success=DEFAULT_MOCK) as message_mocks: 143 | JobAdmin(Job, None).action_delete_job(request, queryset) 144 | 145 | self.assertEqual( 146 | set(Job.objects.all()), 147 | set(to_remain) 148 | ) 149 | self.assertTrue(message_mocks['warning'].called) 150 | self.assertTrue(message_mocks['success'].called) 151 | -------------------------------------------------------------------------------- /snippets/base/tests/test_admin_filters.py: -------------------------------------------------------------------------------- 1 | from snippets.base.admin.adminmodels import JobAdmin 2 | from snippets.base.admin.filters import ChannelFilter 3 | from snippets.base.models import Job 4 | from snippets.base.tests import JobFactory, TargetFactory, TestCase 5 | 6 | 7 | class ChannelFilterTests(TestCase): 8 | def test_job(self): 9 | nightly_snippets = JobFactory.create_batch( 10 | 2, targets=[TargetFactory(channels='nightly')]) 11 | JobFactory.create_batch(2, targets=[TargetFactory(channels='beta')]) 12 | 13 | filtr = ChannelFilter(None, {'channel': 'nightly'}, Job, JobAdmin) 14 | result = filtr.queryset(None, Job.objects.all()) 15 | 16 | self.assertTrue(result.count(), 2) 17 | self.assertEqual(set(result.all()), set(nightly_snippets)) 18 | -------------------------------------------------------------------------------- /snippets/base/tests/test_feed.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import Mock, patch 2 | 3 | from django.test import Client 4 | from django.urls import reverse 5 | 6 | from snippets.base.feed import JobsFeed 7 | from snippets.base.tests import JobFactory, TestCase 8 | 9 | 10 | class JobsFeedTests(TestCase): 11 | def test_base(self): 12 | JobFactory.create_batch(2) 13 | client = Client() 14 | response = client.get(reverse('ical-feed'), follow=True) 15 | self.assertEqual(response.status_code, 200) 16 | 17 | def test_item_filtering(self): 18 | request = Mock() 19 | request.GET = {} 20 | 21 | with patch('snippets.base.feed.models.Job') as JobMock: 22 | with patch('snippets.base.filters.JobFilter') as JobFilterMock: 23 | JobMock.objects.filter.return_value.order_by.return_value = 'foo' 24 | JobsFeed()(request) 25 | 26 | JobMock.objects.filter.assert_called() 27 | JobFilterMock.assert_called_with(request.GET, queryset='foo') 28 | -------------------------------------------------------------------------------- /snippets/base/tests/test_filters.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from django.http.request import QueryDict 4 | 5 | from snippets.base import models 6 | from snippets.base.filters import JobFilter 7 | from snippets.base.tests import JobFactory, TestCase 8 | 9 | 10 | class JobFilterTests(TestCase): 11 | def test_base(self): 12 | job1, job2 = JobFactory.create_batch(2) 13 | job4 = JobFactory.create(publish_end=datetime(2019, 2, 2)) 14 | job3 = JobFactory.create(publish_start=datetime(2019, 1, 1), 15 | publish_end=datetime(2019, 2, 2)) 16 | filtr = JobFilter(QueryDict(query_string='only_scheduled=all'), 17 | queryset=models.Job.objects.all()) 18 | self.assertEqual(set([job1, job2, job3, job4]), set(filtr.qs)) 19 | 20 | def test_only_scheduled_true(self): 21 | job1 = JobFactory.create(publish_end=datetime(2019, 2, 2)) 22 | job2 = JobFactory.create(publish_end=datetime(2019, 2, 2)) 23 | job3 = JobFactory.create(publish_start=datetime(2019, 2, 2), 24 | publish_end=datetime(2019, 2, 3)) 25 | JobFactory.create(publish_start=None, publish_end=None) 26 | filtr = JobFilter(QueryDict(query_string='only_scheduled=true'), 27 | queryset=models.Job.objects.all()) 28 | self.assertEqual(set([job1, job2, job3]), set(filtr.qs)) 29 | 30 | def test_only_scheduled_false(self): 31 | job1 = JobFactory.create(publish_start=None, publish_end=None) 32 | JobFactory.create(publish_end=datetime(2019, 2, 2)) 33 | filtr = JobFilter(QueryDict(query_string='only_scheduled=false'), 34 | queryset=models.Job.objects.all()) 35 | self.assertEqual(set([job1]), set(filtr.qs)) 36 | 37 | def test_only_scheduled_all(self): 38 | job1, job2 = JobFactory.create_batch(2) 39 | job3 = JobFactory.create(publish_end=datetime(2019, 2, 2)) 40 | filtr = JobFilter(QueryDict(query_string='only_scheduled=all'), 41 | queryset=models.Job.objects.all()) 42 | self.assertEqual(set([job1, job2, job3]), set(filtr.qs)) 43 | 44 | def test_name(self): 45 | JobFactory.create(id=2990, snippet__id=20000, snippet__name='foo 1') 46 | JobFactory.create(id=2991, snippet__id=20001, snippet__name='foo 2') 47 | job = JobFactory.create(id=2992, snippet__id=20002, snippet__name='bar 1') 48 | JobFactory.create(id=2993, snippet__name='foo lala foo') 49 | filtr = JobFilter( 50 | QueryDict(query_string='only_scheduled=all&name=bar'), 51 | queryset=models.Job.objects.all() 52 | ) 53 | self.assertEqual(set([job]), set(filtr.qs)) 54 | 55 | # Test search with Job ID 56 | filtr = JobFilter( 57 | QueryDict(query_string=f'only_scheduled=all&name={job.id}'), 58 | queryset=models.Job.objects.all() 59 | ) 60 | 61 | self.assertEqual(set([job]), set(filtr.qs)) 62 | 63 | # Test search with Snippet ID 64 | filtr = JobFilter( 65 | QueryDict(query_string=f'only_scheduled=all&name={job.snippet.id}'), 66 | queryset=models.Job.objects.all() 67 | ) 68 | self.assertEqual(set([job]), set(filtr.qs)) 69 | 70 | def test_locale(self): 71 | job = JobFactory.create(snippet__locale='fr') 72 | JobFactory.create(snippet__locale='de') 73 | JobFactory.create(snippet__locale='xx') 74 | locale = job.snippet.locale 75 | filtr = JobFilter(QueryDict(query_string=f'only_scheduled=all&locale={locale.id}'), 76 | queryset=models.Job.objects.all()) 77 | self.assertEqual(set([job]), set(filtr.qs)) 78 | -------------------------------------------------------------------------------- /snippets/base/tests/test_forms.py: -------------------------------------------------------------------------------- 1 | from snippets.base.forms import TargetAdminForm 2 | from snippets.base.tests import TestCase, TargetFactory 3 | 4 | 5 | class TargetAdminFormTests(TestCase): 6 | def setUp(self): 7 | self.data = { 8 | 'name': 'foo-target', 9 | 'filtr_is_default_browser': 'true', 10 | } 11 | 12 | def test_save(self): 13 | data = self.data.copy() 14 | instance = TargetFactory() 15 | form = TargetAdminForm(data, instance=instance) 16 | 17 | self.assertTrue(form.is_valid()) 18 | form.save() 19 | instance.refresh_from_db() 20 | self.assertEqual(instance.jexl_expr, 'isDefaultBrowser == true') 21 | self.assertTrue(instance.filtr_is_default_browser) 22 | -------------------------------------------------------------------------------- /snippets/base/tests/test_middleware.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import Mock, patch 2 | 3 | from django.test import RequestFactory 4 | 5 | from snippets.base.middleware import FetchSnippetsMiddleware 6 | from snippets.base.tests import TestCase 7 | 8 | 9 | class FetchSnippetsMiddlewareTests(TestCase): 10 | def setUp(self): 11 | self.get_response_mock = Mock() 12 | self.middleware = FetchSnippetsMiddleware(self.get_response_mock) 13 | 14 | @patch('snippets.base.middleware.resolve') 15 | @patch('snippets.base.middleware.fetch_snippets') 16 | def test_resolve_fetch_snippets_match(self, fetch_snippets, resolve): 17 | """ 18 | If resolve returns a match to the fetch_snippets view, return the 19 | result of the view. 20 | """ 21 | request = Mock() 22 | result = resolve.return_value 23 | result.func = fetch_snippets 24 | result.args = (1, 'asdf') 25 | result.kwargs = {'blah': 5} 26 | 27 | self.assertEqual(self.middleware(request), fetch_snippets.return_value) 28 | fetch_snippets.assert_called_with(request, 1, 'asdf', blah=5) 29 | 30 | @patch('snippets.base.middleware.resolve') 31 | @patch('snippets.base.middleware.fetch_snippets') 32 | def test_resolve_no_match(self, fetch_snippets, resolve): 33 | """ 34 | If resolve doesn't return a match to the fetch_snippets view, return 35 | get_response_mock 36 | """ 37 | request = Mock() 38 | result = resolve.return_value 39 | result.func = lambda request: 'asdf' 40 | 41 | self.assertEqual(self.middleware(request), self.get_response_mock()) 42 | 43 | def test_unknown_url(self): 44 | """ 45 | If resolve doesn't return a match a URL, return get_response_mock 46 | """ 47 | request = RequestFactory().get('/admin') 48 | self.assertEqual(self.middleware(request), self.get_response_mock()) 49 | -------------------------------------------------------------------------------- /snippets/base/tests/test_slack.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | from django.test.utils import override_settings 4 | 5 | from snippets.base import slack 6 | from snippets.base.tests import TestCase 7 | 8 | 9 | class SendSlackTests(TestCase): 10 | @override_settings(SLACK_ENABLE=False) 11 | def test_slack_disalbed(self): 12 | with patch('snippets.base.slack.requests') as requests_mock: 13 | slack._send_slack('foo') 14 | self.assertFalse(requests_mock.called) 15 | 16 | @override_settings(SLACK_ENABLE=True, SLACK_WEBHOOK='https://example.com') 17 | def test_slack_enabled(self): 18 | with patch('snippets.base.slack.requests') as requests_mock: 19 | slack._send_slack('foo') 20 | self.assertTrue(requests_mock.post.called) 21 | requests_mock.post.assert_called_with( 22 | 'https://example.com', data=b'foo', timeout=4, 23 | headers={'Content-Type': 'application/json'}) 24 | -------------------------------------------------------------------------------- /snippets/base/tests/test_validators.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | from django.core.exceptions import ValidationError 4 | 5 | from snippets.base import models 6 | from snippets.base.tests import TestCase 7 | from snippets.base.validators import (validate_as_router_fluent_variables, 8 | validate_json_data, validate_jexl) 9 | 10 | 11 | class ASRouterFluentVariablesValidatorTests(TestCase): 12 | @patch('snippets.base.validators.ALLOWED_TAGS', ['a', 'strong']) 13 | def test_valid(self): 14 | obj = models.SimpleTemplate( 15 | text='Link to example.com.', 16 | title='This is important', 17 | ) 18 | self.assertEqual(validate_as_router_fluent_variables(obj, ['text', 'title']), obj) 19 | 20 | @patch('snippets.base.validators.ALLOWED_TAGS', 'a') 21 | def test_invalid_tag(self): 22 | obj = models.SimpleTemplate( 23 | text='Strong text.', 24 | ) 25 | self.assertRaises(ValidationError, validate_as_router_fluent_variables, obj, ['text']) 26 | 27 | def test_invalid_protocol(self): 28 | obj = models.SimpleTemplate( 29 | text='Strong text.', 30 | ) 31 | self.assertRaises(ValidationError, validate_as_router_fluent_variables, obj, ['text']) 32 | 33 | 34 | class ValidateJSONDataTests(TestCase): 35 | def test_base(self): 36 | data = '{"foo": 3}' 37 | self.assertEqual(validate_json_data(data), data) 38 | 39 | def test_invalid_data(self): 40 | data = '{"foo": 3' 41 | self.assertRaises(ValidationError, validate_json_data, data) 42 | 43 | 44 | class ValidatorJEXL(TestCase): 45 | def test_base(self): 46 | data = 'browser.update == True && foo' 47 | self.assertEqual(validate_jexl(data), data) 48 | 49 | def test_invalid_Data(self): 50 | data = '(browser.update == True' 51 | self.assertRaises(ValidationError, validate_jexl, data) 52 | -------------------------------------------------------------------------------- /snippets/base/tests/test_views.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import DEFAULT, patch 2 | 3 | from django.http import Http404 4 | from django.test.client import RequestFactory 5 | from django.test.utils import override_settings 6 | from django.urls import reverse 7 | 8 | import snippets.base.models 9 | from snippets.base import views 10 | from snippets.base.tests import ASRSnippetFactory, TestCase 11 | 12 | snippets.base.models.CHANNELS = ('release', 'beta', 'aurora', 'nightly') 13 | 14 | 15 | class FetchSnippetsTests(TestCase): 16 | def test_base(self): 17 | asrclient_kwargs = dict([ 18 | ('startpage_version', 6), 19 | ('name', 'Firefox'), 20 | ('version', '64.0'), 21 | ('appbuildid', '20190110041606'), 22 | ('build_target', 'Darwin_Universal-gcc3'), 23 | ('locale', 'en-US'), 24 | ('channel', 'release'), 25 | ('os_version', 'Darwin 10.8.0'), 26 | ('distribution', 'default'), 27 | ('distribution_version', 'default_version'), 28 | ]) 29 | request = RequestFactory().get('/') 30 | 31 | with patch.multiple('snippets.base.views', 32 | fetch_snippet_pregen_bundle=DEFAULT) as patches: 33 | views.fetch_snippets(request, **asrclient_kwargs) 34 | self.assertTrue(patches['fetch_snippet_pregen_bundle'].called) 35 | 36 | # Old client. 37 | with patch.multiple('snippets.base.views', 38 | fetch_snippet_pregen_bundle=DEFAULT) as patches: 39 | asrclient_kwargs['startpage_version'] = 5 40 | self.assertRaises(Http404, views.fetch_snippets, request, **asrclient_kwargs) 41 | self.assertFalse(patches['fetch_snippet_pregen_bundle'].called) 42 | 43 | 44 | @override_settings(SITE_URL='http://example.org', 45 | MEDIA_BUNDLES_PREGEN_ROOT='/bundles/pregen/', 46 | INSTANT_BUNDLE_GENERATION=False) 47 | class FetchSnippetPregenBundleTests(TestCase): 48 | def setUp(self): 49 | self.factory = RequestFactory() 50 | self.request = self.factory.get('/') 51 | self.asrclient_kwargs = dict([ 52 | ('startpage_version', 6), 53 | ('name', 'Firefox'), 54 | ('version', '70.0'), 55 | ('appbuildid', '20190110041606'), 56 | ('build_target', 'Darwin_Universal-gcc3'), 57 | ('locale', 'el-GR'), 58 | ('channel', 'default'), 59 | ('os_version', 'Darwin 10.8.0'), 60 | ('distribution', 'other-than-default'), 61 | ('distribution_version', 'default_version'), 62 | ]) 63 | 64 | def test_base(self): 65 | with patch('snippets.base.views.calculate_redirect') as calculate_redirect_mock: 66 | calculate_redirect_mock.return_value = ('el-gr', 'default', 'https://example.com') 67 | response = views.fetch_snippet_pregen_bundle(self.request, **self.asrclient_kwargs) 68 | calculate_redirect_mock.assert_called_with( 69 | locale='el-GR', distribution='other-than-default' 70 | ) 71 | 72 | self.assertEqual(response.url, 'https://example.com') 73 | 74 | @override_settings(INSTANT_BUNDLE_GENERATION=True) 75 | def test_instant_bundle_generation(self): 76 | with patch('snippets.base.views.generate_bundles') as generate_bundles_mock: 77 | generate_bundles_mock.return_value = 'foo=bar' 78 | response = views.fetch_snippet_pregen_bundle(self.request, **self.asrclient_kwargs) 79 | generate_bundles_mock.assert_called_with( 80 | limit_to_locale='el-gr', 81 | limit_to_distribution_bundle='default', 82 | save_to_disk=False, 83 | ) 84 | self.assertEqual(response.status_code, 200) 85 | self.assertEqual(response['Content-Type'], 'application/json') 86 | self.assertEqual(response.content, b'foo=bar') 87 | 88 | 89 | class PreviewASRSnippetTests(TestCase): 90 | def test_base(self): 91 | snippet = ASRSnippetFactory() 92 | url = reverse('asr-preview', kwargs={'uuid': snippet.uuid}) 93 | with patch('snippets.base.views.ASRSnippet.render') as render_mock: 94 | render_mock.return_value = 'foo' 95 | response = self.client.get(url) 96 | render_mock.assert_called_with(preview=True) 97 | self.assertEqual(response.status_code, 200) 98 | self.assertEqual(response['Content-Type'], 'application/json') 99 | 100 | def test_404(self): 101 | url = reverse('asr-preview', kwargs={'uuid': 'foo'}) 102 | response = self.client.get(url) 103 | self.assertEqual(response.status_code, 404) 104 | 105 | url = reverse('asr-preview', kwargs={'uuid': '804c062b-844f-4f33-80d3-9915514a14b4'}) 106 | response = self.client.get(url) 107 | self.assertEqual(response.status_code, 404) 108 | -------------------------------------------------------------------------------- /snippets/base/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | 3 | from watchman import views as watchman_views 4 | 5 | from snippets.base import feed, views 6 | 7 | 8 | urlpatterns = [ 9 | path('', views.HomeView.as_view()), 10 | 11 | # ASRSnippets 12 | path('/////' 13 | '/////', 14 | views.fetch_snippets, name='base.fetch_snippets'), 15 | path('list/jobs/', views.JobListView.as_view(), name='base.list_jobs'), 16 | path('feeds/snippets.ics', feed.JobsFeed(), name='ical-feed'), 17 | 18 | path('preview-asr//', views.preview_asr_snippet, name='asr-preview'), 19 | # Application 20 | path('csp-violation-capture', views.csp_violation_capture, name='csp-violation-capture'), 21 | path('healthz/', watchman_views.ping, name='watchman.ping'), 22 | path('readiness/', watchman_views.status, name='watchman.status'), 23 | ] 24 | -------------------------------------------------------------------------------- /snippets/base/validators.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import bleach 4 | import pyjexl 5 | import django.core.validators as django_validators 6 | from django.core.exceptions import ValidationError 7 | 8 | ALLOWED_TAGS = ['a', 'i', 'b', 'u', 'strong', 'em', 'br'] 9 | ALLOWED_ATTRIBUTES = {'a': ['href', 'data-metric']} 10 | ALLOWED_PROTOCOLS = ['https', 'special'] 11 | 12 | 13 | def validate_as_router_fluent_variables(obj, variables): 14 | for variable in variables: 15 | text = getattr(obj, variable) 16 | bleached_text = bleach.clean( 17 | text, 18 | tags=ALLOWED_TAGS, 19 | attributes=ALLOWED_ATTRIBUTES, 20 | # Allow only secure protocols and custom special links. 21 | protocols=ALLOWED_PROTOCOLS, 22 | ) 23 | # Bleach escapes '&' to '&'. We need to revert back to compare with 24 | # text 25 | bleached_text = bleached_text.replace('&', '&') 26 | 27 | if text != bleached_text: 28 | error_msg = ( 29 | 'Field contains unsupported tags or insecure links. ' 30 | 'Only {} tags and https links are supported.' 31 | ).format(', '.join(ALLOWED_TAGS)) 32 | raise ValidationError({variable: error_msg}) 33 | return obj 34 | 35 | 36 | def validate_json_data(data): 37 | try: 38 | json.loads(data) 39 | except ValueError: 40 | raise ValidationError('Enter valid JSON string.') 41 | return data 42 | 43 | 44 | # URLValidator that also allows `special:*` links 45 | class URLValidator(django_validators.URLValidator): 46 | def __init__(self, schemes=None, **kwargs): 47 | self.schemes = ['https', 'special'] 48 | self.regex = django_validators._lazy_re_compile( 49 | r'^(special:\w+)|(' + self.regex.pattern + ')') 50 | 51 | 52 | def validate_jexl(data): 53 | try: 54 | pyjexl.JEXL().parse(data) 55 | except pyjexl.JEXLException: 56 | raise ValidationError('Enter valid JEXL expression.') 57 | return data 58 | -------------------------------------------------------------------------------- /snippets/base/views.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import sentry_sdk 4 | from django.conf import settings 5 | from django.core.exceptions import ValidationError 6 | from django.http import ( 7 | Http404, 8 | HttpResponse, 9 | HttpResponseBadRequest, 10 | HttpResponseRedirect, 11 | ) 12 | from django.shortcuts import get_object_or_404 13 | from django.views.decorators.cache import cache_control 14 | from django.views.decorators.csrf import csrf_exempt 15 | from django.views.decorators.http import require_POST 16 | from django.utils.decorators import method_decorator 17 | from django.views.generic import TemplateView 18 | from django_filters.views import FilterView 19 | from ratelimit.decorators import ratelimit 20 | 21 | from redirector.redirect import calculate_redirect 22 | from snippets.base.bundles import generate_bundles 23 | from snippets.base.filters import JobFilter 24 | from snippets.base.models import ASRSnippet 25 | 26 | 27 | class HomeView(TemplateView): 28 | template_name = 'base/home.jinja' 29 | 30 | 31 | class JobListView(FilterView): 32 | filterset_class = JobFilter 33 | 34 | @method_decorator( 35 | ratelimit( 36 | rate=settings.RATELIMIT_RATE, 37 | block=True, 38 | key=lambda g, r: r.META.get('HTTP_X_FORWARDED_FOR', r.META['REMOTE_ADDR']) 39 | ) 40 | ) 41 | def get(self, request, **kwargs): 42 | return super().get(request, **kwargs) 43 | 44 | @property 45 | def template_name(self): 46 | if self.request.GET.get('calendar', 'false') == 'true': 47 | return 'base/jobs_list_calendar.jinja' 48 | 49 | return 'base/jobs_list_table.jinja' 50 | 51 | 52 | def fetch_snippets(request, **kwargs): 53 | if kwargs['startpage_version'] != 6: 54 | raise Http404() 55 | 56 | return fetch_snippet_pregen_bundle(request, **kwargs) 57 | 58 | 59 | @cache_control(public=True, max_age=settings.SNIPPET_BUNDLE_PREGEN_REDIRECT_TIMEOUT) 60 | def fetch_snippet_pregen_bundle(request, **kwargs): 61 | locale, distribution, full_url = calculate_redirect(locale=kwargs['locale'], 62 | distribution=kwargs['distribution']) 63 | 64 | if settings.INSTANT_BUNDLE_GENERATION: 65 | content = generate_bundles( 66 | limit_to_locale=locale, 67 | limit_to_distribution_bundle=distribution, 68 | save_to_disk=False 69 | ) 70 | return HttpResponse(status=200, content=content, content_type='application/json') 71 | 72 | return HttpResponseRedirect(full_url) 73 | 74 | 75 | def preview_asr_snippet(request, uuid): 76 | try: 77 | snippet = get_object_or_404(ASRSnippet, uuid=uuid) 78 | except ValidationError: 79 | # Raised when UUID is a badly formed hexadecimal UUID string 80 | raise Http404() 81 | 82 | bundle_content = json.dumps({ 83 | 'messages': [snippet.render(preview=True)], 84 | }) 85 | return HttpResponse(bundle_content, content_type='application/json') 86 | 87 | 88 | @csrf_exempt 89 | @require_POST 90 | def csp_violation_capture(request): 91 | try: 92 | csp_data = json.loads(request.body) 93 | except ValueError: 94 | # Cannot decode CSP violation data, ignore 95 | return HttpResponseBadRequest('Invalid CSP Report') 96 | 97 | try: 98 | blocked_uri = csp_data['csp-report']['blocked-uri'] 99 | except KeyError: 100 | # Incomplete CSP report 101 | return HttpResponseBadRequest('Incomplete CSP Report') 102 | 103 | with sentry_sdk.configure_scope() as scope: 104 | scope.level = 'info' 105 | scope.set_tag('logger', 'csp') 106 | 107 | sentry_sdk.capture_message( 108 | message='CSP Violation: {}'.format(blocked_uri)) 109 | 110 | return HttpResponse('Captured CSP violation, thanks for reporting.') 111 | -------------------------------------------------------------------------------- /snippets/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.contrib import admin 3 | from django.contrib.staticfiles.urls import staticfiles_urlpatterns 4 | from django.http import HttpResponse, HttpResponseForbidden 5 | from django.views.generic import RedirectView 6 | from django.views.static import serve as static_serve 7 | from django.shortcuts import render 8 | from django.urls import include, path, re_path 9 | 10 | import sentry_sdk 11 | from ratelimit.exceptions import Ratelimited 12 | 13 | 14 | def robots_txt(request): 15 | permission = 'Allow' if settings.ENGAGE_ROBOTS else 'Disallow' 16 | return HttpResponse('User-agent: *\n{0}: /'.format(permission), content_type='text/plain') 17 | 18 | 19 | def handler403(request, exception=None): 20 | if isinstance(exception, Ratelimited): 21 | with sentry_sdk.configure_scope() as scope: 22 | scope.level = 'info' 23 | scope.set_tag('logger', 'ratelimited') 24 | sentry_sdk.capture_message(message='Rate limited') 25 | return render(request, template_name='base/ratelimited.jinja', status=429) 26 | return HttpResponseForbidden('Forbidden') 27 | 28 | 29 | urlpatterns = [ 30 | path('', include('snippets.base.urls')), 31 | path('robots.txt', robots_txt), 32 | 33 | # Favicon 34 | re_path(r'^(?Pfavicon\.ico)$', static_serve, {'document_root': settings.STATIC_ROOT}), 35 | # contribute.json url 36 | re_path(r'^(?Pcontribute\.json)$', static_serve, {'document_root': settings.ROOT}), 37 | ] 38 | 39 | if settings.ENABLE_ADMIN: 40 | urlpatterns += [ 41 | re_path(r'^taggit/', include('taggit_selectize.urls')), 42 | path('admin/', admin.site.urls), 43 | ] 44 | admin.site.site_header = settings.SITE_HEADER 45 | admin.site.site_title = settings.SITE_TITLE 46 | 47 | elif settings.ADMIN_REDIRECT_URL: 48 | urlpatterns.append( 49 | path('admin/', RedirectView.as_view(url=f'{settings.ADMIN_REDIRECT_URL}/admin/')) 50 | ) 51 | 52 | if settings.OIDC_ENABLE: 53 | urlpatterns.append(path('oidc/', include('mozilla_django_oidc.urls'))) 54 | 55 | # In DEBUG mode, serve media files through Django. 56 | if settings.DEBUG: 57 | # Use custom serve function that adds necessary headers. 58 | def serve_media(*args, **kwargs): 59 | response = static_serve(*args, **kwargs) 60 | response['Access-Control-Allow-Origin'] = '*' 61 | if (settings.BUNDLE_BROTLI_COMPRESS and 62 | kwargs['path'].startswith(settings.MEDIA_BUNDLES_PREGEN_ROOT)): 63 | response['Content-Encoding'] = 'br' 64 | return response 65 | 66 | urlpatterns += [ 67 | re_path(r'^media/(?P.*)$', serve_media, {'document_root': settings.MEDIA_ROOT}), 68 | ] + staticfiles_urlpatterns() 69 | -------------------------------------------------------------------------------- /snippets/wsgi/__init__.py: -------------------------------------------------------------------------------- 1 | from .app import application # noqa 2 | -------------------------------------------------------------------------------- /snippets/wsgi/app.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for snippets project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.7/howto/deployment/wsgi/ 8 | """ 9 | import newrelic.agent 10 | newrelic.agent.initialize('newrelic.ini') 11 | 12 | import os # NOQA 13 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'snippets.settings') # NOQA 14 | 15 | from django.core.wsgi import get_wsgi_application # NOQA 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /snippets/wsgi/config.py: -------------------------------------------------------------------------------- 1 | # see http://docs.gunicorn.org/en/latest/configure.html#configuration-file 2 | 3 | from os import getenv 4 | 5 | 6 | bind = '0.0.0.0:{}'.format(getenv('PORT', 8000)) 7 | workers = getenv('WSGI_NUM_WORKERS', 2) 8 | accesslog = '-' 9 | errorlog = '-' 10 | loglevel = getenv('WSGI_LOG_LEVEL', 'info') 11 | 12 | # Larger keep-alive values maybe needed when directly talking to ELBs 13 | # See https://github.com/benoitc/gunicorn/issues/1194 14 | keepalive = getenv('WSGI_KEEP_ALIVE', 2) 15 | worker_class = getenv('GUNICORN_WORKER_CLASS', 'meinheld.gmeinheld.MeinheldWorker') 16 | worker_tmp_dir = '/dev/shm' 17 | -------------------------------------------------------------------------------- /test.env: -------------------------------------------------------------------------------- 1 | DEBUG=False 2 | ALLOWED_HOSTS=* 3 | SECRET_KEY=foo 4 | DATABASE_URL=postgres://snippets:snippets@db/snippets 5 | SITE_URL=http://localhost:8000 6 | CACHE_URL=locmem:// 7 | ENABLE_ADMIN=True 8 | SECURE_SSL_REDIRECT=False 9 | --------------------------------------------------------------------------------