├── .circleci └── config.yml ├── .coveragerc ├── .dockerignore ├── .editorconfig ├── .env-dist ├── .gitignore ├── .pyup.yml ├── CHANGELOG.rst ├── CODE_OF_CONDUCT.md ├── Dockerfile ├── Dockerfile.dev ├── LICENSE ├── Makefile ├── NEWS.md ├── Procfile ├── README.md ├── atmo ├── __init__.py ├── apps.py ├── celery.py ├── clusters │ ├── __init__.py │ ├── admin.py │ ├── factories.py │ ├── forms.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ ├── deactivate_clusters.py │ │ │ └── update_clusters.py │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_cluster_emr_release.py │ │ ├── 0003_cluster_master_address.py │ │ ├── 0004_auto_20161002_1841.py │ │ ├── 0005_auto_20161017_0913.py │ │ ├── 0006_auto_20161024_1006.py │ │ ├── 0007_auto_20161102_1053.py │ │ ├── 0008_cluster_expiration_mail_sent.py │ │ ├── 0009_auto_20161108_0933.py │ │ ├── 0010_assign_view_perms.py │ │ ├── 0011_assign_more_perms.py │ │ ├── 0012_cluster_ssh_key.py │ │ ├── 0013_migrate_public_key.py │ │ ├── 0014_remove_cluster_public_key.py │ │ ├── 0015_auto_20170124_1357.py │ │ ├── 0016_auto_20170130_1704.py │ │ ├── 0017_auto_20170222_1457.py │ │ ├── 0018_auto_20170307_1423.py │ │ ├── 0019_auto_20170314_1216.py │ │ ├── 0020_emr_release_model.py │ │ ├── 0021_rename_cluster_emr_release.py │ │ ├── 0022_convert_cluster_emr_release.py │ │ ├── 0023_alter_cluster_emr_release.py │ │ ├── 0024_remove_cluster_emr_release.py │ │ ├── 0025_emrrelease_is_active.py │ │ ├── 0026_auto_20170330_1732.py │ │ ├── 0027_cluster_lifetime.py │ │ ├── 0028_cluster_lifetime_extension_count.py │ │ ├── 0029_remove_start_date.py │ │ ├── 0030_rename_end_date_expires_at.py │ │ ├── 0031_cluster_finished_at.py │ │ ├── 0032_auto_20170519_1336.py │ │ ├── 0033_auto_20170522_1426.py │ │ ├── 0034_auto_20170530_1907.py │ │ ├── 0035_auto_20180814_1710.py │ │ └── __init__.py │ ├── models.py │ ├── provisioners.py │ ├── queries.py │ ├── tasks.py │ ├── urls.py │ └── views.py ├── context_processors.py ├── decorators.py ├── forms │ ├── __init__.py │ ├── cache.py │ ├── fields.py │ ├── mixins.py │ └── widgets.py ├── jobs │ ├── __init__.py │ ├── admin.py │ ├── exceptions.py │ ├── factories.py │ ├── forms.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ └── run_jobs.py │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_auto_20161017_0913.py │ │ ├── 0003_auto_20161019_1245.py │ │ ├── 0004_sparkjob_emr_release.py │ │ ├── 0005_auto_20161102_1049.py │ │ ├── 0006_auto_20161108_0933.py │ │ ├── 0007_assign_view_perms.py │ │ ├── 0008_assign_more_perms.py │ │ ├── 0009_sparkjob_description.py │ │ ├── 0010_auto_20170124_1357.py │ │ ├── 0011_auto_20170130_1704.py │ │ ├── 0012_add_job_run_history.py │ │ ├── 0013_sparkjobrunalert.py │ │ ├── 0014_auto_20170307_1423.py │ │ ├── 0015_auto_20170317_1027.py │ │ ├── 0016_auto_20170320_0943.py │ │ ├── 0017_assign_group_view_perm.py │ │ ├── 0018_rename_add_spark_job_emr_release.py │ │ ├── 0019_convert_spark_job_emr_release.py │ │ ├── 0020_alter_spark_job_emr_release.py │ │ ├── 0021_remove_spark_job_emr_release.py │ │ ├── 0022_sparkjobrun_emr_release_version.py │ │ ├── 0023_sparkjob_expired_date.py │ │ ├── 0024_auto_20170425_1324.py │ │ ├── 0025_populate_job_schedule.py │ │ ├── 0026_sparkjobrun_size.py │ │ ├── 0027_rename_terminated_final_date.py │ │ ├── 0028_auto_20170517_1443.py │ │ ├── 0029_auto_20170519_1336.py │ │ ├── 0030_rename_run_date.py │ │ ├── 0031_update_started_at.py │ │ ├── 0032_sparkjobrun_ready_at.py │ │ ├── 0033_rename_scheduled_date.py │ │ ├── 0034_auto_20170529_1424.py │ │ ├── 0035_auto_20170529_1424.py │ │ ├── 0036_sparkjobrunalert_temp_id.py │ │ ├── 0037_populate_temp_id.py │ │ ├── 0038_auto_20170530_1848.py │ │ ├── 0039_auto_20170530_1848.py │ │ ├── 0040_auto_20170530_1849.py │ │ ├── 0041_auto_20170530_1857.py │ │ ├── 0042_auto_20170530_1903.py │ │ ├── 0043_auto_20170530_1906.py │ │ └── __init__.py │ ├── models.py │ ├── provisioners.py │ ├── queries.py │ ├── schedules.py │ ├── signals.py │ ├── tasks.py │ ├── templatetags │ │ ├── __init__.py │ │ ├── notebook.py │ │ └── status.py │ ├── urls.py │ └── views.py ├── keys │ ├── __init__.py │ ├── admin.py │ ├── factories.py │ ├── forms.py │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_assign_view_perms.py │ │ ├── 0003_auto_20170116_1512.py │ │ ├── 0004_auto_20170519_1336.py │ │ └── __init__.py │ ├── models.py │ ├── urls.py │ ├── utils.py │ └── views.py ├── models.py ├── names.py ├── news │ ├── __init__.py │ ├── urls.py │ └── views.py ├── provisioners.py ├── settings.py ├── static │ ├── css │ │ ├── base.css │ │ ├── fileinput.css │ │ ├── login.css │ │ └── notebook.css │ ├── img │ │ ├── cluster.png │ │ ├── dashboards.png │ │ ├── schedule.png │ │ └── worker.png │ ├── js │ │ ├── base.js │ │ ├── clusters.js │ │ ├── csrf.js │ │ ├── fileinput.js │ │ ├── forms.js │ │ ├── jobs.js │ │ ├── keys.js │ │ └── raven.js │ └── public │ │ ├── favicon.ico │ │ └── robots.txt ├── stats │ ├── __init__.py │ ├── migrations │ │ ├── 0001_initial.py │ │ └── __init__.py │ └── models.py ├── tasks.py ├── templates │ ├── 403.html │ ├── 404.html │ ├── 500.html │ └── atmo │ │ ├── _announcements.html │ │ ├── _form.html │ │ ├── base.html │ │ ├── clusters │ │ ├── detail.html │ │ ├── extend.html │ │ ├── mails │ │ │ └── expiration.mail │ │ ├── new.html │ │ └── terminate.html │ │ ├── dashboard.html │ │ ├── error.html │ │ ├── jobs │ │ ├── delete.html │ │ ├── detail.html │ │ ├── edit.html │ │ ├── mails │ │ │ ├── expired.mail │ │ │ ├── failed_run_alert.mail │ │ │ └── timed_out.mail │ │ ├── new.html │ │ ├── run.html │ │ └── zeppelin_notebook.html │ │ ├── keys │ │ ├── delete.html │ │ ├── detail.html │ │ ├── list.html │ │ └── new.html │ │ ├── news │ │ └── list.html │ │ └── users │ │ └── login.html ├── templatetags.py ├── urls.py ├── users │ ├── __init__.py │ ├── backends.py │ ├── factories.py │ ├── migrations │ │ ├── 0001_initial_site.py │ │ ├── 0002_rewrite_usernames.py │ │ └── __init__.py │ ├── urls.py │ └── utils.py ├── utils.py ├── views.py └── wsgi.py ├── bin ├── build ├── check_license ├── deploy ├── run └── test ├── contribute.json ├── docker-compose.yml ├── docs ├── Makefile ├── _static │ ├── fonts.css │ ├── fonts │ │ ├── ZillaSlab-Bold.woff │ │ ├── ZillaSlab-Bold.woff2 │ │ ├── ZillaSlab-BoldItalic.woff │ │ ├── ZillaSlab-BoldItalic.woff2 │ │ ├── ZillaSlab-Italic.woff │ │ ├── ZillaSlab-Italic.woff2 │ │ ├── ZillaSlab-Light.woff │ │ ├── ZillaSlab-Light.woff2 │ │ ├── ZillaSlab-LightItalic.woff │ │ ├── ZillaSlab-LightItalic.woff2 │ │ ├── ZillaSlab-Medium.woff │ │ ├── ZillaSlab-Medium.woff2 │ │ ├── ZillaSlab-MediumItalic.woff │ │ ├── ZillaSlab-MediumItalic.woff2 │ │ ├── ZillaSlab-Regular.woff │ │ ├── ZillaSlab-Regular.woff2 │ │ ├── ZillaSlab-SemiBold.woff │ │ ├── ZillaSlab-SemiBold.woff2 │ │ ├── ZillaSlab-SemiBoldItalic.woff │ │ ├── ZillaSlab-SemiBoldItalic.woff2 │ │ ├── ZillaSlabHighlight-Bold.woff │ │ ├── ZillaSlabHighlight-Bold.woff2 │ │ ├── ZillaSlabHighlight-Regular.woff │ │ ├── ZillaSlabHighlight-Regular.woff2 │ │ ├── open-sans-v13-latin-700.woff │ │ ├── open-sans-v13-latin-700.woff2 │ │ ├── open-sans-v13-latin-700italic.woff │ │ ├── open-sans-v13-latin-700italic.woff2 │ │ ├── open-sans-v13-latin-italic.woff │ │ ├── open-sans-v13-latin-italic.woff2 │ │ ├── open-sans-v13-latin-regular.woff │ │ ├── open-sans-v13-latin-regular.woff2 │ │ ├── roboto-slab-v6-latin-700.woff │ │ ├── roboto-slab-v6-latin-700.woff2 │ │ ├── roboto-slab-v6-latin-regular.woff │ │ └── roboto-slab-v6-latin-regular.woff2 │ └── style.css ├── changelog.rst ├── conf.py ├── deployment.rst ├── development.rst ├── extensions │ ├── __init__.py │ ├── celery.py │ └── django.py ├── index.rst ├── maintenance.rst ├── overview.rst ├── reference │ ├── atmo.clusters.rst │ ├── atmo.jobs.rst │ ├── atmo.keys.rst │ ├── atmo.news.rst │ ├── atmo.rst │ ├── atmo.settings.rst │ ├── atmo.users.rst │ └── index.rst ├── spelling_wordlist.txt └── workflows.rst ├── manage.py ├── newrelic.ini ├── package-lock.json ├── package.json ├── pyproject.toml ├── pytest.ini ├── renovate.json ├── requirements ├── all.txt ├── build.txt ├── docs.txt └── tests.txt ├── runtime.txt ├── setup.cfg ├── setup.py └── tests ├── blockade.py ├── clusters ├── __init__.py ├── test_admin.py ├── test_forms.py ├── test_models.py ├── test_provisioners.py ├── test_tasks.py └── test_views.py ├── conftest.py ├── jobs ├── __init__.py ├── test_admin.py ├── test_forms.py ├── test_models.py ├── test_provisioners.py ├── test_schedules.py ├── test_tasks.py └── test_views.py ├── messages.py ├── test_dashboard.py ├── test_database.py ├── test_keys.py ├── test_names.py ├── test_stats.py ├── test_templatetags.py └── test_users.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # These environment variables must be set in CircleCI UI 2 | # 3 | # DOCKERHUB_REPO - docker hub repo, format: / 4 | # DOCKER_USER 5 | # DOCKER_PASS 6 | # 7 | 8 | version: 2 9 | jobs: 10 | build: 11 | machine: 12 | enable: true 13 | working_directory: ~/mozilla/telemetry-analysis-service 14 | steps: 15 | - checkout 16 | - run: ./bin/build 17 | 18 | test: 19 | machine: 20 | enable: true 21 | working_directory: ~/mozilla/telemetry-analysis-service 22 | steps: 23 | - checkout 24 | - run: sudo apt-get update 25 | - run: sudo apt-get install python-dev 26 | - run: sudo pip install --upgrade pip 27 | - run: pip install docker-compose 28 | - run: docker info 29 | - run: docker --version 30 | - run: ./bin/test 31 | 32 | deploy: 33 | machine: 34 | enable: true 35 | working_directory: ~/mozilla/telemetry-analysis-service 36 | steps: 37 | - checkout 38 | - run: ./bin/build && ./bin/deploy 39 | 40 | workflows: 41 | version: 2 42 | build-test-deploy: 43 | jobs: 44 | - build: 45 | filters: 46 | tags: 47 | only: /.*/ 48 | 49 | - test: 50 | filters: 51 | tags: 52 | only: /.*/ 53 | 54 | - deploy: 55 | requires: 56 | - test 57 | filters: 58 | tags: 59 | only: /.*/ 60 | branches: 61 | only: master 62 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = atmo 3 | branch = True 4 | data_file = /tmp/.coverage 5 | 6 | [report] 7 | show_missing = True 8 | omit = 9 | tests/* 10 | *migrations* 11 | *management* 12 | *management/commands* 13 | 14 | [xml] 15 | output = /tmp/coverage.xml 16 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .bash_history 3 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # Top-most EditorConfig file ? 2 | root = true 3 | 4 | # Unix-style newlines with a newline ending every file 5 | [*] 6 | end_of_line = lf 7 | insert_final_newline = true 8 | trim_trailing_whitespace = true 9 | 10 | # Indentiation 11 | [*.py] 12 | indent_style = space 13 | indent_size = 4 14 | 15 | [*.{css,js,json,html}] 16 | indent_style = space 17 | indent_size = 2 18 | -------------------------------------------------------------------------------- /.env-dist: -------------------------------------------------------------------------------- 1 | PYTHONUNBUFFERED=1 2 | PYTHONDONTWRITEBYTECODE=1 3 | DATABASE_URL=postgres://postgres@db/postgres 4 | REDIS_URL=redis://redis:6379/0 5 | PORT=8000 6 | DJANGO_DEBUG=True 7 | DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1, 8 | DJANGO_SITE_URL=http://localhost:8000 9 | AWS_ACCESS_KEY_ID= 10 | AWS_SECRET_ACCESS_KEY= 11 | AWS_DEFAULT_REGION=us-west-2 12 | OIDC_RP_CLIENT_ID= 13 | OIDC_RP_CLIENT_SECRET= 14 | OIDC_OP_AUTHORIZATION_ENDPOINT=https://.auth0.com/authorize 15 | OIDC_OP_TOKEN_ENDPOINT=https://.auth0.com/oauth/token 16 | OIDC_OP_USER_ENDPOINT=https://.auth0.com/userinfo 17 | OIDC_OP_DOMAIN=.auth0.com 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .local 3 | *.pyc 4 | .DS_Store 5 | docs/_build 6 | MANIFEST 7 | .coverage 8 | coverage.xml 9 | *.db 10 | dist/ 11 | /static/ 12 | .cache/ 13 | revision.txt 14 | version.json 15 | htmlcov 16 | node_modules/ 17 | .npm/ 18 | .bash_history 19 | coverage 20 | celerybeat.pid 21 | celerymonitor.pid 22 | .psql_history 23 | .python_history 24 | .config/ 25 | .pytest_cache 26 | -------------------------------------------------------------------------------- /.pyup.yml: -------------------------------------------------------------------------------- 1 | search: False 2 | label_prs: update 3 | requirements: 4 | - requirements/build.txt 5 | - requirements/docs.txt 6 | - requirements/tests.txt 7 | assignees: jezdez 8 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Community Participation Guidelines 2 | 3 | This repository is governed by Mozilla's code of conduct and etiquette guidelines. 4 | For more details, please read the 5 | [Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). 6 | 7 | ## How to Report 8 | For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. 9 | -------------------------------------------------------------------------------- /Dockerfile.dev: -------------------------------------------------------------------------------- 1 | ## Node container: 2 | 3 | FROM node:10 as npm 4 | 5 | WORKDIR /opt/npm 6 | COPY package.json package-lock.json /opt/npm/ 7 | RUN npm install 8 | 9 | ## Python container: 10 | 11 | FROM python:3.6-slim 12 | LABEL maintainer="Jannis Leidel " 13 | 14 | ENV PYTHONUNBUFFERED=1 \ 15 | PYTHONPATH=/app/ \ 16 | DJANGO_CONFIGURATION=Dev \ 17 | DEVELOPMENT=1 \ 18 | PORT=8000 19 | 20 | EXPOSE $PORT 21 | 22 | # add a non-privileged user for installing and running the application 23 | # don't use --create-home option to prevent populating with skeleton files 24 | RUN mkdir /app && \ 25 | chown 10001:10001 /app && \ 26 | groupadd --gid 10001 app && \ 27 | useradd --no-create-home --uid 10001 --gid 10001 --home-dir /app app 28 | 29 | # install a few essentials and clean apt caches afterwards 30 | RUN mkdir -p \ 31 | /usr/share/man/man1 \ 32 | /usr/share/man/man2 \ 33 | /usr/share/man/man3 \ 34 | /usr/share/man/man4 \ 35 | /usr/share/man/man5 \ 36 | /usr/share/man/man6 \ 37 | /usr/share/man/man7 \ 38 | /usr/share/man/man8 && \ 39 | apt-get update && \ 40 | apt-get install -y --no-install-recommends \ 41 | apt-transport-https build-essential curl git gnupg libpq-dev \ 42 | postgresql-client gettext sqlite3 libffi-dev \ 43 | graphviz enchant && \ 44 | apt-get autoremove -y && \ 45 | apt-get clean && \ 46 | rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 47 | 48 | # Create static and npm roots 49 | RUN mkdir -p /opt/npm /opt/static && \ 50 | chown -R 10001:10001 /opt 51 | 52 | # Install Python dependencies 53 | COPY requirements/*.txt /tmp/requirements/ 54 | # Switch to /tmp to install dependencies outside home dir 55 | WORKDIR /tmp 56 | RUN pip install --no-cache-dir -r requirements/all.txt 57 | 58 | USER 10001 59 | 60 | # Copy Node dependencies from NPM container 61 | COPY --from=npm /opt/npm /opt/npm 62 | 63 | # Switch back to home directory 64 | WORKDIR /app 65 | 66 | # Using /bin/bash as the entrypoint works around some volume mount issues on Windows 67 | # where volume-mounted files do not have execute bits set. 68 | # https://github.com/docker/compose/issues/2301#issuecomment-154450785 has additional background. 69 | ENTRYPOINT ["/bin/bash", "/app/bin/run"] 70 | 71 | CMD ["web-dev"] 72 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build clean docs live-docs migrate redis-cli revision shell stop test up 2 | 3 | help: 4 | @echo "Welcome to the Telemetry Analysis Service\n" 5 | @echo "The list of commands for local development:\n" 6 | @echo " build Builds the docker images for the docker-compose setup" 7 | @echo " ci Run the test with the CI specific Docker setup" 8 | @echo " clean Stops and removes all docker containers" 9 | @echo " migrate Runs the Django database migrations" 10 | @echo " redis-cli Opens a Redis CLI" 11 | @echo " shell Opens a Bash shell" 12 | @echo " test Runs the Python test suite" 13 | @echo " up Runs the whole stack, served under http://localhost:8000/\n" 14 | @echo " stop Stops the docker containers" 15 | 16 | build: 17 | docker-compose build 18 | 19 | clean: stop 20 | docker-compose rm -f 21 | 22 | migrate: 23 | docker-compose run web \ 24 | python manage.py migrate 25 | 26 | shell: 27 | docker-compose run web bash 28 | 29 | redis-cli: 30 | docker-compose run redis redis-cli -h redis 31 | 32 | stop: 33 | docker-compose stop 34 | 35 | test: 36 | @bin/test 37 | 38 | up: 39 | docker-compose up 40 | 41 | docs: 42 | docker-compose run web \ 43 | python -m sphinx docs docs/_build/html 44 | 45 | live-docs: 46 | docker-compose run --service-ports web \ 47 | sphinx-autobuild \ 48 | -H 0.0.0.0 \ 49 | -p 8000 \ 50 | --watch /app/atmo \ 51 | docs \ 52 | docs/_build/html 53 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: bin/start-stunnel bin/run web 2 | worker: bin/start-stunnel bin/run worker 3 | scheduler: bin/start-stunnel bin/run scheduler 4 | monitor: bin/start-stunnel bin/run monitor 5 | release: bin/pre_deploy 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # atmo - The code for the Telemetry Analysis Service 2 | 3 | [![Documentation Status](https://readthedocs.org/projects/atmo/badge/?version=latest)](https://atmo.readthedocs.io/en/latest/?badge=latest) 4 | [![CircleCI](https://img.shields.io/circleci/project/github/mozilla/telemetry-analysis-service/master.svg)](https://circleci.com/gh/mozilla/telemetry-analysis-service) 5 | [![codecov](https://codecov.io/gh/mozilla/telemetry-analysis-service/branch/master/graph/badge.svg)](https://codecov.io/gh/mozilla/telemetry-analysis-service) 6 | [![Stories ready](https://img.shields.io/waffle/label/mozilla/telemetry-analysis-service/ready.svg)](http://waffle.io/mozilla/telemetry-analysis-service) 7 | [![Stories in progress](https://img.shields.io/waffle/label/mozilla/telemetry-analysis-service/in%20progress.svg)](http://waffle.io/mozilla/telemetry-analysis-service) 8 | [![CalVer - Timely Software Versioning](https://img.shields.io/badge/calver-YY.M.MINOR-22bfda.svg)](https://calver.org/) 9 | 10 | The full documentation can be found on [Read The Docs](https://readthedocs.org/): 11 | 12 | https://atmo.readthedocs.io/ 13 | -------------------------------------------------------------------------------- /atmo/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | default_app_config = "atmo.apps.AtmoAppConfig" 5 | -------------------------------------------------------------------------------- /atmo/apps.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | import logging 5 | 6 | import session_csrf 7 | from django.apps import AppConfig 8 | from django.db.models.signals import post_save, pre_delete 9 | 10 | logger = logging.getLogger("django") 11 | 12 | 13 | class AtmoAppConfig(AppConfig): 14 | name = "atmo" 15 | 16 | def ready(self): 17 | # The app is now ready. Include any monkey patches here. 18 | 19 | # Monkey patch CSRF to switch to session based CSRF. Session 20 | # based CSRF will prevent attacks from apps under the same 21 | # domain. If you're planning to host your app under it's own 22 | # domain you can remove session_csrf and use Django's CSRF 23 | # library. See also 24 | # https://github.com/mozilla/sugardough/issues/38 25 | session_csrf.monkeypatch() 26 | 27 | # Connect signals. 28 | from atmo.jobs.models import SparkJob 29 | from atmo.jobs.signals import assign_group_perm, remove_group_perm 30 | 31 | post_save.connect( 32 | assign_group_perm, 33 | sender=SparkJob, 34 | dispatch_uid="sparkjob_post_save_assign_perm", 35 | ) 36 | pre_delete.connect( 37 | remove_group_perm, 38 | sender=SparkJob, 39 | dispatch_uid="sparkjob_pre_delete_remove_perm", 40 | ) 41 | 42 | 43 | class KeysAppConfig(AppConfig): 44 | name = "atmo.keys" 45 | label = "keys" 46 | verbose_name = "Keys" 47 | -------------------------------------------------------------------------------- /atmo/clusters/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /atmo/clusters/admin.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.contrib import admin 5 | from guardian.admin import GuardedModelAdmin 6 | 7 | from .models import Cluster, EMRRelease 8 | 9 | 10 | def deactivate(modeladmin, request, queryset): 11 | for cluster in queryset: 12 | cluster.deactivate() 13 | 14 | 15 | @admin.register(Cluster) 16 | class ClusterAdmin(GuardedModelAdmin): 17 | list_display = [ 18 | "identifier", 19 | "size", 20 | "lifetime", 21 | "created_by", 22 | "created_at", 23 | "modified_at", 24 | "expires_at", 25 | "started_at", 26 | "ready_at", 27 | "finished_at", 28 | "jobflow_id", 29 | "emr_release", 30 | "most_recent_status", 31 | "lifetime_extension_count", 32 | ] 33 | list_filter = [ 34 | "most_recent_status", 35 | "size", 36 | "lifetime", 37 | "emr_release", 38 | "created_at", 39 | "expires_at", 40 | "started_at", 41 | "ready_at", 42 | "finished_at", 43 | ] 44 | search_fields = ["identifier", "jobflow_id", "created_by__email"] 45 | actions = [deactivate] 46 | 47 | 48 | @admin.register(EMRRelease) 49 | class EMRReleaseAdmin(admin.ModelAdmin): 50 | list_display = [ 51 | "version", 52 | "changelog_url", 53 | "is_active", 54 | "is_experimental", 55 | "is_deprecated", 56 | ] 57 | list_filter = ["is_active", "is_experimental", "is_deprecated"] 58 | search_fields = ["version", "changelog_url", "help_text"] 59 | -------------------------------------------------------------------------------- /atmo/clusters/factories.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | import factory 5 | 6 | from . import models 7 | from .. import names 8 | from ..keys.factories import SSHKeyFactory 9 | from ..users.factories import UserFactory 10 | 11 | 12 | class EMRReleaseFactory(factory.django.DjangoModelFactory): 13 | version = factory.Sequence(lambda n: "1.%s" % n) 14 | changelog_url = factory.LazyAttribute( 15 | lambda emr_release: ( 16 | "https://docs.aws.amazon.com/emr/latest/ReleaseGuide/" 17 | "emr-%s/emr-release-components.html" % emr_release.version 18 | ) 19 | ) 20 | help_text = "just a help text" 21 | is_active = True 22 | is_experimental = False 23 | is_deprecated = False 24 | 25 | class Meta: 26 | model = models.EMRRelease 27 | 28 | 29 | class ClusterFactory(factory.django.DjangoModelFactory): 30 | identifier = factory.LazyFunction(names.random_scientist) 31 | size = 5 32 | lifetime = models.Cluster.DEFAULT_LIFETIME 33 | ssh_key = factory.SubFactory(SSHKeyFactory) 34 | jobflow_id = factory.Sequence(lambda n: "j-%s" % n) 35 | most_recent_status = "" 36 | master_address = "" 37 | expiration_mail_sent = False 38 | created_by = factory.SubFactory(UserFactory) 39 | emr_release = factory.SubFactory(EMRReleaseFactory) 40 | 41 | class Meta: 42 | model = models.Cluster 43 | -------------------------------------------------------------------------------- /atmo/clusters/management/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /atmo/clusters/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /atmo/clusters/management/commands/deactivate_clusters.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.core.management.base import BaseCommand 5 | 6 | from ...jobs import deactivate_clusters 7 | 8 | 9 | class Command(BaseCommand): 10 | help = ( 11 | "Go through expired clusters to deactivate or warn about ones that are expiring" 12 | ) 13 | 14 | def handle(self, *args, **options): 15 | self.stdout.write("Deleting expired clusters...", ending="") 16 | deactivate_clusters() 17 | self.stdout.write("done.") 18 | -------------------------------------------------------------------------------- /atmo/clusters/management/commands/update_clusters.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.core.management.base import BaseCommand 5 | 6 | from ...tasks import update_clusters 7 | 8 | 9 | class Command(BaseCommand): 10 | help = "Go through active clusters and update their status" 11 | 12 | def handle(self, *args, **options): 13 | self.stdout.write("Updating cluster info...", ending="") 14 | update_clusters() 15 | self.stdout.write("done.") 16 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0002_cluster_emr_release.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | # -*- coding: utf-8 -*- 5 | # Generated by Django 1.9.1 on 2016-09-15 12:40 6 | from django.db import migrations, models 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("clusters", "0001_initial")] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name="cluster", 16 | name="emr_release", 17 | field=models.CharField( 18 | choices=[("5.0.0", "5.0.0"), ("4.5.0", "4.5.0")], 19 | default="4.5.0", 20 | help_text="Different EMR versions have different versions of software like Hadoop, Spark, etc", 21 | max_length=50, 22 | ), 23 | ) 24 | ] 25 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0003_cluster_master_address.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | # -*- coding: utf-8 -*- 5 | # Generated by Django 1.9.1 on 2016-09-27 22:29 6 | from django.db import migrations, models 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("clusters", "0002_cluster_emr_release")] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name="cluster", 16 | name="master_address", 17 | field=models.CharField( 18 | default="", 19 | help_text="Public address of the master node.This is only available once the cluster has bootstrapped", 20 | max_length=255, 21 | ), 22 | ) 23 | ] 24 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0004_auto_20161002_1841.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | # -*- coding: utf-8 -*- 5 | # Generated by Django 1.9.1 on 2016-10-02 18:41 6 | from django.db import migrations, models 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("clusters", "0003_cluster_master_address")] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="cluster", 16 | name="master_address", 17 | field=models.CharField( 18 | blank=True, 19 | default="", 20 | help_text="Public address of the master node.This is only available once the cluster has bootstrapped", 21 | max_length=255, 22 | ), 23 | ) 24 | ] 25 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0005_auto_20161017_0913.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.10 on 2016-10-17 09:13 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("clusters", "0004_auto_20161002_1841")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="cluster", 13 | name="emr_release", 14 | field=models.CharField( 15 | choices=[("5.0.0", "5.0.0"), ("4.5.0", "4.5.0")], 16 | default="5.0.0", 17 | help_text="Different EMR versions have different versions of software like Hadoop, Spark, etc", 18 | max_length=50, 19 | ), 20 | ) 21 | ] 22 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0006_auto_20161024_1006.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.10 on 2016-10-24 10:06 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("clusters", "0005_auto_20161017_0913")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="cluster", 13 | name="emr_release", 14 | field=models.CharField( 15 | choices=[("5.0.0", "5.0.0"), ("4.5.0", "4.5.0")], 16 | default="5.0.0", 17 | help_text="Different EMR versions have different versions of software like Hadoop, Spark, etc", 18 | max_length=50, 19 | verbose_name="EMR release version", 20 | ), 21 | ) 22 | ] 23 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0007_auto_20161102_1053.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2016-11-02 10:53 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("clusters", "0006_auto_20161024_1006")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="cluster", 13 | name="most_recent_status", 14 | field=models.CharField( 15 | blank=True, 16 | default="", 17 | help_text="Most recently retrieved AWS status for the cluster.", 18 | max_length=50, 19 | ), 20 | ) 21 | ] 22 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0008_cluster_expiration_mail_sent.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2016-11-04 15:21 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("clusters", "0007_auto_20161102_1053")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="cluster", 13 | name="expiration_mail_sent", 14 | field=models.BooleanField(default=False), 15 | ) 16 | ] 17 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0009_auto_20161108_0933.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2016-11-08 09:33 3 | import django.db.models.deletion 4 | from django.conf import settings 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("clusters", "0008_cluster_expiration_mail_sent")] 11 | 12 | operations = [ 13 | migrations.AlterModelOptions( 14 | name="cluster", 15 | options={"permissions": [("view_cluster", "Can view cluster")]}, 16 | ), 17 | migrations.AlterField( 18 | model_name="cluster", 19 | name="created_by", 20 | field=models.ForeignKey( 21 | help_text="User that created the instance.", 22 | on_delete=django.db.models.deletion.CASCADE, 23 | related_name="created_clusters", 24 | to=settings.AUTH_USER_MODEL, 25 | ), 26 | ), 27 | ] 28 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0010_assign_view_perms.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2016-11-08 11:41 3 | from django.db import migrations 4 | 5 | from atmo.models import PermissionMigrator 6 | 7 | 8 | def assign_cluster_view_permission(apps, schema_editor): 9 | Cluster = apps.get_model("clusters", "Cluster") 10 | PermissionMigrator(apps, Cluster, "view", user_field="created_by").assign() 11 | 12 | 13 | def remove_cluster_view_permission(apps, schema_editor): 14 | Cluster = apps.get_model("clusters", "Cluster") 15 | PermissionMigrator(apps, Cluster, "view", user_field="created_by").remove() 16 | 17 | 18 | class Migration(migrations.Migration): 19 | 20 | dependencies = [ 21 | ("clusters", "0009_auto_20161108_0933"), 22 | ("auth", "0007_alter_validators_add_error_messages"), 23 | ("guardian", "0001_initial"), 24 | ("contenttypes", "0001_initial"), 25 | ] 26 | 27 | operations = [ 28 | migrations.RunPython( 29 | assign_cluster_view_permission, remove_cluster_view_permission 30 | ) 31 | ] 32 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0011_assign_more_perms.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2016-11-08 11:41 3 | from django.db import migrations 4 | 5 | from atmo.models import PermissionMigrator 6 | 7 | 8 | def assign_cluster_more_permission(apps, schema_editor): 9 | Cluster = apps.get_model("clusters", "Cluster") 10 | PermissionMigrator(apps, Cluster, "change", user_field="created_by").assign() 11 | PermissionMigrator(apps, Cluster, "delete", user_field="created_by").assign() 12 | 13 | 14 | def remove_cluster_more_permission(apps, schema_editor): 15 | Cluster = apps.get_model("clusters", "Cluster") 16 | PermissionMigrator(apps, Cluster, "change", user_field="created_by").remove() 17 | PermissionMigrator(apps, Cluster, "delete", user_field="created_by").remove() 18 | 19 | 20 | class Migration(migrations.Migration): 21 | 22 | dependencies = [("clusters", "0010_assign_view_perms")] 23 | 24 | operations = [ 25 | migrations.RunPython( 26 | assign_cluster_more_permission, remove_cluster_more_permission 27 | ) 28 | ] 29 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0012_cluster_ssh_key.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2017-01-16 13:40 3 | import django.db.models.deletion 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ("keys", "0002_assign_view_perms"), 11 | ("clusters", "0011_assign_more_perms"), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name="cluster", 17 | name="ssh_key", 18 | field=models.ForeignKey( 19 | blank=True, 20 | help_text="SSH key to use when launching the cluster.", 21 | null=True, 22 | on_delete=django.db.models.deletion.SET_NULL, 23 | related_name="launched_clusters", 24 | to="keys.SSHKey", 25 | ), 26 | ) 27 | ] 28 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0013_migrate_public_key.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2017-01-16 14:51 3 | from django.db import migrations 4 | 5 | from atmo.models import PermissionMigrator 6 | 7 | from ...keys.utils import calculate_fingerprint 8 | 9 | 10 | def ssh_key_mapping(apps): 11 | "create a mapping of key fingerprint -> list of clusters" 12 | Cluster = apps.get_model("clusters", "Cluster") 13 | ssh_keys = {} 14 | for cluster in Cluster.objects.all(): 15 | fingerprint = calculate_fingerprint(cluster.public_key.strip()) 16 | ssh_keys.setdefault(fingerprint, []).append(cluster) 17 | return ssh_keys 18 | 19 | 20 | def migrate_public_keys(apps, schema_editor): 21 | SSHKey = apps.get_model("keys", "SSHKey") 22 | 23 | # create a mapping of key fingerprint -> list of clusters 24 | ssh_keys = ssh_key_mapping(apps) 25 | 26 | # go through the mapping and create a title for the key 27 | for fingerprint, clusters in list(ssh_keys.items()): 28 | # cut off after 100 characters in case of lots of clusters 29 | title = "key used on: " + ",".join([cluster.identifier for cluster in clusters]) 30 | ssh_key = SSHKey( 31 | title=title[:100], 32 | key=clusters[0].public_key.strip(), 33 | created_by=cluster.created_by, 34 | fingerprint=fingerprint, 35 | ) 36 | ssh_key.save() 37 | 38 | PermissionMigrator(apps, SSHKey, "view", user_field="created_by").assign() 39 | PermissionMigrator(apps, SSHKey, "change", user_field="created_by").assign() 40 | PermissionMigrator(apps, SSHKey, "delete", user_field="created_by").assign() 41 | 42 | 43 | def rollback_public_keys(apps, schema_editor): 44 | SSHKey = apps.get_model("keys", "SSHKey") 45 | ssh_keys = ssh_key_mapping(apps) 46 | SSHKey.objects.filter(fingerprint__in=set(list(ssh_keys.keys()))).delete() 47 | 48 | 49 | class Migration(migrations.Migration): 50 | 51 | dependencies = [ 52 | ("keys", "0002_assign_view_perms"), 53 | ("clusters", "0012_cluster_ssh_key"), 54 | ] 55 | 56 | operations = [migrations.RunPython(migrate_public_keys, rollback_public_keys)] 57 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0014_remove_cluster_public_key.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2017-01-17 09:22 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("clusters", "0013_migrate_public_key")] 9 | 10 | operations = [migrations.RemoveField(model_name="cluster", name="public_key")] 11 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0015_auto_20170124_1357.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2017-01-24 13:57 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("clusters", "0014_remove_cluster_public_key")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="cluster", 13 | name="emr_release", 14 | field=models.CharField( 15 | choices=[("5.2.1", "5.2.1"), ("5.0.0", "5.0.0"), ("4.5.0", "4.5.0")], 16 | default="5.2.1", 17 | help_text="Different EMR versions have different versions of software like Hadoop, Spark, etc", 18 | max_length=50, 19 | verbose_name="EMR release version", 20 | ), 21 | ) 22 | ] 23 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0016_auto_20170130_1704.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2017-01-30 17:04 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("clusters", "0015_auto_20170124_1357")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="cluster", 13 | name="emr_release", 14 | field=models.CharField( 15 | choices=[("5.2.1", "5.2.1"), ("5.0.0", "5.0.0"), ("4.5.0", "4.5.0")], 16 | default="5.2.1", 17 | help_text='Different AWS EMR versions have different versions of software like Hadoop, Spark, etc. See what\'s new in each.', 18 | max_length=50, 19 | verbose_name="EMR release", 20 | ), 21 | ) 22 | ] 23 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0017_auto_20170222_1457.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2017-02-22 14:57 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("clusters", "0016_auto_20170130_1704")] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="cluster", 15 | name="expiration_mail_sent", 16 | field=models.BooleanField( 17 | default=False, help_text="Whether the expiration mail were sent." 18 | ), 19 | ) 20 | ] 21 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0018_auto_20170307_1423.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2017-03-07 14:23 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("clusters", "0017_auto_20170222_1457")] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="cluster", 15 | name="emr_release", 16 | field=models.CharField( 17 | choices=[("5.2.1", "5.2.1"), ("5.0.0", "5.0.0")], 18 | default="5.2.1", 19 | help_text='Different AWS EMR versions have different versions of software like Hadoop, Spark, etc. See what\'s new in each.', 20 | max_length=50, 21 | verbose_name="EMR release", 22 | ), 23 | ) 24 | ] 25 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0019_auto_20170314_1216.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2017-03-14 12:16 3 | from __future__ import unicode_literals 4 | 5 | import django.utils.timezone 6 | from django.db import migrations, models 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("clusters", "0018_auto_20170307_1423")] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name="cluster", 16 | name="created_at", 17 | field=models.DateTimeField( 18 | auto_now_add=True, default=django.utils.timezone.now 19 | ), 20 | preserve_default=False, 21 | ), 22 | migrations.AddField( 23 | model_name="cluster", 24 | name="modified_at", 25 | field=models.DateTimeField( 26 | auto_now=True, default=django.utils.timezone.now 27 | ), 28 | preserve_default=False, 29 | ), 30 | ] 31 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0021_rename_cluster_emr_release.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-21 12:36 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("clusters", "0020_emr_release_model")] 12 | 13 | operations = [ 14 | migrations.RenameField( 15 | model_name="cluster", old_name="emr_release", new_name="emr_release_version" 16 | ), 17 | migrations.AddField( 18 | model_name="cluster", 19 | name="emr_release", 20 | field=models.ForeignKey( 21 | blank=True, 22 | help_text='Different AWS EMR versions have different versions of software like Hadoop, Spark, etc. See what\'s new in each.', 23 | null=True, 24 | on_delete=django.db.models.deletion.SET_NULL, 25 | related_name="created_clusters", 26 | to="clusters.EMRRelease", 27 | verbose_name="EMR release", 28 | ), 29 | ), 30 | ] 31 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0022_convert_cluster_emr_release.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-21 12:36 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | CURRENT_EMR_RELEASES = ("5.2.1", "5.0.0") 9 | 10 | 11 | def convert_emr_releases(apps, schema_editor): 12 | EMRRelease = apps.get_model("clusters", "EMRRelease") 13 | Cluster = apps.get_model("clusters", "Cluster") 14 | 15 | for cluster in Cluster.objects.all(): 16 | emr_release, created = EMRRelease.objects.get_or_create( 17 | version=cluster.emr_release_version, 18 | defaults={ 19 | "changelog_url": "https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-%s/emr-release-components.html" 20 | % cluster.emr_release_version, 21 | "is_deprecated": cluster.emr_release_version 22 | not in CURRENT_EMR_RELEASES, 23 | }, 24 | ) 25 | cluster.emr_release = emr_release 26 | cluster.save() 27 | 28 | 29 | def revert_emr_releases(apps, schema_editor): 30 | EMRRelease = apps.get_model("clusters", "EMRRelease") 31 | Cluster = apps.get_model("clusters", "Cluster") 32 | for cluster in Cluster.objects.all(): 33 | cluster.emr_release = None 34 | cluster.save() 35 | 36 | 37 | class Migration(migrations.Migration): 38 | 39 | dependencies = [("clusters", "0021_rename_cluster_emr_release")] 40 | 41 | operations = [migrations.RunPython(convert_emr_releases, revert_emr_releases)] 42 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0023_alter_cluster_emr_release.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-21 12:36 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("clusters", "0022_convert_cluster_emr_release")] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="cluster", 16 | name="emr_release", 17 | field=models.ForeignKey( 18 | help_text='Different AWS EMR versions have different versions of software like Hadoop, Spark, etc. See what\'s new in each.', 19 | on_delete=django.db.models.deletion.PROTECT, 20 | related_name="created_clusters", 21 | to="clusters.EMRRelease", 22 | verbose_name="EMR release", 23 | ), 24 | ) 25 | ] 26 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0024_remove_cluster_emr_release.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-21 12:36 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("clusters", "0023_alter_cluster_emr_release")] 12 | 13 | operations = [ 14 | migrations.RemoveField(model_name="cluster", name="emr_release_version") 15 | ] 16 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0025_emrrelease_is_active.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-29 21:21 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("clusters", "0024_remove_cluster_emr_release")] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="emrrelease", 15 | name="is_active", 16 | field=models.BooleanField( 17 | default=True, 18 | help_text="Whether this version should be shown to the user at all.", 19 | ), 20 | ) 21 | ] 22 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0026_auto_20170330_1732.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-30 17:32 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("clusters", "0025_emrrelease_is_active")] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="cluster", 15 | name="size", 16 | field=models.IntegerField( 17 | help_text="Number of computers used in the cluster." 18 | ), 19 | ) 20 | ] 21 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0027_cluster_lifetime.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-30 18:03 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("clusters", "0026_auto_20170330_1732")] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="cluster", 15 | name="lifetime", 16 | field=models.PositiveSmallIntegerField( 17 | default=8, 18 | help_text="Lifetime of the cluster after which it's automatically terminated, in hours.", 19 | ), 20 | ) 21 | ] 22 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0028_cluster_lifetime_extension_count.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-04-03 18:26 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("clusters", "0027_cluster_lifetime")] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="cluster", 15 | name="lifetime_extension_count", 16 | field=models.PositiveSmallIntegerField( 17 | default=0, help_text="Number of lifetime extensions." 18 | ), 19 | ) 20 | ] 21 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0029_remove_start_date.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-17 13:51 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("clusters", "0028_cluster_lifetime_extension_count")] 11 | 12 | operations = [migrations.RemoveField(model_name="cluster", name="start_date")] 13 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0030_rename_end_date_expires_at.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-17 14:03 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("clusters", "0029_remove_start_date")] 11 | 12 | operations = [ 13 | migrations.RenameField( 14 | model_name="cluster", old_name="end_date", new_name="expires_at" 15 | ) 16 | ] 17 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0031_cluster_finished_at.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-17 14:43 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("clusters", "0030_rename_end_date_expires_at")] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="cluster", 15 | name="finished_at", 16 | field=models.DateTimeField( 17 | blank=True, 18 | help_text="Date/time when the cluster was terminated or failed.", 19 | null=True, 20 | ), 21 | ) 22 | ] 23 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0032_auto_20170519_1336.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-19 13:36 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.utils.timezone 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("clusters", "0031_cluster_finished_at")] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="cluster", 16 | name="created_at", 17 | field=models.DateTimeField( 18 | blank=True, default=django.utils.timezone.now, editable=False 19 | ), 20 | ), 21 | migrations.AlterField( 22 | model_name="cluster", 23 | name="modified_at", 24 | field=models.DateTimeField( 25 | blank=True, default=django.utils.timezone.now, editable=False 26 | ), 27 | ), 28 | migrations.AlterField( 29 | model_name="emrrelease", 30 | name="created_at", 31 | field=models.DateTimeField( 32 | blank=True, default=django.utils.timezone.now, editable=False 33 | ), 34 | ), 35 | migrations.AlterField( 36 | model_name="emrrelease", 37 | name="modified_at", 38 | field=models.DateTimeField( 39 | blank=True, default=django.utils.timezone.now, editable=False 40 | ), 41 | ), 42 | ] 43 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0033_auto_20170522_1426.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-22 14:26 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("clusters", "0032_auto_20170519_1336")] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="cluster", 15 | name="ready_at", 16 | field=models.DateTimeField( 17 | blank=True, 18 | help_text="Date/time when the cluster was ready to run steps on AWS EMR.", 19 | null=True, 20 | ), 21 | ), 22 | migrations.AddField( 23 | model_name="cluster", 24 | name="started_at", 25 | field=models.DateTimeField( 26 | blank=True, 27 | help_text="Date/time when the cluster was started on AWS EMR.", 28 | null=True, 29 | ), 30 | ), 31 | migrations.AlterField( 32 | model_name="cluster", 33 | name="finished_at", 34 | field=models.DateTimeField( 35 | blank=True, 36 | help_text="Date/time when the cluster was terminated or failed on AWS EMR.", 37 | null=True, 38 | ), 39 | ), 40 | ] 41 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0034_auto_20170530_1907.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-30 19:07 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("clusters", "0033_auto_20170522_1426")] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="cluster", 15 | name="most_recent_status", 16 | field=models.CharField( 17 | blank=True, 18 | db_index=True, 19 | default="", 20 | help_text="Most recently retrieved AWS status for the cluster.", 21 | max_length=50, 22 | ), 23 | ) 24 | ] 25 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/0035_auto_20180814_1710.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.15 on 2018-08-14 17:10 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("clusters", "0034_auto_20170530_1907")] 11 | 12 | operations = [ 13 | migrations.AlterModelOptions( 14 | name="cluster", 15 | options={ 16 | "permissions": [ 17 | ("view_cluster", "Can view cluster"), 18 | ("maintain_cluster", "Can maintain cluster"), 19 | ] 20 | }, 21 | ) 22 | ] 23 | -------------------------------------------------------------------------------- /atmo/clusters/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /atmo/clusters/queries.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.db import models 5 | 6 | 7 | class EMRReleaseQuerySet(models.QuerySet): 8 | """ 9 | A Django queryset for the :class:`~atmo.clusters.models.EMRRelease` model. 10 | """ 11 | 12 | def natural_sort_by_version(self): 13 | """ 14 | Sorts this queryset by the EMR version naturally (human-readable). 15 | """ 16 | return self.extra( 17 | select={"natural_version": "string_to_array(version, '.')::int[]"} 18 | ).order_by("-natural_version") 19 | 20 | def active(self): 21 | return self.filter(is_active=True) 22 | 23 | def stable(self): 24 | """ 25 | The EMR releases that are considered stable. 26 | """ 27 | return self.filter(is_experimental=False, is_deprecated=False, is_active=True) 28 | 29 | def experimental(self): 30 | """ 31 | The EMR releases that are considered experimental. 32 | """ 33 | return self.filter(is_experimental=True, is_active=True) 34 | 35 | def deprecated(self): 36 | """ 37 | The EMR releases that are deprecated. 38 | """ 39 | return self.filter(is_deprecated=True, is_active=True) 40 | 41 | 42 | class ClusterQuerySet(models.QuerySet): 43 | """A Django queryset that filters by cluster status. 44 | 45 | Used by the :class:`~atmo.clusters.models.Cluster` model. 46 | """ 47 | 48 | def active(self): 49 | """ 50 | The clusters that have an active status. 51 | """ 52 | return self.filter(most_recent_status__in=self.model.ACTIVE_STATUS_LIST) 53 | 54 | def terminated(self): 55 | """ 56 | The clusters that have an terminated status. 57 | """ 58 | return self.filter(most_recent_status__in=self.model.TERMINATED_STATUS_LIST) 59 | 60 | def failed(self): 61 | """ 62 | The clusters that have an failed status. 63 | """ 64 | return self.filter(most_recent_status__in=self.model.FAILED_STATUS_LIST) 65 | -------------------------------------------------------------------------------- /atmo/clusters/urls.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.conf.urls import url 5 | 6 | from . import views 7 | 8 | urlpatterns = [ 9 | url(r"^new/$", views.new_cluster, name="clusters-new"), 10 | url(r"^(?P\d+)/extend/$", views.extend_cluster, name="clusters-extend"), 11 | url( 12 | r"^(?P\d+)/terminate/$", views.terminate_cluster, name="clusters-terminate" 13 | ), 14 | url(r"^(?P\d+)/$", views.detail_cluster, name="clusters-detail"), 15 | ] 16 | -------------------------------------------------------------------------------- /atmo/context_processors.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.conf import settings as django_settings 5 | from django.contrib import messages 6 | from django.utils.safestring import mark_safe 7 | 8 | 9 | def settings(request): 10 | """ 11 | Adds the Django settings object to the template context. 12 | """ 13 | return {"settings": django_settings} 14 | 15 | 16 | def version(request): 17 | """ 18 | Adds version-related context variables to the context. 19 | """ 20 | response = {} 21 | if django_settings.VERSION: 22 | response = {"version": django_settings.VERSION.get("version", None)} 23 | commit = django_settings.VERSION.get("commit") 24 | if commit: 25 | response["commit"] = commit[:7] 26 | return response 27 | 28 | 29 | def alerts(request): 30 | """ 31 | Here be dragons, for who are bold enough to break systems and lose data 32 | 33 | This adds an alert to requests in stage and development environments. 34 | """ 35 | host = request.get_host() 36 | warning = """ 37 |

Here be dragons!

38 | This service is currently under development and may not be stable.""" 39 | if any(hint in host for hint in ["stag", "localhost", "dev"]): 40 | messages.warning(request, mark_safe(warning)) 41 | return {} 42 | -------------------------------------------------------------------------------- /atmo/forms/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /atmo/forms/fields.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django import forms 5 | 6 | 7 | class CachedFileField(forms.FileField): 8 | """ 9 | A custom FileField class for use in conjunction with CachedFileModelFormMixin 10 | that allows storing uploaded file in a cache for re-submission. 11 | 12 | That requires moving the "required" validation into the form's clean 13 | method instead of handling it on field level. 14 | """ 15 | 16 | def __init__(self, *args, **kwargs): 17 | self.real_required = kwargs.pop("required", True) 18 | kwargs["required"] = False 19 | super().__init__(*args, **kwargs) 20 | -------------------------------------------------------------------------------- /atmo/forms/widgets.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django import forms 5 | from django.utils.html import conditional_escape 6 | from django.utils.safestring import mark_safe 7 | 8 | from .cache import CachedFileCache 9 | 10 | 11 | class CachedFileHiddenInput(forms.HiddenInput): 12 | template_with_cachekey = """ 13 |
14 | 15 | Just uploaded file: %(file_name)s 16 |

This file will be used when the form is successfully submitted

17 |
18 | %(cachekey_field)s 19 | """ 20 | 21 | def __init__(self, *args, **kwargs): 22 | super().__init__(*args, **kwargs) 23 | self.cache = CachedFileCache() 24 | 25 | def render(self, name, value, attrs=None): 26 | # render the hidden input first 27 | cachekey_field = super().render(name, value, attrs) 28 | 29 | # check if there is a cached file 30 | metadata = self.cache.metadata(value) 31 | if metadata is None: 32 | # if not, just return the hidden input 33 | return cachekey_field 34 | 35 | # or render the additional cached file 36 | return mark_safe( 37 | self.template_with_cachekey 38 | % { 39 | "file_name": conditional_escape(metadata["name"]), 40 | "cachekey_field": cachekey_field, 41 | } 42 | ) 43 | -------------------------------------------------------------------------------- /atmo/jobs/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /atmo/jobs/admin.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.contrib import admin 5 | from guardian.admin import GuardedModelAdmin 6 | 7 | from .models import SparkJob, SparkJobRun, SparkJobRunAlert 8 | 9 | 10 | def run_now(modeladmin, request, queryset): 11 | for job in queryset: 12 | job.run() 13 | 14 | 15 | class SparkJobRunInline(admin.TabularInline): 16 | model = SparkJobRun 17 | 18 | extra = 0 19 | fields = [ 20 | "jobflow_id", 21 | "scheduled_at", 22 | "started_at", 23 | "ready_at", 24 | "finished_at", 25 | "status", 26 | ] 27 | readonly_fields = [ 28 | "jobflow_id", 29 | "scheduled_at", 30 | "started_at", 31 | "ready_at", 32 | "finished_at", 33 | "status", 34 | ] 35 | 36 | 37 | @admin.register(SparkJob) 38 | class SparkJobAdmin(GuardedModelAdmin): 39 | actions = [run_now] 40 | inlines = [SparkJobRunInline] 41 | list_display = [ 42 | "identifier", 43 | "size", 44 | "created_by", 45 | "start_date", 46 | "end_date", 47 | "is_enabled", 48 | "emr_release", 49 | ] 50 | list_filter = [ 51 | "size", 52 | "is_enabled", 53 | "emr_release", 54 | "start_date", 55 | "end_date", 56 | "interval_in_hours", 57 | "runs__scheduled_at", 58 | "runs__finished_at", 59 | "runs__status", 60 | ] 61 | search_fields = [ 62 | "identifier", 63 | "description", 64 | "created_by__email", 65 | "runs__jobflow_id", 66 | "runs__status", 67 | ] 68 | 69 | 70 | @admin.register(SparkJobRunAlert) 71 | class SparkJobRunAlertAdmin(admin.ModelAdmin): 72 | list_display = ["run", "reason_code", "reason_message", "mail_sent_date"] 73 | list_filter = [ 74 | "reason_code", 75 | "mail_sent_date", 76 | "run__scheduled_at", 77 | "run__finished_at", 78 | "run__status", 79 | ] 80 | search_fields = ["reason_code", "reason_message"] 81 | -------------------------------------------------------------------------------- /atmo/jobs/exceptions.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | 6 | class SparkJobException(Exception): 7 | pass 8 | 9 | 10 | class SparkJobNotFound(SparkJobException): 11 | pass 12 | 13 | 14 | class SparkJobNotEnabled(SparkJobException): 15 | pass 16 | -------------------------------------------------------------------------------- /atmo/jobs/factories.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | import factory 5 | from django.utils import timezone 6 | 7 | from . import models 8 | from .. import names 9 | from ..clusters.factories import EMRReleaseFactory 10 | from ..users.factories import UserFactory 11 | 12 | 13 | class SparkJobFactory(factory.django.DjangoModelFactory): 14 | identifier = factory.LazyFunction(names.random_scientist) 15 | description = "some description" 16 | notebook_s3_key = "jobs/test-spark-job/test-notebook.ipynb" 17 | result_visibility = models.SparkJob.RESULT_PRIVATE 18 | size = 5 19 | interval_in_hours = models.SparkJob.INTERVAL_DAILY 20 | job_timeout = 12 21 | start_date = factory.LazyFunction(timezone.now) 22 | end_date = None 23 | is_enabled = True 24 | created_by = factory.SubFactory(UserFactory) 25 | emr_release = factory.SubFactory(EMRReleaseFactory) 26 | 27 | class Meta: 28 | model = models.SparkJob 29 | 30 | 31 | class SparkJobRunFactory(factory.django.DjangoModelFactory): 32 | spark_job = factory.SubFactory(SparkJobFactory) 33 | jobflow_id = factory.Sequence(lambda n: "j-%s" % n) 34 | emr_release_version = factory.LazyAttribute( 35 | lambda run: run.spark_job.emr_release.version 36 | ) 37 | size = 5 38 | status = models.DEFAULT_STATUS 39 | scheduled_at = factory.LazyFunction(timezone.now) 40 | started_at = None 41 | ready_at = None 42 | finished_at = None 43 | created_at = factory.LazyFunction(timezone.now) 44 | 45 | class Meta: 46 | model = models.SparkJobRun 47 | 48 | 49 | class SparkJobWithRunFactory(SparkJobFactory): 50 | """ 51 | A SparkJob factory that automatically creates a SparkJobRun 52 | """ 53 | 54 | notebook_s3_key = factory.LazyAttributeSequence( 55 | lambda job, n: "jobs/%s/test-notebook-%s.ipynb" % (job.identifier, n) 56 | ) 57 | run = factory.RelatedFactory(SparkJobRunFactory, "spark_job") 58 | -------------------------------------------------------------------------------- /atmo/jobs/management/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /atmo/jobs/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /atmo/jobs/management/commands/run_jobs.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.core.management.base import BaseCommand 5 | 6 | from ... import jobs 7 | 8 | 9 | class Command(BaseCommand): 10 | help = "Run scheduled jobs if necessary" 11 | 12 | def handle(self, *args, **options): 13 | self.stdout.write("Running scheduled jobs ", ending="") 14 | run_jobs = jobs.run_jobs() 15 | self.stdout.write(", ".join(run_jobs)) 16 | self.stdout.write(" done.") 17 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0002_auto_20161017_0913.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.10 on 2016-10-17 09:13 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("jobs", "0001_initial")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="sparkjob", 13 | name="interval_in_hours", 14 | field=models.IntegerField( 15 | choices=[(24, "Daily"), (168, "Weekly"), (720, "Monthly")], 16 | default=24, 17 | help_text="Interval at which the job should run, in hours.", 18 | ), 19 | ), 20 | migrations.AlterField( 21 | model_name="sparkjob", 22 | name="result_visibility", 23 | field=models.CharField( 24 | choices=[ 25 | ( 26 | "private", 27 | "Private: results output to an S3 bucket, viewable with AWS credentials", 28 | ), 29 | ( 30 | "public", 31 | "Public: results output to a public S3 bucket, viewable by anyone", 32 | ), 33 | ], 34 | default="private", 35 | help_text="Whether notebook results are uploaded to a public or private bucket", 36 | max_length=50, 37 | ), 38 | ), 39 | ] 40 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0003_auto_20161019_1245.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.10 on 2016-10-19 12:45 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("jobs", "0002_auto_20161017_0913")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="sparkjob", 13 | name="identifier", 14 | field=models.CharField( 15 | help_text="Job name, used to uniqely identify individual jobs.", 16 | max_length=100, 17 | unique=True, 18 | ), 19 | ) 20 | ] 21 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0004_sparkjob_emr_release.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.10 on 2016-10-24 09:28 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("jobs", "0003_auto_20161019_1245")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="sparkjob", 13 | name="emr_release", 14 | field=models.CharField( 15 | choices=[("5.0.0", "5.0.0"), ("4.5.0", "4.5.0")], 16 | default="5.0.0", 17 | help_text="Different EMR versions have different versions of software like Hadoop, Spark, etc", 18 | max_length=50, 19 | verbose_name="EMR release version", 20 | ), 21 | ) 22 | ] 23 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0005_auto_20161102_1049.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2016-11-02 10:49 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("jobs", "0004_sparkjob_emr_release")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="sparkjob", 13 | name="most_recent_status", 14 | field=models.CharField(blank=True, default="", max_length=50), 15 | ) 16 | ] 17 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0006_auto_20161108_0933.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2016-11-08 09:33 3 | import django.db.models.deletion 4 | from django.conf import settings 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0005_auto_20161102_1049")] 11 | 12 | operations = [ 13 | migrations.AlterModelOptions( 14 | name="sparkjob", 15 | options={"permissions": [("view_sparkjob", "Can view Spark job")]}, 16 | ), 17 | migrations.AlterField( 18 | model_name="sparkjob", 19 | name="created_by", 20 | field=models.ForeignKey( 21 | help_text="User that created the instance.", 22 | on_delete=django.db.models.deletion.CASCADE, 23 | related_name="created_sparkjobs", 24 | to=settings.AUTH_USER_MODEL, 25 | ), 26 | ), 27 | ] 28 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0007_assign_view_perms.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2016-11-08 13:32 3 | from django.db import migrations 4 | 5 | from atmo.models import PermissionMigrator 6 | 7 | 8 | def assign_spark_job_view_permission(apps, schema_editor): 9 | SparkJob = apps.get_model("jobs", "SparkJob") 10 | PermissionMigrator(apps, SparkJob, "view", user_field="created_by").assign() 11 | 12 | 13 | def remove_spark_job_view_permission(apps, schema_editor): 14 | SparkJob = apps.get_model("jobs", "SparkJob") 15 | PermissionMigrator(apps, SparkJob, "view", user_field="created_by").remove() 16 | 17 | 18 | class Migration(migrations.Migration): 19 | 20 | dependencies = [ 21 | ("jobs", "0006_auto_20161108_0933"), 22 | ("auth", "0007_alter_validators_add_error_messages"), 23 | ("guardian", "0001_initial"), 24 | ("contenttypes", "0001_initial"), 25 | ] 26 | 27 | operations = [ 28 | migrations.RunPython( 29 | assign_spark_job_view_permission, remove_spark_job_view_permission 30 | ) 31 | ] 32 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0008_assign_more_perms.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2016-11-08 13:32 3 | from django.db import migrations 4 | 5 | from atmo.models import PermissionMigrator 6 | 7 | 8 | def assign_spark_job_more_permission(apps, schema_editor): 9 | SparkJob = apps.get_model("jobs", "SparkJob") 10 | PermissionMigrator(apps, SparkJob, "change", user_field="created_by").assign() 11 | PermissionMigrator(apps, SparkJob, "delete", user_field="created_by").assign() 12 | 13 | 14 | def remove_spark_job_more_permission(apps, schema_editor): 15 | SparkJob = apps.get_model("jobs", "SparkJob") 16 | PermissionMigrator(apps, SparkJob, "change", user_field="created_by").remove() 17 | PermissionMigrator(apps, SparkJob, "delete", user_field="created_by").remove() 18 | 19 | 20 | class Migration(migrations.Migration): 21 | 22 | dependencies = [("jobs", "0007_assign_view_perms")] 23 | 24 | operations = [ 25 | migrations.RunPython( 26 | assign_spark_job_more_permission, remove_spark_job_more_permission 27 | ) 28 | ] 29 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0009_sparkjob_description.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2017-01-13 23:58 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("jobs", "0008_assign_more_perms")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="sparkjob", 13 | name="description", 14 | field=models.TextField(default="", help_text="Job description."), 15 | ) 16 | ] 17 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0010_auto_20170124_1357.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2017-01-24 13:57 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("jobs", "0009_sparkjob_description")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="sparkjob", 13 | name="emr_release", 14 | field=models.CharField( 15 | choices=[("5.2.1", "5.2.1"), ("5.0.0", "5.0.0"), ("4.5.0", "4.5.0")], 16 | default="5.2.1", 17 | help_text="Different EMR versions have different versions of software like Hadoop, Spark, etc", 18 | max_length=50, 19 | verbose_name="EMR release version", 20 | ), 21 | ) 22 | ] 23 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0011_auto_20170130_1704.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2017-01-30 17:04 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("jobs", "0010_auto_20170124_1357")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="sparkjob", 13 | name="emr_release", 14 | field=models.CharField( 15 | choices=[("5.2.1", "5.2.1"), ("5.0.0", "5.0.0"), ("4.5.0", "4.5.0")], 16 | default="5.2.1", 17 | help_text='Different AWS EMR versions have different versions of software like Hadoop, Spark, etc. See what\'s new in each.', 18 | max_length=50, 19 | verbose_name="EMR release", 20 | ), 21 | ), 22 | migrations.AlterField( 23 | model_name="sparkjob", 24 | name="result_visibility", 25 | field=models.CharField( 26 | choices=[("private", "Private"), ("public", "Public")], 27 | default="private", 28 | help_text="Whether notebook results are uploaded to a public or private bucket", 29 | max_length=50, 30 | ), 31 | ), 32 | ] 33 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0013_sparkjobrunalert.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2017-02-22 15:02 3 | from __future__ import unicode_literals 4 | 5 | import django.db.models.deletion 6 | from django.db import migrations, models 7 | 8 | import atmo.models 9 | 10 | 11 | class Migration(migrations.Migration): 12 | 13 | dependencies = [("jobs", "0012_add_job_run_history")] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name="SparkJobRunAlert", 18 | fields=[ 19 | ("created_at", models.DateTimeField(auto_now_add=True)), 20 | ("modified_at", models.DateTimeField(auto_now=True)), 21 | ( 22 | "run", 23 | models.OneToOneField( 24 | on_delete=django.db.models.deletion.CASCADE, 25 | primary_key=True, 26 | related_name="alert", 27 | serialize=False, 28 | to="jobs.SparkJobRun", 29 | ), 30 | ), 31 | ( 32 | "reason", 33 | models.CharField( 34 | blank=True, 35 | help_text="The reason for the creation of the alert.", 36 | max_length=50, 37 | null=True, 38 | ), 39 | ), 40 | ( 41 | "mail_sent_date", 42 | models.DateTimeField( 43 | blank=True, 44 | help_text="The datetime the alert email was sent.", 45 | null=True, 46 | ), 47 | ), 48 | ], 49 | options={ 50 | "abstract": False, 51 | "get_latest_by": "modified_at", 52 | "ordering": ("-modified_at", "-created_at"), 53 | }, 54 | ) 55 | ] 56 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0014_auto_20170307_1423.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2017-03-07 14:23 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0013_sparkjobrunalert")] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="sparkjob", 15 | name="emr_release", 16 | field=models.CharField( 17 | choices=[("5.2.1", "5.2.1"), ("5.0.0", "5.0.0")], 18 | default="5.2.1", 19 | help_text='Different AWS EMR versions have different versions of software like Hadoop, Spark, etc. See what\'s new in each.', 20 | max_length=50, 21 | verbose_name="EMR release", 22 | ), 23 | ) 24 | ] 25 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0015_auto_20170317_1027.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-17 10:27 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0014_auto_20170307_1423")] 11 | 12 | operations = [ 13 | migrations.RenameField( 14 | model_name="sparkjobrunalert", old_name="reason", new_name="reason_code" 15 | ), 16 | migrations.AlterField( 17 | model_name="sparkjobrunalert", 18 | name="reason_code", 19 | field=models.CharField( 20 | blank=True, 21 | help_text="The reason code for the creation of the alert.", 22 | max_length=50, 23 | null=True, 24 | ), 25 | ), 26 | migrations.AddField( 27 | model_name="sparkjobrunalert", 28 | name="reason_message", 29 | field=models.CharField( 30 | blank=True, 31 | help_text="The reason message for the creation of the alert.", 32 | max_length=50, 33 | null=True, 34 | ), 35 | ), 36 | ] 37 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0016_auto_20170320_0943.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-20 09:43 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0015_auto_20170317_1027")] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="sparkjobrunalert", 15 | name="reason_message", 16 | field=models.TextField( 17 | default="", 18 | help_text="The reason message for the creation of the alert.", 19 | ), 20 | ) 21 | ] 22 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0017_assign_group_view_perm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-20 23:26 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | from atmo.models import PermissionMigrator 8 | 9 | 10 | def assign_spark_job_view_permission_to_group(apps, schema_editor): 11 | SparkJob = apps.get_model("jobs", "SparkJob") 12 | Group = apps.get_model("auth", "Group") 13 | 14 | group, created = Group.objects.get_or_create(name="Spark job maintainers") 15 | PermissionMigrator(apps, SparkJob, "view", group=group).assign() 16 | 17 | 18 | def remove_spark_job_view_permission_to_group(apps, schema_editor): 19 | SparkJob = apps.get_model("jobs", "SparkJob") 20 | Group = apps.get_model("auth", "Group") 21 | 22 | group = Group.objects.get(name="Spark job maintainers") 23 | PermissionMigrator(apps, SparkJob, "view", group=group).remove() 24 | 25 | 26 | class Migration(migrations.Migration): 27 | 28 | dependencies = [("jobs", "0016_auto_20170320_0943")] 29 | 30 | operations = [ 31 | migrations.RunPython( 32 | assign_spark_job_view_permission_to_group, 33 | remove_spark_job_view_permission_to_group, 34 | ) 35 | ] 36 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0018_rename_add_spark_job_emr_release.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-21 12:15 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ("clusters", "0020_emr_release_model"), 13 | ("jobs", "0017_assign_group_view_perm"), 14 | ] 15 | 16 | operations = [ 17 | migrations.RenameField( 18 | model_name="sparkjob", 19 | old_name="emr_release", 20 | new_name="emr_release_version", 21 | ), 22 | migrations.AddField( 23 | model_name="sparkjob", 24 | name="emr_release", 25 | field=models.ForeignKey( 26 | blank=True, 27 | help_text='Different AWS EMR versions have different versions of software like Hadoop, Spark, etc. See what\'s new in each.', 28 | null=True, 29 | on_delete=django.db.models.deletion.SET_NULL, 30 | related_name="created_sparkjobs", 31 | to="clusters.EMRRelease", 32 | verbose_name="EMR release", 33 | ), 34 | ), 35 | ] 36 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0019_convert_spark_job_emr_release.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-21 12:15 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | CURRENT_EMR_RELEASES = ("5.2.1", "5.0.0") 9 | 10 | 11 | def convert_emr_releases(apps, schema_editor): 12 | EMRRelease = apps.get_model("clusters", "EMRRelease") 13 | SparkJob = apps.get_model("jobs", "SparkJob") 14 | 15 | for spark_job in SparkJob.objects.all(): 16 | emr_release, created = EMRRelease.objects.get_or_create( 17 | version=spark_job.emr_release_version, 18 | defaults={ 19 | "changelog_url": "https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-%s/emr-release-components.html" 20 | % spark_job.emr_release_version, 21 | "is_deprecated": spark_job.emr_release_version 22 | not in CURRENT_EMR_RELEASES, 23 | }, 24 | ) 25 | spark_job.emr_release = emr_release 26 | spark_job.save() 27 | 28 | 29 | def revert_emr_releases(apps, schema_editor): 30 | EMRRelease = apps.get_model("clusters", "EMRRelease") 31 | SparkJob = apps.get_model("jobs", "SparkJob") 32 | for spark_job in SparkJob.objects.all(): 33 | spark_job.emr_release = None 34 | spark_job.save() 35 | 36 | 37 | class Migration(migrations.Migration): 38 | 39 | dependencies = [("jobs", "0018_rename_add_spark_job_emr_release")] 40 | 41 | operations = [migrations.RunPython(convert_emr_releases, revert_emr_releases)] 42 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0020_alter_spark_job_emr_release.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-21 12:15 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("jobs", "0019_convert_spark_job_emr_release")] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="sparkjob", 16 | name="emr_release", 17 | field=models.ForeignKey( 18 | help_text='Different AWS EMR versions have different versions of software like Hadoop, Spark, etc. See what\'s new in each.', 19 | on_delete=django.db.models.deletion.PROTECT, 20 | related_name="created_sparkjobs", 21 | to="clusters.EMRRelease", 22 | verbose_name="EMR release", 23 | ), 24 | ) 25 | ] 26 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0021_remove_spark_job_emr_release.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-21 12:15 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("jobs", "0020_alter_spark_job_emr_release")] 12 | 13 | operations = [ 14 | migrations.RemoveField(model_name="sparkjob", name="emr_release_version") 15 | ] 16 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0022_sparkjobrun_emr_release_version.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-03-21 15:07 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | def port_emr_release_version(apps, schema_editor): 9 | SparkJobRun = apps.get_model("jobs", "SparkJobRun") 10 | 11 | for run in SparkJobRun.objects.all(): 12 | run.emr_release_version = run.spark_job.emr_release.version 13 | run.save() 14 | 15 | 16 | def revert_emr_release_version(apps, schema_editor): 17 | SparkJobRun = apps.get_model("jobs", "SparkJobRun") 18 | 19 | for run in SparkJobRun.objects.all(): 20 | run.emr_release_version = None 21 | run.save() 22 | 23 | 24 | class Migration(migrations.Migration): 25 | 26 | dependencies = [("jobs", "0021_remove_spark_job_emr_release")] 27 | 28 | operations = [ 29 | migrations.AddField( 30 | model_name="sparkjobrun", 31 | name="emr_release_version", 32 | field=models.CharField(blank=True, max_length=50, null=True), 33 | ), 34 | migrations.RunPython(port_emr_release_version, revert_emr_release_version), 35 | ] 36 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0023_sparkjob_expired_date.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.12 on 2017-04-12 13:24 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0022_sparkjobrun_emr_release_version")] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="sparkjob", 15 | name="expired_date", 16 | field=models.DateTimeField( 17 | blank=True, help_text="Date/time that the job was expired.", null=True 18 | ), 19 | ) 20 | ] 21 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0024_auto_20170425_1324.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.13 on 2017-04-25 13:24 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.utils.timezone 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("jobs", "0023_sparkjob_expired_date")] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name="sparkjob", 16 | name="created_at", 17 | field=models.DateTimeField( 18 | auto_now_add=True, default=django.utils.timezone.now 19 | ), 20 | preserve_default=False, 21 | ), 22 | migrations.AddField( 23 | model_name="sparkjob", 24 | name="modified_at", 25 | field=models.DateTimeField( 26 | auto_now=True, default=django.utils.timezone.now 27 | ), 28 | preserve_default=False, 29 | ), 30 | ] 31 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0025_populate_job_schedule.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.13 on 2017-04-26 09:09 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | from atmo.jobs.schedules import SparkJobSchedule 7 | 8 | 9 | def save_schedules(apps, schema_editor): 10 | SparkJob = apps.get_model("jobs", "SparkJob") 11 | 12 | for job in SparkJob.objects.all(): 13 | SparkJobSchedule(job).add() 14 | 15 | 16 | def delete_schedules(apps, schema_editor): 17 | SparkJob = apps.get_model("jobs", "SparkJob") 18 | 19 | for job in SparkJob.objects.all(): 20 | SparkJobSchedule(job).delete() 21 | 22 | 23 | class Migration(migrations.Migration): 24 | 25 | dependencies = [("jobs", "0024_auto_20170425_1324")] 26 | 27 | operations = [migrations.RunPython(save_schedules, delete_schedules)] 28 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0026_sparkjobrun_size.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-19 18:26 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0025_populate_job_schedule")] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="sparkjobrun", 15 | name="size", 16 | field=models.IntegerField( 17 | blank=True, 18 | help_text="Number of computers used to run the job.", 19 | null=True, 20 | ), 21 | ) 22 | ] 23 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0027_rename_terminated_final_date.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-17 13:39 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0026_sparkjobrun_size")] 11 | 12 | operations = [ 13 | migrations.RenameField( 14 | model_name="sparkjobrun", old_name="terminated_date", new_name="finished_at" 15 | ) 16 | ] 17 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0028_auto_20170517_1443.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-17 14:43 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0027_rename_terminated_final_date")] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="sparkjobrun", 15 | name="finished_at", 16 | field=models.DateTimeField( 17 | blank=True, 18 | help_text="Date/time that the job was terminated or failed.", 19 | null=True, 20 | ), 21 | ) 22 | ] 23 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0029_auto_20170519_1336.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-19 13:36 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.utils.timezone 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("jobs", "0028_auto_20170517_1443")] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="sparkjob", 16 | name="created_at", 17 | field=models.DateTimeField( 18 | blank=True, default=django.utils.timezone.now, editable=False 19 | ), 20 | ), 21 | migrations.AlterField( 22 | model_name="sparkjob", 23 | name="modified_at", 24 | field=models.DateTimeField( 25 | blank=True, default=django.utils.timezone.now, editable=False 26 | ), 27 | ), 28 | migrations.AlterField( 29 | model_name="sparkjobrun", 30 | name="created_at", 31 | field=models.DateTimeField( 32 | blank=True, default=django.utils.timezone.now, editable=False 33 | ), 34 | ), 35 | migrations.AlterField( 36 | model_name="sparkjobrun", 37 | name="modified_at", 38 | field=models.DateTimeField( 39 | blank=True, default=django.utils.timezone.now, editable=False 40 | ), 41 | ), 42 | migrations.AlterField( 43 | model_name="sparkjobrunalert", 44 | name="created_at", 45 | field=models.DateTimeField( 46 | blank=True, default=django.utils.timezone.now, editable=False 47 | ), 48 | ), 49 | migrations.AlterField( 50 | model_name="sparkjobrunalert", 51 | name="modified_at", 52 | field=models.DateTimeField( 53 | blank=True, default=django.utils.timezone.now, editable=False 54 | ), 55 | ), 56 | ] 57 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0030_rename_run_date.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-23 13:26 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0029_auto_20170519_1336")] 11 | 12 | operations = [ 13 | migrations.RenameField( 14 | model_name="sparkjobrun", old_name="run_date", new_name="started_at" 15 | ) 16 | ] 17 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0031_update_started_at.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-23 13:28 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0030_rename_run_date")] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="sparkjobrun", 15 | name="started_at", 16 | field=models.DateTimeField( 17 | blank=True, 18 | help_text="Date/time when the cluster was started on AWS EMR.", 19 | null=True, 20 | ), 21 | ) 22 | ] 23 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0032_sparkjobrun_ready_at.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-23 13:29 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0031_update_started_at")] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="sparkjobrun", 15 | name="ready_at", 16 | field=models.DateTimeField( 17 | blank=True, 18 | help_text="Date/time when the cluster was ready to run steps on AWS EMR.", 19 | null=True, 20 | ), 21 | ) 22 | ] 23 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0033_rename_scheduled_date.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-23 13:30 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0032_sparkjobrun_ready_at")] 11 | 12 | operations = [ 13 | migrations.RenameField( 14 | model_name="sparkjobrun", old_name="scheduled_date", new_name="scheduled_at" 15 | ) 16 | ] 17 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0034_auto_20170529_1424.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-29 14:24 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("jobs", "0033_rename_scheduled_date")] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="sparkjobrunalert", 16 | name="run", 17 | field=models.ForeignKey( 18 | on_delete=django.db.models.deletion.CASCADE, 19 | primary_key=True, 20 | related_name="alerts", 21 | serialize=False, 22 | to="jobs.SparkJobRun", 23 | ), 24 | ) 25 | ] 26 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0035_auto_20170529_1424.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-29 14:24 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0034_auto_20170529_1424")] 11 | 12 | operations = [ 13 | migrations.AlterModelOptions( 14 | name="sparkjobrun", 15 | options={"get_latest_by": "created_at", "ordering": ["-created_at"]}, 16 | ) 17 | ] 18 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0036_sparkjobrunalert_temp_id.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-30 18:43 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0035_auto_20170529_1424")] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="sparkjobrunalert", 15 | name="temp_id", 16 | field=models.IntegerField(null=True), 17 | ) 18 | ] 19 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0037_populate_temp_id.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-30 18:43 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | def populate_temp_id(apps, schema_editor): 9 | SparkJobRunAlert = apps.get_model("jobs", "SparkJobRunAlert") 10 | alerts = SparkJobRunAlert.objects.all() 11 | for index, alert in enumerate(alerts, start=1): # pks starts at 1, not 0 12 | alert.temp_id = index 13 | alert.save() 14 | 15 | 16 | def delete_temp_id(apps, schema_editor): 17 | SparkJobRunAlert = apps.get_model("jobs", "SparkJobRunAlert") 18 | 19 | for alert in SparkJobRunAlert.objects.all(): 20 | alert.temp_id = None 21 | alert.save() 22 | 23 | 24 | class Migration(migrations.Migration): 25 | 26 | dependencies = [("jobs", "0036_sparkjobrunalert_temp_id")] 27 | 28 | operations = [migrations.RunPython(populate_temp_id, delete_temp_id)] 29 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0038_auto_20170530_1848.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-30 18:48 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("jobs", "0037_populate_temp_id")] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="sparkjobrunalert", 16 | name="run", 17 | field=models.ForeignKey( 18 | on_delete=django.db.models.deletion.CASCADE, 19 | related_name="alerts", 20 | to="jobs.SparkJobRun", 21 | ), 22 | ), 23 | migrations.AlterField( 24 | model_name="sparkjobrunalert", 25 | name="temp_id", 26 | field=models.IntegerField(primary_key=True, serialize=False), 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0039_auto_20170530_1848.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-30 18:48 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0038_auto_20170530_1848")] 11 | 12 | operations = [ 13 | migrations.RenameField( 14 | model_name="sparkjobrunalert", old_name="temp_id", new_name="id" 15 | ) 16 | ] 17 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0040_auto_20170530_1849.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-30 18:49 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0039_auto_20170530_1848")] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="sparkjobrunalert", 15 | name="id", 16 | field=models.AutoField( 17 | auto_created=True, primary_key=True, serialize=False, verbose_name="ID" 18 | ), 19 | ) 20 | ] 21 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0041_auto_20170530_1857.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-30 18:57 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0040_auto_20170530_1849")] 11 | 12 | operations = [ 13 | migrations.AlterModelOptions(name="sparkjobrunalert", options={}), 14 | migrations.AlterUniqueTogether( 15 | name="sparkjobrunalert", 16 | unique_together=set([("run", "reason_code", "reason_message")]), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0042_auto_20170530_1903.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-30 19:03 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0041_auto_20170530_1857")] 11 | 12 | operations = [ 13 | migrations.AlterIndexTogether( 14 | name="sparkjobrunalert", 15 | index_together=set([("reason_code", "mail_sent_date")]), 16 | ) 17 | ] 18 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/0043_auto_20170530_1906.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-30 19:06 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("jobs", "0042_auto_20170530_1903")] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="sparkjob", 15 | name="expired_date", 16 | field=models.DateTimeField( 17 | blank=True, 18 | db_index=True, 19 | help_text="Date/time that the job was expired.", 20 | null=True, 21 | ), 22 | ), 23 | migrations.AlterField( 24 | model_name="sparkjob", 25 | name="identifier", 26 | field=models.CharField( 27 | db_index=True, 28 | help_text="Job name, used to uniqely identify individual jobs.", 29 | max_length=100, 30 | unique=True, 31 | ), 32 | ), 33 | migrations.AlterField( 34 | model_name="sparkjobrun", 35 | name="status", 36 | field=models.CharField( 37 | blank=True, db_index=True, default="", max_length=50 38 | ), 39 | ), 40 | ] 41 | -------------------------------------------------------------------------------- /atmo/jobs/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /atmo/jobs/queries.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.db import models 5 | from django.utils import timezone 6 | 7 | from ..clusters.models import Cluster 8 | 9 | 10 | class SparkJobQuerySet(models.QuerySet): 11 | def with_runs(self): 12 | """ 13 | The Spark jobs with runs. 14 | """ 15 | return self.filter(runs__isnull=False) 16 | 17 | def active(self): 18 | """ 19 | The Spark jobs that have an active cluster status. 20 | """ 21 | return self.filter(runs__status__in=Cluster.ACTIVE_STATUS_LIST) 22 | 23 | def terminated(self): 24 | """ 25 | The Spark jobs that have a terminated cluster status. 26 | """ 27 | return self.filter(runs__status__in=Cluster.TERMINATED_STATUS_LIST) 28 | 29 | def failed(self): 30 | """ 31 | The Spark jobs that have a failed cluster status. 32 | """ 33 | return self.filter(runs__status__in=Cluster.FAILED_STATUS_LIST) 34 | 35 | def lapsed(self): 36 | """ 37 | The Spark jobs that have passed their end dates 38 | but haven't been expired yet. 39 | """ 40 | return self.filter(end_date__lte=timezone.now(), expired_date__isnull=True) 41 | 42 | 43 | class SparkJobRunQuerySet(models.QuerySet): 44 | def active(self): 45 | """ 46 | The Spark jobs that have an active cluster status. 47 | """ 48 | return self.filter(status__in=Cluster.ACTIVE_STATUS_LIST) 49 | -------------------------------------------------------------------------------- /atmo/jobs/signals.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.contrib.auth.models import Group 5 | 6 | from guardian.shortcuts import assign_perm, remove_perm 7 | 8 | 9 | def assign_group_perm(sender, instance, created, **kwargs): 10 | if created: 11 | group, _ = Group.objects.get_or_create(name="Spark job maintainers") 12 | assign_perm("jobs.view_sparkjob", group, instance) 13 | 14 | 15 | def remove_group_perm(sender, instance, **kwargs): 16 | group, _ = Group.objects.get_or_create(name="Spark job maintainers") 17 | remove_perm("jobs.view_sparkjob", group, instance) 18 | -------------------------------------------------------------------------------- /atmo/jobs/templatetags/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /atmo/jobs/templatetags/notebook.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django import template 5 | from django.template.defaultfilters import stringfilter 6 | 7 | 8 | register = template.Library() 9 | 10 | 11 | @register.filter 12 | @stringfilter 13 | def is_jupyter_notebook(value): 14 | return value.endswith(".ipynb") 15 | 16 | 17 | @register.filter 18 | @stringfilter 19 | def is_zeppelin_notebook(value): 20 | return value.endswith(".json") 21 | -------------------------------------------------------------------------------- /atmo/jobs/templatetags/status.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django import template 5 | from django.template.defaultfilters import stringfilter 6 | 7 | from atmo.clusters.models import Cluster 8 | 9 | 10 | register = template.Library() 11 | 12 | 13 | @register.filter 14 | @stringfilter 15 | def status_icon(status): 16 | if status in Cluster.ACTIVE_STATUS_LIST: 17 | return "glyphicon-play" 18 | elif status in Cluster.TERMINATED_STATUS_LIST: 19 | return "glyphicon-stop" 20 | elif status in Cluster.FAILED_STATUS_LIST: 21 | return "glyphicon-exclamation-sign" 22 | 23 | 24 | @register.filter 25 | @stringfilter 26 | def status_color(status): 27 | if status in Cluster.ACTIVE_STATUS_LIST: 28 | return "status-running" 29 | elif status in Cluster.FAILED_STATUS_LIST: 30 | return "status-errors" 31 | -------------------------------------------------------------------------------- /atmo/jobs/urls.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.conf.urls import url 5 | 6 | from . import views 7 | 8 | urlpatterns = [ 9 | url(r"^new/", views.new_spark_job, name="jobs-new"), 10 | url( 11 | r"^identifier-available/", 12 | views.check_identifier_available, 13 | name="jobs-identifier-available", 14 | ), 15 | url(r"^(?P\d+)/delete/", views.delete_spark_job, name="jobs-delete"), 16 | url(r"^(?P\d+)/download/", views.download_spark_job, name="jobs-download"), 17 | url(r"^(?P\d+)/edit/", views.edit_spark_job, name="jobs-edit"), 18 | url(r"^(?P\d+)/run/", views.run_spark_job, name="jobs-run"), 19 | url(r"^(?P\d+)/$", views.detail_spark_job, name="jobs-detail"), 20 | url(r"^(?P\d+)/zeppelin/", views.detail_zeppelin_job, name="jobs-zeppelin"), 21 | ] 22 | -------------------------------------------------------------------------------- /atmo/keys/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /atmo/keys/admin.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.contrib import admin 5 | 6 | from .models import SSHKey 7 | 8 | 9 | @admin.register(SSHKey) 10 | class SSHKeyAdmin(admin.ModelAdmin): 11 | list_display = ["title", "created_by", "fingerprint", "created_at", "modified_at"] 12 | list_filter = ["created_at", "modified_at"] 13 | search_fields = ["title", "fingerprint", "created_by__email"] 14 | -------------------------------------------------------------------------------- /atmo/keys/factories.py: -------------------------------------------------------------------------------- 1 | import factory 2 | from cryptography.hazmat.backends import default_backend as crypto_default_backend 3 | from cryptography.hazmat.primitives import serialization as crypto_serialization 4 | from cryptography.hazmat.primitives.asymmetric import rsa 5 | 6 | from . import models 7 | 8 | from ..users.factories import UserFactory 9 | 10 | 11 | def rsa_key(): 12 | key = rsa.generate_private_key( 13 | backend=crypto_default_backend(), public_exponent=65537, key_size=2048 14 | ) 15 | return ( 16 | key.public_key() 17 | .public_bytes( 18 | crypto_serialization.Encoding.OpenSSH, 19 | crypto_serialization.PublicFormat.OpenSSH, 20 | ) 21 | .decode("utf-8") 22 | ) 23 | 24 | 25 | class SSHKeyFactory(factory.django.DjangoModelFactory): 26 | class Meta: 27 | model = models.SSHKey 28 | 29 | title = "id_rsa" 30 | key = factory.LazyFunction(rsa_key) 31 | fingerprint = "50:a2:40:cb:2d:a2:38:64:66:ec:40:c7:a2:86:97:18" 32 | 33 | created_by = factory.SubFactory(UserFactory) 34 | -------------------------------------------------------------------------------- /atmo/keys/migrations/0002_assign_view_perms.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | # -*- coding: utf-8 -*- 5 | # Generated by Django 1.9.11 on 2017-01-13 15:38 6 | from django.db import migrations 7 | 8 | from atmo.models import PermissionMigrator 9 | 10 | 11 | def assign_sshkey_view_permission(apps, schema_editor): 12 | SSHKey = apps.get_model("keys", "SSHKey") 13 | PermissionMigrator(apps, SSHKey, "view", user_field="created_by").assign() 14 | PermissionMigrator(apps, SSHKey, "change", user_field="created_by").assign() 15 | PermissionMigrator(apps, SSHKey, "delete", user_field="created_by").assign() 16 | 17 | 18 | def remove_sshkey_view_permission(apps, schema_editor): 19 | SSHKey = apps.get_model("jobs", "SSHKey") 20 | PermissionMigrator(apps, SSHKey, "view", user_field="created_by").remove() 21 | PermissionMigrator(apps, SSHKey, "change", user_field="created_by").remove() 22 | PermissionMigrator(apps, SSHKey, "delete", user_field="created_by").remove() 23 | 24 | 25 | class Migration(migrations.Migration): 26 | 27 | dependencies = [("guardian", "0001_initial"), ("keys", "0001_initial")] 28 | 29 | operations = [ 30 | migrations.RunPython( 31 | assign_sshkey_view_permission, remove_sshkey_view_permission 32 | ) 33 | ] 34 | -------------------------------------------------------------------------------- /atmo/keys/migrations/0003_auto_20170116_1512.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.11 on 2017-01-16 15:12 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("keys", "0002_assign_view_perms")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="sshkey", 13 | name="fingerprint", 14 | field=models.CharField(blank=True, max_length=48), 15 | ), 16 | migrations.AlterUniqueTogether( 17 | name="sshkey", unique_together=set([("created_by", "fingerprint")]) 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /atmo/keys/migrations/0004_auto_20170519_1336.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-19 13:36 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.utils.timezone 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("keys", "0003_auto_20170116_1512")] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="sshkey", 16 | name="created_at", 17 | field=models.DateTimeField( 18 | blank=True, default=django.utils.timezone.now, editable=False 19 | ), 20 | ), 21 | migrations.AlterField( 22 | model_name="sshkey", 23 | name="modified_at", 24 | field=models.DateTimeField( 25 | blank=True, default=django.utils.timezone.now, editable=False 26 | ), 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /atmo/keys/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /atmo/keys/models.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from autorepr import autorepr, autostr 5 | from django.db import models 6 | 7 | from ..models import CreatedByModel, EditedAtModel, URLActionModel 8 | from .utils import calculate_fingerprint 9 | 10 | 11 | class SSHKey(CreatedByModel, EditedAtModel, URLActionModel): 12 | """ 13 | A Django data model to store public SSH keys for logged-in users 14 | to be used in the :mod:`on-demand clusters `. 15 | """ 16 | 17 | #: The list of valid SSH key data prefixes, will be validated 18 | #: on save. 19 | VALID_PREFIXES = [ 20 | "ssh-rsa", 21 | "ssh-dss", 22 | "ecdsa-sha2-nistp256", 23 | "ecdsa-sha2-nistp384", 24 | "ecdsa-sha2-nistp521", 25 | ] 26 | 27 | title = models.CharField( 28 | max_length=100, help_text="Name to give to this public key" 29 | ) 30 | key = models.TextField( 31 | help_text="Should start with one of the following prefixes: %s" 32 | % ", ".join(VALID_PREFIXES) 33 | ) 34 | fingerprint = models.CharField(max_length=48, blank=True) 35 | 36 | class Meta: 37 | permissions = [("view_sshkey", "Can view SSH key")] 38 | unique_together = ("created_by", "fingerprint") 39 | 40 | __str__ = autostr("{self.title}") 41 | 42 | __repr__ = autorepr(["title", "fingerprint"]) 43 | 44 | url_prefix = "keys" 45 | url_actions = ["detail", "delete", "raw"] 46 | 47 | def get_absolute_url(self): 48 | return self.urls.detail 49 | 50 | @property 51 | def prefix(self): 52 | """ 53 | The prefix of the key data, 54 | one of the :data:`~atmo.keys.models.SSHKey.VALID_PREFIXES`. 55 | """ 56 | return self.key.strip().split()[0] 57 | 58 | def save(self, *args, **kwargs): 59 | self.fingerprint = calculate_fingerprint(self.key) 60 | super().save(*args, **kwargs) 61 | -------------------------------------------------------------------------------- /atmo/keys/urls.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.conf.urls import url 5 | from . import views 6 | 7 | 8 | urlpatterns = [ 9 | url(r"^new/", views.new_key, name="keys-new"), 10 | url(r"^(?P\d+)/delete/$", views.delete_key, name="keys-delete"), 11 | url(r"^(?P\d+)/raw/$", views.detail_key, {"raw": True}, name="keys-raw"), 12 | url(r"^(?P\d+)/$", views.detail_key, name="keys-detail"), 13 | url(r"^$", views.list_keys, name="keys-list"), 14 | ] 15 | -------------------------------------------------------------------------------- /atmo/keys/utils.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | import base64 5 | import hashlib 6 | 7 | 8 | def calculate_fingerprint(data): 9 | """ 10 | Calculate the hexadecimal fingerprint for the given key data. 11 | 12 | :param data: str - The key data to calculate the fingerprint for. 13 | :return: The fingerprint. 14 | :rtype: str 15 | """ 16 | key_data = data.strip().split()[1] 17 | decoded_key_data = base64.b64decode(key_data) 18 | fingerprint = hashlib.md5(decoded_key_data).hexdigest() 19 | return ":".join(fingerprint[ i: i + 2] for i in range(0, len(fingerprint), 2)) 20 | -------------------------------------------------------------------------------- /atmo/news/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /atmo/news/urls.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.conf.urls import url 5 | 6 | from . import views 7 | 8 | urlpatterns = [ 9 | url(r"^list/$", views.list_news, name="news-list"), 10 | url(r"^check/$", views.check_news, name="news-check"), 11 | ] 12 | -------------------------------------------------------------------------------- /atmo/static/css/base.css: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 | /* don't let the top navigation bar cover any content */ 5 | body { 6 | padding-bottom: 90px; 7 | } 8 | 9 | nav.navbar-dev { 10 | background-color: #00539F; 11 | -webkit-background-size: 40px 40px; 12 | -moz-background-size: 40px 40px; 13 | background-size: 40px 40px; 14 | background-image: 15 | linear-gradient(-45deg, rgba(255, 255, 255, .1) 25%, transparent 25%, 16 | transparent 50%, rgba(255, 255, 255, .1) 50%, rgba(255, 255, 255, .1) 75%, 17 | transparent 75%, transparent); 18 | } 19 | .navbar-dev a, 20 | .navbar-inverse .navbar-nav > li > a, 21 | .navbar-inverse .navbar-brand, 22 | .navbar-inverse .navbar-text, 23 | .navbar-inverse .navbar-text { 24 | color: white; 25 | } 26 | 27 | .alert-dragons { 28 | margin-top: 20px; 29 | margin-bottom: 0; 30 | } 31 | 32 | .alert-dragons:first-of-type { 33 | margin-top: 0px; 34 | margin-bottom: 0; 35 | } 36 | 37 | label span.optional-label { 38 | font-style: italic; 39 | font-weight: 400; 40 | color: #737373; 41 | } 42 | 43 | .page-header { 44 | margin-top: 20px; 45 | } 46 | 47 | .modal-body h1:first-child, 48 | .modal-body h2:first-child, 49 | .modal-body h3:first-child, 50 | .modal-body h4:first-child, 51 | .modal-body h5:first-child { 52 | margin-top: 0px; 53 | } 54 | 55 | /* Job status colors */ 56 | .status-running { 57 | color: #007E33; 58 | font-weight: bold; 59 | } 60 | .status-errors { 61 | color: #CC0000; 62 | font-weight: bold; 63 | } 64 | 65 | /* The max width is dependant on the container (more info below) */ 66 | .popover, .confirmation { 67 | max-width: 100%; /* Max Width of the popover (depending on the container!) */ 68 | min-width: 130px; 69 | min-height: 75px; 70 | } 71 | 72 | /* So the buttons aren't attached to each other on small screen */ 73 | .btn-toolbar > .btn, .btn-toolbar > .btn-group, .btn-toolbar > .input-group { 74 | margin-top: 5px; 75 | } 76 | -------------------------------------------------------------------------------- /atmo/static/css/login.css: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 | 5 | .jumbotron { 6 | margin: 1em; 7 | } 8 | 9 | .login-button { 10 | margin: 1em; 11 | } -------------------------------------------------------------------------------- /atmo/static/css/notebook.css: -------------------------------------------------------------------------------- 1 | /*Original from https://github.com/jsvine/nbpreview/blob/9da3f2dad5fa3cc1d38702fdc62df80c75baf664/css/vendor/notebook.css 2 | Copyright (c) 2015, Jeremy Singer-Vine*/ 3 | .nb-notebook { 4 | line-height: 1.5; 5 | } 6 | 7 | .nb-stdout, .nb-stderr { 8 | white-space: pre-wrap; 9 | margin: 1em 0; 10 | padding: 0.1em 0.5em; 11 | } 12 | 13 | .nb-stderr { 14 | background-color: #FAA; 15 | } 16 | 17 | .nb-cell + .nb-cell { 18 | margin-top: 0.5em; 19 | } 20 | 21 | .nb-output table { 22 | border: 1px solid #000; 23 | border-collapse: collapse; 24 | } 25 | 26 | .nb-output th { 27 | font-weight: bold; 28 | } 29 | 30 | .nb-output th, .nb-output td { 31 | border: 1px solid #000; 32 | padding: 0.25em; 33 | text-align: left; 34 | vertical-align: middle; 35 | border-collapse: collapse; 36 | } 37 | 38 | .nb-cell { 39 | position: relative; 40 | } 41 | 42 | .nb-raw-cell { 43 | white-space: pre-wrap; 44 | background-color: #f5f2f0; 45 | font-family: Consolas, Monaco, 'Andale Mono', monospace; 46 | padding: 1em; 47 | margin: .5em 0; 48 | } 49 | 50 | .nb-output { 51 | min-height: 1em; 52 | width: 100%; 53 | overflow-x: scroll; 54 | border-right: 1px dotted #CCC; 55 | } 56 | 57 | .nb-output img { 58 | max-width: 100%; 59 | } 60 | 61 | .nb-output:before, .nb-input:before { 62 | position: absolute; 63 | font-family: monospace; 64 | color: #999; 65 | left: -7.5em; 66 | width: 7em; 67 | text-align: right; 68 | } 69 | 70 | .nb-input:before { 71 | content: "In [" attr(data-prompt-number) "]:"; 72 | } 73 | .nb-output:before { 74 | content: "Out [" attr(data-prompt-number) "]:"; 75 | } 76 | 77 | // Fix pandas dataframe formatting 78 | div[style="max-height:1000px;max-width:1500px;overflow:auto;"] { 79 | max-height: none !important; 80 | } 81 | -------------------------------------------------------------------------------- /atmo/static/img/cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/atmo/static/img/cluster.png -------------------------------------------------------------------------------- /atmo/static/img/dashboards.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/atmo/static/img/dashboards.png -------------------------------------------------------------------------------- /atmo/static/img/schedule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/atmo/static/img/schedule.png -------------------------------------------------------------------------------- /atmo/static/img/worker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/atmo/static/img/worker.png -------------------------------------------------------------------------------- /atmo/static/js/clusters.js: -------------------------------------------------------------------------------- 1 | $(function() { 2 | var instructionFlipper = function() { 3 | if ($(this).hasClass("active")) { 4 | $(".with-nfs").hide(); 5 | $(".without-nfs").show(); 6 | $(this).removeClass("active"); 7 | } else { 8 | $(".with-nfs").removeClass("hidden"); 9 | $(".with-nfs").show(); 10 | $(".without-nfs").hide(); 11 | $(this).addClass("active"); 12 | } 13 | }; 14 | AtmoCallbacks.add(function() { 15 | $('#instruction-flipper').on('click', instructionFlipper); 16 | }); 17 | }); 18 | -------------------------------------------------------------------------------- /atmo/static/js/csrf.js: -------------------------------------------------------------------------------- 1 | $(function() { 2 | // Ensure that all AJAX requests sent with jQuery have CSRF tokens 3 | var csrfToken = jQuery("input[name=csrfmiddlewaretoken]").val(); 4 | $.ajaxSetup({ 5 | beforeSend: function(xhr, settings) { 6 | // non-CSRF-safe method that isn't cross domain 7 | if (["GET", "HEAD", "OPTIONS", "TRACE"].indexOf(settings.Type) < 0 && !this.crossDomain) { 8 | xhr.setRequestHeader("X-CSRFToken", csrfToken); 9 | } 10 | } 11 | }); 12 | }); 13 | -------------------------------------------------------------------------------- /atmo/static/js/jobs.js: -------------------------------------------------------------------------------- 1 | $(function() { 2 | $.fn.atmoNotebook = function() { 3 | var container = this, 4 | content_given = container.attr('data-content-given'), 5 | download_url = container.attr('data-download-url'), 6 | zeppelin_url = container.attr('data-zeppelin-url'); 7 | 8 | var fail = function() { 9 | container.html('

Apologies, we could not load Notebook content.

'); 10 | }; 11 | var current_script_path = function() { 12 | var scripts = $('script[src]'); 13 | var current_script = scripts[scripts.length - 1].src; 14 | var current_script_chunks = current_script.split('/'); 15 | var current_script_file = current_script_chunks[current_script_chunks.length - 1]; 16 | return current_script.replace(current_script_file, ''); 17 | }; 18 | var render = function(data) { 19 | if (data) { 20 | container.empty(); 21 | if (!data['nbformat']) { 22 | $.ajax({ 23 | url: zeppelin_url, 24 | type:'GET', 25 | success: function(data){ 26 | var md = new Remarkable(); 27 | container.html(md.render($(data).find('#markdown').html())); 28 | } 29 | }); 30 | container.html('

Please download the Zeppelin notebook to view its contents.

') 31 | } else { 32 | var notebook = nb.parse(data); 33 | container.append(notebook.render()); 34 | Prism.plugins.autoloader.languages_path = current_script_path() + '../npm/prismjs/components/'; 35 | Prism.highlightAll(); 36 | } 37 | }; 38 | } 39 | if (content_given == 'true') { 40 | var content = container.children().filter('textarea').val(); 41 | render(JSON.parse(content)); 42 | } else if (jQuery.type(download_url) !== 'undefined') { 43 | $.get(download_url).done(render).fail(fail); 44 | } 45 | }; 46 | 47 | $.fn.atmoRenderZeppelin = function() { 48 | var md = new Remarkable(); 49 | var renderedHTML = md.render(this.text()); 50 | $('#renderedHTML').append(renderedHTML); 51 | }; 52 | 53 | AtmoCallbacks.add(function() { 54 | $('#notebook-content').atmoNotebook(); 55 | $('#markdown').atmoRenderZeppelin(); 56 | }); 57 | }); 58 | -------------------------------------------------------------------------------- /atmo/static/js/raven.js: -------------------------------------------------------------------------------- 1 | $(function() { 2 | $(document).ready( function() { 3 | Raven.config($('body').attr('data-sentry-public-dsn')).install(); 4 | }); 5 | }); 6 | -------------------------------------------------------------------------------- /atmo/static/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/atmo/static/public/favicon.ico -------------------------------------------------------------------------------- /atmo/static/public/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: / 3 | -------------------------------------------------------------------------------- /atmo/stats/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/atmo/stats/__init__.py -------------------------------------------------------------------------------- /atmo/stats/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.3 on 2017-07-19 21:23 3 | from __future__ import unicode_literals 4 | 5 | import django.contrib.postgres.fields.jsonb 6 | import django.core.serializers.json 7 | from django.db import migrations, models 8 | import django.utils.timezone 9 | 10 | 11 | class Migration(migrations.Migration): 12 | 13 | initial = True 14 | 15 | dependencies = [] 16 | 17 | operations = [ 18 | migrations.CreateModel( 19 | name="Metric", 20 | fields=[ 21 | ( 22 | "id", 23 | models.AutoField( 24 | auto_created=True, 25 | primary_key=True, 26 | serialize=False, 27 | verbose_name="ID", 28 | ), 29 | ), 30 | ( 31 | "created_at", 32 | models.DateTimeField( 33 | blank=True, 34 | db_index=True, 35 | default=django.utils.timezone.now, 36 | editable=False, 37 | ), 38 | ), 39 | ( 40 | "key", 41 | models.CharField( 42 | db_index=True, 43 | help_text="Name of the metric being recorded", 44 | max_length=100, 45 | ), 46 | ), 47 | ( 48 | "value", 49 | models.PositiveIntegerField( 50 | help_text="Integer value of the metric" 51 | ), 52 | ), 53 | ( 54 | "data", 55 | django.contrib.postgres.fields.jsonb.JSONField( 56 | blank=True, 57 | encoder=django.core.serializers.json.DjangoJSONEncoder, 58 | help_text="Extra data about this metric", 59 | null=True, 60 | ), 61 | ), 62 | ], 63 | ) 64 | ] 65 | -------------------------------------------------------------------------------- /atmo/stats/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/atmo/stats/migrations/__init__.py -------------------------------------------------------------------------------- /atmo/stats/models.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.contrib.postgres.fields import JSONField 5 | from django.core.serializers.json import DjangoJSONEncoder 6 | from django.db import models 7 | from django.utils import timezone 8 | 9 | 10 | class Metric(models.Model): 11 | created_at = models.DateTimeField( 12 | editable=False, blank=True, default=timezone.now, db_index=True 13 | ) 14 | key = models.CharField( 15 | max_length=100, db_index=True, help_text="Name of the metric being recorded" 16 | ) 17 | value = models.PositiveIntegerField(help_text="Integer value of the metric") 18 | data = JSONField( 19 | encoder=DjangoJSONEncoder, 20 | blank=True, 21 | null=True, 22 | help_text="Extra data about this metric", 23 | ) 24 | 25 | @classmethod 26 | def record(cls, key, value=1, **kwargs): 27 | """ 28 | Create a new entry in the ``Metric`` table. 29 | 30 | :param key: 31 | The metric key name. 32 | 33 | :param value: 34 | The metric value as an integer. 35 | 36 | :param data: 37 | Any extra data to be stored with this record as a dictionary. 38 | 39 | """ 40 | created_at = kwargs.pop("created_at", None) or timezone.now() 41 | data = kwargs.pop("data", None) 42 | 43 | cls.objects.create(created_at=created_at, key=key, value=value, data=data) 44 | -------------------------------------------------------------------------------- /atmo/tasks.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from celery.utils.log import get_task_logger 5 | from guardian.utils import clean_orphan_obj_perms 6 | 7 | from .celery import celery 8 | 9 | logger = get_task_logger(__name__) 10 | 11 | 12 | @celery.task() 13 | def cleanup_permissions(): 14 | "A Celery task that cleans up old django-guardian object permissions." 15 | clean_orphan_obj_perms() 16 | -------------------------------------------------------------------------------- /atmo/templates/403.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/error.html" %} 2 | 3 | {% block head_title %}Permission denied{% endblock %} 4 | 5 | {% block error_name %}Permission denied{% endblock %} 6 | 7 | {% block error_code %}403{% endblock %} 8 | 9 | {% block error_excuse %} 10 | Apologies but you're not allowed to view this page. 11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /atmo/templates/404.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/error.html" %} 2 | 3 | {% block head_title %}Page not found{% endblock %} 4 | 5 | {% block error_name %}Page not found{% endblock %} 6 | 7 | {% block error_code %}404{% endblock %} 8 | 9 | {% block error_excuse %} 10 | Apologies but we cannot find the page you requested. 11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /atmo/templates/500.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/error.html" %} 2 | 3 | {% block head_title %}Server Error{% endblock %} 4 | 5 | {% block error_name %}Server Error{% endblock %} 6 | 7 | {% block error_code %}500{% endblock %} 8 | 9 | {% block error_excuse %}Apologies but we encountered an internal server error.{% endblock %} 10 | -------------------------------------------------------------------------------- /atmo/templates/atmo/_announcements.html: -------------------------------------------------------------------------------- 1 | {% load atmo %} 2 | {% if config.ANNOUNCEMENT_ENABLED %} 3 | 11 | {% endif %} 12 | -------------------------------------------------------------------------------- /atmo/templates/atmo/clusters/extend.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/base.html" %} 2 | {% load static %} 3 | 4 | {% block page_title %}Extend Spark cluster {{ cluster }}{% endblock %} 5 | 6 | {% block content %} 7 |
8 |

Extend Spark cluster {{ cluster }}

9 |
10 |
11 |
12 |
13 | {% csrf_token %} 14 | {% include "atmo/_form.html" %} 15 | 19 | 20 | Cancel 21 |
22 |
23 |
24 | {% endblock %} 25 | -------------------------------------------------------------------------------- /atmo/templates/atmo/clusters/mails/expiration.mail: -------------------------------------------------------------------------------- 1 | {% load atmo %} 2 | 3 | {% block subject %}{{ settings.EMAIL_SUBJECT_PREFIX }}Cluster {{ cluster.identifier }} is expiring soon!{% endblock subject %} 4 | 5 | {% block to %}{{ cluster.created_by.email }}{% endblock to %} 6 | 7 | {% block body %} 8 | Your cluster "{{ cluster.identifier }}" will be terminated in roughly one hour, 9 | around {{ deadline }} UTC. 10 | 11 | The URL of the cluster is: {{ cluster.urls.detail|full_url }} 12 | 13 | To extend the cluster lifetime please go here: {{ cluster.urls.extend|full_url }} 14 | 15 | This is an automated message sent by the Telemetry Analysis service. 16 | See {{ settings.SITE_URL }} for more details. 17 | {% endblock body %} 18 | -------------------------------------------------------------------------------- /atmo/templates/atmo/clusters/new.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/base.html" %} 2 | {% load static %} 3 | 4 | {% block page_title %}New Spark cluster{% endblock %} 5 | 6 | {% block content %} 7 |
8 |

New Spark cluster

9 |
10 |
11 |
12 |
13 | {% csrf_token %} 14 | {% include "atmo/_form.html" %} 15 | 19 | 20 | Cancel 21 |
22 |
23 |
24 | {% endblock %} 25 | -------------------------------------------------------------------------------- /atmo/templates/atmo/clusters/terminate.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/base.html" %} 2 | {% load static %} 3 | 4 | {% block page_title %}Terminate Spark cluster {{ cluster }}{% endblock %} 5 | 6 | {% block content %} 7 |
8 |

Terminate Spark cluster {{ cluster }}

9 |
10 |
11 |
12 |

Do you really want to terminate the cluster?

13 |

14 | Any data that was created on the filesystem of the cluster will be deleted. 15 |

16 |
17 | {% csrf_token %} 18 | 22 | 23 | Cancel 24 |
25 |
26 |
27 | {% endblock content %} 28 | -------------------------------------------------------------------------------- /atmo/templates/atmo/error.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/base.html" %} 2 | 3 | {% block head_title %}Error {% endblock %} 4 | 5 | {% block content %} 6 |
7 |
8 |
9 |

10 | {% block error_name %}{% endblock %} 11 | {% block error_code %}{% endblock %} 12 |

13 |
14 |

15 | {% block error_excuse %}{% endblock %} 16 |

17 |

18 | If the error shows up a lot, please open a 19 | GitHub issue{% if request.sentry.id %} 20 | and reference this error with Sentry error ID 21 | {{ request.sentry.id }}{% endif %}. 22 |

23 |
24 | {% endblock %} 25 | -------------------------------------------------------------------------------- /atmo/templates/atmo/jobs/delete.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/base.html" %} 2 | {% load static %} 3 | 4 | {% block page_title %}Delete Spark job {{ spark_job }}{% endblock %} 5 | 6 | {% block content %} 7 |
8 |

Delete Spark job {{ spark_job }}

9 |
10 |
11 |
12 |

Do you really want to delete the Spark job?

13 |
14 | {% csrf_token %} 15 | 19 | 20 | Cancel 21 |
22 |
23 |
24 | {% endblock content %} 25 | -------------------------------------------------------------------------------- /atmo/templates/atmo/jobs/edit.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/base.html" %} 2 | {% load static %} 3 | 4 | {% block page_title %}Edit Spark job {{ form.instance }}{% endblock %} 5 | 6 | {% block content %} 7 |
8 |

Edit Spark job {{ form.instance }}

9 |
10 |
11 |
12 |
13 | {% csrf_token %} 14 | {% include "atmo/_form.html" %} 15 | 19 | 20 | Cancel 21 |
22 |
23 |
24 | {% endblock %} 25 | -------------------------------------------------------------------------------- /atmo/templates/atmo/jobs/mails/expired.mail: -------------------------------------------------------------------------------- 1 | {% load atmo %} 2 | 3 | {% block subject %}{{ settings.EMAIL_SUBJECT_PREFIX }}Spark job {{ spark_job.identifier }} expired{% endblock subject %} 4 | 5 | {% block to %}{{ spark_job.created_by.email }}{% endblock to %} 6 | 7 | {% block cc %}{{ settings.DEFAULT_FROM_EMAIL }}{% endblock cc %} 8 | 9 | {% block body %} 10 | Your Spark job "{{ spark_job.identifier }}" has expired. 11 | 12 | We marked it as expired at {{ spark_job.expired_date }} UTC but the 13 | end date you originally set was {{ spark_job.end_date }} UTC. 14 | 15 | The Spark job was originally created at {{ spark_job.created_at }} UTC and 16 | last modified at {{ spark_job.modified_at }} UTC. 17 | {% if spark_job.latest_run %} 18 | The last run of the Spark job was scheduled at approximately 19 | {{ spark_job.latest_run.scheduled_at }} UTC. 20 | {% endif %} 21 | 22 | The URL of the Spark job is: {{ spark_job.urls.detail|full_url }} 23 | 24 | To reschedule the Spark job please go here and edit the end date accordingly: {{ spark_job.urls.edit|full_url }} 25 | 26 | This is an automated message sent by the Telemetry Analysis service. 27 | See {{ settings.SITE_URL }} for more details. 28 | {% endblock body %} 29 | -------------------------------------------------------------------------------- /atmo/templates/atmo/jobs/mails/failed_run_alert.mail: -------------------------------------------------------------------------------- 1 | {% load atmo %} 2 | 3 | {% block subject %}{{ settings.EMAIL_SUBJECT_PREFIX }}Running Spark job {{ alert.run.spark_job.identifier }} failed{% endblock subject %} 4 | 5 | {% block to %}{{ alert.run.spark_job.created_by.email }}{% endblock to %} 6 | 7 | {% block cc %}{{ settings.DEFAULT_FROM_EMAIL }}{% endblock cc %} 8 | 9 | {% block body %} 10 | Your scheduled Spark job "{{ alert.run.spark_job.identifier }}" has failed 11 | at approximately {{ alert.run.created_at }} UTC. 12 | 13 | Description: {{ alert.run.spark_job.description }} 14 | 15 | You may want to check the logs to see what failed in the Spark job. 16 | 17 | {% if alert.reason_message %}The reason for the failure that AWS reported was: {{ alert.reason_message }} {% if alert.reason_code %}(Error code {{ alert.reason_code }}){% endif %}{% endif %} 18 | 19 | The URL of the Spark job is: {{ alert.run.spark_job.urls.detail|full_url }} 20 | 21 | This is an automated message sent by the Telemetry Analysis service. 22 | See {{ settings.SITE_URL }} for more details. 23 | {% endblock body %} 24 | -------------------------------------------------------------------------------- /atmo/templates/atmo/jobs/mails/timed_out.mail: -------------------------------------------------------------------------------- 1 | {% load atmo %} 2 | 3 | {% block subject %}{{ settings.EMAIL_SUBJECT_PREFIX }}Spark job {{ spark_job.identifier }} timed out{% endblock subject %} 4 | 5 | {% block to %}{{ spark_job.created_by.email }}{% endblock to %} 6 | 7 | {% block cc %}{{ settings.DEFAULT_FROM_EMAIL }}{% endblock cc %} 8 | 9 | {% block body %} 10 | Your Spark job "{{ spark_job.identifier }}" has timed out. 11 | 12 | This has most likely happened because it ran longer than the maximum 13 | run time of {{ spark_job.job_timeout }} hours. 14 | 15 | Please fix this by increasing the size of the Spark job cluster 16 | or improve the Spark job code to run less than the maximum run time. 17 | 18 | The Spark job was originally created at {{ spark_job.created_at }} UTC and 19 | last modified at {{ spark_job.modified_at }} UTC. 20 | {% if spark_job.latest_run %} 21 | The last run of the Spark job was scheduled at approximately 22 | {{ spark_job.latest_run.scheduled_at }} UTC. 23 | {% endif %} 24 | 25 | The URL of the Spark job is: {{ spark_job.urls.detail|full_url }} 26 | 27 | To update the Spark job please go here: {{ spark_job.urls.edit|full_url }} 28 | 29 | This is an automated message sent by the Telemetry Analysis service. 30 | See {{ settings.SITE_URL }} for more details. 31 | {% endblock body %} 32 | -------------------------------------------------------------------------------- /atmo/templates/atmo/jobs/new.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/base.html" %} 2 | {% load static %} 3 | 4 | {% block page_title %}New Spark job{% endblock %} 5 | 6 | {% block content %} 7 |
8 |

New Spark job

9 |
10 |
11 |
12 |
13 | {% csrf_token %} 14 | {% include "atmo/_form.html" %} 15 | 19 | 20 | Cancel 21 |
22 |
23 |
24 | {% endblock %} 25 | -------------------------------------------------------------------------------- /atmo/templates/atmo/jobs/run.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/base.html" %} 2 | {% load staticfiles %} 3 | 4 | {% block page_title %}Run Spark job {{ spark_job }}{% endblock %} 5 | 6 | {% block content %} 7 |
8 |

Run Spark job {{ spark_job }}

9 |
10 |
11 |
12 |

Do you really want to run the Spark job?

13 |

14 | The schedule of the Spark job is not being changed by this run. 15 |

16 |

17 | Please make sure the job will finish before its next scheduled run 18 | or it may be skipped automatically. 19 |

20 |
21 | {% csrf_token %} 22 | 26 | 27 | Cancel 28 |
29 |
30 |
31 | {% endblock content %} 32 | -------------------------------------------------------------------------------- /atmo/templates/atmo/jobs/zeppelin_notebook.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/base.html" %} 2 | {% load static %} 3 | {% load notebook %} 4 | 5 | {% block head_title %}Spark job {{ spark_job }}{% endblock %} 6 | 7 | {% if modified_date %} 8 | {% block body_attrs %}data-modified-date="{{ modified_date.isoformat }}"{% endblock %} 9 | {% endif %} 10 | 11 | {% block modified_date_title %}Spark job status outdated{% endblock modified_date_title %} 12 | {% block modified_date_description %}The Spark job was updated on the server.{% endblock modified_date_description %} 13 | 14 | {% block footer_extra %} 15 | 16 | {% endblock %} 17 | 18 | {% block content %} 19 |
20 |

Zeppelin Notebook

21 |
22 | 23 |
24 | {% endblock content %} 25 | -------------------------------------------------------------------------------- /atmo/templates/atmo/keys/delete.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/base.html" %} 2 | 3 | {% block page_title %}Delete SSH key {{ ssh_key }}{% endblock %} 4 | 5 | {% block content %} 6 |
7 |

Delete SSH key {{ ssh_key }}

8 |
9 |
10 |
11 |

Do you really want to delete the SSH key?

12 |
13 | {% csrf_token %} 14 | 18 | 19 | Cancel 20 |
21 |
22 |
23 | {% endblock content %} 24 | -------------------------------------------------------------------------------- /atmo/templates/atmo/keys/detail.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/base.html" %} 2 | 3 | {% block head_title %}SSH key {{ ssh_key }}{% endblock %} 4 | 5 | {% block content %} 6 |
7 |

SSH key {{ ssh_key }}

8 | 32 |
33 |
34 |
35 |

36 | Fingerprint: 37 | {{ ssh_key.fingerprint }} 38 |

39 | 40 |
41 |
42 |

43 | {% with cluster_count=ssh_key.launched_clusters.active.count %} 44 | This key is used by {{ cluster_count }} active cluster{{ cluster_count|pluralize }}. 45 | {% endwith %} 46 |

47 |
48 |
Created at
49 |
{{ ssh_key.created_at }} UTC
50 |
Created by
51 |
{{ ssh_key.created_by.username }}
52 |
Key prefix
53 |
{{ ssh_key.prefix }}
54 |
55 |
56 |
57 | {% endblock content %} 58 | -------------------------------------------------------------------------------- /atmo/templates/atmo/keys/new.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/base.html" %} 2 | {% load static %} 3 | 4 | {% block page_title %}New SSH key{% endblock %} 5 | 6 | {% block footer_extra %} 7 | 8 | {% endblock %} 9 | 10 | {% block content %} 11 |
12 |

13 | New SSH key 14 |

15 |
16 |
17 |
18 |
19 | {% csrf_token %} 20 | {% include "atmo/_form.html" %} 21 | 25 | 26 | Cancel 27 |
28 |
29 |
30 | {% endblock %} 31 | -------------------------------------------------------------------------------- /atmo/templates/atmo/news/list.html: -------------------------------------------------------------------------------- 1 | {% extends "atmo/base.html" %} 2 | {% load static %} 3 | 4 | {% block page_title %}What's new{% endblock %} 5 | 6 | {% block content %} 7 |
8 |

What's new

9 |
10 |
11 |
12 | {{ news.render|safe }} 13 | Not all code changes are listed here. More details in the 14 | changelog. 15 |
16 |
17 | {% endblock %} 18 | -------------------------------------------------------------------------------- /atmo/templatetags.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from urllib.parse import urljoin 5 | 6 | from django import template 7 | from django.conf import settings 8 | 9 | import commonmark 10 | from furl import furl 11 | 12 | 13 | register = template.Library() 14 | 15 | 16 | @register.simple_tag 17 | def url_update(url, **kwargs): 18 | """ 19 | A Django template tag to update the query parameters for the given URL. 20 | """ 21 | if kwargs: 22 | new_url = furl(url) 23 | new_url.args.update(kwargs) 24 | return new_url.url 25 | 26 | return url 27 | 28 | 29 | @register.filter 30 | def full_url(url): 31 | """ 32 | A Django template filter to prepend the given URL path with the full 33 | site URL. 34 | """ 35 | return urljoin(settings.SITE_URL, url) 36 | 37 | 38 | @register.filter 39 | def markdown(content): 40 | """ 41 | A Django template filter to render the given content as Markdown. 42 | """ 43 | return commonmark.commonmark(content) 44 | -------------------------------------------------------------------------------- /atmo/urls.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.conf import settings 5 | from django.conf.urls import include, url 6 | from django.contrib import admin 7 | from django.contrib.auth.decorators import login_required 8 | from django.views import generic, static 9 | 10 | from . import views 11 | 12 | handler403 = "atmo.views.permission_denied" 13 | handler500 = "atmo.views.server_error" 14 | 15 | admin.site.login = login_required(admin.site.login) 16 | 17 | urlpatterns = [ 18 | url(r"^$", views.DashboardView.as_view(), name="dashboard"), 19 | url(r"^admin/", include(admin.site.urls)), 20 | url(r"clusters/", include("atmo.clusters.urls")), 21 | url(r"jobs/", include("atmo.jobs.urls")), 22 | url(r"keys/", include("atmo.keys.urls")), 23 | url(r"news/", include("atmo.news.urls")), 24 | url(r"users/", include("atmo.users.urls")), 25 | url(r"oidc/", include("mozilla_django_oidc.urls")), 26 | # contribute.json url 27 | url( 28 | r"^(?Pcontribute\.json)$", 29 | static.serve, 30 | {"document_root": settings.BASE_DIR}, 31 | ), 32 | url(r"^404/$", generic.TemplateView.as_view(template_name="404.html")), 33 | url(r"^500/$", generic.TemplateView.as_view(template_name="500.html")), 34 | ] 35 | -------------------------------------------------------------------------------- /atmo/users/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /atmo/users/backends.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | from mozilla_django_oidc.auth import OIDCAuthenticationBackend 4 | 5 | 6 | class AtmoOIDCAuthenticationBackend(OIDCAuthenticationBackend): 7 | def verify_claims(self, claims): 8 | """ 9 | See if the claims contain a list of user groups (in various forms) 10 | and then check it against a configured list of allowed groups. 11 | """ 12 | # shortcut in case remote groups aren't enabled 13 | if not settings.REMOTE_GROUPS_ENABLED: 14 | return True 15 | remote_groups = set( 16 | claims.get("groups") 17 | or claims.get("https://sso.mozilla.com/claim/groups") 18 | or [] 19 | ) 20 | allowed_groups = settings.REMOTE_GROUPS_ALLOWED 21 | return bool(allowed_groups.intersection(remote_groups)) 22 | -------------------------------------------------------------------------------- /atmo/users/factories.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | import factory 5 | 6 | from django.contrib.auth.models import User, Group 7 | from django.contrib.auth.hashers import make_password 8 | 9 | 10 | class GroupFactory(factory.django.DjangoModelFactory): 11 | name = factory.Sequence(lambda n: "group#%s" % n) 12 | 13 | class Meta: 14 | model = Group 15 | 16 | @factory.post_generation 17 | def permissions(self, create, extracted, **kwargs): 18 | if not create: 19 | # Simple build, do nothing. 20 | return 21 | 22 | if extracted: 23 | # A list of groups were passed in, use them 24 | for permission in extracted: 25 | self.permissions.add(permission) 26 | 27 | 28 | class UserFactory(factory.django.DjangoModelFactory): 29 | username = factory.Sequence(lambda n: "user%s" % n) 30 | first_name = factory.Sequence(lambda n: "user %03d" % n) 31 | email = "test@example.com" 32 | 33 | class Meta: 34 | model = User 35 | 36 | @factory.post_generation 37 | def password(self, create, extracted, **kwargs): 38 | if not create: 39 | return 40 | return make_password("password") 41 | 42 | @factory.post_generation 43 | def groups(self, create, extracted, **kwargs): 44 | if not create: 45 | # Simple build, do nothing. 46 | return 47 | 48 | if extracted: 49 | # A list of groups were passed in, use them 50 | for group in extracted: 51 | self.groups.add(group) 52 | -------------------------------------------------------------------------------- /atmo/users/migrations/0001_initial_site.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.1 on 2016-09-20 09:12 3 | from urllib.parse import urlparse 4 | 5 | from django.conf import settings 6 | from django.db import migrations 7 | 8 | 9 | def add_site(apps, schema_editor): 10 | Site = apps.get_model("sites", "Site") 11 | db_alias = schema_editor.connection.alias 12 | domain = urlparse(settings.SITE_URL) 13 | Site.objects.using(db_alias).get_or_create( 14 | id=settings.SITE_ID, defaults={"domain": domain.netloc, "name": domain.netloc} 15 | ) 16 | 17 | 18 | def delete_site(apps, schema_editor): 19 | Site = apps.get_model("sites", "Site") 20 | db_alias = schema_editor.connection.alias 21 | Site.objects.using(db_alias).filter(id=settings.SITE_ID).delete() 22 | 23 | 24 | class Migration(migrations.Migration): 25 | 26 | dependencies = [("sites", "0002_alter_domain_unique")] 27 | 28 | operations = [migrations.RunPython(add_site, delete_site)] 29 | -------------------------------------------------------------------------------- /atmo/users/migrations/0002_rewrite_usernames.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.13 on 2017-04-26 09:39 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | def fix_usernames(apps, schema_editor): 9 | pass 10 | 11 | 12 | def revert_usernames(apps, schema_editor): 13 | pass 14 | 15 | 16 | class Migration(migrations.Migration): 17 | 18 | dependencies = [("users", "0001_initial_site")] 19 | 20 | operations = [migrations.RunPython(fix_usernames, revert_usernames)] 21 | -------------------------------------------------------------------------------- /atmo/users/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/atmo/users/migrations/__init__.py -------------------------------------------------------------------------------- /atmo/users/urls.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from django.conf.urls import url 5 | from django.views import generic 6 | 7 | 8 | urlpatterns = [ 9 | url( 10 | r"login/$", 11 | generic.TemplateView.as_view(template_name="atmo/users/login.html"), 12 | name="users-login", 13 | ) 14 | ] 15 | -------------------------------------------------------------------------------- /atmo/users/utils.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | 6 | def generate_username_from_email(email): 7 | """ 8 | Use the unique part of the email as the username for mozilla.com 9 | and the full email address for all other users. 10 | """ 11 | if "@" in email and email.endswith("@mozilla.com"): 12 | return email.split("@")[0] 13 | else: 14 | return email 15 | -------------------------------------------------------------------------------- /atmo/utils.py: -------------------------------------------------------------------------------- 1 | from environ import Env 2 | 3 | cache_url_config = Env.cache_url_config 4 | -------------------------------------------------------------------------------- /atmo/wsgi.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | """ 5 | WSGI config for atmo project. 6 | 7 | It exposes the WSGI callable as a module-level variable named ``application``. 8 | 9 | For more information on this file, see 10 | https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/ 11 | """ 12 | import os 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "atmo.settings") 15 | os.environ.setdefault("DJANGO_CONFIGURATION", "Dev") 16 | 17 | from configurations.wsgi import get_wsgi_application # noqa 18 | 19 | application = get_wsgi_application() 20 | 21 | if "SENTRY_DSN" in os.environ: 22 | from raven.contrib.django.raven_compat.middleware.wsgi import Sentry 23 | 24 | application = Sentry(application) 25 | -------------------------------------------------------------------------------- /bin/build: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | 4 | # create a version.json 5 | [ -z $CIRCLE_TAG ] || printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ 6 | "$CIRCLE_SHA1" \ 7 | "$CIRCLE_TAG" \ 8 | "$CIRCLE_PROJECT_USERNAME" \ 9 | "$CIRCLE_PROJECT_REPONAME" \ 10 | "$CIRCLE_BUILD_URL" \ 11 | > version.json 12 | 13 | echo "Building the docker image with the tag app:build" 14 | docker build -t app:build . 15 | -------------------------------------------------------------------------------- /bin/check_license: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | HEADER="\ 4 | # This Source Code Form is subject to the terms of the Mozilla Public 5 | # License, v. 2.0. If a copy of the MPL was not distributed with this 6 | # file, you can obtain one at http://mozilla.org/MPL/2.0/." 7 | 8 | EXIT=0 9 | FILES=$(find . -name "*.py" | grep -e ^./atmo -e ^./tests --exclude "./atmo/*/migrations/*") 10 | 11 | echo "Checking which files are missing the MPL 2.0 header.." 12 | 13 | for FILE in $FILES; do 14 | BLOCK=$(head -n3 $FILE); 15 | if [ "$BLOCK" != "$HEADER" ]; then 16 | echo "$FILE"; 17 | EXIT=1; 18 | fi 19 | done 20 | 21 | echo "Done." 22 | exit $EXIT 23 | -------------------------------------------------------------------------------- /bin/deploy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | 4 | # default variables 5 | : "${CIRCLE_TAG:=latest}" 6 | 7 | # Usage: retry MAX CMD... 8 | # Retry CMD up to MAX times. If it fails MAX times, returns failure. 9 | # Example: retry 3 docker push "mozilla/telemetry-analysis-service:$TAG" 10 | function retry() { 11 | max=$1 12 | shift 13 | count=1 14 | until "$@"; do 15 | count=$((count + 1)) 16 | if [[ $count -gt $max ]]; then 17 | return 1 18 | fi 19 | echo "$count / $max" 20 | done 21 | return 0 22 | } 23 | 24 | echo "Logging into Docker hub" 25 | retry 3 docker login -u="$DOCKER_USER" -p="$DOCKER_PASS" 26 | 27 | echo "Tagging app:build with $CIRCLE_TAG" 28 | docker tag app:build "$DOCKERHUB_REPO:$CIRCLE_TAG" || 29 | (echo "Couldn't tag app:build as $DOCKERHUB_REPO:$CIRCLE_TAG" && false) 30 | 31 | echo "Pushing tag $CIRCLE_TAG to $DOCKERHUB_REPO" 32 | retry 3 docker push "$DOCKERHUB_REPO:$CIRCLE_TAG" || 33 | (echo "Couldn't push $DOCKERHUB_REPO:$CIRCLE_TAG" && false) 34 | 35 | echo "Pushed $DOCKERHUB_REPO:$TAG" 36 | -------------------------------------------------------------------------------- /bin/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | 4 | # default variables 5 | : "${PORT:=8000}" 6 | : "${SLEEP:=1}" 7 | : "${TRIES:=60}" 8 | : "${MONITOR_PIDFILE:=/app/celerymonitor.pid}" 9 | 10 | usage() { 11 | echo "usage: bin/run web|web-dev|worker|scheduler|test" 12 | exit 1 13 | } 14 | 15 | wait_for() { 16 | tries=0 17 | echo "Waiting for $1 to listen on $2..." 18 | while true; do 19 | [[ $tries -lt $TRIES ]] || return 20 | (echo > /dev/tcp/$1/$2) >/dev/null 2>&1 21 | result= 22 | [[ $? -eq 0 ]] && return 23 | sleep $SLEEP 24 | tries=$((tries + 1)) 25 | done 26 | } 27 | 28 | [ $# -lt 1 ] && usage 29 | 30 | # Only wait for backend services in development 31 | # http://stackoverflow.com/a/13864829 32 | [ ! -z ${DEVELOPMENT+check} ] && wait_for db 5432 && wait_for redis 6379 33 | 34 | case $1 in 35 | web) 36 | newrelic-admin run-python manage.py migrate --noinput 37 | exec newrelic-admin run-program gunicorn atmo.wsgi:application -b 0.0.0.0:${PORT} --workers 4 --access-logfile - 38 | ;; 39 | web-dev) 40 | python manage.py migrate --noinput 41 | exec python manage.py runserver 0.0.0.0:${PORT} 42 | ;; 43 | worker) 44 | exec newrelic-admin run-program celery -A atmo.celery:celery worker -l info -O fair --events 45 | ;; 46 | scheduler) 47 | python manage.py migrate --noinput 48 | exec newrelic-admin run-program celery -A atmo.celery:celery \ 49 | beat -l info --pidfile /tmp/celerybeat.pid 50 | ;; 51 | monitor) 52 | [ -f ${MONITOR_PIDFILE} ] && rm ${MONITOR_PIDFILE} 53 | exec newrelic-admin run-program celery -A atmo.celery:celery \ 54 | events -l info -c django_celery_monitor.camera.Camera --frequency=2.0 \ 55 | --pidfile ${MONITOR_PIDFILE} 56 | ;; 57 | test) 58 | pytest 59 | if [[ ! -z ${CI+check} ]]; then 60 | # submit coverage 61 | bash <(curl -s https://codecov.io/bash) -s /tmp 62 | fi 63 | ;; 64 | *) 65 | exec "$@" 66 | ;; 67 | esac 68 | -------------------------------------------------------------------------------- /bin/test: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | 4 | # copy the .env template to .env if not there already 5 | [ ! -f .env ] && cp .env-dist .env 6 | 7 | # default variables 8 | export DJANGO_CONFIGURATION=Test 9 | 10 | # pass CI env vars into docker containers for codecov submission 11 | [ ! -z ${CI+check} ] && \ 12 | echo "Getting Codecov environment variables" && \ 13 | export CI_ENV=`bash <(curl -s https://codecov.io/env)` 14 | 15 | # run docker compose with the given environment variables 16 | docker-compose run -e DJANGO_CONFIGURATION $CI_ENV web test 17 | -------------------------------------------------------------------------------- /contribute.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "atmo", 3 | "description": "The Telemetry Analysis Service", 4 | "participate": { 5 | "irc": "irc://irc.mozilla.org/#telemetry", 6 | "irc-contacts": [ 7 | "jezdez", 8 | "mdoglio", 9 | "rvitillo" 10 | ] 11 | }, 12 | "repository": { 13 | "url": "https://github.com/mozilla/telemetry-analysis-service/", 14 | "license": "MPL 2.0", 15 | "tests": "https://circleci.com/gh/mozilla/telemetry-analysis-service" 16 | }, 17 | "bugs": { 18 | "list": "https://github.com/mozilla/telemetry-analysis-service/issues", 19 | "report": "https://github.com/mozilla/telemetry-analysis-service/issues/new" 20 | }, 21 | "urls": { 22 | "prod": "https://analysis.telemetry.mozilla.org/", 23 | "staging": "https://atmo.stage.mozaws.net/" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | x-app: &app 4 | build: 5 | context: . 6 | dockerfile: Dockerfile.dev 7 | environment: 8 | - DJANGO_CONFIGURATION 9 | env_file: 10 | - .env 11 | volumes: 12 | - $PWD:/app 13 | links: 14 | - db 15 | - redis 16 | command: web-dev 17 | 18 | services: 19 | db: 20 | image: postgres:9.4 21 | ports: 22 | - "15432:5432" 23 | redis: 24 | image: redis:3.2 25 | 26 | web: 27 | <<: *app 28 | ports: 29 | - "8000:8000" 30 | 31 | worker: 32 | <<: *app 33 | command: worker 34 | 35 | scheduler: 36 | <<: *app 37 | command: scheduler 38 | 39 | monitor: 40 | <<: *app 41 | command: monitor 42 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = python -msphinx 7 | SPHINXPROJ = ATMO 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-Bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-Bold.woff -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-Bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-Bold.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-BoldItalic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-BoldItalic.woff -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-BoldItalic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-BoldItalic.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-Italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-Italic.woff -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-Italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-Italic.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-Light.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-Light.woff -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-Light.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-Light.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-LightItalic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-LightItalic.woff -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-LightItalic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-LightItalic.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-Medium.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-Medium.woff -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-Medium.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-Medium.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-MediumItalic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-MediumItalic.woff -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-MediumItalic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-MediumItalic.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-Regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-Regular.woff -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-Regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-Regular.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-SemiBold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-SemiBold.woff -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-SemiBold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-SemiBold.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-SemiBoldItalic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-SemiBoldItalic.woff -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlab-SemiBoldItalic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlab-SemiBoldItalic.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlabHighlight-Bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlabHighlight-Bold.woff -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlabHighlight-Bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlabHighlight-Bold.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlabHighlight-Regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlabHighlight-Regular.woff -------------------------------------------------------------------------------- /docs/_static/fonts/ZillaSlabHighlight-Regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/ZillaSlabHighlight-Regular.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/open-sans-v13-latin-700.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/open-sans-v13-latin-700.woff -------------------------------------------------------------------------------- /docs/_static/fonts/open-sans-v13-latin-700.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/open-sans-v13-latin-700.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/open-sans-v13-latin-700italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/open-sans-v13-latin-700italic.woff -------------------------------------------------------------------------------- /docs/_static/fonts/open-sans-v13-latin-700italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/open-sans-v13-latin-700italic.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/open-sans-v13-latin-italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/open-sans-v13-latin-italic.woff -------------------------------------------------------------------------------- /docs/_static/fonts/open-sans-v13-latin-italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/open-sans-v13-latin-italic.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/open-sans-v13-latin-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/open-sans-v13-latin-regular.woff -------------------------------------------------------------------------------- /docs/_static/fonts/open-sans-v13-latin-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/open-sans-v13-latin-regular.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/roboto-slab-v6-latin-700.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/roboto-slab-v6-latin-700.woff -------------------------------------------------------------------------------- /docs/_static/fonts/roboto-slab-v6-latin-700.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/roboto-slab-v6-latin-700.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/roboto-slab-v6-latin-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/roboto-slab-v6-latin-regular.woff -------------------------------------------------------------------------------- /docs/_static/fonts/roboto-slab-v6-latin-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/_static/fonts/roboto-slab-v6-latin-regular.woff2 -------------------------------------------------------------------------------- /docs/_static/style.css: -------------------------------------------------------------------------------- 1 | /* 2 | This Source Code Form is subject to the terms of the Mozilla Public 3 | License, v. 2.0. If a copy of the MPL was not distributed with this 4 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | */ 6 | html { 7 | font-size: 100%; 8 | background: #fff; 9 | } 10 | 11 | body { 12 | font-size: 100%; 13 | background: #fff; 14 | border-top: 2px solid #fff; 15 | color: #000; 16 | font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; 17 | line-height: 1.5; 18 | } 19 | 20 | code, pre, tt { 21 | font-family: "Roboto Mono", Monaco, Consolas, "Lucida Console", monospace; 22 | font-size: 0.95rem; 23 | line-height: 1.3rem; 24 | } 25 | 26 | code { 27 | font-weight: bold; 28 | } 29 | 30 | pre { 31 | color: #484848; 32 | background-color: #eee; 33 | } 34 | 35 | 36 | h1, h2, h3, h4, h5, h6, legend { 37 | font-weight: bold; 38 | line-height: 1.1; 39 | margin: 0 0 .25em; 40 | font-family: "Zilla Slab", "Helvetica Neue", Helvetica, Arial, sans-serif; 41 | } 42 | 43 | h1 { 44 | font-size: 2.375rem; 45 | font-family: "Zilla Slab Highlight", "Helvetica Neue", Helvetica, Arial, sans-serif; 46 | } 47 | h2 { font-size: 1.75rem;} 48 | h3 { font-size: 1.3125rem;} 49 | h4 { font-size: 1rem;} 50 | h5 { font-size: 1rem;} 51 | h6 { font-size: 1rem;} 52 | 53 | 54 | a { 55 | color: #00a7e0; 56 | text-decoration: none; 57 | } 58 | 59 | a:visited { 60 | color: #00a7e0; 61 | text-decoration: none; 62 | } 63 | 64 | 65 | strong { 66 | font-weight: bold; 67 | } 68 | 69 | div.bodywrapper { 70 | padding-right: 1rem; 71 | } 72 | 73 | ul.search li div.context { 74 | color: #666; 75 | margin: 2px 0 0 30px; 76 | text-align: left; 77 | } 78 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGELOG.rst 2 | -------------------------------------------------------------------------------- /docs/deployment.rst: -------------------------------------------------------------------------------- 1 | Deployment 2 | ========== 3 | 4 | Releasing ATMO happens by tagging a CalVer_ based Git tag with the following 5 | pattern: 6 | 7 | YYYY.M.N 8 | 9 | ``YYYY`` is the four-digit year number, ``M`` is a single-digit month number 10 | and ``N`` is a single-digit zero-based counter which does **NOT** relate to 11 | the day of the release. Valid versions numbers are: 12 | 13 | - 2017.10.0 14 | 15 | - 2018.1.0 16 | 17 | - 2018.12.12 18 | 19 | - 1970.1.1 20 | 21 | Once the Git tag has been pushed to the main GitHub repository using 22 | ``git push origin --tags``, Circle CI will automatically build a tagged 23 | Docker image after the tests have passed and push it to Docker Hub. 24 | From there the Mozilla CloudOPs team has configured a stage/prod deployment 25 | pipeline. 26 | 27 | Stage deployments happen automatically when a new release is made. 28 | Prod deployments happen on demand by the CloudOPs team. 29 | 30 | .. _CalVer: http://calver.org/ 31 | -------------------------------------------------------------------------------- /docs/extensions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/telemetry-analysis-service/17d79360bd70b9cfb9695a57591c6b8c39e38c6c/docs/extensions/__init__.py -------------------------------------------------------------------------------- /docs/extensions/celery.py: -------------------------------------------------------------------------------- 1 | from celery.contrib.sphinx import TaskDocumenter, TaskDirective 2 | 3 | 4 | class AtmoTaskDocumenter(TaskDocumenter): 5 | """ 6 | A TaskDocument subclass to fix 7 | https://github.com/celery/celery/issues/4072 temporarily. 8 | """ 9 | 10 | def check_module(self): 11 | """Normally checks if *self.object* is really defined in the module 12 | given by *self.modname*. But since functions decorated with the @task 13 | decorator are instances living in the celery.local module we're 14 | checking for that and simply agree to document those then. 15 | """ 16 | modname = self.get_attr(self.object, "__module__", None) 17 | if modname and modname == "celery.local": 18 | return True 19 | return super(TaskDocumenter, self).check_module() 20 | 21 | 22 | def setup(app): 23 | """Setup Sphinx extension.""" 24 | app.add_autodocumenter(AtmoTaskDocumenter) 25 | app.add_directive_to_domain("py", "task", TaskDirective) 26 | app.add_config_value("celery_task_prefix", "(task)", True) 27 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Mozilla ATMO 2 | ============ 3 | 4 | Welcome to the documentation of **ATMO**, the code that runs Mozilla_'s 5 | `Telementry Analysis Service`_. 6 | 7 | ATMO is a self-service portal to launch on-demand `AWS EMR`_ clusters with 8 | `Apache Spark`_, `Apache Zeppelin`_ and Jupyter_ installed. 9 | Additionally it allows to schedule Spark jobs to run regularly based on 10 | uploaded Jupyter (and soon Zeppelin) notebooks. 11 | 12 | It provides a management UI for public SSH keys when launching on-demand 13 | clusters, login via Google auth and flexible adminstration interfaces for 14 | users and admins. 15 | 16 | Behind the scenes it's shipped as Docker_ images and uses Python_ 3.6 for 17 | the web UI (Django_) and the task management (Celery_). 18 | 19 | .. _`Mozilla`: https://www.mozilla.org/ 20 | .. _`Telementry Analysis Service`: https://analysis.telemetry.mozilla.org/ 21 | .. _`AWS EMR`: https://aws.amazon.com/emr/ 22 | .. _`Apache Spark`: https://spark.apache.org/ 23 | .. _`Apache Zeppelin`: https://zeppelin.apache.org/ 24 | .. _`Jupyter`: https://jupyter.org/ 25 | .. _Docker: https://www.docker.com/ 26 | .. _Python: https://python.org/ 27 | .. _Django: https://www.djangoproject.com/ 28 | .. _Celery: https://www.celeryproject.org/ 29 | 30 | .. toctree:: 31 | :maxdepth: 2 32 | :caption: Contents: 33 | 34 | overview 35 | workflows 36 | maintenance 37 | development 38 | deployment 39 | reference/index 40 | changelog 41 | 42 | Indices and tables 43 | ================== 44 | 45 | * :ref:`genindex` 46 | * :ref:`modindex` 47 | * :ref:`search` 48 | -------------------------------------------------------------------------------- /docs/overview.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Overview 3 | ======== 4 | 5 | This is a quick overview of how ATMO works, both from the perspective of 6 | code structure as well as an architecture overview. 7 | -------------------------------------------------------------------------------- /docs/reference/atmo.clusters.rst: -------------------------------------------------------------------------------- 1 | atmo.clusters 2 | ============= 3 | 4 | .. module:: atmo.clusters 5 | 6 | The code base to manage AWS EMR clusters. 7 | 8 | atmo.clusters.forms 9 | ------------------- 10 | 11 | .. automodule:: atmo.clusters.forms 12 | :members: 13 | 14 | atmo.clusters.models 15 | -------------------- 16 | 17 | .. automodule:: atmo.clusters.models 18 | :members: 19 | 20 | atmo.clusters.provisioners 21 | -------------------------- 22 | 23 | .. automodule:: atmo.clusters.provisioners 24 | :members: 25 | 26 | atmo.clusters.queries 27 | --------------------- 28 | 29 | .. automodule:: atmo.clusters.queries 30 | :members: 31 | :undoc-members: 32 | 33 | atmo.clusters.tasks 34 | ------------------- 35 | 36 | .. automodule:: atmo.clusters.tasks 37 | :members: 38 | :undoc-members: 39 | 40 | atmo.clusters.views 41 | ------------------- 42 | 43 | .. automodule:: atmo.clusters.views 44 | :members: 45 | :undoc-members: 46 | -------------------------------------------------------------------------------- /docs/reference/atmo.jobs.rst: -------------------------------------------------------------------------------- 1 | atmo.jobs 2 | ========= 3 | 4 | .. module:: atmo.jobs 5 | 6 | The code base to manage scheduled Spark job via AWS EMR clusters. 7 | 8 | atmo.jobs.forms 9 | --------------- 10 | 11 | .. automodule:: atmo.jobs.forms 12 | :members: 13 | 14 | atmo.jobs.models 15 | ---------------- 16 | 17 | .. automodule:: atmo.jobs.models 18 | :members: 19 | 20 | atmo.jobs.provisioners 21 | ---------------------- 22 | 23 | .. automodule:: atmo.jobs.provisioners 24 | :members: 25 | 26 | atmo.jobs.queries 27 | ----------------- 28 | 29 | .. automodule:: atmo.jobs.queries 30 | :members: 31 | 32 | atmo.jobs.tasks 33 | --------------- 34 | 35 | .. automodule:: atmo.jobs.tasks 36 | :members: 37 | 38 | atmo.jobs.views 39 | --------------- 40 | 41 | .. automodule:: atmo.jobs.views 42 | :members: 43 | -------------------------------------------------------------------------------- /docs/reference/atmo.keys.rst: -------------------------------------------------------------------------------- 1 | atmo.keys 2 | ========= 3 | 4 | .. module:: atmo.keys 5 | 6 | The code base to manage public SSH keys to be used with 7 | :mod:`ATMO clusters `. 8 | 9 | atmo.keys.forms 10 | --------------- 11 | 12 | .. automodule:: atmo.keys.forms 13 | :members: 14 | 15 | atmo.keys.models 16 | ---------------- 17 | 18 | .. automodule:: atmo.keys.models 19 | :members: 20 | 21 | .. autoattribute:: atmo.keys.models.SSHKey.VALID_PREFIXES 22 | 23 | atmo.keys.utils 24 | --------------- 25 | 26 | .. automodule:: atmo.keys.utils 27 | :members: 28 | 29 | atmo.keys.views 30 | --------------- 31 | 32 | .. automodule:: atmo.keys.views 33 | :members: 34 | -------------------------------------------------------------------------------- /docs/reference/atmo.news.rst: -------------------------------------------------------------------------------- 1 | atmo.news 2 | ========= 3 | 4 | .. module:: atmo.news 5 | 6 | The code base to show the "News" section to users. 7 | 8 | atmo.news.views 9 | --------------- 10 | 11 | .. automodule:: atmo.news.views 12 | :members: 13 | -------------------------------------------------------------------------------- /docs/reference/atmo.rst: -------------------------------------------------------------------------------- 1 | atmo 2 | ==== 3 | 4 | These are the submodules of the ``atmo`` package that don't quite fit 5 | "topics", like the :mod:`atmo.clusters`, :mod:`atmo.jobs` and 6 | :mod:`atmo.users` packages. 7 | 8 | atmo.celery 9 | ----------- 10 | 11 | .. automodule:: atmo.celery 12 | :members: 13 | 14 | atmo.context_processors 15 | ----------------------- 16 | 17 | .. automodule:: atmo.context_processors 18 | :members: 19 | 20 | atmo.decorators 21 | --------------- 22 | 23 | .. automodule:: atmo.decorators 24 | :members: 25 | 26 | atmo.models 27 | ----------- 28 | 29 | .. automodule:: atmo.models 30 | :members: 31 | 32 | atmo.names 33 | ---------- 34 | 35 | .. automodule:: atmo.names 36 | :members: 37 | 38 | atmo.provisioners 39 | ----------------- 40 | 41 | .. automodule:: atmo.provisioners 42 | :members: 43 | 44 | atmo.tasks 45 | ---------- 46 | 47 | .. automodule:: atmo.tasks 48 | :members: 49 | :undoc-members: 50 | 51 | atmo.templatetags 52 | ----------------- 53 | 54 | .. automodule:: atmo.templatetags 55 | :members: 56 | 57 | atmo.views 58 | ---------- 59 | 60 | .. automodule:: atmo.views 61 | :members: 62 | -------------------------------------------------------------------------------- /docs/reference/atmo.settings.rst: -------------------------------------------------------------------------------- 1 | atmo.settings 2 | ============= 3 | 4 | .. automodule:: atmo.settings 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/reference/atmo.users.rst: -------------------------------------------------------------------------------- 1 | atmo.users 2 | ========== 3 | 4 | .. module:: atmo.users 5 | 6 | The code base to handle user sign ups and logins. 7 | 8 | atmo.users.utils 9 | ---------------- 10 | 11 | .. automodule:: atmo.users.utils 12 | :members: 13 | -------------------------------------------------------------------------------- /docs/reference/index.rst: -------------------------------------------------------------------------------- 1 | Reference 2 | ========= 3 | 4 | Here you'll find the automated code documentation for the 5 | ATMO code: 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | :caption: Modules: 10 | 11 | atmo 12 | atmo.clusters 13 | atmo.jobs 14 | atmo.keys 15 | atmo.news 16 | atmo.settings 17 | atmo.users 18 | -------------------------------------------------------------------------------- /docs/spelling_wordlist.txt: -------------------------------------------------------------------------------- 1 | adminstration 2 | atmo 3 | ATMO 4 | autodiscovers 5 | Boto 6 | datetimes 7 | denormalizes 8 | dev 9 | Django 10 | dss 11 | ecdsa 12 | Emr 13 | frontend 14 | google 15 | Hadoop 16 | Heroku 17 | Http 18 | infos 19 | jittered 20 | jobflow 21 | Jobflow 22 | Jupyter 23 | kb 24 | login 25 | logins 26 | nistp 27 | OAuth 28 | openid 29 | provisioners 30 | queryset 31 | radioset 32 | rsa 33 | sha 34 | signup 35 | Subclasses 36 | Telementry 37 | templatetags 38 | timestamps 39 | tracebacks 40 | uniqely 41 | username 42 | utils 43 | -------------------------------------------------------------------------------- /docs/workflows.rst: -------------------------------------------------------------------------------- 1 | Workflows 2 | ========= 3 | 4 | There are a few workflows in the ATMO code base that are of interest and 5 | decide how it works: 6 | 7 | - Adding SSH keys 8 | 9 | - Creating an on-demand cluster 10 | 11 | - Scheduling a Spark job 12 | 13 | 14 | Adding SSH keys 15 | --------------- 16 | 17 | 18 | 19 | Creating an on-demand cluster 20 | ----------------------------- 21 | 22 | 23 | Scheduling a Spark job 24 | ---------------------- 25 | 26 | .. graphviz:: 27 | 28 | digraph runjob { 29 | job [label="Run job task"]; 30 | getrun [shape=diamond, label="Get run"]; 31 | hasrun [shape=diamond, label="Has Run?"]; 32 | logandreturn [shape=box, label="Log and return"]; 33 | isenabled [shape=diamond, label="Is enabled?"]; 34 | sync [shape=box, label="Sync run"]; 35 | isrunnable [shape=diamond, label="Is runnable?"]; 36 | hastimedout [shape=diamond, label="Timed out?"]; 37 | isdue [shape=diamond, label="Is due?"]; 38 | retryin10mins [shape=box, label="Retry in 10 mins"]; 39 | notifyowner [shape=box, label="Notify owner" ]; 40 | terminatejob [shape=box, label="Terminate last run" ]; 41 | unschedule_and_expire [shape=box, label="Unschedule and expire" ]; 42 | provisioncluster [shape=box, label="Provision cluster" ]; 43 | 44 | job -> getrun; 45 | getrun -> hasrun [label="FOUND"]; 46 | getrun -> logandreturn [label="NOT FOUND"]; 47 | hasrun -> isenabled [label="NO"]; 48 | hasrun -> sync [label="YES"]; 49 | sync -> isenabled; 50 | isenabled -> logandreturn [label="NO"]; 51 | isenabled -> isrunnable [label="YES"]; 52 | isrunnable -> hastimedout [label="NO"]; 53 | isrunnable -> isdue [label="YES"]; 54 | hastimedout -> retryin10mins [ label="NO" ]; 55 | hastimedout -> notifyowner [ label="YES" ]; 56 | notifyowner -> terminatejob [ label="Job ABC timed out too early..."]; 57 | isdue -> unschedule_and_expire [ label="NO" ]; 58 | isdue -> provisioncluster [ label="YES" ]; 59 | } 60 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "atmo.settings") 7 | os.environ.setdefault("DJANGO_CONFIGURATION", "Dev") 8 | 9 | from configurations.management import execute_from_command_line 10 | 11 | execute_from_command_line(sys.argv) 12 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "atmo", 3 | "description": "Telemetry Analysis Service", 4 | "private": true, 5 | "repository": { 6 | "type": "git", 7 | "url": "git://github.com/mozilla/telemetry-analysis-service.git" 8 | }, 9 | "license": "MPL-2.0", 10 | "dependencies": { 11 | "ansi_up": "3.0.0", 12 | "bootstrap": "3.4.0", 13 | "bootstrap-confirmation2": "2.4.2", 14 | "bootstrap-datetime-picker": "2.4.4", 15 | "clipboard": "2.0.4", 16 | "jquery": "3.3.1", 17 | "marked": "0.6.0", 18 | "moment": "2.23.0", 19 | "moment-timezone": "0.5.23", 20 | "notebookjs": "0.3.2", 21 | "parsleyjs": "2.8.1", 22 | "prismjs": "1.15.0", 23 | "raven-js": "3.27.0", 24 | "remarkable": "1.7.1" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # Example configuration for Black. 2 | 3 | # NOTE: you have to use single-quoted strings in TOML for regular expressions. 4 | # It's the equivalent of r-strings in Python. Multiline strings are treated as 5 | # verbose regular expressions by Black. Use [ ] to denote a significant space 6 | # character. 7 | 8 | [tool.black] 9 | line-length = 88 10 | py36 = true 11 | include = ''' 12 | /( 13 | | atmo 14 | | docs 15 | | tests 16 | )/ 17 | ''' 18 | exclude = ''' 19 | /( 20 | | \.local 21 | | node_modules 22 | )/ 23 | ''' 24 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | norecursedirs = .git .* static 3 | addopts = -rsxX --showlocals --tb=native --nomigrations --flake8 --staticfiles --cov-report term --cov-report xml --cov atmo --black -p no:cacheprovider 4 | DJANGO_SETTINGS_MODULE = atmo.settings 5 | DJANGO_CONFIGURATION = Test 6 | blockade = True 7 | ; blockade-http-whitelist = accounts.google.com 8 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base" 4 | ], 5 | "labels": [ 6 | "update" 7 | ], 8 | "docker": { 9 | "enabled": false 10 | }, 11 | "python": { 12 | "enabled": false 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /requirements/all.txt: -------------------------------------------------------------------------------- 1 | -r build.txt 2 | -r docs.txt 3 | -r tests.txt 4 | -------------------------------------------------------------------------------- /runtime.txt: -------------------------------------------------------------------------------- 1 | python-3.6.1 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length=100 3 | exclude= 4 | # ignore the migrations since they are created faulty by default 5 | atmo/*/migrations/*, 6 | # No use in checking the Node modules 7 | node_modules/*/*/*, 8 | # No need to traverse our git directory 9 | .git, 10 | # There's no value in checking cache directories 11 | __pycache__, 12 | atmo/names.py, 13 | 14 | # ignore spaces around keyword arguments and dict entries, 15 | # which are very useful for alignment 16 | ignore=E201,E221,E251,E241,E501,W503 17 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="mozilla-atmo", 5 | use_scm_version=True, 6 | setup_requires=["setuptools_scm"], 7 | packages=find_packages(exclude=["tests", "tests/*"]), 8 | description="The code of the Telemetry Analysis Service", 9 | author="Mozilla Foundation", 10 | author_email="telemetry-analysis-service@mozilla.org", 11 | url="https://github.com/mozilla/telemetry-analysis-service", 12 | license="MPL 2.0", 13 | classifiers=[ 14 | "Development Status :: 5 - Production/Stable", 15 | "Environment :: Web Environment :: Mozilla", 16 | "Framework :: Django", 17 | "Intended Audience :: Developers", 18 | "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", 19 | "Programming Language :: Python", 20 | "Programming Language :: Python :: 3", 21 | "Programming Language :: Python :: 3.6", 22 | "Topic :: Internet :: WWW/HTTP", 23 | "Topic :: Scientific/Engineering :: Information Analysis", 24 | ], 25 | zip_safe=False, 26 | ) 27 | -------------------------------------------------------------------------------- /tests/clusters/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /tests/clusters/test_admin.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from atmo.clusters.admin import deactivate as deactivate_action 5 | from atmo.clusters.models import Cluster 6 | 7 | 8 | def test_deactivate_action(mocker, cluster_factory): 9 | deactivate_method = mocker.patch("atmo.clusters.models.Cluster.deactivate") 10 | cluster_factory.create_batch(5) 11 | deactivate_action(None, None, Cluster.objects.all()) 12 | assert deactivate_method.call_count == 5 13 | -------------------------------------------------------------------------------- /tests/clusters/test_forms.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from atmo.clusters.forms import EMRReleaseChoiceField 5 | 6 | 7 | def test_emr_release_choice_field(emr_release_factory): 8 | def make_label(label, text): 9 | return '%s' % (label, text) 10 | 11 | regular = emr_release_factory() 12 | inactive = emr_release_factory(is_active=False) 13 | deprecated = emr_release_factory(is_deprecated=True) 14 | experimental = emr_release_factory(is_experimental=True) 15 | 16 | choice_field = EMRReleaseChoiceField() 17 | result = choice_field.widget.render("test", regular.pk) 18 | assert inactive.version not in result 19 | assert "%s" % regular.version in result 20 | assert ( 21 | "%s %s" % (deprecated.version, make_label("warning", "deprecated")) 22 | in result 23 | ) 24 | assert ( 25 | "%s %s" % (experimental.version, make_label("info", "experimental")) 26 | in result 27 | ) 28 | -------------------------------------------------------------------------------- /tests/jobs/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /tests/jobs/test_admin.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from atmo.jobs.admin import run_now 5 | from atmo.jobs.models import SparkJob 6 | 7 | 8 | def test_run_now_action(mocker, spark_job_factory): 9 | run = mocker.patch("atmo.jobs.models.SparkJob.run") 10 | spark_job_factory.create_batch(5) 11 | run_now(None, None, SparkJob.objects.all()) 12 | assert run.call_count == 5 13 | -------------------------------------------------------------------------------- /tests/jobs/test_forms.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from atmo.jobs.forms import NewSparkJobForm 5 | 6 | 7 | BASE_DATA = { 8 | # Replace these `None` values with pytest fixtures. 9 | "new-notebook": None, 10 | "new-emr_release": None, 11 | "new-identifier": "test-spark-job", 12 | "new-description": "A description", 13 | "new-notebook-cache": "some-random-hash", 14 | "new-result_visibility": "private", 15 | "new-size": 5, 16 | "new-interval_in_hours": 24, 17 | "new-job_timeout": 12, 18 | "new-start_date": "2016-04-05 13:25:47", 19 | } 20 | 21 | 22 | def test_new_sparkjob_form(user, emr_release, notebook_maker): 23 | 24 | data = BASE_DATA.copy() 25 | data.update({"new-emr_release": emr_release.version}) 26 | file_data = {"new-notebook": notebook_maker()} 27 | form = NewSparkJobForm(user, data, file_data) 28 | assert form.is_valid(), form.errors 29 | 30 | file_data = {"new-notebook": notebook_maker(extension="json")} 31 | form = NewSparkJobForm(user, data, file_data) 32 | assert form.is_valid(), form.errors 33 | 34 | 35 | def test_new_sparkjob_form_bad_notebook_extension(user, emr_release, notebook_maker): 36 | data = BASE_DATA.copy() 37 | data.update({"new-emr_release": emr_release.version}) 38 | file_data = {"new-notebook": notebook_maker(extension="foo")} 39 | form = NewSparkJobForm(user, data, file_data) 40 | 41 | assert not form.is_valid() 42 | assert "notebook" in form.errors 43 | -------------------------------------------------------------------------------- /tests/jobs/test_schedules.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from datetime import timedelta 5 | 6 | from redbeat.schedulers import RedBeatSchedulerEntry 7 | 8 | from atmo.jobs.schedules import SparkJobSchedule 9 | 10 | 11 | def test_init(spark_job): 12 | assert isinstance(spark_job.schedule, SparkJobSchedule) 13 | assert spark_job.schedule.spark_job == spark_job 14 | assert isinstance(spark_job.schedule.run_every, timedelta) 15 | assert spark_job.schedule.name == "atmo.jobs.tasks.run_job:%s" % spark_job.pk 16 | 17 | 18 | def test_added_on_save(spark_job_factory, user, emr_release): 19 | spark_job = spark_job_factory.build(created_by=user, emr_release=emr_release) 20 | assert spark_job.schedule.get() is None 21 | spark_job.save() 22 | assert isinstance(spark_job.schedule.get(), RedBeatSchedulerEntry) 23 | 24 | 25 | def test_get(spark_job): 26 | assert isinstance(spark_job.schedule.get(), RedBeatSchedulerEntry) 27 | assert repr(spark_job.schedule.get()) == repr(spark_job.schedule.create()) 28 | 29 | 30 | def test_add(mocker, spark_job): 31 | # entry exists due to save() call 32 | spark_job.schedule.delete() 33 | assert spark_job.schedule.get() is None 34 | mocker.spy(SparkJobSchedule, "create") 35 | returned_entry = spark_job.schedule.add() 36 | assert spark_job.schedule.create.call_count == 1 37 | fetched_entry = spark_job.schedule.get() 38 | assert isinstance(fetched_entry, RedBeatSchedulerEntry) 39 | assert repr(returned_entry) == repr(fetched_entry) 40 | 41 | 42 | def test_delete(spark_job): 43 | # entry exists due to save() call 44 | deleted = spark_job.schedule.delete() 45 | assert deleted 46 | deleted = spark_job.schedule.delete() 47 | assert not deleted 48 | assert spark_job.schedule.get() is None 49 | -------------------------------------------------------------------------------- /tests/test_database.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from io import StringIO 5 | 6 | import pytest 7 | from django.core.management import call_command 8 | 9 | 10 | def test_for_missing_migrations(): 11 | output = StringIO() 12 | try: 13 | call_command("makemigrations", interactive=False, check=True, stdout=output) 14 | except SystemExit as exc: 15 | # The exit code will be 0 when there are no missing migrations 16 | assert exc.code == 1 17 | pytest.fail("There are missing migrations:\n %s" % output.getvalue()) 18 | -------------------------------------------------------------------------------- /tests/test_names.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from atmo import names 3 | 4 | 5 | @pytest.mark.parametrize("separator", ["-", "_"]) 6 | def test_names(separator): 7 | name = names.random_scientist(separator=separator) 8 | adjective, noun, suffix = name.split(separator) 9 | assert adjective in names.adjectives 10 | assert noun in names.scientists 11 | assert len(suffix) == 4 12 | assert int(suffix) 13 | -------------------------------------------------------------------------------- /tests/test_stats.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from atmo.stats.models import Metric 5 | 6 | 7 | def test_metrics_record(now, one_hour_ago): 8 | Metric.record("metric-key-1") 9 | Metric.record("metric-key-2", 500) 10 | Metric.record("metric-key-3", data={"other-value": "test"}) 11 | Metric.record("metric-key-4", created_at=one_hour_ago, data={"other-value-2": 100}) 12 | 13 | m = Metric.objects.get(key="metric-key-1") 14 | assert m.value == 1 15 | assert m.created_at.replace(microsecond=0) == now 16 | assert m.data is None 17 | 18 | m = Metric.objects.get(key="metric-key-2") 19 | assert m.value == 500 20 | assert m.created_at.replace(microsecond=0) == now 21 | assert m.data is None 22 | 23 | m = Metric.objects.get(key="metric-key-3") 24 | assert m.value == 1 25 | assert m.created_at.replace(microsecond=0) == now 26 | assert m.data == {"other-value": "test"} 27 | 28 | m = Metric.objects.get(key="metric-key-4") 29 | assert m.value == 1 30 | assert m.created_at.replace(microsecond=0) == one_hour_ago 31 | assert m.data == {"other-value-2": 100} 32 | -------------------------------------------------------------------------------- /tests/test_templatetags.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from atmo.clusters.models import Cluster 5 | from atmo.jobs.templatetags.notebook import is_jupyter_notebook, is_zeppelin_notebook 6 | from atmo.jobs.templatetags.status import status_color, status_icon 7 | from atmo.templatetags import full_url, markdown, url_update 8 | 9 | 10 | def test_is_notebook(): 11 | assert is_jupyter_notebook("test.ipynb") 12 | assert is_zeppelin_notebook("test.json") 13 | assert not is_jupyter_notebook("test.txt") 14 | assert not is_zeppelin_notebook("test.txt") 15 | 16 | 17 | def test_url_update(): 18 | url = "/test/?foo=bar" 19 | 20 | assert url_update(url) == url 21 | assert url_update(url, fizz="buzz") == "/test/?foo=bar&fizz=buzz" 22 | 23 | 24 | def test_get_full_url(): 25 | assert full_url("/test/") == "http://localhost:8000/test/" 26 | 27 | 28 | def test_status_icon(): 29 | for status in Cluster.ACTIVE_STATUS_LIST: 30 | assert status_icon(status) == "glyphicon-play" 31 | for status in Cluster.TERMINATED_STATUS_LIST: 32 | assert status_icon(status) == "glyphicon-stop" 33 | for status in Cluster.FAILED_STATUS_LIST: 34 | assert status_icon(status) == "glyphicon-exclamation-sign" 35 | 36 | 37 | def test_status_color(): 38 | for status in Cluster.ACTIVE_STATUS_LIST: 39 | assert status_color(status) == "status-running" 40 | for status in Cluster.TERMINATED_STATUS_LIST: 41 | assert status_color(status) is None 42 | for status in Cluster.FAILED_STATUS_LIST: 43 | assert status_color(status) == "status-errors" 44 | 45 | 46 | def test_markdown(): 47 | assert markdown("**test**") == "

test

\n" 48 | -------------------------------------------------------------------------------- /tests/test_users.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/. 4 | from urllib.parse import urlparse 5 | from django.contrib.sites.models import Site 6 | 7 | 8 | def test_initial_site(settings): 9 | "Test if the site migration uses the env var SITE_URL" 10 | domain = urlparse(settings.SITE_URL) 11 | assert Site.objects.get_current().domain, domain.netloc 12 | --------------------------------------------------------------------------------