├── kuma ├── api │ ├── __init__.py │ ├── v1 │ │ ├── __init__.py │ │ ├── plus │ │ │ ├── __init__.py │ │ │ └── landing_page.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── test_subscriptions.py │ │ │ ├── test_landing_page.py │ │ │ ├── test_watched_items.py │ │ │ ├── test_forms.py │ │ │ ├── test_views.py │ │ │ └── test_search.py │ │ ├── forms.py │ │ ├── urls.py │ │ ├── api.py │ │ ├── decorators.py │ │ ├── pagination.py │ │ ├── auth.py │ │ ├── search │ │ │ └── forms.py │ │ └── views.py │ ├── tests │ │ ├── __init__.py │ │ └── test_admin.py │ ├── urls.py │ ├── admin_urls.py │ └── apps.py ├── core │ ├── __init__.py │ ├── exceptions.py │ ├── tests │ │ ├── logging_urls.py │ │ ├── test_settings.py │ │ ├── test_views.py │ │ ├── test_validators.py │ │ ├── __init__.py │ │ ├── test_decorators.py │ │ └── test_utils.py │ ├── views.py │ ├── admin.py │ ├── apps.py │ ├── tasks.py │ ├── decorators.py │ └── urlresolvers.py ├── health │ ├── __init__.py │ ├── tests │ │ └── __init__.py │ └── urls.py ├── users │ ├── __init__.py │ ├── migrations │ │ ├── __init__.py │ │ ├── 0004_remove_userprofile_fxa_uid.py │ │ ├── 0005_userprofile_is_subscriber.py │ │ ├── 0002_auto_20210922_1159.py │ │ ├── 0003_auto_20211014_1214.py │ │ ├── 0007_userprofile_subscription_type.py │ │ ├── 0001_initial.py │ │ └── 0006_accountevent.py │ ├── apps.py │ ├── utils.py │ ├── tests │ │ ├── test_auth.py │ │ ├── test_views.py │ │ ├── test_checks.py │ │ └── test_tasks.py │ ├── middleware.py │ ├── models.py │ ├── tasks.py │ └── checks.py ├── bookmarks │ ├── __init__.py │ ├── migrations │ │ ├── __init__.py │ │ ├── 0002_remove_bookmark_notes.py │ │ ├── 0003_auto_20220118_0409.py │ │ └── 0001_initial.py │ ├── apps.py │ └── models.py ├── settings │ ├── __init__.py │ ├── local.py │ ├── prod.py │ └── pytest.py ├── version │ ├── __init__.py │ ├── tests │ │ ├── __init__.py │ │ └── test_views.py │ ├── urls.py │ └── views.py ├── documenturls │ ├── __init__.py │ ├── migrations │ │ ├── __init__.py │ │ └── 0001_initial.py │ ├── apps.py │ ├── admin.py │ └── models.py ├── notifications │ ├── __init__.py │ ├── migrations │ │ ├── __init__.py │ │ ├── 0004_notificationdata_data.py │ │ ├── 0010_notification_deleted.py │ │ ├── 0003_watch_title.py │ │ ├── 0009_notificationdata_page_url.py │ │ ├── 0011_auto_20220210_0921.py │ │ ├── 0007_watch_users.py │ │ ├── 0002_auto_20211105_1529.py │ │ ├── 0006_auto_20220105_0138.py │ │ ├── 0005_auto_20211215_2004.py │ │ ├── 0008_default_watch.py │ │ └── 0001_initial.py │ ├── apps.py │ ├── admin.py │ ├── management │ │ └── commands │ │ │ └── extract_notifications.py │ ├── browsers.py │ └── models.py ├── plus │ ├── migrations │ │ ├── __init__.py │ │ ├── 0002_auto_20210723_1901.py │ │ └── 0001_initial.py │ ├── apps.py │ ├── models.py │ └── admin.py ├── attachments │ ├── tests │ │ ├── __init__.py │ │ ├── conftest.py │ │ └── test_views.py │ ├── __init__.py │ ├── apps.py │ ├── urls.py │ ├── utils.py │ └── views.py ├── __init__.py ├── wsgi.py ├── celery.py ├── urls.py └── conftest.py ├── Jenkinsfiles ├── prod-push.yml ├── stage-push.yml ├── standby-push.yml ├── default.groovy ├── main.groovy └── push.groovy ├── tmp └── emails │ └── .keep ├── docs ├── images │ └── rendering.png ├── elasticsearch.rst ├── index.rst ├── requirements.txt ├── docker.rst ├── documentation.rst ├── troubleshooting.rst └── tests.rst ├── .gitmodules ├── .github ├── merge-when-green.yml ├── dependabot.yml ├── ISSUE_TEMPLATE │ ├── support-request.md │ ├── bug_report.md │ ├── content-issue.md │ ├── change-request.md │ └── feature_request.md └── workflows │ ├── documentation-build.yml │ ├── python-lints.yml │ └── docker.yml ├── pytest.ini ├── docker ├── docker-mysql.repo └── images │ ├── kuma │ ├── Dockerfile │ └── README.rst │ └── kuma_base │ ├── README.rst │ └── Dockerfile ├── scripts ├── ci-codecovsubmit ├── ci-localerefresh ├── start-worker ├── compile-mo.sh ├── ci-python └── slack-notify.sh ├── .flake8 ├── templates └── admin │ └── search │ └── index │ └── change_list.html ├── .editorconfig ├── jinja2 └── includes │ └── translate_locales.html ├── SECURITY.md ├── renovate.json ├── manage.py ├── CODE_OF_CONDUCT.md ├── contribute.json ├── Jenkinsfile ├── .env-dist.dev ├── .gitignore ├── .dockerignore ├── pyproject.toml ├── Makefile ├── README.rst └── docker-compose.yml /kuma/api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/api/v1/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/health/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/users/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/api/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/api/v1/plus/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/bookmarks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/settings/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/version/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/api/v1/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/documenturls/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/health/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/notifications/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/plus/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/users/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/version/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/attachments/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/bookmarks/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/documenturls/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kuma/notifications/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Jenkinsfiles/prod-push.yml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | enabled: true 3 | script: "push" 4 | -------------------------------------------------------------------------------- /Jenkinsfiles/stage-push.yml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | enabled: true 3 | script: "push" 4 | -------------------------------------------------------------------------------- /Jenkinsfiles/standby-push.yml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | enabled: true 3 | script: "push" 4 | -------------------------------------------------------------------------------- /tmp/emails/.keep: -------------------------------------------------------------------------------- 1 | This folder contains emails generated by the development environment 2 | -------------------------------------------------------------------------------- /docs/images/rendering.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/kuma/main/docs/images/rendering.png -------------------------------------------------------------------------------- /kuma/attachments/__init__.py: -------------------------------------------------------------------------------- 1 | """kuma.attachments handles user-uploaded page attachments.""" 2 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "locale"] 2 | path = locale 3 | url = https://github.com/mozilla-l10n/mdn-l10n.git 4 | -------------------------------------------------------------------------------- /kuma/api/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import include, re_path 2 | 3 | urlpatterns = [ 4 | re_path("^v1/", include("kuma.api.v1.urls")), 5 | ] 6 | -------------------------------------------------------------------------------- /kuma/plus/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class PlusConfig(AppConfig): 5 | name = "kuma.plus" 6 | verbose_name = "Plus" 7 | -------------------------------------------------------------------------------- /.github/merge-when-green.yml: -------------------------------------------------------------------------------- 1 | # From https://github.com/phstc/probot-merge-when-green#configuration 2 | mergeMethod: squash 3 | requireApprovalFromRequestedReviewers: true 4 | -------------------------------------------------------------------------------- /kuma/bookmarks/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class BookmarksConfig(AppConfig): 5 | name = "kuma.bookmarks" 6 | verbose_name = "Bookmarks" 7 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = -rsxX --tb=native 3 | testpaths = kuma 4 | python_files = test*.py 5 | DJANGO_SETTINGS_MODULE = kuma.settings.pytest 6 | filterwarnings = default 7 | -------------------------------------------------------------------------------- /docker/docker-mysql.repo: -------------------------------------------------------------------------------- 1 | [mysql-community-releases] 2 | name=Mysql Community Packages 3 | baseurl=http://repo.mysql.com/yum/mysql-community/el/6/$basearch/ 4 | gpgcheck=0 5 | enabled=1 6 | -------------------------------------------------------------------------------- /kuma/documenturls/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class DocumentURLsConfig(AppConfig): 5 | name = "kuma.documenturls" 6 | verbose_name = "Document URLs" 7 | -------------------------------------------------------------------------------- /kuma/__init__.py: -------------------------------------------------------------------------------- 1 | # This will make sure the app is always imported when 2 | # Django starts so that shared_task will use this app. 3 | from .celery import app as celery_app 4 | 5 | __all__ = ("celery_app",) 6 | -------------------------------------------------------------------------------- /kuma/notifications/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class NotificationsConfig(AppConfig): 5 | default_auto_field = "django.db.models.BigAutoField" 6 | name = "kuma.notifications" 7 | -------------------------------------------------------------------------------- /kuma/version/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import re_path 2 | 3 | from . import views 4 | 5 | urlpatterns = [ 6 | # Serve the revision hashes. 7 | re_path(r"^media/revision.txt$", views.revision_hash, name="version.kuma"), 8 | ] 9 | -------------------------------------------------------------------------------- /kuma/api/admin_urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | 3 | from kuma.api.v1.api import admin_api 4 | from kuma.api.v1.plus.notifications import admin_router 5 | 6 | admin_api.add_router("/", admin_router) 7 | 8 | urlpatterns = [ 9 | path("", admin_api.urls), 10 | ] 11 | -------------------------------------------------------------------------------- /scripts/ci-codecovsubmit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e # Exit on non-zero status 3 | set -u # Treat unset variables as an error 4 | 5 | # FYI, as of April 2020, you get version 7.47.0 of curl 6 | # and `--retry-connrefused` was added in curl 7.52.0 :( 7 | 8 | bash <(curl -s --retry 3 https://codecov.io/bash) 9 | -------------------------------------------------------------------------------- /docs/elasticsearch.rst: -------------------------------------------------------------------------------- 1 | ====== 2 | Search 3 | ====== 4 | Kuma uses Elasticsearch_ to power its on-site search. 5 | 6 | .. _Elasticsearch: https://www.elastic.co/elasticsearch/ 7 | 8 | .. _indexing-documents: 9 | 10 | Indexing documents 11 | ================== 12 | Indexing is done outside Kuma. It's done by the Deployer in Yari. 13 | -------------------------------------------------------------------------------- /kuma/health/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import re_path 2 | 3 | from . import views 4 | 5 | urlpatterns = [ 6 | re_path(r"^healthz/?$", views.liveness, name="health.liveness"), 7 | re_path(r"^readiness/?$", views.readiness, name="health.readiness"), 8 | re_path(r"^_kuma_status.json$", views.status, name="health.status"), 9 | ] 10 | -------------------------------------------------------------------------------- /kuma/core/exceptions.py: -------------------------------------------------------------------------------- 1 | class ProgrammingError(Exception): 2 | """Somebody made a mistake in the code.""" 3 | 4 | 5 | class DateTimeFormatError(Exception): 6 | """Called by the datetimeformat function when receiving invalid format.""" 7 | 8 | pass 9 | 10 | 11 | class FixtureMissingError(Exception): 12 | """Raise this if a fixture is missing""" 13 | -------------------------------------------------------------------------------- /Jenkinsfiles/default.groovy: -------------------------------------------------------------------------------- 1 | stage('Build base') { 2 | utils.sh_with_notify( 3 | 'make build-base VERSION=latest', 4 | 'Build of latest-tagged Kuma base image' 5 | ) 6 | } 7 | 8 | stage('Build & push images') { 9 | utils.sh_with_notify( 10 | 'make build-kuma push-kuma', 11 | "Build & push of commit-tagged Kuma image" 12 | ) 13 | } 14 | -------------------------------------------------------------------------------- /kuma/core/tests/logging_urls.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import SuspiciousOperation 2 | from django.urls import re_path 3 | 4 | from kuma.core.urlresolvers import i18n_patterns 5 | 6 | 7 | def suspicious(request): 8 | raise SuspiciousOperation("Raising exception to test logging.") 9 | 10 | 11 | urlpatterns = i18n_patterns(re_path(r"^suspicious/$", suspicious, name="suspicious")) 12 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | 2 | ==== 3 | Kuma 4 | ==== 5 | 6 | .. include:: ../README.rst 7 | :start-after: .. Omit badges from docs 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | installation 15 | development 16 | troubleshooting 17 | tests 18 | tests-functional 19 | 20 | celery 21 | elasticsearch 22 | documentation 23 | 24 | docker 25 | deploy 26 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | updates: 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "weekly" 8 | 9 | - package-ecosystem: "pip" 10 | directory: "/" 11 | schedule: 12 | interval: "weekly" 13 | 14 | - package-ecosystem: "docker" 15 | directory: "/docker/images/kuma_base" 16 | schedule: 17 | interval: "weekly" 18 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # This is a standalone requirements file for the documentation requirements 2 | # used by ReadTheDocs to set up the documentation rendering environment 3 | 4 | # These should stay synced with other requirements files 5 | Babel==2.9.1 6 | docutils==0.17.1 7 | Jinja2==3.0.1 8 | pytz==2018.5 9 | 10 | # These requirements are unique to docs 11 | Pygments==2.10.0 12 | Sphinx==4.2.0 13 | snowballstemmer==2.1.0 14 | -------------------------------------------------------------------------------- /kuma/attachments/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | from django.utils.translation import gettext_lazy as _ 3 | 4 | 5 | class AttachmentsConfig(AppConfig): 6 | """ 7 | The Django App Config class to store information about the users app 8 | and do startup time things. 9 | """ 10 | 11 | name = "kuma.attachments" 12 | verbose_name = _("Attachments") 13 | 14 | def ready(self): 15 | pass 16 | -------------------------------------------------------------------------------- /kuma/bookmarks/migrations/0002_remove_bookmark_notes.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.10 on 2022-01-18 04:08 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("bookmarks", "0001_initial"), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name="bookmark", 15 | name="notes", 16 | ), 17 | ] 18 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | # Black recommends 88-char lines and ignoring the following lints: 3 | # - E203 - whitespace before ':' 4 | # - E501 - line too long 5 | # - W503 - line break before binary operator 6 | max-line-length=88 7 | ignore = E203, E501, W503 8 | 9 | # Allow star imports in config files: 10 | per-file-ignores = 11 | kuma/settings/local.py:F403,F405 12 | kuma/settings/prod.py:F403,F405 13 | kuma/settings/pytest.py:F403,F405 14 | -------------------------------------------------------------------------------- /kuma/notifications/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from kuma.notifications.models import Notification, NotificationData, Watch 4 | 5 | 6 | @admin.register(Notification) 7 | class NotificationAdmin(admin.ModelAdmin): 8 | ... 9 | 10 | 11 | @admin.register(NotificationData) 12 | class NotificationDataAdmin(admin.ModelAdmin): 13 | ... 14 | 15 | 16 | @admin.register(Watch) 17 | class WatchAdmin(admin.ModelAdmin): 18 | ... 19 | -------------------------------------------------------------------------------- /kuma/users/migrations/0004_remove_userprofile_fxa_uid.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2021-10-20 07:25 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("users", "0003_auto_20211014_1214"), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name="userprofile", 15 | name="fxa_uid", 16 | ), 17 | ] 18 | -------------------------------------------------------------------------------- /kuma/users/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | from django.core.checks import register 3 | from django.utils.translation import gettext_lazy as _ 4 | 5 | 6 | class UsersConfig(AppConfig): 7 | name = "kuma.users" 8 | verbose_name = _("Users") 9 | 10 | def ready(self): 11 | self.register_checks() 12 | 13 | def register_checks(self): 14 | from .checks import oidc_config_check 15 | 16 | register(oidc_config_check) 17 | -------------------------------------------------------------------------------- /kuma/version/views.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.http import HttpResponse 3 | from django.views.decorators.cache import never_cache 4 | from django.views.decorators.http import require_safe 5 | 6 | 7 | @never_cache 8 | @require_safe 9 | def revision_hash(request): 10 | """ 11 | Return the kuma revision hash. 12 | """ 13 | return HttpResponse( 14 | settings.REVISION_HASH, content_type="text/plain; charset=utf-8" 15 | ) 16 | -------------------------------------------------------------------------------- /kuma/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for kuma project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "kuma.settings.local") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /templates/admin/search/index/change_list.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/change_list.html" %} 2 | {% load i18n %} 3 | 4 | {% block content_title %} 5 | {{ block.super }} 6 | 7 | {% if settings.ES_URLS %} 8 |
9 | {# 'safe_es_urls' comes from a context processor #} 10 |

{% blocktrans with backend=safe_es_urls|join:", " %}Elasticsearch server: {{ backend }}{% endblocktrans %}

11 |
12 | {% endif %} 13 | {% endblock %} 14 | -------------------------------------------------------------------------------- /kuma/attachments/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import re_path 2 | 3 | from . import views 4 | 5 | urlpatterns = [ 6 | re_path( 7 | r"^files/(?P\d+)/(?P.+)$", 8 | views.raw_file, 9 | name="attachments.raw_file", 10 | ), 11 | re_path( 12 | r"^@api/deki/files/(?P\d+)/=(?P.+)$", 13 | views.mindtouch_file_redirect, 14 | name="attachments.mindtouch_file_redirect", 15 | ), 16 | ] 17 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # Top-most EditorConfig file ? 2 | root = true 3 | 4 | # Unix-style newlines with a newline ending every file 5 | [*] 6 | end_of_line = lf 7 | insert_final_newline = true 8 | trim_trailing_whitespace = true 9 | 10 | # Indentation 11 | indent_style = space 12 | indent_size = 4 13 | 14 | # YAML and Jenkins use two spaces 15 | [{*.yml,Jenkinsfile}] 16 | indent_size = 2 17 | 18 | # git and Makefile use the superior tabs 19 | [{.git*,Makefile,make.bat}] 20 | indent_style = tab 21 | -------------------------------------------------------------------------------- /kuma/core/views.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpResponse 2 | 3 | from kuma.core.decorators import shared_cache_control 4 | 5 | 6 | @shared_cache_control(s_maxage=60 * 60 * 24 * 30) 7 | def humans_txt(request): 8 | """We no longer maintain an actual /humans.txt endpoint but to avoid the 9 | sad 404 we instead now just encourage people to go and use the GitHub 10 | UI to see the contributors.""" 11 | return HttpResponse("See https://github.com/mdn/kuma/graphs/contributors\n") 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/support-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Support request 3 | about: Request a support task from the MDN team. 4 | title: '' 5 | labels: 'status: needs triage' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Summary** 11 | _What support task would you like completed by the team?_ 12 | 13 | 14 | **Rationale** 15 | _What problems would this solve?_ 16 | 17 | 18 | **Due Date** 19 | _When would you like this task completed by?_ 20 | 21 | 22 | **Additional context** 23 | _Is there anything else we should know?_ 24 | -------------------------------------------------------------------------------- /kuma/notifications/migrations/0004_notificationdata_data.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2021-12-13 15:05 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("notifications", "0003_watch_title"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="notificationdata", 15 | name="data", 16 | field=models.JSONField(default="{}"), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /jinja2/includes/translate_locales.html: -------------------------------------------------------------------------------- 1 | This template is automatically generated by *manage.py translate_locales_name* in order to make the languages name localizable. Do not edit it manually 2 | Background: https://bugzil.la/859499#c11 3 | {{ _('Chinese (Simplified)') }} 4 | {{ _('Chinese (Traditional)') }} 5 | {{ _('English (US)') }} 6 | {{ _('French') }} 7 | {{ _('German') }} 8 | {{ _('Japanese') }} 9 | {{ _('Korean') }} 10 | {{ _('Polish') }} 11 | {{ _('Portuguese (Brazilian)') }} 12 | {{ _('Russian') }} 13 | {{ _('Spanish') }} 14 | -------------------------------------------------------------------------------- /kuma/users/migrations/0005_userprofile_is_subscriber.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2021-12-15 12:31 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("users", "0004_remove_userprofile_fxa_uid"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="userprofile", 15 | name="is_subscriber", 16 | field=models.BooleanField(default=False), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /kuma/notifications/migrations/0010_notification_deleted.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.10 on 2022-01-27 14:36 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("notifications", "0009_notificationdata_page_url"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="notification", 15 | name="deleted", 16 | field=models.BooleanField(default=False), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /kuma/settings/local.py: -------------------------------------------------------------------------------- 1 | from .common import * 2 | 3 | # Settings for Docker Development 4 | 5 | INTERNAL_IPS = ("127.0.0.1",) 6 | 7 | # Default DEBUG to True, and recompute derived settings 8 | DEBUG = config("DEBUG", default=True, cast=bool) 9 | DEBUG_PROPAGATE_EXCEPTIONS = config( 10 | "DEBUG_PROPAGATE_EXCEPTIONS", default=DEBUG, cast=bool 11 | ) 12 | 13 | PROTOCOL = config("PROTOCOL", default="https://") 14 | DOMAIN = config("DOMAIN", default="developer-local.allizom.org") 15 | SITE_URL = config("SITE_URL", default=PROTOCOL + DOMAIN) 16 | -------------------------------------------------------------------------------- /kuma/notifications/management/commands/extract_notifications.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.core.management.base import BaseCommand 4 | 5 | from kuma.notifications.utils import process_changes 6 | 7 | 8 | class Command(BaseCommand): 9 | help = "Extracts notifications from a changes.json file" 10 | 11 | def add_arguments(self, parser): 12 | parser.add_argument("file", type=open) 13 | 14 | def handle(self, *args, **options): 15 | changes = json.loads(options["file"].read()) 16 | process_changes(changes) 17 | -------------------------------------------------------------------------------- /kuma/notifications/migrations/0003_watch_title.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2021-11-06 13:06 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("notifications", "0002_auto_20211105_1529"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="watch", 15 | name="title", 16 | field=models.CharField(default="", max_length=2048), 17 | preserve_default=False, 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | 5 | To report a suspected security-related vulnerability within [MDN Web Docs](https://developer.mozilla.org), please use 6 | [this Bugzilla link](https://bugzilla.mozilla.org/enter_bug.cgi?product=developer.mozilla.org&component=Security), **and before submitting**, it is important to click the checkbox, just above the `Submit` button, entitled: 7 | 8 | ``` 9 | This report is about a problem that is putting users at risk. It should be kept hidden from the public until it is resolved. 10 | ``` 11 | -------------------------------------------------------------------------------- /docker/images/kuma/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mdnwebdocs/kuma_base:latest 2 | 3 | ARG REVISION_HASH 4 | # Make the git commit hash permanently available within this image. 5 | ENV REVISION_HASH=$REVISION_HASH \ 6 | DJANGO_SETTINGS_MODULE=kuma.settings.prod 7 | 8 | COPY --chown=kuma:kuma . /app 9 | 10 | # Temporarily enable candidate languages so assets are built for all 11 | # environments, but still defaults to disabled in production. 12 | # Also generate react.po translation files for beta. 13 | RUN ENABLE_CANDIDATE_LANGUAGES=True \ 14 | make localecompile build-static 15 | -------------------------------------------------------------------------------- /kuma/notifications/migrations/0009_notificationdata_page_url.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2022-01-18 12:52 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("notifications", "0008_default_watch"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="notificationdata", 15 | name="page_url", 16 | field=models.TextField(default="/"), 17 | preserve_default=False, 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /scripts/ci-localerefresh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e # Exit on non-zero status 3 | set -u # Treat unset variables as an error 4 | 5 | # XXX Instead of using the "make-bundle" `localerefresh` we spell out 6 | # each and every one of these. Otherwise it causes to build-static again. 7 | ## docker-compose exec -T web make localerefresh 8 | docker-compose exec -T web make localeextract 9 | docker-compose exec -T web make localetest 10 | docker-compose exec -T web make localecompile 11 | 12 | cd locale 13 | export GIT_PAGER=cat 14 | git diff -G "^msgid " templates/LC_MESSAGES 15 | -------------------------------------------------------------------------------- /Jenkinsfiles/main.groovy: -------------------------------------------------------------------------------- 1 | stage('Build base') { 2 | utils.sh_with_notify( 3 | 'make build-base VERSION=latest', 4 | 'Build of latest-tagged Kuma base image' 5 | ) 6 | } 7 | 8 | stage('Test') { 9 | utils.compose_test() 10 | } 11 | 12 | stage('Build & push images') { 13 | utils.sh_with_notify( 14 | 'make build-kuma push-kuma', 15 | "Build & push of commit-tagged Kuma image" 16 | ) 17 | utils.sh_with_notify( 18 | 'make push-base VERSION=latest', 19 | 'Push of latest-tagged Kuma base image' 20 | ) 21 | } 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Help us fix things that are broken 4 | title: '' 5 | labels: 'status: needs triage' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Summary** 11 | _What is the problem?_ 12 | 13 | 14 | **Steps To Reproduce (STR)** 15 | _How can we reproduce the problem?_ 16 | 17 | 1. 18 | 2. 19 | 3. 20 | 21 | 22 | **Actual behavior** 23 | _What actually happened?_ 24 | 25 | 26 | **Expected behavior** 27 | _What did you expect to happen?_ 28 | 29 | 30 | **Additional context** 31 | _Is there anything else we should know?_ 32 | -------------------------------------------------------------------------------- /kuma/api/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | from django.conf import settings 3 | from elasticsearch_dsl.connections import connections 4 | 5 | 6 | class APIConfig(AppConfig): 7 | """ 8 | The Django App Config class to store information about the API app 9 | and do startup time things. 10 | """ 11 | 12 | name = "kuma.api" 13 | verbose_name = "API" 14 | 15 | def ready(self): 16 | # Configure Elasticsearch connections for connection pooling. 17 | connections.configure( 18 | default={"hosts": settings.ES_URLS}, 19 | ) 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/content-issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Content request 3 | about: Suggest a change to MDN Content 4 | title: '' 5 | labels: 'meta: content' 6 | assignees: 'chrisdavidmills' 7 | 8 | --- 9 | 10 | 11 | # Content Issue 12 | 13 | - [x] Please close this issue, I accidentally submitted it without adding any details 14 | - [ ] New documentation 15 | - [ ] Correction or update 16 | 17 | # URL of page you are seeing the problem on: 18 | 19 | # Details 20 | 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/change-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Change request 3 | about: Suggest a change to how MDN currently works 4 | title: '' 5 | labels: 'status: needs triage' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Summary** 11 | _What should be changed? (Please provide a URL if applicable)_ 12 | 13 | 14 | **Rationale** 15 | _What problems would this solve?_ 16 | 17 | 18 | **Audience** 19 | _Who would use this changed feature?_ 20 | 21 | 22 | **Proposal** 23 | _What would users see and do? What would happen as a result?_ 24 | 25 | 26 | **Additional context** 27 | _Is there anything else we should know?_ 28 | -------------------------------------------------------------------------------- /kuma/notifications/migrations/0011_auto_20220210_0921.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.10 on 2022-02-10 09:21 2 | 3 | from django.db import migrations 4 | 5 | 6 | def delete_watchers(apps, schema_editor): 7 | Watch = apps.get_model("notifications", "Watch") 8 | Watch.objects.all().delete() 9 | 10 | 11 | class Migration(migrations.Migration): 12 | 13 | dependencies = [ 14 | ("notifications", "0010_notification_deleted"), 15 | ] 16 | 17 | operations = [ 18 | migrations.RunPython( 19 | code=delete_watchers, reverse_code=migrations.RunPython.noop 20 | ) 21 | ] 22 | -------------------------------------------------------------------------------- /scripts/start-worker: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e # Exit on non-zero status 3 | set -u # Treat unset variables as an error 4 | 5 | # This will make sure we don't even attempt to start the worker 6 | # until the $CELERY_BROKER_URL (i.e. Redis) is up and running. 7 | urlwait ${CELERY_BROKER_URL} 10 8 | 9 | # Note, in production you'll want to run a separate process with 10 | # just `celery -A kuma.celery:app beat ...`. But for docker-compose 11 | # it's fine to run it from the regular worker. 12 | exec celery -A kuma.celery:app worker -l info --beat --concurrency=${CELERY_WORKERS:-4} -Q mdn_purgeable,mdn_search,mdn_emails,mdn_wiki,mdn_api,celery 13 | -------------------------------------------------------------------------------- /scripts/compile-mo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # http://www.davidpashley.com/articles/writing-robust-shell-scripts/ 3 | set -u # Exit with error on unset variables 4 | set -e # Exit with error if a command fails 5 | 6 | # syntax: 7 | # compile-mo.sh locale-dir/ 8 | 9 | function usage() { 10 | echo "syntax:" 11 | echo "compile.sh locale-dir/" 12 | exit 1 13 | } 14 | 15 | # check if file and dir are there 16 | if [[ ($# -ne 1) || (! -d "$1") ]]; then usage; fi 17 | 18 | for lang in `find $1 -type f -name "*.po"`; do 19 | dir=`dirname $lang` 20 | stem=`basename $lang .po` 21 | msgfmt --check-header -o ${dir}/${stem}.mo $lang 22 | done 23 | -------------------------------------------------------------------------------- /kuma/api/v1/forms.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.conf import settings 3 | 4 | 5 | class AccountSettingsForm(forms.Form): 6 | locale = forms.ChoiceField( 7 | required=False, 8 | # The `settings.LANGUAGES` looks like this: 9 | # [('en-US', 'English (US)'), ...] 10 | # But the valid choices actually come from Yari which is the only 11 | # one that knows which ones are really valid. 12 | # Here in kuma we can't enforce it as string. But at least we 13 | # have a complete list of all possible languages 14 | choices=[(code, name) for code, name in settings.LANGUAGES], 15 | ) 16 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base" 4 | ], 5 | "ignorePaths": [ 6 | "package.json", 7 | "assets/ckeditor4/source/plugins/descriptionlist/package.json", 8 | "kuma/static/js/libs/jquery-ui-1.10.3.custom/development-bundle/package.json", 9 | "kumascript/package.json", 10 | "docker/images/kuma/Dockerfile" 11 | ], 12 | "masterIssue": true, 13 | "dockerfile": { 14 | "pinDigests": true 15 | }, 16 | "docker-compose": { 17 | "enabled": false 18 | }, 19 | "python": { 20 | "enabled": false 21 | }, 22 | "rebaseStalePrs": true, 23 | "timezone": "UTC", 24 | "schedule": ["before 13:00 on mondays"] 25 | } 26 | -------------------------------------------------------------------------------- /kuma/users/utils.py: -------------------------------------------------------------------------------- 1 | from kuma.users.models import UserProfile 2 | 3 | 4 | # Finds union of valid subscription id's from input and UserProfile.SubscriptionType.values 5 | # Should only be mdn_plus + one of 'SubscriptionType.values' 6 | def get_valid_subscription_type_or_none(input): 7 | subscription_types = list(set(input) & set(UserProfile.SubscriptionType.values)) 8 | if len(subscription_types) > 1: 9 | print("Multiple subscriptions found in update %s" % subscription_types) 10 | # Sort array lexicographically. At least makes wrong answer consistent. 11 | subscription_types.sort() 12 | 13 | return subscription_types[0] if len(subscription_types) > 0 else "" 14 | -------------------------------------------------------------------------------- /kuma/plus/migrations/0002_auto_20210723_1901.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.5 on 2021-07-23 19:01 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("plus", "0001_initial"), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name="landingpagesurvey", 15 | name="email", 16 | ), 17 | migrations.RemoveField( 18 | model_name="landingpagesurvey", 19 | name="user", 20 | ), 21 | migrations.RemoveField( 22 | model_name="landingpagesurvey", 23 | name="variant", 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /kuma/notifications/migrations/0007_watch_users.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2022-01-05 19:37 2 | 3 | from django.conf import settings 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 11 | ("notifications", "0006_auto_20220105_0138"), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name="watch", 17 | name="users", 18 | field=models.ManyToManyField( 19 | through="notifications.UserWatch", to=settings.AUTH_USER_MODEL 20 | ), 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /kuma/bookmarks/migrations/0003_auto_20220118_0409.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.10 on 2022-01-18 04:09 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("bookmarks", "0002_remove_bookmark_notes"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="bookmark", 15 | name="custom_name", 16 | field=models.CharField(blank=True, max_length=500), 17 | ), 18 | migrations.AddField( 19 | model_name="bookmark", 20 | name="notes", 21 | field=models.CharField(blank=True, max_length=500), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest a new idea for MDN 4 | title: '' 5 | labels: 'status: needs triage' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Summary** 11 | _What problem would this solve?_ 12 | 13 | 14 | **Audience** 15 | _Who has this problem? Everyone? Article authors? Etc._ 16 | 17 | 18 | **Rationale** 19 | _How do you know that people identified above actually have this problem?_ 20 | 21 | 22 | **Workaround** 23 | _How are the people identified above currently handling this problem?_ 24 | 25 | 26 | **Proposal** 27 | _Do you have suggestions for solving this problem?_ 28 | 29 | 30 | **Additional context** 31 | _Is there anything else we should know?_ 32 | -------------------------------------------------------------------------------- /.github/workflows/documentation-build.yml: -------------------------------------------------------------------------------- 1 | name: Documentation Build 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - "docs/**" 7 | - .github/workflows/documentation-build.yml 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v3 15 | 16 | - uses: actions/setup-python@v3 17 | with: 18 | python-version: "3.8" 19 | 20 | - name: Install dependencies 21 | run: | 22 | python -m pip install --upgrade pip 23 | pip install --disable-pip-version-check -r docs/requirements.txt 24 | 25 | - name: Build docs 26 | run: | 27 | sphinx-build -b html -d doctrees -W docs html 28 | -------------------------------------------------------------------------------- /docker/images/kuma/README.rst: -------------------------------------------------------------------------------- 1 | kuma 2 | ---- 3 | The kuma Docker image builds on the kuma_base image, installing a kuma branch 4 | and building the assets needed for running as a webservice. The environment 5 | can be customized for different deployments. 6 | 7 | The image can be recreated locally with ``make build-kuma``. 8 | 9 | The image tagged ``latest`` is used by default for development. It can be 10 | created locally with ``make build-kuma VERSION=latest``. The official latest 11 | image is created from the main branch in Jenkins__ and published to 12 | DockerHub__. 13 | 14 | .. __: https://ci.us-west-2.mdn.mozit.cloud/blue/organizations/jenkins/kuma/branches/ 15 | .. __: https://hub.docker.com/r/mdnwebdocs/kuma/ 16 | -------------------------------------------------------------------------------- /kuma/plus/models.py: -------------------------------------------------------------------------------- 1 | from uuid import uuid4 2 | 3 | from django.db import models 4 | 5 | 6 | class LandingPageSurvey(models.Model): 7 | uuid = models.UUIDField(default=uuid4, editable=False, primary_key=True) 8 | # An insecure random string so that when a survey is submitted with more information 9 | # it can not easily be guessed and the ratelimit will make it impossible to try 10 | # all combinations. 11 | response = models.JSONField(editable=False, null=True) 12 | geo_information = models.TextField(editable=False, null=True) 13 | created = models.DateTimeField(auto_now_add=True) 14 | updated = models.DateTimeField(auto_now=True) 15 | 16 | def __str__(self): 17 | return str(self.uuid) 18 | -------------------------------------------------------------------------------- /docker/images/kuma_base/README.rst: -------------------------------------------------------------------------------- 1 | kuma_base 2 | --------- 3 | The kuma_base Docker image contains the OS and libraries (C, Python, and 4 | Node.js) that support the kuma project. The kuma image extends this by 5 | installing the kuma source and building assets needed for production. 6 | 7 | The image can be recreated locally with ``make build-base``. 8 | 9 | The image tagged ``latest`` is used by default for development. It can be 10 | created locally with ``make build-base VERSION=latest``. The official 11 | latest image is created from the master branch in Jenkins__ and published to 12 | DockerHub__ 13 | 14 | .. __: https://ci.us-west-2.mdn.mozit.cloud/blue/organizations/jenkins/kuma/branches/ 15 | .. __: https://hub.docker.com/r/mdnwebdocs/kuma_base/ 16 | -------------------------------------------------------------------------------- /kuma/attachments/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import json 2 | from pathlib import Path 3 | 4 | import pytest 5 | 6 | 7 | @pytest.fixture 8 | def sample_attachment_redirect(): 9 | redirects_file = Path(__file__).parent.parent / "redirects.json" 10 | with open(redirects_file) as f: 11 | redirects = json.load(f) 12 | first = list(redirects.keys())[0] 13 | return {"id": int(first), "url": redirects[first]} 14 | 15 | 16 | @pytest.fixture 17 | def sample_mindtouch_attachment_redirect(): 18 | redirects_file = Path(__file__).parent.parent / "mindtouch_redirects.json" 19 | with open(redirects_file) as f: 20 | redirects = json.load(f) 21 | first = list(redirects.keys())[0] 22 | return {"id": int(first), "url": redirects[first]} 23 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | """Run administrative tasks.""" 9 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "kuma.settings.local") 10 | try: 11 | from django.core.management import execute_from_command_line 12 | except ImportError as exc: 13 | raise ImportError( 14 | "Couldn't import Django. Are you sure it's installed and " 15 | "available on your PYTHONPATH environment variable? Did you " 16 | "forget to activate a virtual environment?" 17 | ) from exc 18 | execute_from_command_line(sys.argv) 19 | 20 | 21 | if __name__ == "__main__": 22 | main() 23 | -------------------------------------------------------------------------------- /kuma/users/migrations/0002_auto_20210922_1159.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2021-09-22 11:59 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("users", "0001_initial"), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name="userprofile", 15 | name="is_subscriber", 16 | ), 17 | migrations.RemoveField( 18 | model_name="userprofile", 19 | name="subscriber_number", 20 | ), 21 | migrations.AddField( 22 | model_name="userprofile", 23 | name="fxa_uid", 24 | field=models.CharField(blank=True, max_length=255, null=True, unique=True), 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /kuma/api/v1/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | 3 | from . import search 4 | from .api import api 5 | from .plus.bookmarks import router as bookmarks_router 6 | from .plus.landing_page import router as landing_page_router 7 | from .plus.notifications import notifications_router, watch_router 8 | from .views import settings_router 9 | 10 | api.add_router("/settings", settings_router) 11 | api.add_router("/plus/notifications/", notifications_router) 12 | api.add_router("/plus/", watch_router) 13 | api.add_router("/plus/collection/", bookmarks_router) 14 | api.add_router("/plus/landing-page/", landing_page_router) 15 | 16 | urlpatterns = [ 17 | path("", api.urls), 18 | path("search/", search.search, name="api.v1.search_legacy"), 19 | path("search", search.search, name="api.v1.search"), 20 | ] 21 | -------------------------------------------------------------------------------- /kuma/users/tests/test_auth.py: -------------------------------------------------------------------------------- 1 | from django.test import RequestFactory 2 | 3 | from kuma.users.auth import logout_url 4 | 5 | 6 | # TODO: Check which new tests are needed. 7 | def test_logout_url(settings): 8 | request = RequestFactory().get("/some/path") 9 | request.session = {} 10 | url = logout_url(request) 11 | assert url == "/" 12 | 13 | request = RequestFactory().get("/some/path?next=/docs") 14 | request.session = {} 15 | url = logout_url(request) 16 | assert url == "/docs" 17 | 18 | settings.LOGOUT_REDIRECT_URL = "/loggedout" 19 | request = RequestFactory().get("/some/path") 20 | request.session = {} 21 | url = logout_url(request) 22 | assert url == "/loggedout" 23 | 24 | request.session["oidc_login_next"] = "/original" 25 | url = logout_url(request) 26 | assert url == "/original" 27 | -------------------------------------------------------------------------------- /kuma/users/tests/test_views.py: -------------------------------------------------------------------------------- 1 | import urllib 2 | 3 | import pytest 4 | from django.urls import reverse 5 | 6 | 7 | @pytest.mark.django_db 8 | @pytest.mark.parametrize( 9 | "email", (None, "ringo@beatles.com"), ids=("without-email", "with-email") 10 | ) 11 | def test_no_prompt_login(client, settings, email): 12 | params = {} 13 | if email: 14 | params.update(email=email) 15 | response = client.get(reverse("no_prompt_login"), data=params) 16 | assert response.status_code == 302 17 | location = response.headers.get("location") 18 | assert location 19 | location = urllib.parse.unquote(location) 20 | assert settings.OIDC_OP_AUTHORIZATION_ENDPOINT in location 21 | assert "next=/en-US/plus" in location 22 | if email: 23 | assert "prompt=none" in location 24 | assert f"login_hint={email}" in location 25 | -------------------------------------------------------------------------------- /kuma/notifications/migrations/0002_auto_20211105_1529.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2021-11-05 15:29 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("notifications", "0001_initial"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="watch", 15 | field=models.SlugField(blank=True), 16 | name="slug", 17 | preserve_default=False, 18 | ), 19 | migrations.RemoveField( 20 | model_name="watch", 21 | name="slug", 22 | ), 23 | migrations.AddField( 24 | model_name="watch", 25 | name="url", 26 | field=models.TextField(default=""), 27 | preserve_default=False, 28 | ), 29 | ] 30 | -------------------------------------------------------------------------------- /kuma/users/migrations/0003_auto_20211014_1214.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2021-10-14 12:14 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("users", "0002_auto_20210922_1159"), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name="userprofile", 15 | name="claims", 16 | ), 17 | migrations.AddField( 18 | model_name="userprofile", 19 | name="avatar", 20 | field=models.URLField(blank=True, default="", max_length=512), 21 | ), 22 | migrations.AddField( 23 | model_name="userprofile", 24 | name="fxa_refresh_token", 25 | field=models.CharField(blank=True, default="", max_length=128), 26 | ), 27 | ] 28 | -------------------------------------------------------------------------------- /kuma/api/v1/api.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpResponse 2 | from django.utils.cache import add_never_cache_headers 3 | from ninja import NinjaAPI 4 | from ratelimit.exceptions import Ratelimited 5 | 6 | from .auth import admin_auth, profile_auth 7 | 8 | 9 | class NoCacheNinjaAPI(NinjaAPI): 10 | def create_response(self, *args, **kwargs) -> HttpResponse: 11 | response = super().create_response(*args, **kwargs) 12 | add_never_cache_headers(response) 13 | return response 14 | 15 | 16 | api = NoCacheNinjaAPI(auth=profile_auth, csrf=True, version="v1") 17 | 18 | admin_api = NoCacheNinjaAPI( 19 | auth=admin_auth, csrf=False, version="v1", urls_namespace="admin_api" 20 | ) 21 | 22 | 23 | @api.exception_handler(Ratelimited) 24 | def rate_limited(request, exc): 25 | return api.create_response(request, {"error": "Too many requests"}, status=429) 26 | -------------------------------------------------------------------------------- /kuma/notifications/migrations/0006_auto_20220105_0138.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2022-01-05 01:38 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("notifications", "0005_auto_20211215_2004"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="userwatch", 15 | name="browser_compatibility", 16 | field=models.JSONField(default=list), 17 | ), 18 | migrations.AddField( 19 | model_name="userwatch", 20 | name="content_updates", 21 | field=models.BooleanField(default=True), 22 | ), 23 | migrations.AddField( 24 | model_name="userwatch", 25 | name="custom", 26 | field=models.BooleanField(default=False), 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Mozilla Community Participation Guidelines 2 | 3 | This repository is governed by Mozilla's code of conduct and etiquette guidelines. 4 | For more details, please read the 5 | [Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). 6 | 7 | ## How to Report 8 | 9 | For more information on how to report violations of the CPG, please read our 10 | [How to Report](https://www.mozilla.org/en-US/about/governance/policies/participation/reporting/) 11 | page. 12 | 13 | ## Project Specific Etiquette 14 | 15 | See the [Contributing Guide][cg] for code standards, and the 16 | [Bugzilla Developer Etiquette Guidelines](bdeg) for general tips for being a 17 | helpful participant in an open source project. 18 | 19 | [cg]: https://github.com/mdn/kuma/blob/main/CONTRIBUTING.md 20 | [bdeg]: https://bugzilla.mozilla.org/page.cgi?id=etiquette.html 21 | -------------------------------------------------------------------------------- /kuma/settings/prod.py: -------------------------------------------------------------------------------- 1 | from .common import * 2 | 3 | ATTACHMENT_HOST = config("ATTACHMENT_HOST", default="mdn.mozillademos.org") 4 | ALLOW_ROBOTS = config("ALLOW_ROBOTS", default=True, cast=bool) 5 | 6 | # Email 7 | DEFAULT_FROM_EMAIL = config( 8 | "DEFAULT_FROM_EMAIL", default="no-reply@developer.mozilla.org" 9 | ) 10 | SERVER_EMAIL = config("SERVER_EMAIL", default="mdn-prod-noreply@mozilla.com") 11 | 12 | # Cache 13 | CACHES["default"]["TIMEOUT"] = 60 * 60 * 24 14 | 15 | MEDIA_URL = config("MEDIA_URL", default="https://developer.cdn.mozilla.net/media/") 16 | 17 | CELERY_TASK_ALWAYS_EAGER = config("CELERY_TASK_ALWAYS_EAGER", False, cast=bool) 18 | CELERYD_MAX_TASKS_PER_CHILD = ( 19 | config("CELERYD_MAX_TASKS_PER_CHILD", default=500, cast=int) or None 20 | ) 21 | 22 | ES_INDEX_PREFIX = config("ES_INDEX_PREFIX", default="mdnprod") 23 | ES_LIVE_INDEX = config("ES_LIVE_INDEX", default=True, cast=bool) 24 | -------------------------------------------------------------------------------- /kuma/users/migrations/0007_userprofile_subscription_type.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.10 on 2022-03-03 13:45 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("users", "0006_accountevent"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="userprofile", 15 | name="subscription_type", 16 | field=models.CharField( 17 | blank=True, 18 | choices=[ 19 | ("mdn_plus_5m", "MDN Plus 5M"), 20 | ("mdn_plus_5y", "MDN Plus 5Y"), 21 | ("mdn_plus_10m", "MDN Plus 10M"), 22 | ("mdn_plus_10y", "MDN Plus 10Y"), 23 | ("", "None"), 24 | ], 25 | default="", 26 | max_length=512, 27 | ), 28 | ), 29 | ] 30 | -------------------------------------------------------------------------------- /kuma/version/tests/test_views.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from kuma.core.tests import assert_no_cache_header 4 | from kuma.core.urlresolvers import reverse 5 | 6 | 7 | @pytest.mark.parametrize("method", ["get", "head"]) 8 | def test_revision_hash(client, db, method, settings): 9 | settings.REVISION_HASH = "the_revision_hash" 10 | response = getattr(client, method)(reverse("version.kuma")) 11 | assert response.status_code == 200 12 | assert response["Content-Type"] == "text/plain; charset=utf-8" 13 | assert_no_cache_header(response) 14 | if method == "get": 15 | assert response.content.decode() == "the_revision_hash" 16 | 17 | 18 | @pytest.mark.parametrize("method", ["post", "put", "delete", "options", "patch"]) 19 | def test_revision_hash_405s(client, db, method): 20 | response = getattr(client, method)(reverse("version.kuma")) 21 | assert response.status_code == 405 22 | assert_no_cache_header(response) 23 | -------------------------------------------------------------------------------- /docs/docker.rst: -------------------------------------------------------------------------------- 1 | ====== 2 | Docker 3 | ====== 4 | 5 | Docker__ is used for development and for deployment. 6 | 7 | .. __: https://www.docker.com 8 | 9 | Docker Images 10 | ============= 11 | Docker images are used in development, usually with the local 12 | working files mounted in the images to set behaviour. 13 | 14 | Images are built by Jenkins__, after tests pass, and are 15 | published to DockerHub__. We try to 16 | `store the configuration in the environment`_, so that the 17 | published images can be used in deployments by setting 18 | environment variables to deployment-specific values. 19 | 20 | .. __: https://ci.us-west-2.mdn.mozit.cloud 21 | .. __: https://hub.docker.com/r/mdnwebdocs/kuma/ 22 | .. _`store the configuration in the environment`: https://12factor.net/config 23 | 24 | Here are some of the images used in the Kuma project: 25 | 26 | .. Published images 27 | .. include:: ../docker/images/kuma/README.rst 28 | .. include:: ../docker/images/kuma_base/README.rst 29 | -------------------------------------------------------------------------------- /kuma/core/tests/test_settings.py: -------------------------------------------------------------------------------- 1 | """Check that settings are consistent.""" 2 | 3 | 4 | import pytest 5 | from django.conf import settings 6 | 7 | 8 | def test_accepted_locales(): 9 | """Check for a consistent ACCEPTED_LOCALES.""" 10 | assert len(settings.ACCEPTED_LOCALES) == len(set(settings.ACCEPTED_LOCALES)) 11 | assert settings.ACCEPTED_LOCALES[0] == settings.LANGUAGE_CODE 12 | 13 | 14 | @pytest.mark.parametrize( 15 | "primary,secondary", 16 | (("zh-CN", "zh-TW"),), 17 | ) 18 | def test_preferred_locale_codes(primary, secondary): 19 | assert settings.ACCEPTED_LOCALES.index(primary) < settings.ACCEPTED_LOCALES.index( 20 | secondary 21 | ) 22 | 23 | 24 | @pytest.mark.parametrize("alias,locale", settings.LOCALE_ALIASES.items()) 25 | def test_locale_aliases(alias, locale): 26 | """Check that each locale alias matches a supported locale.""" 27 | assert alias not in settings.ACCEPTED_LOCALES 28 | assert alias == alias.lower() 29 | assert locale in settings.ACCEPTED_LOCALES 30 | -------------------------------------------------------------------------------- /.github/workflows/python-lints.yml: -------------------------------------------------------------------------------- 1 | name: Python Lints 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - "kuma/**/*.py" 7 | - .github/workflows/python-lints.yml 8 | # This is in case Dependabot updates 'black' 9 | - pyproject.toml 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v3 17 | 18 | - uses: actions/setup-python@v3 19 | with: 20 | python-version: "3.8" 21 | 22 | - name: Install dependencies 23 | run: | 24 | python -m pip install --upgrade pip 25 | pip install --disable-pip-version-check black flake8 flake8-isort 26 | echo "Version of black installed:" 27 | black --version 28 | echo "Version of flake8 installed:" 29 | flake8 --version 30 | 31 | - name: Lint with flake8 32 | run: | 33 | flake8 kuma docs 34 | 35 | - name: Lint with black 36 | run: | 37 | black --check --diff kuma docs 38 | -------------------------------------------------------------------------------- /contribute.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Kuma", 3 | "description": "The backend app powering MDN Web Docs.", 4 | "repository": { 5 | "url": "https://github.com/mdn/kuma", 6 | "license": "MPL-2.0", 7 | "tests": "https://github.com/mdn/kuma/actions" 8 | }, 9 | "participate": { 10 | "home": "https://wiki.mozilla.org/MDN", 11 | "docs": "https://kuma.readthedocs.io/", 12 | "mailing-list": "https://discourse.mozilla.org/c/mdn", 13 | "chat": { 14 | "url": "https://mozilla.slack.com/archives/CLTBV3W9X" 15 | } 16 | }, 17 | "bugs": { 18 | "list": "https://github.com/mdn/kuma/issues", 19 | "report": "https://github.com/mdn/kuma/issues/new/choose" 20 | }, 21 | "urls": { 22 | "prod": "https://developer.mozilla.org/", 23 | "stage": "https://developer.allizom.org/" 24 | }, 25 | "keywords": [ 26 | "python", 27 | "django", 28 | "docker", 29 | "mysql", 30 | "elasticsearch" 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /kuma/notifications/migrations/0005_auto_20211215_2004.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2021-12-15 20:04 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("notifications", "0004_notificationdata_data"), 10 | ] 11 | 12 | operations = [ 13 | migrations.DeleteModel( 14 | name="CompatibilityData", 15 | ), 16 | migrations.AddField( 17 | model_name="notification", 18 | name="starred", 19 | field=models.BooleanField(default=False), 20 | ), 21 | migrations.AddField( 22 | model_name="notificationdata", 23 | name="type", 24 | field=models.CharField( 25 | choices=[("content", "content"), ("compat", "compat")], 26 | default="compat", 27 | max_length=32, 28 | ), 29 | ), 30 | migrations.AlterField( 31 | model_name="notificationdata", 32 | name="data", 33 | field=models.JSONField(default=dict), 34 | ), 35 | ] 36 | -------------------------------------------------------------------------------- /kuma/celery.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from celery import Celery 4 | 5 | # set the default Django settings module for the 'celery' program. 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "kuma.settings.local") 7 | 8 | app = Celery("kuma") 9 | 10 | # Using a string here means the worker doesn't have to serialize 11 | # the configuration object to child processes. 12 | # - namespace='CELERY' means all celery-related configuration keys 13 | # should have a `CELERY_` prefix. 14 | app.config_from_object("django.conf:settings", namespace="CELERY") 15 | 16 | # Load task modules from all registered Django app configs. 17 | app.autodiscover_tasks() 18 | 19 | 20 | @app.task(bind=True) 21 | def debug_task(self): 22 | print("Request: {0!r}".format(self.request)) 23 | 24 | 25 | @app.task() 26 | def debug_task_returning(a, b): 27 | """Useful to see if the results backend is working. 28 | And it also checks that called with a `datetime.date` 29 | it gets that as parameters in the task.""" 30 | import datetime 31 | 32 | assert isinstance(a, datetime.date), type(a) 33 | assert isinstance(b, datetime.date), type(b) 34 | return a < b 35 | -------------------------------------------------------------------------------- /Jenkinsfiles/push.groovy: -------------------------------------------------------------------------------- 1 | stage("Announce") { 2 | utils.announce_push() 3 | } 4 | 5 | stage("Check Pull") { 6 | // Check that the image can be successfully pulled from the registry. 7 | utils.ensure_pull() 8 | } 9 | 10 | stage("Prepare Infra") { 11 | // Checkout the "mdn/infra" repo's "main" branch into the 12 | // "infra" sub-directory of the current working directory. 13 | utils.checkout_repo('https://github.com/mdn/infra', 'main', 'infra') 14 | } 15 | 16 | stage('Push') { 17 | dir('infra/apps/mdn/mdn-aws/k8s') { 18 | def current_revision_hash = utils.get_revision_hash() 19 | withEnv(["TO_REVISION_HASH=${env.GIT_COMMIT}", 20 | "FROM_REVISION_HASH=${current_revision_hash}"]) { 21 | // Run the database migrations. 22 | utils.migrate_db() 23 | // Start a rolling update of the Kuma-based deployments. 24 | utils.rollout() 25 | // Monitor the rollout until it has completed. 26 | utils.monitor_rollout() 27 | // Record the rollout in external services like New-Relic. 28 | utils.record_rollout() 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /kuma/notifications/browsers.py: -------------------------------------------------------------------------------- 1 | browsers = { 2 | "chrome": { 3 | "name": "Chrome", 4 | "preview_name": "Canary", 5 | }, 6 | "chrome_android": { 7 | "name": "Chrome Android", 8 | }, 9 | "deno": { 10 | "name": "Deno", 11 | }, 12 | "edge": { 13 | "name": "Edge", 14 | }, 15 | "firefox": { 16 | "name": "Firefox", 17 | "preview_name": "Nightly", 18 | }, 19 | "firefox_android": { 20 | "name": "Firefox for Android", 21 | "pref_url": "about:config", 22 | }, 23 | "ie": { 24 | "name": "Internet Explorer", 25 | }, 26 | "nodejs": { 27 | "name": "Node.js", 28 | }, 29 | "opera": { 30 | "name": "Opera", 31 | }, 32 | "opera_android": { 33 | "name": "Opera Android", 34 | }, 35 | "safari": { 36 | "name": "Safari", 37 | "preview_name": "TP", 38 | }, 39 | "safari_ios": { 40 | "name": "Safari on iOS", 41 | }, 42 | "samsunginternet_android": { 43 | "name": "Samsung Internet", 44 | }, 45 | "webview_android": { 46 | "name": "WebView Android", 47 | }, 48 | } 49 | -------------------------------------------------------------------------------- /kuma/bookmarks/models.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import User 2 | from django.db import models 3 | 4 | from kuma.documenturls.models import DocumentURL 5 | 6 | 7 | class Bookmark(models.Model): 8 | documenturl = models.ForeignKey( 9 | DocumentURL, on_delete=models.CASCADE, verbose_name="Document URL" 10 | ) 11 | user = models.ForeignKey(User, on_delete=models.CASCADE) 12 | custom_name = models.CharField(max_length=500, blank=True) 13 | notes = models.CharField(max_length=500, blank=True) 14 | deleted = models.DateTimeField(null=True) 15 | created = models.DateTimeField(auto_now_add=True) 16 | modified = models.DateTimeField(auto_now=True) 17 | 18 | class Meta: 19 | verbose_name = "Bookmark" 20 | unique_together = ["documenturl", "user_id"] 21 | 22 | def save(self, *args, **kwargs): 23 | if self.custom_name == self.documenturl.metadata["title"]: 24 | self.custom_name = "" 25 | super().save(*args, **kwargs) 26 | 27 | def __str__(self): 28 | return self.documenturl.uri 29 | 30 | @property 31 | def title(self): 32 | return self.custom_name or self.documenturl.metadata["title"] 33 | -------------------------------------------------------------------------------- /docker/images/kuma_base/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10.1-slim@sha256:9f702aa0f2bd7fe7a43bcf46e487040ba3237f2115994ae74ea7b270479ea8f3 2 | 3 | # Set the environment variables 4 | ENV PYTHONDONTWRITEBYTECODE=1 \ 5 | PYTHONUNBUFFERED=1 \ 6 | PIP_DISABLE_PIP_VERSION_CHECK=1 \ 7 | # gunicorn concurrency 8 | WEB_CONCURRENCY=4 9 | 10 | RUN set -x \ 11 | && apt-get update \ 12 | && apt-get install -y --no-install-recommends \ 13 | curl \ 14 | dirmngr \ 15 | # Needed for pytz to be able to install 16 | libsasl2-modules \ 17 | gettext \ 18 | build-essential \ 19 | # Needed for Python to build cffi 20 | libffi-dev 21 | 22 | # add non-privileged user 23 | RUN useradd --uid 1000 --shell /bin/bash --create-home kuma \ 24 | && mkdir -p app \ 25 | && chown kuma:kuma /app \ 26 | && chmod 775 /app 27 | 28 | # install Python libraries 29 | WORKDIR /app 30 | COPY --chown=kuma:kuma ./pyproject.toml ./poetry.lock /app/ 31 | RUN pip install poetry~=1.1.12 \ 32 | && POETRY_VIRTUALENVS_CREATE=false poetry install --no-root \ 33 | && rm -rf ~/.cache/pip ~/.cache/pypoetry/cache 34 | 35 | # setup default run parameters 36 | USER kuma 37 | WORKDIR /app 38 | EXPOSE 8000 39 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Docker testing 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | env: 15 | UID: 0 16 | 17 | steps: 18 | - uses: actions/checkout@v3 19 | 20 | - name: Environment info 21 | run: | 22 | docker-compose --version 23 | 24 | - name: Build docker 25 | run: docker-compose build 26 | 27 | - name: DB migrations 28 | run: | 29 | docker-compose run -T testing urlwait postgresql://kuma:kuma@postgres:5432/developer_mozilla_org 30 30 | docker-compose run -T testing ./manage.py migrate 31 | # Essentially compares **/models.py with **/migrations/*.py and 32 | # makes sure the developer didn't forget to create a new migration. 33 | docker-compose run -T testing ./manage.py makemigrations --check --dry-run 34 | 35 | - name: Run Python tests 36 | run: | 37 | docker-compose run -T testing make coveragetest 38 | 39 | - name: Submit code coverage 40 | run: | 41 | bash <(curl -s --retry 3 --retry-connrefused https://codecov.io/bash) 42 | -------------------------------------------------------------------------------- /kuma/api/v1/decorators.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | 3 | from django.http import HttpResponseForbidden 4 | 5 | from .auth import NotASubscriber, is_subscriber 6 | 7 | 8 | def allow_CORS_GET(func): 9 | """Decorator to allow CORS for GET requests""" 10 | 11 | @wraps(func) 12 | def inner(request, *args, **kwargs): 13 | response = func(request, *args, **kwargs) 14 | if "GET" == request.method: 15 | response["Access-Control-Allow-Origin"] = "*" 16 | return response 17 | 18 | return inner 19 | 20 | 21 | def require_subscriber(view_function): 22 | """Check if a user is authorized to retrieve a resource. 23 | 24 | * Check if a user is logged in through kuma (django session). 25 | * Check if there is a bearer token in the request header. 26 | - Validate the token. 27 | - Create a new user if there is not one 28 | - Retrieve the resource for that user 29 | """ 30 | 31 | @wraps(view_function) 32 | def is_authorized(request, *args, **kwargs): 33 | try: 34 | assert is_subscriber(request, raise_error=True) 35 | except NotASubscriber as e: 36 | return HttpResponseForbidden(e) 37 | return view_function(request, *args, **kwargs) 38 | 39 | return is_authorized 40 | -------------------------------------------------------------------------------- /kuma/documenturls/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from .models import DocumentURL, DocumentURLCheck 4 | 5 | 6 | @admin.register(DocumentURL) 7 | class DocumentURLAdmin(admin.ModelAdmin): 8 | list_display = ( 9 | "uri", 10 | "absolute_url", 11 | "is_valid", 12 | "modified", 13 | ) 14 | readonly_fields = ( 15 | "metadata", 16 | "invalid", 17 | "created", 18 | "modified", 19 | ) 20 | 21 | list_filter = ("invalid",) 22 | search_fields = ("uri",) 23 | ordering = ("-created",) 24 | list_per_page = 10 25 | 26 | def is_valid(self, obj): 27 | return not obj.invalid 28 | 29 | 30 | @admin.register(DocumentURLCheck) 31 | class DocumentURLCheckAdmin(admin.ModelAdmin): 32 | list_display = ( 33 | "_document_url", 34 | "http_error", 35 | "created", 36 | ) 37 | readonly_fields = ( 38 | "document_url", 39 | "http_error", 40 | "headers", 41 | "created", 42 | ) 43 | 44 | list_filter = ("http_error",) 45 | search_fields = ("document_url__uri", "document_url__absolute_url") 46 | ordering = ("-created",) 47 | list_per_page = 10 48 | 49 | def _document_url(self, obj): 50 | return obj.document_url.absolute_url 51 | -------------------------------------------------------------------------------- /kuma/core/admin.py: -------------------------------------------------------------------------------- 1 | # from django.contrib import admin 2 | 3 | # from rest_framework.authtoken.admin import TokenAdmin 4 | 5 | # from kuma.core.models import IPBan 6 | 7 | 8 | class DisabledDeleteActionMixin(object): 9 | def get_actions(self, request): 10 | """ 11 | Remove the built-in delete action, since it bypasses the model 12 | delete() method (bad) and we want people using the non-admin 13 | deletion UI anyway. 14 | """ 15 | actions = super(DisabledDeleteActionMixin, self).get_actions(request) 16 | if "delete_selected" in actions: 17 | del actions["delete_selected"] 18 | return actions 19 | 20 | 21 | class DisabledDeletionMixin(DisabledDeleteActionMixin): 22 | def has_delete_permission(self, request, obj=None): 23 | """ 24 | Disable deletion of individual Documents, by always returning 25 | False for the permission check. 26 | """ 27 | return False 28 | 29 | 30 | # @admin.register(IPBan) 31 | # class IPBanAdmin(admin.ModelAdmin): 32 | # # Remove list delete action to enforce model soft delete in admin site 33 | # actions = None 34 | # readonly_fields = ("deleted",) 35 | # list_display = ("ip", "created", "deleted") 36 | 37 | 38 | # TokenAdmin.raw_id_fields = ["user"] 39 | -------------------------------------------------------------------------------- /kuma/api/v1/tests/test_subscriptions.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import pytest 4 | from django.urls import reverse 5 | 6 | from kuma.users import models 7 | 8 | 9 | @pytest.mark.django_db 10 | def test_subscriptions(subscriber_client, wiki_user): 11 | response = subscriber_client.get(reverse("api-v1:whoami")) 12 | assert response.status_code == 200 13 | assert json.loads(response.content) == { 14 | "username": wiki_user.username, 15 | "is_authenticated": True, 16 | "email": wiki_user.email, 17 | "is_subscriber": True, 18 | "subscription_type": "", 19 | "avatar_url": "", 20 | } 21 | 22 | # Add Subscription and save model back to db 23 | profile = models.UserProfile.objects.get(user=wiki_user) 24 | profile.subscription_type = models.UserProfile.SubscriptionType.MDN_PLUS_10Y 25 | profile.is_subscriber = True 26 | profile.save() 27 | 28 | # Assert subscription type present in response 29 | response = subscriber_client.get(reverse("api-v1:whoami")) 30 | assert response.status_code == 200 31 | assert json.loads(response.content) == { 32 | "username": wiki_user.username, 33 | "is_authenticated": True, 34 | "email": wiki_user.email, 35 | "is_subscriber": True, 36 | "subscription_type": models.UserProfile.SubscriptionType.MDN_PLUS_10Y.value, 37 | "avatar_url": "", 38 | } 39 | -------------------------------------------------------------------------------- /kuma/core/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | from django.conf import settings 3 | from django.utils.functional import cached_property 4 | from django.utils.translation import gettext_lazy as _ 5 | 6 | from kuma.celery import app 7 | 8 | 9 | class CoreConfig(AppConfig): 10 | """ 11 | The Django App Config class to store information about the core app 12 | and do startup time things. 13 | """ 14 | 15 | name = "kuma.core" 16 | verbose_name = _("Core") 17 | 18 | def ready(self): 19 | """Configure kuma.core after models are loaded.""" 20 | self.add_periodc_tasks() 21 | 22 | def add_periodc_tasks(self): 23 | from kuma.core.tasks import clean_sessions, clear_old_notifications 24 | 25 | # Clean up expired sessions every 60 minutes 26 | app.add_periodic_task(60 * 60, clean_sessions.s()) 27 | # Delete old notifications every month 28 | app.add_periodic_task(60 * 60 * 24 * 30, clear_old_notifications.s()) 29 | 30 | @cached_property 31 | def language_mapping(self): 32 | """ 33 | a static mapping of lower case language names and their native names 34 | """ 35 | # LANGUAGES settings return a list of tuple with language code and their native name 36 | # Make the language code lower and convert the tuple to dictionary 37 | return {lang[0].lower(): lang[1] for lang in settings.LANGUAGES} 38 | -------------------------------------------------------------------------------- /kuma/core/tests/test_views.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.core import mail 4 | from django.test import override_settings 5 | from django.utils.log import AdminEmailHandler 6 | 7 | from . import KumaTestCase 8 | 9 | 10 | @override_settings( 11 | DEBUG=False, 12 | DEBUG_PROPAGATE_EXCEPTIONS=False, 13 | ADMINS=(("admin", "admin@example.com"),), 14 | ROOT_URLCONF="kuma.core.tests.logging_urls", 15 | ) 16 | class LoggingTests(KumaTestCase): 17 | logger = logging.getLogger("django.security") 18 | suspicous_path = "/en-US/suspicious/" 19 | 20 | def setUp(self): 21 | super(LoggingTests, self).setUp() 22 | self.old_handlers = self.logger.handlers[:] 23 | 24 | def tearDown(self): 25 | super(LoggingTests, self).tearDown() 26 | self.logger.handlers = self.old_handlers 27 | 28 | def test_no_mail_handler(self): 29 | self.logger.handlers = [logging.NullHandler()] 30 | response = self.client.get(self.suspicous_path) 31 | assert 400 == response.status_code 32 | assert 0 == len(mail.outbox) 33 | 34 | def test_mail_handler(self): 35 | self.logger.handlers = [AdminEmailHandler()] 36 | response = self.client.get(self.suspicous_path) 37 | assert 400 == response.status_code 38 | assert 1 == len(mail.outbox) 39 | 40 | assert "admin@example.com" in mail.outbox[0].to 41 | assert self.suspicous_path in mail.outbox[0].body 42 | -------------------------------------------------------------------------------- /kuma/api/v1/plus/landing_page.py: -------------------------------------------------------------------------------- 1 | from uuid import UUID 2 | 3 | from django.middleware.csrf import get_token 4 | from django.shortcuts import get_object_or_404 5 | from ninja import Form, Router 6 | from pydantic import Json 7 | from ratelimit.decorators import ratelimit 8 | 9 | from kuma.api.v1.plus.notifications import Ok 10 | from kuma.plus.models import LandingPageSurvey 11 | 12 | router = Router() 13 | 14 | 15 | @router.post("/survey/", response=Ok, url_name="plus.landing_page.survey", auth=None) 16 | @ratelimit(group="landing_page_survey", key="user_or_ip", rate="100/m", block=True) 17 | def post_survey(request, uuid: str = Form(...), response: Json = Form(...)): 18 | survey = get_object_or_404(LandingPageSurvey, uuid=uuid) 19 | survey.response = response 20 | survey.save() 21 | return True 22 | 23 | 24 | @router.get("/survey/", url_name="plus.landing_page.survey", auth=None) 25 | @ratelimit(group="landing_page_survey", key="user_or_ip", rate="100/m", block=True) 26 | def get_survey(request, uuid: UUID = None): 27 | # Inspired by https://github.com/mdn/kuma/pull/7849/files 28 | if uuid: 29 | survey = get_object_or_404(LandingPageSurvey, uuid=uuid) 30 | else: 31 | geo_information = request.META.get("HTTP_CLOUDFRONT_VIEWER_COUNTRY_NAME") or "" 32 | survey = LandingPageSurvey.objects.create( 33 | geo_information=geo_information, 34 | ) 35 | return {"uuid": survey.uuid, "csrfmiddlewaretoken": get_token(request)} 36 | -------------------------------------------------------------------------------- /kuma/plus/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from .models import LandingPageSurvey 4 | 5 | 6 | class HasResponseFilter(admin.SimpleListFilter): 7 | title = "Has response" 8 | parameter_name = "has_response" 9 | 10 | def lookups(self, request, model_admin): 11 | return ( 12 | ("true", "Has response"), 13 | ("false", "No response"), 14 | ) 15 | 16 | def queryset(self, request, queryset): 17 | if self.value() == "true": 18 | return queryset.filter(response__isnull=False) 19 | if self.value() == "false": 20 | return queryset.filter(response__isnull=True) 21 | 22 | 23 | @admin.register(LandingPageSurvey) 24 | class LandingPageSurveyAdmin(admin.ModelAdmin): 25 | list_display = ( 26 | "uuid", 27 | "geo_information", 28 | "has_response", 29 | "created", 30 | ) 31 | fields = ( 32 | "geo_information", 33 | "response", 34 | ) 35 | readonly_fields = ( 36 | "geo_information", 37 | "response", 38 | ) 39 | 40 | list_filter = (HasResponseFilter,) 41 | search_fields = ("response", "uuid", "geo_information") 42 | ordering = ("-created",) 43 | list_per_page = 10 44 | 45 | def has_email(self, obj): 46 | return bool(obj.email) 47 | 48 | def has_response(self, obj): 49 | return bool(obj.response) 50 | 51 | def signed_in(self, obj): 52 | return bool(obj.user) 53 | -------------------------------------------------------------------------------- /kuma/api/v1/tests/test_landing_page.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from kuma.core.urlresolvers import reverse 4 | from kuma.plus.models import LandingPageSurvey 5 | 6 | 7 | @pytest.mark.django_db 8 | def test_ping_landing_page_survey_bad_request(client): 9 | url = reverse("api-v1:plus.landing_page.survey") 10 | 11 | # Not a valid UUID 12 | response = client.get(url, {"uuid": "xxx"}) 13 | assert response.status_code == 422 14 | 15 | # Not a recognized UUID 16 | response = client.get(url, {"uuid": "88f7a689-454a-4647-99bf-d62fa66da24a"}) 17 | assert response.status_code == 404 18 | 19 | # No UUID in post 20 | response = client.post(url) 21 | assert response.status_code == 422 22 | 23 | response = client.get(url, {"variant": "1"}) 24 | assert response.status_code == 200 25 | # Invalid JSON 26 | response = client.post(url, {"uuid": response.json()["uuid"], "response": "{{{{"}) 27 | assert response.status_code == 422 28 | 29 | 30 | @pytest.mark.django_db 31 | def test_ping_landing_page_survey_reuse_uuid(client): 32 | url = reverse("api-v1:plus.landing_page.survey") 33 | response1 = client.get(url, HTTP_CLOUDFRONT_VIEWER_COUNTRY_NAME="Sweden") 34 | assert response1.status_code == 200 35 | assert LandingPageSurvey.objects.all().count() == 1 36 | response2 = client.get( 37 | url, 38 | {"uuid": response1.json()["uuid"]}, 39 | HTTP_CLOUDFRONT_VIEWER_COUNTRY_NAME="USA", 40 | ) 41 | assert response2.json()["uuid"] == response1.json()["uuid"] 42 | assert LandingPageSurvey.objects.all().count() == 1 43 | -------------------------------------------------------------------------------- /kuma/api/tests/test_admin.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.conf import settings 4 | from django.urls import reverse 5 | from model_bakery import baker 6 | 7 | from kuma.notifications import models 8 | 9 | 10 | def test_admin_update_content(user_client, wiki_user): 11 | # Prepare: Watch page. 12 | page_title = ": The Dialog element" 13 | page_url = "/en-us/docs/web/html/element/dialog" 14 | baker.make(models.Watch, users=[wiki_user], title=page_title, url=page_url) 15 | 16 | # Test: Trigger content update. 17 | url = reverse("admin_api:admin.update_content") 18 | auth_headers = { 19 | "HTTP_AUTHORIZATION": f"Bearer {settings.NOTIFICATIONS_ADMIN_TOKEN}", 20 | } 21 | response = user_client.post( 22 | url, 23 | json.dumps( 24 | { 25 | "page": "/en-US/docs/Web/HTML/Element/dialog", 26 | "pr": "https://github.com/mdn/content/pull/14607", 27 | } 28 | ), 29 | content_type="application/json", 30 | **auth_headers, 31 | ) 32 | 33 | assert response.status_code == 200 34 | 35 | # Verify: Notification was created. 36 | url = reverse("api-v1:plus.notifications") 37 | response = user_client.get(url) 38 | assert response.status_code == 200 39 | 40 | notifications = json.loads(response.content)["items"] 41 | assert len(notifications) == 1 42 | 43 | notification = notifications[0] 44 | assert notification["title"] == page_title 45 | assert notification["url"] == page_url 46 | assert notification["text"] == "Page updated (see PR!mdn/content!14607!!)" 47 | -------------------------------------------------------------------------------- /kuma/notifications/migrations/0008_default_watch.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2022-01-06 03:06 2 | 3 | import django.db.models.deletion 4 | from django.conf import settings 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 12 | ("notifications", "0007_watch_users"), 13 | ] 14 | 15 | operations = [ 16 | migrations.AddField( 17 | model_name="userwatch", 18 | name="custom_default", 19 | field=models.BooleanField(default=False), 20 | ), 21 | migrations.CreateModel( 22 | name="DefaultWatch", 23 | fields=[ 24 | ( 25 | "id", 26 | models.BigAutoField( 27 | auto_created=True, 28 | primary_key=True, 29 | serialize=False, 30 | verbose_name="ID", 31 | ), 32 | ), 33 | ("content_updates", models.BooleanField(default=True)), 34 | ("browser_compatibility", models.JSONField(default=list)), 35 | ( 36 | "user", 37 | models.OneToOneField( 38 | on_delete=django.db.models.deletion.CASCADE, 39 | to=settings.AUTH_USER_MODEL, 40 | ), 41 | ), 42 | ], 43 | options={ 44 | "abstract": False, 45 | }, 46 | ), 47 | ] 48 | -------------------------------------------------------------------------------- /kuma/api/v1/tests/test_watched_items.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.urls import reverse 4 | from model_bakery import baker 5 | 6 | from kuma.notifications import models 7 | 8 | 9 | def test_unwatch_manys(user_client, wiki_user): 10 | url = reverse("api-v1:watching") 11 | user_watched_items = [] 12 | 13 | for i in range(10): 14 | mock_watch = baker.make(models.Watch, users=[wiki_user]) 15 | user_watch = baker.make( 16 | models.UserWatch, user=wiki_user, id=i, watch=mock_watch 17 | ) 18 | user_watched_items.append(user_watch) 19 | 20 | response = user_client.get(url, {"limit": 10}) 21 | assert response.status_code == 200 22 | items_json = json.loads(response.content)["items"] 23 | assert len(items_json) == 10 24 | 25 | unwatch_many_url = reverse("api-v1:unwatch_many") 26 | 27 | # Given 6 items are deleted. 28 | del1 = user_watched_items[0].watch.url 29 | del2 = user_watched_items[1].watch.url 30 | response = user_client.post( 31 | unwatch_many_url, 32 | json.dumps( 33 | { 34 | "unwatch": [ 35 | del1, 36 | del2, 37 | ] 38 | } 39 | ), 40 | content_type="application/json", 41 | ) 42 | assert response.status_code == 200 43 | # Refetch 44 | response = user_client.get(url, {"limit": 10}) 45 | items_json = json.loads(response.content)["items"] 46 | filtered = filter( 47 | lambda item: item["url"] == del1 or item["url"] == del2, items_json 48 | ) 49 | # Assert deleted no longer there :) 50 | assert len(list(filtered)) == 0 51 | -------------------------------------------------------------------------------- /kuma/plus/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.5 on 2021-07-09 21:00 2 | 3 | import uuid 4 | 5 | import django.db.models.deletion 6 | from django.conf import settings 7 | from django.db import migrations, models 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | initial = True 13 | 14 | dependencies = [ 15 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 16 | ] 17 | 18 | operations = [ 19 | migrations.CreateModel( 20 | name="LandingPageSurvey", 21 | fields=[ 22 | ( 23 | "uuid", 24 | models.UUIDField( 25 | default=uuid.uuid4, 26 | editable=False, 27 | primary_key=True, 28 | serialize=False, 29 | ), 30 | ), 31 | ("email", models.CharField(blank=True, max_length=100)), 32 | ("variant", models.PositiveIntegerField()), 33 | ("response", models.JSONField(editable=False, null=True)), 34 | ("geo_information", models.TextField(editable=False, null=True)), 35 | ("created", models.DateTimeField(auto_now_add=True)), 36 | ("updated", models.DateTimeField(auto_now=True)), 37 | ( 38 | "user", 39 | models.ForeignKey( 40 | null=True, 41 | on_delete=django.db.models.deletion.CASCADE, 42 | to=settings.AUTH_USER_MODEL, 43 | ), 44 | ), 45 | ], 46 | ), 47 | ] 48 | -------------------------------------------------------------------------------- /kuma/users/middleware.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from django.conf import settings 4 | from django.contrib.auth import logout 5 | from django.core.exceptions import MiddlewareNotUsed 6 | from mozilla_django_oidc.middleware import SessionRefresh 7 | 8 | from kuma.users.auth import KumaOIDCAuthenticationBackend 9 | 10 | 11 | class ValidateAccessTokenMiddleware(SessionRefresh): 12 | """Validate the access token every hour. 13 | 14 | Verify that the access token has not been invalidated 15 | by the user through the Firefox Accounts web interface. 16 | """ 17 | 18 | def __init__(self, *args, **kwargs): 19 | if settings.DEV and settings.DEBUG: 20 | raise MiddlewareNotUsed 21 | super().__init__(*args, **kwargs) 22 | 23 | def process_request(self, request): 24 | 25 | if not self.is_refreshable_url(request): 26 | return 27 | 28 | expiration = request.session.get("oidc_id_token_expiration", 0) 29 | now = time.time() 30 | access_token = request.session.get("oidc_access_token") 31 | profile = request.user.userprofile 32 | 33 | if access_token and expiration < now: 34 | 35 | token_info = KumaOIDCAuthenticationBackend.refresh_access_token( 36 | profile.fxa_refresh_token 37 | ) 38 | new_access_token = token_info.get("access_token") 39 | if new_access_token: 40 | request.session["oidc_access_token"] = new_access_token 41 | request.session["oidc_id_token_expiration"] = ( 42 | now + settings.FXA_TOKEN_EXPIRY 43 | ) 44 | else: 45 | profile.fxa_refresh_token = "" 46 | profile.save() 47 | logout(request) 48 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | #!groovy 2 | 3 | @Library('github.com/mozmeao/jenkins-pipeline@master') 4 | 5 | def loadBranch(String branch) { 6 | utils = load 'Jenkinsfiles/utils.groovy' 7 | if (fileExists("./Jenkinsfiles/${branch}.yml")) { 8 | config = readYaml file: "./Jenkinsfiles/${branch}.yml" 9 | println "config ==> ${config}" 10 | } else { 11 | config = [] 12 | } 13 | 14 | if (config && config.pipeline && config.pipeline.enabled == false) { 15 | println "Pipeline disabled." 16 | } else { 17 | if (config && config.pipeline && config.pipeline.script) { 18 | println "Loading ./Jenkinsfiles/${config.pipeline.script}.groovy" 19 | load "./Jenkinsfiles/${config.pipeline.script}.groovy" 20 | } else { 21 | println "Loading ./Jenkinsfiles/${branch}.groovy" 22 | load "./Jenkinsfiles/${branch}.groovy" 23 | } 24 | } 25 | } 26 | 27 | node { 28 | stage("Prepare") { 29 | checkout scm 30 | sh 'git submodule sync' 31 | sh 'git submodule update --init --recursive' 32 | setGitEnvironmentVariables() 33 | // Set UID to jenkins 34 | env['UID'] = sh(returnStdout: true, script: 'id -u jenkins').trim() 35 | // Prepare for junit test results 36 | sh "mkdir -p test_results" 37 | sh "rm -f test_results/*.xml" 38 | 39 | // When checking in a file exists in another directory start with './' or 40 | // prepare to fail. 41 | try { 42 | if (fileExists("./Jenkinsfiles/${env.BRANCH_NAME}.groovy") || fileExists("./Jenkinsfiles/${env.BRANCH_NAME}.yml")) { 43 | loadBranch(env.BRANCH_NAME) 44 | } else { 45 | loadBranch("default") 46 | } 47 | } 48 | finally { 49 | if (findFiles(glob: 'test_results/*.xml')) { 50 | junit 'test_results/*.xml' 51 | } 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /kuma/users/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.5 on 2021-08-05 12:47 2 | 3 | import django.db.models.deletion 4 | from django.conf import settings 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name="UserProfile", 19 | fields=[ 20 | ( 21 | "id", 22 | models.BigAutoField( 23 | auto_created=True, 24 | primary_key=True, 25 | serialize=False, 26 | verbose_name="ID", 27 | ), 28 | ), 29 | ("claims", models.JSONField(default=dict)), 30 | ("locale", models.CharField(max_length=6, null=True)), 31 | ("is_subscriber", models.DateTimeField(null=True)), 32 | ( 33 | "subscriber_number", 34 | models.PositiveIntegerField(null=True, unique=True), 35 | ), 36 | ("created", models.DateTimeField(auto_now_add=True)), 37 | ("modified", models.DateTimeField(auto_now=True)), 38 | ( 39 | "user", 40 | models.OneToOneField( 41 | on_delete=django.db.models.deletion.CASCADE, 42 | to=settings.AUTH_USER_MODEL, 43 | ), 44 | ), 45 | ], 46 | options={ 47 | "verbose_name": "User profile", 48 | }, 49 | ), 50 | ] 51 | -------------------------------------------------------------------------------- /scripts/ci-python: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e # Exit on non-zero status 3 | set -u # Treat unset variables as an error 4 | 5 | # First install docker-compose (right version of it) 6 | if [ -x $(command -v docker-compose) ] 7 | then 8 | echo "Overwriting existing docker-compose." 9 | docker-compose -v 10 | else 11 | echo "Installing docker-compose ${DOCKER_COMPOSE_VERSION}." 12 | fi 13 | mkdir -p downloads 14 | DOCKER_COMPOSE_FILE=downloads/docker-compose-${DOCKER_COMPOSE_VERSION} 15 | if [ ! -f $DOCKER_COMPOSE_FILE ] 16 | then 17 | wget -q --waitretry=1 --retry-connrefused -T 10 \ 18 | -O downloads/docker-compose-${DOCKER_COMPOSE_VERSION} \ 19 | https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` 20 | fi 21 | chmod +x $DOCKER_COMPOSE_FILE 22 | sudo cp $DOCKER_COMPOSE_FILE /usr/local/bin/docker-compose 23 | 24 | 25 | # Now build the Docker container 26 | 27 | VERSION=latest make build-base 28 | 29 | # This "make build-kuma" step is done here only to test that the kuma docker 30 | # image can be built without error, not because it's needed by docker-compose. 31 | VERSION=latest make build-kuma 32 | 33 | docker-compose --version 34 | docker-compose up -d 35 | 36 | # Hang tight until MySQL appears to be working 37 | docker-compose exec -T web urlwait mysql://root:kuma@mysql:3306/developer_mozilla_org 30 38 | # Hang tight until Elasticsearch appears to be working 39 | docker-compose exec -T web urlwait http://elasticsearch:9200 30 40 | docker-compose exec -T web make clean 41 | # Needed for Django's check_for_language 42 | docker-compose exec -T web make localecompile 43 | docker-compose exec -T web ./manage.py migrate 44 | # This checks that running `./manage.py makemigrations` wasn't forgotten 45 | docker-compose exec -T web ./manage.py makemigrations --check --dry-run 46 | docker-compose exec -T web make coveragetest 47 | -------------------------------------------------------------------------------- /.env-dist.dev: -------------------------------------------------------------------------------- 1 | # Sample .env file for local development - copy to .env and customize 2 | 3 | # User ID to use in Docker containers 4 | # Linux users should set this value to avoid permissions issues. Use "id" or 5 | # "echo $UID" to determine the user ID. 6 | # MacOS and Windows users can use the default of 1000 (kuma), and the Docker 7 | # file system layer will translate to your local user permissions. 8 | #UID=1000 9 | 10 | # Enable the Django Debug Toolbar 11 | # Provides useful troubleshooting data, but adds 3+ seconds to response time 12 | #DEBUG_TOOLBAR=False 13 | 14 | # Enable Maintenance Mode 15 | # https://kuma.readthedocs.io/en/latest/development.html#maintenance-mode 16 | #MAINTENANCE_MODE=True 17 | #DATABASE_USER=kuma_ro 18 | 19 | # Local development of Interactive Examples 20 | # See https://github.com/mdn/interactive-examples/ 21 | #INTERACTIVE_EXAMPLES_BASE=http://localhost:9090 22 | 23 | # Set the level of the ElasticSearch logger in Kuma 24 | #ES_LOG_LEVEL=ERROR # Default, never logs 25 | #ES_LOG_LEVEL=WARNING # Logs HTTP method and path of ElasticSearch requests 26 | #ES_LOG_LEVEL=DEBUG # Logs the body (usually JSON) of the ES requests and responses 27 | 28 | # Set the level of the ElasticSearch trace logging in Kuma 29 | # ES_TRACE_LOG_LEVEL=ERROR # Default, never logs 30 | # ES_TRACE_LOG_LEVEL=INFO # Logs pretty-printed curl variant of the request 31 | # ES_TRACE_LOG_LEVEL=DEBUG # Logs pretty-printed JSON response 32 | 33 | # Switch to SSL configuration at https://developer-local.allizom.org 34 | #COMPOSE_FILE=docker-compose.yml:docker-compose.ssl.yml 35 | 36 | ENABLE_RESTRICTIONS_BY_HOST=True 37 | DOMAIN=localhost.org 38 | ATTACHMENT_HOST=demos:8000 39 | SITE_URL=http://localhost.org:8000 40 | STATIC_URL=http://localhost.org:8000/static/ 41 | ALLOW_ROBOTS_WEB_DOMAINS=localhost.org:8000 42 | FXA_TOKEN_ISSUER=https://accounts.stage.mozaws.net 43 | FXA_VERIFY_URL=https://oauth.stage.mozaws.net/v1/verify -------------------------------------------------------------------------------- /kuma/api/v1/pagination.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Generic, TypeVar 4 | 5 | from django.conf import settings 6 | from django.db.models import QuerySet 7 | from django.http import HttpRequest 8 | from django.middleware.csrf import get_token 9 | from ninja import Field 10 | from ninja.pagination import LimitOffsetPagination 11 | from ninja.schema import Schema 12 | from pydantic.generics import GenericModel 13 | 14 | ItemSchema = TypeVar("ItemSchema") 15 | 16 | 17 | class PaginationInput(Schema): 18 | page: int = Field(1, gt=0) 19 | per_page: int = Field(settings.API_V1_PAGE_SIZE, ge=0, le=100) 20 | 21 | 22 | class LimitOffsetInput(Schema): 23 | limit: int = Field(20, gt=0) 24 | offset: int = Field(0, ge=1) 25 | 26 | 27 | class PaginatedMetadata(Schema): 28 | total: int 29 | page: int 30 | per_page: int 31 | max_non_subscribed: int 32 | 33 | 34 | class PaginatedResponse(Schema, GenericModel, Generic[ItemSchema]): 35 | items: list[ItemSchema] 36 | metadata: PaginatedMetadata 37 | csrfmiddlewaretoken: str 38 | 39 | 40 | class LimitOffsetPaginatedData: 41 | items: QuerySet | list 42 | csrfmiddlewaretoken: str 43 | 44 | def __init__(self, items: QuerySet | list, csrfmiddlewaretoken: str): 45 | self.items = items 46 | self.csrfmiddlewaretoken = csrfmiddlewaretoken 47 | 48 | 49 | class LimitOffsetPaginatedResponse(Schema, GenericModel, Generic[ItemSchema]): 50 | items: list[ItemSchema] 51 | csrfmiddlewaretoken: str 52 | 53 | 54 | class LimitOffsetPaginationWithMeta(LimitOffsetPagination): 55 | def paginate_queryset( 56 | self, items: QuerySet, request: HttpRequest, **params 57 | ) -> LimitOffsetPaginatedData: 58 | paginated_items = super().paginate_queryset(items, request, **params) 59 | return LimitOffsetPaginatedData( 60 | paginated_items, csrfmiddlewaretoken=get_token(request) 61 | ) 62 | -------------------------------------------------------------------------------- /kuma/bookmarks/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.5 on 2021-07-12 19:04 2 | 3 | import django.db.models.deletion 4 | from django.conf import settings 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 14 | ("documenturls", "0001_initial"), 15 | ] 16 | 17 | operations = [ 18 | migrations.CreateModel( 19 | name="Bookmark", 20 | fields=[ 21 | ( 22 | "id", 23 | models.BigAutoField( 24 | auto_created=True, 25 | primary_key=True, 26 | serialize=False, 27 | verbose_name="ID", 28 | ), 29 | ), 30 | ("notes", models.JSONField(default=list)), 31 | ("deleted", models.DateTimeField(null=True)), 32 | ("created", models.DateTimeField(auto_now_add=True)), 33 | ("modified", models.DateTimeField(auto_now=True)), 34 | ( 35 | "documenturl", 36 | models.ForeignKey( 37 | on_delete=django.db.models.deletion.CASCADE, 38 | to="documenturls.documenturl", 39 | verbose_name="Document URL", 40 | ), 41 | ), 42 | ( 43 | "user", 44 | models.ForeignKey( 45 | on_delete=django.db.models.deletion.CASCADE, 46 | to=settings.AUTH_USER_MODEL, 47 | ), 48 | ), 49 | ], 50 | options={ 51 | "verbose_name": "Bookmark", 52 | "unique_together": {("documenturl", "user_id")}, 53 | }, 54 | ), 55 | ] 56 | -------------------------------------------------------------------------------- /docs/documentation.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | Documentation 3 | ============= 4 | This documentation is generated and published at 5 | `Read the Docs`_ whenever the master branch is updated. 6 | 7 | GitHub can render our ``.rst`` documents as ReStructuredText_, which is 8 | close enough to Sphinx_ for most code reviews, without features like links 9 | between documents. 10 | 11 | It is occasionally necessary to generate the documentation locally. It is 12 | easiest to do this with a virtualenv_ on the host system, using Docker only to 13 | regenerate the MDN Sphinx template. If you are not comfortable with that style 14 | of development, it can be done entirely in Docker using ``docker-compose``. 15 | 16 | .. _`Read the Docs`: https://kuma.readthedocs.io/en/latest/ 17 | .. _ReStructuredText: https://en.wikipedia.org/wiki/ReStructuredText 18 | .. _Sphinx: https://en.wikipedia.org/wiki/Sphinx_(documentation_generator) 19 | .. _virtualenv: https://virtualenv.pypa.io/en/stable/ 20 | 21 | Generating documentation 22 | ------------------------ 23 | Sphinx uses a ``Makefile`` in the ``docs`` subfolder to build documentation in 24 | several formats. MDN only uses the HTML format, and the generated document 25 | index is at ``docs/_build/html/index.html``. 26 | 27 | To generate the documentation in a virtualenv on the host machine, first 28 | install the requirements:: 29 | 30 | pip install -r docs/requirements.txt 31 | 32 | Then switch to the ``docs`` folder to use the ``Makefile``:: 33 | 34 | cd docs 35 | make html 36 | python -m webbrowser file://${PWD}/_build/html/index.html 37 | 38 | 39 | To generate the documentation with Docker:: 40 | 41 | docker-compose run --rm web sh -c "\ 42 | python -m venv /tmp/.venvs/docs && \ 43 | . /tmp/.venvs/docs/bin/activate && \ 44 | pip install -r /app/docs/requirements.txt && \ 45 | cd /app/docs && \ 46 | make html" 47 | python -m webbrowser file://${PWD}/docs/_build/html/index.html 48 | 49 | A ``virtualenv`` is required, to avoid a ``pip`` bug when changing the version 50 | of a system-installed package. 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Modeled after https://github.com/github/gitignore/blob/master/Python.gitignore 2 | # .gitignore directory matching: 3 | # /foo/ - Directory foo in the root of the project 4 | # foo/ - Directory foo anywhere in the project 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | 10 | # Distribution / packaging 11 | /build/ 12 | *.egg-info/ 13 | 14 | # Installer logs 15 | pip-log.txt 16 | 17 | # Unit test / coverage reports 18 | htmlcov/ 19 | .tox 20 | .coverage 21 | .coverage.* 22 | .cache 23 | .noseids 24 | .noserc 25 | .pytest_cache 26 | nosetests.xml 27 | coverage.xml 28 | /test_results/ 29 | 30 | # Atom 31 | .ropeproject 32 | 33 | # VSCode 34 | .vscode 35 | 36 | # PyCharm 37 | .idea 38 | 39 | # Sphinx documentation 40 | /docs/_build/ 41 | 42 | # pyenv 43 | .python-version 44 | 45 | # Celery 46 | celerybeat-schedule 47 | celeryev.pid 48 | 49 | # .dotenv 50 | .env 51 | 52 | # Virtualenv 53 | .venv 54 | venv/ 55 | ENV/ 56 | 57 | # Swapfiles 58 | *.sw? 59 | 60 | # macOS finder files 61 | .DS_Store 62 | 63 | # Kuma-specfic 64 | *devmo*.sql* 65 | assets/ckeditor4/source/ckbuilder/ 66 | bower_components 67 | build.py 68 | developer.mozilla.org.tar.gz 69 | docker-compose.dev.yml 70 | docker-compose.locust.yml 71 | docker-compose.override.yml 72 | etc/nginx/ssl/developer.127.0.0.1.nip.io.crt 73 | etc/nginx/ssl/developer.127.0.0.1.nip.io.key 74 | james.ini 75 | mdn_sample_db.sql 76 | mdn_sample_db.sql.gz 77 | /mdntests/ 78 | /media/attachments 79 | /media/uploads 80 | /media/revision.txt 81 | /media/kumascript-revision.txt 82 | node_modules/ 83 | /kuma/static/ 84 | kuma/javascript/dist/ 85 | /tmp/emails/*.log 86 | npm-debug.log 87 | dist/ 88 | 89 | # Not an actual file any more but kept for simplicity. 90 | humans.txt 91 | 92 | # Selenium 93 | geckodriver.log 94 | 95 | # Legacy 96 | .awsconfig 97 | /logs/ 98 | /xfers/ 99 | wheelhouse 100 | /uploads/ 101 | kumascript.log 102 | kuma/static/js/libs/ckeditor/source/ckeditor/ 103 | kuma/static/js/libs/ckeditor/source/ckbuilder/ 104 | 105 | # Emacs backup files 106 | *~ 107 | junit.xml 108 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # Files excluded from Docker context 2 | # 3 | # This is useful to test changes, find files to exclude: 4 | # https://stackoverflow.com/questions/38946683/how-to-test-dockerignore-file 5 | # 6 | # Patterns from kuma .gitignore, but with changes for 7 | # .dockerignore directory matching: 8 | # foo/ - Directory foo in the root of the repo 9 | # **/foo/ - Directory foo anywhere in the repo 10 | # foo/* - Files under foo in the root of the repo, keep directory 11 | # **/foo/* - Files under foo anywhere in the repo, keep directory 12 | # 13 | 14 | # Byte-compiled / optimized / DLL files 15 | **/__pycache__/ 16 | **/*.py[cod] 17 | 18 | # Distribution / packaging 19 | build/ 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | htmlcov/ 26 | .coverage 27 | .coverage.* 28 | .cache 29 | coverage.xml 30 | test_results/ 31 | 32 | # Sphinx documentation 33 | docs/_build/ 34 | 35 | # pyenv 36 | .python-version 37 | 38 | # Celery 39 | celerybeat-schedule 40 | celeryev.pid 41 | 42 | # .dotenv 43 | .env 44 | 45 | # Virtualenv 46 | .venv 47 | **/venv/ 48 | **/ENV/ 49 | 50 | # Swapfiles 51 | *.sw? 52 | 53 | # macOS finder files 54 | .DS_Store 55 | 56 | # Kuma-specfic 57 | *devmo*.sql* 58 | assets/ckeditor4/source/ckbuilder/ 59 | bower_components 60 | build.py 61 | developer.mozilla.org.tar.gz 62 | docker-compose.dev.yml 63 | docker-compose.locust.yml 64 | james.ini 65 | mdn_sample_db.sql 66 | mdn_sample_db.sql.gz 67 | mdntests/ 68 | media/attachments 69 | media/uploads 70 | media/revision.txt 71 | media/kumascript-revision.txt 72 | **/node_modules/ 73 | static/ 74 | tmp 75 | 76 | # Not an actual file any more but kept for simplicity. 77 | humans.txt 78 | 79 | # Selenium 80 | geckodriver.log 81 | 82 | # Legacy 83 | .awsconfig 84 | logs/ 85 | xfers/ 86 | wheelhouse 87 | uploads/ 88 | kumascript.log 89 | kuma/static/js/libs/ckeditor/source/ckeditor/ 90 | kuma/static/js/libs/ckeditor/source/ckbuilder/ 91 | 92 | # 93 | # End adjusted kuma .gitignore 94 | # 95 | 96 | 97 | # Additional ignores 98 | .git 99 | .gitignore 100 | k8s/ 101 | docker/ 102 | docker-compose* 103 | # Rebuilt in Docker 104 | locale/*/LC_MESSAGES/*.mo 105 | *.pot 106 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "kuma" 3 | version = "0.0.0" 4 | license = "MPL-2.0" 5 | description = "The MDN Web Docs site" 6 | authors = ["MDN Devs "] 7 | homepage = "https://developer.mozilla.org" 8 | repository = "https://github.com/mdn/kuma" 9 | documentation = "https://kuma.readthedocs.io" 10 | 11 | [tool.poetry.dependencies] 12 | python = "^3.10" 13 | 14 | celery = "^4.4.7" 15 | dj-database-url = "^0.5.0" 16 | dj-email-url = "^1" 17 | django = "^3" 18 | django-decorator-include = "^3.0" 19 | django-extensions = "^2.2.6" 20 | django-ratelimit = "2.0.0" 21 | django-redis = "^5.0.0" # (Django cache backend) 22 | elasticsearch = "^7.14.1" 23 | elasticsearch-dsl = "^7.4.0" 24 | gunicorn = "^20.1.0" 25 | newrelic = "6.8.1.164" 26 | python-decouple = "^3.4" 27 | pytz = "^2021.1" 28 | redo = "^2.0.4" 29 | requests = "^2.26.0" 30 | urlwait = "^1.0" 31 | gevent = {extras = ["gevent"], version = "^21.8.0"} 32 | psycopg2-binary = "^2.9.1" 33 | sentry-sdk = "^1.3.1" 34 | whitenoise = "^5.3.0" 35 | mozilla-django-oidc = "^2.0.0" 36 | django-ninja = "^0.16.1" 37 | 38 | [tool.poetry.dev-dependencies] 39 | # Development Tools 40 | werkzeug = "^1.0" # Enables runserver_plus from django-extensions 41 | 42 | # Testing 43 | braceexpand = "^0.1.7" 44 | pytest = "~6.2" 45 | pytest-base-url = "^1.4.2" 46 | pytest-cov = "~3.0.0" 47 | pytest-django = "~4.5.2" 48 | pytest-metadata = "^1.11.0" 49 | pytest-rerunfailures = "^10.2" 50 | requests-mock = "^1.9.3" 51 | 52 | ElasticMock = "^1.8.0" 53 | 54 | # Linting 55 | black = "^22.1.0" 56 | flake8 = "^3.9.2" 57 | flake8-isort = "^4.1.1" 58 | dennis = "^0.9" # Used by `make localetest` to lint po files 59 | isort = "^5.10.1" 60 | 61 | # Pinned Dependencies 62 | coverage = {extras = ["toml"], version = "^6"} # Use optional toml support 63 | pytest-watch = "^4.2.0" 64 | honcho = "^1.0.1" 65 | ipdb = "^0.13.9" 66 | ipython = "^7.31.1" 67 | model-bakery = "^1.4.0" 68 | 69 | [tool.black] 70 | target-version = ["py38"] 71 | 72 | [tool.coverage.run] 73 | source = ["kuma"] 74 | branch = true 75 | dynamic_context = "test_function" 76 | 77 | [tool.coverage.report] 78 | omit = ["*migrations*", "*/management/commands/*"] 79 | 80 | [tool.coverage.html] 81 | show_contexts = true 82 | 83 | [tool.isort] 84 | profile = "black" 85 | 86 | [build-system] 87 | requires = ["poetry>=0.12"] 88 | build-backend = "poetry.masonry.api" 89 | -------------------------------------------------------------------------------- /kuma/users/migrations/0006_accountevent.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.10 on 2021-12-27 09:56 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("users", "0005_userprofile_is_subscriber"), 10 | ] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name="AccountEvent", 15 | fields=[ 16 | ( 17 | "id", 18 | models.BigAutoField( 19 | auto_created=True, 20 | primary_key=True, 21 | serialize=False, 22 | verbose_name="ID", 23 | ), 24 | ), 25 | ("created_at", models.DateTimeField(auto_now_add=True)), 26 | ("modified_at", models.DateTimeField(auto_now=True)), 27 | ("fxa_uid", models.CharField(blank=True, default="", max_length=128)), 28 | ("payload", models.TextField(blank=True, default="", max_length=2048)), 29 | ( 30 | "event_type", 31 | models.IntegerField( 32 | blank=True, 33 | choices=[ 34 | (1, "Password Changed"), 35 | (2, "Profile Changed"), 36 | (3, "Subscription Changed"), 37 | (4, "Profile Deleted"), 38 | ], 39 | default="", 40 | ), 41 | ), 42 | ( 43 | "status", 44 | models.IntegerField( 45 | choices=[ 46 | (1, "Processed"), 47 | (2, "Pending"), 48 | (3, "Ignored"), 49 | (4, "Not Implemented"), 50 | ], 51 | default=2, 52 | ), 53 | ), 54 | ("jwt_id", models.CharField(blank=True, default="", max_length=256)), 55 | ("issued_at", models.CharField(blank=True, default="", max_length=32)), 56 | ], 57 | options={ 58 | "ordering": ["-modified_at"], 59 | }, 60 | ), 61 | ] 62 | -------------------------------------------------------------------------------- /kuma/api/v1/auth.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Any 4 | 5 | from django.conf import settings 6 | from django.http import HttpRequest 7 | from ninja.security import HttpBearer, SessionAuth 8 | 9 | from kuma.users.auth import KumaOIDCAuthenticationBackend, is_authorized_request 10 | from kuma.users.models import UserProfile 11 | 12 | 13 | class NotASubscriber(Exception): 14 | pass 15 | 16 | 17 | def is_subscriber(request, raise_error=False) -> bool: 18 | try: 19 | user = request.user 20 | if user.is_authenticated: 21 | return True 22 | if access_token := request.META.get("HTTP_AUTHORIZATION"): 23 | payload = is_authorized_request(access_token) 24 | 25 | if error := payload.get("error"): 26 | raise NotASubscriber(error) 27 | 28 | # create user if there is not one 29 | request.user = KumaOIDCAuthenticationBackend.create_or_update_subscriber( 30 | payload 31 | ) 32 | return True 33 | raise NotASubscriber("not a subscriber") 34 | except NotASubscriber: 35 | if raise_error: 36 | raise 37 | return False 38 | 39 | 40 | class SubscriberAuth(SessionAuth): 41 | def authenticate(self, request: HttpRequest, key: str | None) -> Any: 42 | if is_subscriber(request): 43 | return request.user 44 | 45 | return None 46 | 47 | 48 | subscriber_auth = SubscriberAuth() 49 | 50 | 51 | class AdminAuth(HttpBearer): 52 | def authenticate(self, request: HttpRequest, token: str) -> Any: 53 | return token == settings.NOTIFICATIONS_ADMIN_TOKEN 54 | 55 | 56 | admin_auth = AdminAuth() 57 | 58 | 59 | class ProfileAuth(SessionAuth): 60 | """ 61 | Requires a Django authenticated user. 62 | 63 | Does not actually *require* a profile, but for users without a profile will 64 | rather return the unsaved profile ready for saving (available in 65 | ``request.auth``). 66 | """ 67 | 68 | def authenticate(self, request: HttpRequest, key: str | None) -> Any: 69 | user = request.user 70 | if not user.is_authenticated: 71 | return None 72 | try: 73 | return UserProfile.objects.get(user=user) 74 | except UserProfile.DoesNotExist: 75 | return UserProfile(user=user) 76 | 77 | 78 | profile_auth = ProfileAuth() 79 | -------------------------------------------------------------------------------- /kuma/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.contrib import admin 3 | from django.urls import include, path, re_path 4 | from django.views.decorators.cache import never_cache 5 | from django.views.generic import RedirectView 6 | 7 | from kuma.attachments import views as attachment_views 8 | from kuma.core import views as core_views 9 | from kuma.users import views as users_views 10 | 11 | DAY = 60 * 60 * 24 12 | MONTH = DAY * 30 13 | 14 | admin.autodiscover() 15 | 16 | urlpatterns = [re_path("", include("kuma.health.urls"))] 17 | 18 | if settings.MAINTENANCE_MODE: 19 | urlpatterns.append( 20 | re_path( 21 | r"^admin/.*", 22 | never_cache(RedirectView.as_view(url="/", permanent=False)), 23 | ) 24 | ) 25 | else: 26 | # Django admin: 27 | urlpatterns += [ 28 | # We don't worry about decorating the views within django.contrib.admin 29 | # with "never_cache", since most have already been decorated, and the 30 | # remaining can be safely cached. 31 | re_path(r"^admin/", admin.site.urls), 32 | ] 33 | 34 | urlpatterns += [re_path("", include("kuma.attachments.urls"))] 35 | urlpatterns += [ 36 | path("users/fxa/login/", include("mozilla_django_oidc.urls")), 37 | path( 38 | "users/fxa/login/no-prompt/", 39 | users_views.no_prompt_login, 40 | name="no_prompt_login", 41 | ), 42 | path( 43 | "events/fxa", 44 | users_views.WebhookView.as_view(), 45 | name="fxa_webhook", 46 | ), 47 | ] 48 | 49 | urlpatterns += [ 50 | # Services and sundry. 51 | re_path("^admin-api/", include("kuma.api.admin_urls")), 52 | re_path("^api/", include("kuma.api.urls")), 53 | re_path("", include("kuma.version.urls")), 54 | re_path(r"^humans.txt$", core_views.humans_txt, name="humans_txt"), 55 | # We use our own views for setting language in cookies. But to just align with django, set it like this. 56 | # re_path(r"^i18n/setlang/", core_views.set_language, name="set-language-cookie"), 57 | ] 58 | 59 | 60 | # Legacy MindTouch redirects. These go last so that they don't mess 61 | # with local instances' ability to serve media. 62 | urlpatterns += [ 63 | re_path( 64 | r"^@api/deki/files/(?P\d+)/=(?P.+)$", 65 | attachment_views.mindtouch_file_redirect, 66 | name="attachments.mindtouch_file_redirect", 67 | ), 68 | ] 69 | -------------------------------------------------------------------------------- /kuma/api/v1/tests/test_forms.py: -------------------------------------------------------------------------------- 1 | from django.test import RequestFactory 2 | 3 | from kuma.api.v1.search.forms import SearchForm 4 | 5 | 6 | def test_search_form_locale_happy_path(): 7 | """The way the form handles 'locale' is a bit overly complicated. 8 | These unit tests focuses exclusively on that and when the form is valid.""" 9 | 10 | initial = {"page": 1, "size": 10} 11 | request = RequestFactory().get("/api/v1/search?q=foo") 12 | form = SearchForm(request.GET, initial=initial) 13 | assert form.is_valid() 14 | assert form.cleaned_data["locale"] == [] 15 | 16 | request = RequestFactory().get("/api/v1/search?q=foo") 17 | initial["locale"] = "ja" 18 | form = SearchForm(request.GET, initial=initial) 19 | assert form.is_valid() 20 | assert form.cleaned_data["locale"] == ["ja"] 21 | 22 | request = RequestFactory().get("/api/v1/search?q=foo&locale=Fr") 23 | form = SearchForm(request.GET, initial=initial) 24 | assert form.is_valid() 25 | assert form.cleaned_data["locale"] == ["Fr"] 26 | 27 | request = RequestFactory().get("/api/v1/search?q=foo&locale=Fr&locale=de") 28 | form = SearchForm(request.GET, initial=initial) 29 | assert form.is_valid() 30 | assert form.cleaned_data["locale"] == ["Fr", "de"] 31 | 32 | # Note, same as the initial default 33 | request = RequestFactory().get("/api/v1/search?q=foo&locale=ja") 34 | form = SearchForm(request.GET, initial=initial) 35 | assert form.is_valid() 36 | assert form.cleaned_data["locale"] == ["ja"] 37 | 38 | request = RequestFactory().get("/api/v1/search?q=foo&locale=ja&locale=fr") 39 | form = SearchForm(request.GET, initial=initial) 40 | assert form.is_valid() 41 | assert form.cleaned_data["locale"] == ["ja", "fr"] 42 | 43 | 44 | def test_search_form_locale_validation_error(): 45 | """The way the form handles 'locale' is a bit overly complicated. 46 | These unit tests focuses exclusively on that and when the form is NOT valid.""" 47 | 48 | initial = {"page": 1, "size": 10} 49 | request = RequestFactory().get("/api/v1/search?q=foo&locale=xxx") 50 | form = SearchForm(request.GET, initial=initial) 51 | assert not form.is_valid() 52 | assert form.errors["locale"] 53 | 54 | request = RequestFactory().get("/api/v1/search?q=foo&locale=") 55 | form = SearchForm(request.GET, initial=initial) 56 | assert not form.is_valid() 57 | assert form.errors["locale"] 58 | -------------------------------------------------------------------------------- /kuma/attachments/utils.py: -------------------------------------------------------------------------------- 1 | import calendar 2 | import hashlib 3 | from datetime import datetime 4 | 5 | from django.conf import settings 6 | from django.urls import reverse 7 | from django.utils import timezone 8 | from django.utils.http import http_date 9 | 10 | 11 | def full_attachment_url(attachment_id, filename): 12 | path = reverse( 13 | "attachments.raw_file", 14 | kwargs={"attachment_id": attachment_id, "filename": filename}, 15 | ) 16 | return f"{settings.PROTOCOL}{settings.ATTACHMENT_HOST}{path}" 17 | 18 | 19 | def full_mindtouch_attachment_url(file_id, filename): 20 | path = reverse( 21 | "attachments.mindtouch_file_redirect", 22 | kwargs={"file_id": file_id, "filename": filename}, 23 | ) 24 | return f"{settings.PROTOCOL}{settings.ATTACHMENT_HOST}{path}" 25 | 26 | 27 | def convert_to_utc(dt): 28 | """ 29 | Given a timezone naive or aware datetime return it converted to UTC. 30 | """ 31 | # Check if the given dt is timezone aware and if not make it aware. 32 | if timezone.is_naive(dt): 33 | default_timezone = timezone.get_default_timezone() 34 | dt = timezone.make_aware(dt, default_timezone) 35 | 36 | # Convert the datetime to UTC. 37 | return dt.astimezone(timezone.utc) 38 | 39 | 40 | def convert_to_http_date(dt): 41 | """ 42 | Given a timezone naive or aware datetime return the HTTP date-formatted 43 | string to be used in HTTP response headers. 44 | """ 45 | # Convert the datetime to UTC. 46 | utc_dt = convert_to_utc(dt) 47 | # Convert the UTC datetime to seconds since the epoch. 48 | epoch_dt = calendar.timegm(utc_dt.utctimetuple()) 49 | # Format the thing as a RFC1123 datetime. 50 | return http_date(epoch_dt) 51 | 52 | 53 | def attachment_upload_to(instance, filename): 54 | """ 55 | Generate a path to store a file attachment. 56 | """ 57 | # For now, the filesystem storage path will look like this: 58 | # 59 | # attachments////// 60 | # 61 | # The md5 hash here is of the full timestamp, down to the 62 | # microsecond, of when the path is generated. 63 | now = datetime.now() 64 | return "attachments/%(date)s/%(id)s/%(md5)s/%(filename)s" % { 65 | "date": now.strftime("%Y/%m/%d"), 66 | "id": instance.attachment.id, 67 | "md5": hashlib.md5(str(now).encode()).hexdigest(), 68 | "filename": filename, 69 | } 70 | -------------------------------------------------------------------------------- /docs/troubleshooting.rst: -------------------------------------------------------------------------------- 1 | .. _Troubleshooting: 2 | 3 | Troubleshooting 4 | =============== 5 | 6 | Kuma has many components. Even core developers need reminders of how to keep 7 | them all working together. This doc outlines some problems and potential 8 | solutions running Kuma. 9 | 10 | Kuma "Reset" 11 | ------------ 12 | These commands will reset your environment to a "fresh" version, with the 13 | current third-party libraries, while retaining the database:: 14 | 15 | cd /path/to/kuma 16 | docker-compose down 17 | make clean 18 | git submodule sync --recursive && git submodule update --init --recursive 19 | docker-compose pull 20 | docker-compose build --pull 21 | docker-compose up 22 | 23 | Reset a corrupt database 24 | ------------------------ 25 | The Kuma database can become corrupted if the system runs out of disk space, 26 | or is unexpectedly shutdown. MySQL can repair some issues, but sometimes you 27 | have to start from scratch. When this happens, pass an extra argument 28 | ``--volumes`` to ``down``:: 29 | 30 | cd /path/to/kuma 31 | docker-compose down --volumes 32 | make clean 33 | docker-compose pull 34 | docker-compose build --pull 35 | docker-compose up -d mysql 36 | sleep 20 # Wait for Postgres to initialize. See notes below 37 | docker-compose up 38 | 39 | The ``--volumes`` flag will remove the named MySQL database volume, which will 40 | be recreated when you run ``docker-compose up``. 41 | 42 | Run alternate services 43 | ---------------------- 44 | Docker services run as containers. To change the commands or environments of 45 | services, it is easiest to add an override configuration file, as documented in 46 | :ref:`advanced_config_docker`. 47 | 48 | Linux file permissions 49 | ---------------------- 50 | On Linux, it is common that files created inside a Docker container are owned 51 | by the root user on the host system. This can cause problems when trying to 52 | work with them after creation. We are investigating solutions to create files 53 | as the developer's user. 54 | 55 | In some cases, you can specify the host user ID when running commands:: 56 | 57 | docker-compose run --rm --user $(id -u) web ./manage.py collectstatic 58 | 59 | In other cases, the command requires root permissions inside the container, and 60 | this trick can't be used. 61 | 62 | Another option is to allow the files to be created as root, and then change 63 | them to your user on the host system:: 64 | 65 | find . -user root -exec sudo chown $(id -u):$(id -g) \{\} \; 66 | 67 | .. _more-help: 68 | -------------------------------------------------------------------------------- /kuma/documenturls/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.5 on 2021-07-12 19:04 2 | 3 | import django.db.models.deletion 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | initial = True 10 | 11 | dependencies = [] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name="DocumentURL", 16 | fields=[ 17 | ( 18 | "id", 19 | models.BigAutoField( 20 | auto_created=True, 21 | primary_key=True, 22 | serialize=False, 23 | verbose_name="ID", 24 | ), 25 | ), 26 | ( 27 | "uri", 28 | models.CharField(max_length=250, unique=True, verbose_name="URI"), 29 | ), 30 | ("absolute_url", models.URLField(verbose_name="Absolute URL")), 31 | ("metadata", models.JSONField(null=True)), 32 | ("invalid", models.DateTimeField(null=True)), 33 | ("created", models.DateTimeField(auto_now_add=True)), 34 | ("modified", models.DateTimeField(auto_now=True)), 35 | ], 36 | options={ 37 | "verbose_name": "Document URL", 38 | }, 39 | ), 40 | migrations.CreateModel( 41 | name="DocumentURLCheck", 42 | fields=[ 43 | ( 44 | "id", 45 | models.BigAutoField( 46 | auto_created=True, 47 | primary_key=True, 48 | serialize=False, 49 | verbose_name="ID", 50 | ), 51 | ), 52 | ("http_error", models.IntegerField(verbose_name="HTTP error")), 53 | ("headers", models.JSONField(default=dict)), 54 | ("created", models.DateTimeField(auto_now_add=True)), 55 | ( 56 | "document_url", 57 | models.ForeignKey( 58 | on_delete=django.db.models.deletion.CASCADE, 59 | to="documenturls.documenturl", 60 | verbose_name="Document URL", 61 | ), 62 | ), 63 | ], 64 | options={ 65 | "verbose_name": "Document URL Check", 66 | }, 67 | ), 68 | ] 69 | -------------------------------------------------------------------------------- /kuma/core/tasks.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | 3 | from celery.task import task 4 | from django.contrib.sessions.models import Session 5 | 6 | from ..notifications.models import Notification, NotificationData 7 | from .decorators import skip_in_maintenance_mode 8 | 9 | LOCK_ID = "clean-sessions-lock" 10 | LOCK_EXPIRE = 60 * 5 11 | 12 | 13 | def get_expired_sessions(now): 14 | return Session.objects.filter(expire_date__lt=now).order_by("expire_date") 15 | 16 | 17 | @task 18 | @skip_in_maintenance_mode 19 | def clean_sessions(): 20 | """ 21 | Queue deleting expired session items without breaking poor MySQL 22 | """ 23 | import warnings 24 | 25 | warnings.warn( 26 | "clean_sessions() is disabled at the moment because depends " 27 | "doing raw SQL queries which might not make sense if you start " 28 | "with a completely empty database." 29 | ) 30 | # now = timezone.now() 31 | # logger = clean_sessions.get_logger() 32 | # chunk_size = settings.SESSION_CLEANUP_CHUNK_SIZE 33 | 34 | # if cache.add(LOCK_ID, now.strftime("%c"), LOCK_EXPIRE): 35 | # total_count = get_expired_sessions(now).count() 36 | # delete_count = 0 37 | # logger.info( 38 | # "Deleting the %s of %s oldest expired sessions" % (chunk_size, total_count) 39 | # ) 40 | # try: 41 | # cursor = connection.cursor() 42 | # delete_count = cursor.execute( 43 | # """ 44 | # DELETE 45 | # FROM django_session 46 | # WHERE expire_date < NOW() 47 | # ORDER BY expire_date ASC 48 | # LIMIT %s; 49 | # """, 50 | # [chunk_size], 51 | # ) 52 | # finally: 53 | # logger.info("Deleted %s expired sessions" % delete_count) 54 | # cache.delete(LOCK_ID) 55 | # expired_sessions = get_expired_sessions(now) 56 | # if expired_sessions.exists(): 57 | # clean_sessions.apply_async() 58 | # else: 59 | # logger.error( 60 | # "The clean_sessions task is already running since %s" % cache.get(LOCK_ID) 61 | # ) 62 | 63 | 64 | @task 65 | @skip_in_maintenance_mode 66 | def clear_old_notifications(): 67 | """ 68 | Delete old notifications from the database 69 | """ 70 | NotificationData.objects.filter( 71 | created__lt=datetime.now() - timedelta(days=6 * 30) 72 | ).delete() 73 | Notification.objects.filter(deleted=True).delete() 74 | -------------------------------------------------------------------------------- /kuma/core/decorators.py: -------------------------------------------------------------------------------- 1 | from functools import partial, wraps 2 | 3 | from django.conf import settings 4 | from django.shortcuts import redirect 5 | 6 | from .utils import add_shared_cache_control 7 | 8 | 9 | def shared_cache_control(func=None, **kwargs): 10 | """ 11 | Decorator to set Cache-Control header for shared caches like CDNs. 12 | 13 | This duplicates Django's cache-control decorators, but avoids changing the 14 | header if a "no-cache" header has already been applied. The cache-control 15 | decorator changes in Django 2.0 to remove Python 2 workarounds. 16 | 17 | Default settings (which can be overridden or extended): 18 | - max-age=0 - Don't use browser cache without asking if still valid 19 | - s-maxage=CACHE_CONTROL_DEFAULT_SHARED_MAX_AGE - Cache in the shared 20 | cache for the default perioid of time 21 | - public - Allow intermediate proxies to cache response 22 | """ 23 | 24 | def _shared_cache_controller(viewfunc): 25 | @wraps(viewfunc) 26 | def _cache_controlled(request, *args, **kw): 27 | response = viewfunc(request, *args, **kw) 28 | add_shared_cache_control(response, **kwargs) 29 | return response 30 | 31 | return _cache_controlled 32 | 33 | if func: 34 | return _shared_cache_controller(func) 35 | return _shared_cache_controller 36 | 37 | 38 | def skip_in_maintenance_mode(func): 39 | """ 40 | Decorator for Celery task functions. If we're in MAINTENANCE_MODE, skip 41 | the call to the decorated function. Otherwise, call the decorated function 42 | as usual. 43 | """ 44 | 45 | @wraps(func) 46 | def wrapped(*args, **kwargs): 47 | if settings.MAINTENANCE_MODE: 48 | return 49 | return func(*args, **kwargs) 50 | 51 | return wrapped 52 | 53 | 54 | def redirect_in_maintenance_mode(func=None, methods=None): 55 | """ 56 | Decorator for view functions. If we're in MAINTENANCE_MODE, redirect 57 | to the home page on requests using the given HTTP "methods" (or all 58 | HTTP methods if "methods" is None). Otherwise, call the wrapped view 59 | function as usual. 60 | """ 61 | if not func: 62 | return partial(redirect_in_maintenance_mode, methods=methods) 63 | 64 | @wraps(func) 65 | def wrapped(request, *args, **kwargs): 66 | if settings.MAINTENANCE_MODE and ( 67 | (methods is None) or (request.method in methods) 68 | ): 69 | locale = getattr(request, "LANGUAGE_CODE", None) 70 | return redirect(f"/{locale}/" if locale else "/") 71 | return func(request, *args, **kwargs) 72 | 73 | return wrapped 74 | -------------------------------------------------------------------------------- /kuma/core/tests/test_validators.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from ..validators import valid_javascript_identifier, valid_jsonp_callback_value 4 | 5 | 6 | class ValidatorTest(TestCase): 7 | def test_valid_javascript_identifier(self): 8 | """ 9 | The function ``valid_javascript_identifier`` validates a given identifier 10 | according to the latest draft of the ECMAScript 5 Specification 11 | """ 12 | self.assertTrue(valid_javascript_identifier(b"hello")) 13 | 14 | self.assertFalse(valid_javascript_identifier(b"alert()")) 15 | 16 | self.assertFalse(valid_javascript_identifier(b"a-b")) 17 | 18 | self.assertFalse(valid_javascript_identifier(b"23foo")) 19 | 20 | self.assertTrue(valid_javascript_identifier(b"foo23")) 21 | 22 | self.assertTrue(valid_javascript_identifier(b"$210")) 23 | 24 | self.assertTrue(valid_javascript_identifier("Stra\u00dfe")) 25 | 26 | self.assertTrue(valid_javascript_identifier(rb"\u0062")) # 'b' 27 | 28 | self.assertFalse(valid_javascript_identifier(rb"\u62")) 29 | 30 | self.assertFalse(valid_javascript_identifier(rb"\u0020")) 31 | 32 | self.assertTrue(valid_javascript_identifier(b"_bar")) 33 | 34 | self.assertTrue(valid_javascript_identifier(b"some_var")) 35 | 36 | self.assertTrue(valid_javascript_identifier(b"$")) 37 | 38 | def test_valid_jsonp_callback_value(self): 39 | """ 40 | But ``valid_jsonp_callback_value`` is the function you want to use for 41 | validating JSON-P callback parameter values: 42 | """ 43 | 44 | self.assertTrue(valid_jsonp_callback_value("somevar")) 45 | 46 | self.assertFalse(valid_jsonp_callback_value("function")) 47 | 48 | self.assertFalse(valid_jsonp_callback_value(" somevar")) 49 | 50 | # It supports the possibility of '.' being present in the callback name, e.g. 51 | 52 | self.assertTrue(valid_jsonp_callback_value("$.ajaxHandler")) 53 | 54 | self.assertFalse(valid_jsonp_callback_value("$.23")) 55 | 56 | # As well as the pattern of providing an array index lookup, e.g. 57 | 58 | self.assertTrue(valid_jsonp_callback_value("array_of_functions[42]")) 59 | 60 | self.assertTrue(valid_jsonp_callback_value("array_of_functions[42][1]")) 61 | 62 | self.assertTrue(valid_jsonp_callback_value("$.ajaxHandler[42][1].foo")) 63 | 64 | self.assertFalse(valid_jsonp_callback_value("array_of_functions[42]foo[1]")) 65 | 66 | self.assertFalse(valid_jsonp_callback_value("array_of_functions[]")) 67 | 68 | self.assertFalse(valid_jsonp_callback_value('array_of_functions["key"]')) 69 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ifeq ($(shell which git),) 2 | # git is not available 3 | VERSION ?= undefined 4 | KS_VERSION ?= undefined 5 | export KUMA_REVISION_HASH ?= undefined 6 | else 7 | # git is available 8 | VERSION ?= $(shell git describe --tags --exact-match 2>/dev/null || git rev-parse --short HEAD) 9 | export KUMA_REVISION_HASH ?= $(shell git rev-parse HEAD) 10 | endif 11 | BASE_IMAGE_NAME ?= kuma_base 12 | KUMA_IMAGE_NAME ?= kuma 13 | IMAGE_PREFIX ?= mdnwebdocs 14 | BASE_IMAGE ?= ${IMAGE_PREFIX}/${BASE_IMAGE_NAME}\:${VERSION} 15 | BASE_IMAGE_LATEST ?= ${IMAGE_PREFIX}/${BASE_IMAGE_NAME}\:latest 16 | KUMA_IMAGE ?= ${IMAGE_PREFIX}/${KUMA_IMAGE_NAME}\:${VERSION} 17 | KUMA_IMAGE_LATEST ?= ${IMAGE_PREFIX}/${KUMA_IMAGE_NAME}\:latest 18 | 19 | target = kuma 20 | 21 | # Note: these targets should be run from the kuma vm 22 | test: 23 | py.test $(target) 24 | 25 | coveragetest: clean 26 | py.test --cov=$(target) --no-cov-on-fail $(target) 27 | # Generate the coverage.xml file from the .coverage file 28 | # so we don't need to `pip install codecov`. 29 | coverage xml 30 | 31 | coveragetesthtml: coveragetest 32 | coverage html 33 | 34 | clean: 35 | rm -rf .coverage build/ tmp/emails/*.log 36 | find . \( -name \*.pyc -o -name \*.pyo -o -name __pycache__ \) -delete 37 | 38 | collectstatic: 39 | @ echo "## Collecting static files ##" 40 | @ python manage.py collectstatic --noinput 41 | 42 | build-static: collectstatic 43 | 44 | pull-base: 45 | docker pull ${BASE_IMAGE} 46 | 47 | pull-kuma: 48 | docker pull ${KUMA_IMAGE} 49 | 50 | pull-base-latest: 51 | docker pull ${BASE_IMAGE_LATEST} 52 | 53 | pull-kuma-latest: 54 | docker pull ${KUMA_IMAGE_LATEST} 55 | 56 | pull-latest: pull-base-latest pull-kuma-latest 57 | 58 | build-base: 59 | docker build -f docker/images/kuma_base/Dockerfile -t ${BASE_IMAGE} . 60 | 61 | build-kuma: 62 | docker build --build-arg REVISION_HASH=${KUMA_REVISION_HASH} \ 63 | -f docker/images/kuma/Dockerfile -t ${KUMA_IMAGE} . 64 | 65 | build: build-base build-kuma 66 | 67 | push-base: 68 | docker push ${BASE_IMAGE} 69 | 70 | push-kuma: 71 | docker push ${KUMA_IMAGE} 72 | 73 | push: push-base push-kuma 74 | 75 | tag-latest: 76 | docker tag ${BASE_IMAGE} ${BASE_IMAGE_LATEST} 77 | docker tag ${KUMA_IMAGE} ${KUMA_IMAGE_LATEST} 78 | 79 | push-latest: push tag-latest 80 | docker push ${BASE_IMAGE_LATEST} 81 | docker push ${KUMA_IMAGE_LATEST} 82 | 83 | up: 84 | docker-compose up -d 85 | 86 | bash: up 87 | docker-compose exec web bash 88 | 89 | shell_plus: up 90 | docker-compose exec web ./manage.py shell_plus 91 | 92 | pythonlint: 93 | flake8 kuma docs 94 | 95 | lint: pythonlint 96 | 97 | # Those tasks don't have file targets 98 | .PHONY: test coveragetest clean locale localetest localeextract localecompile localerefresh 99 | -------------------------------------------------------------------------------- /kuma/core/urlresolvers.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.exceptions import ImproperlyConfigured 3 | from django.urls import LocalePrefixPattern, URLResolver 4 | from django.urls import reverse as django_reverse 5 | from django.utils import translation 6 | 7 | from .i18n import get_language 8 | 9 | 10 | class KumaLocalePrefixPattern(LocalePrefixPattern): 11 | """ 12 | A prefix pattern for localized URLs that uses Kuma's case-sensitive locale 13 | codes instead of Django's, which are all lowercase. 14 | 15 | We do this via a customized get_language function in kuma/core/i18n.py. 16 | 17 | NOTE: See upstream LocalePrefixPattern for Django 2.2 / 3.0: 18 | https://github.com/django/django/blob/3.0/django/urls/resolvers.py#L288-L319 19 | """ 20 | 21 | @property 22 | def language_prefix(self): 23 | language_code = get_language() or settings.LANGUAGE_CODE 24 | return "%s/" % language_code 25 | 26 | 27 | def i18n_patterns(*urls): 28 | """ 29 | Add the language code prefix to every URL pattern within this function. 30 | This may only be used in the root URLconf, not in an included URLconf. 31 | 32 | NOTE: Modified from i18n_patterns in Django 2.2 / 3.0, see: 33 | https://github.com/django/django/blob/3.0/django/conf/urls/i18n.py#L8-L20 34 | 35 | Modifications: 36 | - Raises ImproperlyConfigured if settings.USE_I18N is False 37 | - Forces prefix_default_language to True, so urls always include the locale 38 | - Does not accept prefix_default_language as a kwarg, due to the above 39 | - Uses our custom URL prefix pattern, to support our locale codes 40 | """ 41 | if not settings.USE_I18N: 42 | raise ImproperlyConfigured("Kuma requires settings.USE_I18N to be True.") 43 | return [URLResolver(KumaLocalePrefixPattern(), list(urls))] 44 | 45 | 46 | def reverse( 47 | viewname, urlconf=None, args=None, kwargs=None, current_app=None, locale=None 48 | ): 49 | """Wraps Django's reverse to prepend the requested locale. 50 | Keyword Arguments: 51 | * locale - Use this locale prefix rather than the current active locale. 52 | Keyword Arguments passed to Django's reverse: 53 | * viewname 54 | * urlconf 55 | * args 56 | * kwargs 57 | * current_app 58 | """ 59 | if locale: 60 | with translation.override(locale): 61 | return django_reverse( 62 | viewname, 63 | urlconf=urlconf, 64 | args=args, 65 | kwargs=kwargs, 66 | current_app=current_app, 67 | ) 68 | else: 69 | return django_reverse( 70 | viewname, urlconf=urlconf, args=args, kwargs=kwargs, current_app=current_app 71 | ) 72 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | Kuma (deprecated) 3 | ================= 4 | 5 | **Note**: In July 2022, Kuma was superseded by `Rumba`_. 6 | 7 | .. _Rumba: https://github.com/mdn/rumba 8 | 9 | .. image:: https://github.com/mdn/kuma/workflows/Docker%20testing/badge.svg 10 | :target: https://github.com/mdn/kuma/actions?query=workflow%3A%22Docker+testing%22 11 | :alt: Docker testing 12 | 13 | .. image:: https://github.com/mdn/kuma/workflows/Python%20Lints/badge.svg 14 | :target: https://github.com/mdn/kuma/actions?query=workflow%3A%22Python+Lints%22 15 | :alt: Python Lints 16 | 17 | .. image:: https://github.com/mdn/kuma/workflows/JavaScript%20and%20SASS%20Lints/badge.svg 18 | :target: https://github.com/mdn/kuma/actions?query=workflow%3A%22JavaScript+Lints%22 19 | :alt: JavaScript Lints 20 | 21 | .. image:: https://github.com/mdn/kuma/workflows/Documentation%20Build/badge.svg 22 | :target: https://github.com/mdn/kuma/actions?query=workflow%3A%22Documentation+Build%22 23 | :alt: Documentation Build 24 | 25 | .. image:: https://codecov.io/github/mdn/kuma/coverage.svg?branch=main 26 | :target: https://codecov.io/github/mdn/kuma?branch=main 27 | :alt: Code Coverage Status 28 | 29 | .. image:: http://img.shields.io/badge/license-MPL2-blue.svg 30 | :target: https://raw.githubusercontent.com/mdn/kuma/main/LICENSE 31 | :alt: License 32 | 33 | .. image:: https://img.shields.io/badge/whatsdeployed-stage,prod-green.svg 34 | :target: https://whatsdeployed.io/s/HC0/mdn/kuma 35 | :alt: What's deployed on stage,prod? 36 | 37 | .. Omit badges from docs 38 | 39 | Kuma is the platform that powers `MDN (developer.mozilla.org) 40 | `_ 41 | 42 | Development 43 | =========== 44 | 45 | :Code: https://github.com/mdn/kuma 46 | :Issues: `P1 Bugs`_ (to be fixed ASAP) 47 | 48 | `P2 Bugs`_ (to be fixed in 180 days) 49 | 50 | :Dev Docs: https://kuma.readthedocs.io/en/latest/installation.html 51 | :Forum: https://discourse.mozilla.org/c/mdn 52 | :Matrix: `#mdn room`_ 53 | :Servers: `What's Deployed on MDN?`_ 54 | 55 | https://developer.allizom.org/ (stage) 56 | 57 | https://developer.mozilla.org/ (prod) 58 | 59 | .. _`P1 Bugs`: https://github.com/mdn/kuma/issues?q=is%3Aopen+is%3Aissue+label%3Ap1 60 | .. _`P2 Bugs`: https://github.com/mdn/kuma/issues?q=is%3Aopen+is%3Aissue+label%3Ap2 61 | .. _`What's Deployed on MDN?`: https://whatsdeployed.io/s/HC0/mdn/kuma 62 | .. _`#mdn room`: https://chat.mozilla.org/#/room/#mdn:mozilla.org 63 | 64 | 65 | Getting started 66 | =============== 67 | 68 | Want to help make MDN great? Our `contribution guide 69 | `_ lists some good 70 | first projects and offers direction on submitting code. 71 | -------------------------------------------------------------------------------- /kuma/api/v1/search/forms.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.conf import settings 3 | from django.utils.datastructures import MultiValueDict 4 | 5 | 6 | class TypedMultipleValueField(forms.TypedMultipleChoiceField): 7 | """Unlike TypedMultipleChoiceField we don't care what the individual values 8 | are as long as they're not empty.""" 9 | 10 | def valid_value(self, value): 11 | # Basically, as long as it's not empty, it's fine 12 | return bool(value) 13 | 14 | 15 | class MultipleChoiceFieldICase(forms.MultipleChoiceField): 16 | """Just like forms.MultipleChoiceField but everything's case insentive. 17 | 18 | For simplicity, this field assumes that each choice is a tuple where 19 | the first element is always a string. 20 | """ 21 | 22 | def valid_value(self, value): 23 | return str(value).lower() in [x[0].lower() for x in self.choices] 24 | 25 | 26 | class SearchForm(forms.Form): 27 | q = forms.CharField(max_length=settings.ES_Q_MAXLENGTH) 28 | locale = MultipleChoiceFieldICase( 29 | required=False, 30 | # The `settings.LANGUAGES` looks like this: 31 | # [('en-US', 'English (US)'), ...] 32 | # But all locales are stored in lowercase in Elasticsearch, so 33 | # force everything to lowercase. 34 | choices=[(code, name) for code, name in settings.LANGUAGES], 35 | ) 36 | 37 | SORT_CHOICES = ("best", "relevance", "popularity") 38 | sort = forms.ChoiceField(required=False, choices=[(x, x) for x in SORT_CHOICES]) 39 | 40 | size = forms.IntegerField(required=True, min_value=1, max_value=100) 41 | page = forms.IntegerField(required=True, min_value=1, max_value=10) 42 | 43 | slug_prefix = TypedMultipleValueField(required=False) 44 | 45 | def __init__(self, data, **kwargs): 46 | initial = kwargs.get("initial", {}) 47 | # This makes it possible to supply `initial={some dict}` to the form 48 | # and have its values become part of the default. Normally, in Django, 49 | # the `SomeForm(data, initial={...})` is just used to prepopulate the 50 | # HTML generated form widgets. 51 | # See https://www.peterbe.com/plog/initial-values-bound-django-form-rendered 52 | data = MultiValueDict({**{k: [v] for k, v in initial.items()}, **data}) 53 | 54 | # If, for keys we have an initial value for, it was passed an empty string, 55 | # then swap it for the initial value. 56 | # For example `?q=searching&page=` you probably meant to omit it 57 | # but "allowing" it to be an empty string makes it convenient for the client. 58 | for key, values in data.items(): 59 | if key in initial and values == "": 60 | data[key] = initial[key] 61 | 62 | super().__init__(data, **kwargs) 63 | -------------------------------------------------------------------------------- /kuma/users/tests/test_checks.py: -------------------------------------------------------------------------------- 1 | from kuma.users.checks import ( 2 | MISSING_OIDC_CONFIGURATION_ERROR, 3 | MISSING_OIDC_RP_CLIENT_ID_ERROR, 4 | MISSING_OIDC_RP_CLIENT_SECRET_ERROR, 5 | oidc_config_check, 6 | ) 7 | 8 | 9 | def test_happy_path_config_check(mock_requests, settings): 10 | mock_requests.register_uri( 11 | "GET", 12 | settings.OIDC_CONFIGURATION_URL + "/.well-known/openid-configuration", 13 | json={ 14 | "scopes_supported": settings.OIDC_RP_SCOPES.split(), 15 | "id_token_signing_alg_values_supported": [settings.OIDC_RP_SIGN_ALGO], 16 | "authorization_endpoint": settings.OIDC_OP_AUTHORIZATION_ENDPOINT, 17 | "userinfo_endpoint": settings.OIDC_OP_USER_ENDPOINT, 18 | "token_endpoint": settings.OIDC_OP_TOKEN_ENDPOINT, 19 | }, 20 | ) 21 | 22 | errors = oidc_config_check(None) 23 | assert not errors 24 | 25 | 26 | def test_disable_checks(settings): 27 | # Note! No mock_requests in this test 28 | settings.OIDC_CONFIGURATION_CHECK = False 29 | errors = oidc_config_check(None) 30 | assert not errors 31 | 32 | 33 | def test_not_happy_path_config_check(mock_requests, settings): 34 | settings.OIDC_CONFIGURATION_URL += "/" 35 | mock_requests.register_uri( 36 | "GET", 37 | settings.OIDC_CONFIGURATION_URL + ".well-known/openid-configuration", 38 | json={ 39 | "scopes_supported": ["foo"], 40 | "id_token_signing_alg_values_supported": ["XXX"], 41 | "authorization_endpoint": "authorization?", 42 | "token_endpoint": "token?", 43 | }, 44 | ) 45 | 46 | errors = oidc_config_check(None) 47 | assert errors 48 | # TODO Put back to 5 when missing OIDC Scopes are correct. See checks.py 49 | assert len(errors) == 4 50 | ids = [error.id for error in errors] 51 | assert ids == [MISSING_OIDC_CONFIGURATION_ERROR] * len(errors) 52 | 53 | 54 | def test_missing_important_rp_client_credentials(mock_requests, settings): 55 | settings.OIDC_CONFIGURATION_URL += "/.well-known/openid-configuration" 56 | mock_requests.register_uri( 57 | "GET", 58 | settings.OIDC_CONFIGURATION_URL, 59 | json={ 60 | "scopes_supported": settings.OIDC_RP_SCOPES.split(), 61 | "id_token_signing_alg_values_supported": [settings.OIDC_RP_SIGN_ALGO], 62 | "authorization_endpoint": settings.OIDC_OP_AUTHORIZATION_ENDPOINT, 63 | "userinfo_endpoint": settings.OIDC_OP_USER_ENDPOINT, 64 | "token_endpoint": settings.OIDC_OP_TOKEN_ENDPOINT, 65 | }, 66 | ) 67 | 68 | settings.OIDC_RP_CLIENT_ID = None 69 | settings.OIDC_RP_CLIENT_SECRET = "" 70 | 71 | errors = oidc_config_check(None) 72 | assert errors 73 | ids = [error.id for error in errors] 74 | assert ids == [MISSING_OIDC_RP_CLIENT_ID_ERROR, MISSING_OIDC_RP_CLIENT_SECRET_ERROR] 75 | -------------------------------------------------------------------------------- /kuma/settings/pytest.py: -------------------------------------------------------------------------------- 1 | from .local import * 2 | 3 | DEBUG = False 4 | ENABLE_RESTRICTIONS_BY_HOST = False 5 | TEMPLATES[0]["OPTIONS"]["debug"] = True # Enable recording of templates 6 | CELERY_TASK_ALWAYS_EAGER = True 7 | CELERY_EAGER_PROPAGATES_EXCEPTIONS = True 8 | ES_LIVE_INDEX = config("ES_LIVE_INDEX", default=False, cast=bool) 9 | 10 | # Always make sure we never test against a real Elasticsearch server 11 | ES_URLS = ["1.2.3.4:9200"] 12 | # This makes sure that if we ever fail to mock the connection, 13 | # it won't retry for many many seconds. 14 | ES_RETRY_SLEEPTIME = 0 15 | ES_RETRY_ATTEMPTS = 1 16 | ES_RETRY_JITTER = 0 17 | 18 | # SHA1 because it is fast, and hard-coded in the test fixture JSON. 19 | PASSWORD_HASHERS = ("django.contrib.auth.hashers.SHA1PasswordHasher",) 20 | 21 | LOGGING["loggers"].update( 22 | { 23 | "django.db.backends": { 24 | "handlers": ["console"], 25 | "propagate": True, 26 | "level": "WARNING", 27 | }, 28 | "kuma.search.utils": {"handlers": [], "propagate": False, "level": "CRITICAL"}, 29 | } 30 | ) 31 | 32 | 33 | # Change the cache key prefix for tests, to avoid overwriting runtime. 34 | for cache_settings in CACHES.values(): 35 | current_prefix = cache_settings.get("KEY_PREFIX", "") 36 | cache_settings["KEY_PREFIX"] = "test." + current_prefix 37 | 38 | 39 | # Always assume we prefer https. 40 | PROTOCOL = "https://" 41 | 42 | # Never rely on the .env 43 | GOOGLE_ANALYTICS_ACCOUNT = None 44 | 45 | # Silence warnings about defaults that change in django-storages 2.0 46 | AWS_BUCKET_ACL = None 47 | AWS_DEFAULT_ACL = None 48 | 49 | # To make absolutely sure we never accidentally trigger the GA tracking 50 | # within tests to the actual (and default) www.google-analytics.com this is 51 | # an extra safeguard. 52 | GOOGLE_ANALYTICS_TRACKING_URL = "https://thisllneverwork.example.com/collect" 53 | 54 | # Because that's what all the tests presume. 55 | SITE_ID = 1 56 | 57 | # For legacy reasons, the tests assume these are always true so don't 58 | # let local overrides take effect. 59 | INDEX_HTML_ATTRIBUTES = True 60 | INDEX_CSS_CLASSNAMES = True 61 | 62 | # Amount for the monthly subscription. 63 | # It's hardcoded here in case some test depends on the number and it futureproofs 64 | # our tests to not deviate when the actual number changes since that number 65 | # change shouldn't affect the tests. 66 | CONTRIBUTION_AMOUNT_USD = 4.99 67 | 68 | # So it never accidentally actually uses the real value 69 | BOOKMARKS_BASE_URL = "https://developer.example.com" 70 | 71 | # OIDC related 72 | OIDC_CONFIGURATION_CHECK = True 73 | OIDC_RP_CLIENT_ID = "123456789" 74 | OIDC_RP_CLIENT_SECRET = "xyz-secret-123" 75 | OIDC_CONFIGURATION_URL = "https://accounts.examples.com" 76 | OIDC_OP_AUTHORIZATION_ENDPOINT = f"{OIDC_CONFIGURATION_URL}/authorization" 77 | 78 | OIDC_OP_TOKEN_ENDPOINT = f"{OIDC_CONFIGURATION_URL}/v1/token" 79 | OIDC_OP_USER_ENDPOINT = f"{OIDC_CONFIGURATION_URL}/v1/profile" 80 | OIDC_OP_JWKS_ENDPOINT = f"{OIDC_CONFIGURATION_URL}/v1/jwks" 81 | OIDC_RP_SIGN_ALGO = "XYZ" 82 | OIDC_USE_NONCE = False 83 | OIDC_RP_SCOPES = "openid profile email" 84 | -------------------------------------------------------------------------------- /kuma/users/models.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.contrib.auth import get_user_model 4 | from django.db import models 5 | 6 | 7 | class UserProfile(models.Model): 8 | class SubscriptionType(models.TextChoices): 9 | """Choices for MDN Subscription Types, add new subscription plans to be supported here""" 10 | 11 | MDN_PLUS_5M = "mdn_plus_5m", "MDN Plus 5M" 12 | MDN_PLUS_5Y = "mdn_plus_5y", "MDN Plus 5Y" 13 | MDN_PLUS_10M = "mdn_plus_10m", "MDN Plus 10M" 14 | MDN_PLUS_10Y = "mdn_plus_10y", "MDN Plus 10Y" 15 | NONE = "", "None" 16 | 17 | user = models.OneToOneField(get_user_model(), on_delete=models.CASCADE) 18 | locale = models.CharField(max_length=6, null=True) 19 | created = models.DateTimeField(auto_now_add=True) 20 | modified = models.DateTimeField(auto_now=True) 21 | avatar = models.URLField(max_length=512, blank=True, default="") 22 | fxa_refresh_token = models.CharField(blank=True, default="", max_length=128) 23 | is_subscriber = models.BooleanField(default=False) 24 | subscription_type = models.CharField( 25 | max_length=512, 26 | blank=True, 27 | choices=SubscriptionType.choices, 28 | default=SubscriptionType.NONE, 29 | ) 30 | 31 | class Meta: 32 | verbose_name = "User profile" 33 | 34 | def __str__(self): 35 | return json.dumps( 36 | { 37 | "uid": self.user.username, 38 | "is_subscriber": self.is_subscriber, 39 | "subscription_type": self.subscription_type, 40 | "email": self.user.email, 41 | "avatar": self.avatar, 42 | } 43 | ) 44 | 45 | 46 | class AccountEvent(models.Model): 47 | """Stores the Events received from Firefox Accounts. 48 | 49 | Each event is processed by Celery and stored in this table. 50 | """ 51 | 52 | class EventType(models.IntegerChoices): 53 | """Type of event received from Firefox Accounts.""" 54 | 55 | PASSWORD_CHANGED = 1 56 | PROFILE_CHANGED = 2 57 | SUBSCRIPTION_CHANGED = 3 58 | PROFILE_DELETED = 4 59 | 60 | class EventStatus(models.IntegerChoices): 61 | """Status of each event received from Firefox Accounts.""" 62 | 63 | PROCESSED = 1 64 | PENDING = 2 65 | IGNORED = 3 66 | NOT_IMPLEMENTED = 4 67 | 68 | created_at = models.DateTimeField(auto_now_add=True) 69 | modified_at = models.DateTimeField(auto_now=True) 70 | # This could be a ForeignKey to the UserProfile table (username). Decoupled for now 71 | fxa_uid = models.CharField(max_length=128, blank=True, default="") 72 | payload = models.TextField(max_length=2048, blank=True, default="") 73 | event_type = models.IntegerField(choices=EventType.choices, default="", blank=True) 74 | status = models.IntegerField( 75 | choices=EventStatus.choices, default=EventStatus.PENDING 76 | ) 77 | jwt_id = models.CharField(max_length=256, blank=True, default="") 78 | issued_at = models.CharField(max_length=32, default="", blank=True) 79 | 80 | class Meta: 81 | ordering = ["-modified_at"] 82 | -------------------------------------------------------------------------------- /scripts/slack-notify.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | usage() { 4 | echo -en "Usage: $0 --stage --message --status \n\n" 5 | echo -en "Options: \n" 6 | echo -en " --debug Debug basically sets set -x in bash \n" 7 | echo -en " --message Slack message to send\n" 8 | echo -en " --stage Job stage \n" 9 | echo -en " --status Job status\n" 10 | echo -en " --hook Slack incoming webhook url\n" 11 | } 12 | 13 | return_status() { 14 | local STATUS=$1 15 | 16 | if [[ -n "$STATUS" ]]; then 17 | STATUS=$(echo "$STATUS" | tr '[:lower:]' '[:upper:]') 18 | case "$STATUS" in 19 | 'SUCCESS') 20 | STATUS_PREFIX=":tada:" 21 | COLOR="good" 22 | ;; 23 | 'SHIPPED') 24 | STATUS_PREFIX=":ship:" 25 | COLOR="good" 26 | ;; 27 | 'WARNING') 28 | STATUS_PREFIX=":warning:" 29 | COLOR="warning" 30 | ;; 31 | 'FAILURE') 32 | STATUS_PREFIX=":rotating_light:" 33 | COLOR="danger" 34 | ;; 35 | *) 36 | STATUS_PREFIX=":sparkles:" 37 | COLOR="good" 38 | ;; 39 | esac 40 | STATUS="${STATUS_PREFIX} *${STATUS}* " 41 | echo "STATUS=${STATUS}" 42 | echo "COLOR=${COLOR}" 43 | fi 44 | } 45 | 46 | 47 | while [ "$1" != "" ]; do 48 | case $1 in 49 | -h | --help ) 50 | usage 51 | exit 0 52 | ;; 53 | -x | --setx | --debug ) 54 | set -x 55 | ;; 56 | --stage ) 57 | STAGE="${2}" 58 | shift 59 | ;; 60 | --status ) 61 | STATUS=$(return_status "${2}" | grep "STATUS" | sed 's/.*=//') 62 | COLOR=$(return_status "${2}" | grep "COLOR" | sed 's/.*=//') 63 | shift 64 | ;; 65 | --hook ) 66 | HOOK="${2}" 67 | shift 68 | ;; 69 | -m | --message) 70 | MESSAGE="${2}" 71 | shift 72 | ;; 73 | esac 74 | shift 75 | done 76 | 77 | if [[ -n "$STAGE" ]]; then 78 | MESSAGE="${STATUS}${STAGE}" 79 | elif [[ -n "$MESSAGE" ]]; then 80 | MESSAGE="${STATUS}${MESSAGE}" 81 | else 82 | usage 83 | exit 1 84 | fi 85 | 86 | if [ -z "${HOOK}" ]; then 87 | echo "[ERROR]: --hook or webhook is not set" 88 | exit 1 89 | fi 90 | 91 | read -r -d '' payload <" 55 | 56 | 57 | class CustomBaseModel(models.Model): 58 | content_updates = models.BooleanField(default=True) 59 | browser_compatibility = models.JSONField(default=list) 60 | 61 | class Meta: 62 | abstract = True 63 | 64 | def custom_serialize(self): 65 | return { 66 | "content": self.content_updates, 67 | "compatibility": self.browser_compatibility, 68 | } 69 | 70 | 71 | class UserWatch(CustomBaseModel): 72 | user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) 73 | watch = models.ForeignKey(Watch, on_delete=models.CASCADE) 74 | custom = models.BooleanField(default=False) 75 | custom_default = models.BooleanField(default=False) 76 | 77 | class Meta: 78 | db_table = "notifications_watch_users" 79 | 80 | def serialize(self): 81 | return { 82 | "title": self.watch.title, 83 | "url": self.watch.url, 84 | "path": self.watch.path, 85 | "custom": self.watch.custom, 86 | "custom_default": self.watch.custom_default, 87 | } 88 | 89 | def __str__(self): 90 | return f"User {self.user_id} watching {self.watch_id}" 91 | 92 | 93 | class DefaultWatch(CustomBaseModel): 94 | user = models.OneToOneField(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE) 95 | -------------------------------------------------------------------------------- /kuma/core/tests/test_decorators.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from django.http import HttpResponse 3 | from django.views.decorators.cache import never_cache 4 | 5 | from ..decorators import ( 6 | redirect_in_maintenance_mode, 7 | shared_cache_control, 8 | skip_in_maintenance_mode, 9 | ) 10 | from . import assert_no_cache_header 11 | 12 | 13 | def simple_view(request): 14 | return HttpResponse() 15 | 16 | 17 | @pytest.mark.parametrize("maintenance_mode", [False, True]) 18 | def test_skip_in_maintenance_mode(settings, maintenance_mode): 19 | @skip_in_maintenance_mode 20 | def func(*args, **kwargs): 21 | return (args, sorted(kwargs.items())) 22 | 23 | settings.MAINTENANCE_MODE = maintenance_mode 24 | 25 | if maintenance_mode: 26 | assert func(1, 2, x=3, y=4) is None 27 | else: 28 | assert func(1, 2, x=3, y=4) == ((1, 2), [("x", 3), ("y", 4)]) 29 | 30 | 31 | @pytest.mark.parametrize( 32 | "maintenance_mode, request_method, methods, expected_status_code", 33 | [ 34 | (True, "get", None, 302), 35 | (True, "post", None, 302), 36 | (False, "get", None, 200), 37 | (False, "post", None, 200), 38 | (True, "get", ("PUT", "POST"), 200), 39 | (True, "put", ("PUT", "POST"), 302), 40 | (True, "post", ("PUT", "POST"), 302), 41 | (False, "get", ("PUT", "POST"), 200), 42 | (False, "put", ("PUT", "POST"), 200), 43 | (False, "post", ("PUT", "POST"), 200), 44 | (False, "post", ("PUT", "POST"), 200), 45 | ], 46 | ) 47 | def test_redirect_in_maintenance_mode_decorator( 48 | rf, settings, maintenance_mode, request_method, methods, expected_status_code 49 | ): 50 | request = getattr(rf, request_method)("/foo") 51 | settings.MAINTENANCE_MODE = maintenance_mode 52 | if methods is None: 53 | deco = redirect_in_maintenance_mode 54 | else: 55 | deco = redirect_in_maintenance_mode(methods=methods) 56 | resp = deco(simple_view)(request) 57 | assert resp.status_code == expected_status_code 58 | 59 | 60 | def test_shared_cache_control_decorator_with_defaults(rf, settings): 61 | settings.CACHE_CONTROL_DEFAULT_SHARED_MAX_AGE = 777 62 | request = rf.get("/foo") 63 | response = shared_cache_control(simple_view)(request) 64 | assert response.status_code == 200 65 | assert "public" in response["Cache-Control"] 66 | assert "max-age=0" in response["Cache-Control"] 67 | assert "s-maxage=777" in response["Cache-Control"] 68 | 69 | 70 | def test_shared_cache_control_decorator_with_overrides(rf, settings): 71 | settings.CACHE_CONTROL_DEFAULT_SHARED_MAX_AGE = 777 72 | request = rf.get("/foo") 73 | deco = shared_cache_control(max_age=999, s_maxage=0) 74 | response = deco(simple_view)(request) 75 | assert response.status_code == 200 76 | assert "public" in response["Cache-Control"] 77 | assert "max-age=999" in response["Cache-Control"] 78 | assert "s-maxage=0" in response["Cache-Control"] 79 | 80 | 81 | def test_shared_cache_control_decorator_keeps_no_cache(rf, settings): 82 | request = rf.get("/foo") 83 | response = shared_cache_control(never_cache(simple_view))(request) 84 | assert response.status_code == 200 85 | assert "public" not in response["Cache-Control"] 86 | assert "s-maxage" not in response["Cache-Control"] 87 | assert_no_cache_header(response) 88 | -------------------------------------------------------------------------------- /kuma/core/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from django.core import mail 3 | from django.core.mail.backends.locmem import EmailBackend 4 | from requests.exceptions import ConnectionError 5 | 6 | from kuma.core.utils import ( 7 | EmailMultiAlternativesRetrying, 8 | order_params, 9 | requests_retry_session, 10 | send_mail_retrying, 11 | ) 12 | 13 | 14 | @pytest.mark.parametrize( 15 | "original,expected", 16 | ( 17 | ("https://example.com", "https://example.com"), 18 | ("http://example.com?foo=bar&foo=", "http://example.com?foo=&foo=bar"), 19 | ("http://example.com?foo=bar&bar=baz", "http://example.com?bar=baz&foo=bar"), 20 | ), 21 | ) 22 | def test_order_params(original, expected): 23 | assert order_params(original) == expected 24 | 25 | 26 | def test_requests_retry_session(mock_requests): 27 | def absolute_url(uri): 28 | return "http://example.com" + uri 29 | 30 | mock_requests.get(absolute_url("/a/ok"), text="hi") 31 | mock_requests.get(absolute_url("/oh/noes"), text="bad!", status_code=504) 32 | mock_requests.get(absolute_url("/oh/crap"), exc=ConnectionError) 33 | 34 | session = requests_retry_session(status_forcelist=(504,)) 35 | response_ok = session.get(absolute_url("/a/ok")) 36 | assert response_ok.status_code == 200 37 | 38 | response_bad = session.get(absolute_url("/oh/noes")) 39 | assert response_bad.status_code == 504 40 | 41 | with pytest.raises(ConnectionError): 42 | session.get(absolute_url("/oh/crap")) 43 | 44 | 45 | class SomeException(Exception): 46 | """Just a custom exception class.""" 47 | 48 | 49 | class SMTPFlakyEmailBackend(EmailBackend): 50 | """doc string""" 51 | 52 | def send_messages(self, messages): 53 | self._attempts = getattr(self, "_attempts", 0) + 1 54 | if self._attempts < 2: 55 | raise SomeException("Oh noes!") 56 | return super(SMTPFlakyEmailBackend, self).send_messages(messages) 57 | 58 | 59 | def test_send_mail_retrying(settings): 60 | settings.EMAIL_BACKEND = "kuma.core.tests.test_utils.SMTPFlakyEmailBackend" 61 | 62 | send_mail_retrying( 63 | "Subject", 64 | "Message", 65 | "from@example.com", 66 | ["to@example.com"], 67 | retry_options={ 68 | "retry_exceptions": (SomeException,), 69 | # Overriding defaults to avoid the test being slow. 70 | "sleeptime": 0.02, 71 | "jitter": 0.01, 72 | }, 73 | ) 74 | sent = mail.outbox[-1] 75 | # sanity check 76 | assert sent.subject == "Subject" 77 | 78 | 79 | def test_EmailMultiAlternativesRetrying(settings): 80 | settings.EMAIL_BACKEND = "kuma.core.tests.test_utils.SMTPFlakyEmailBackend" 81 | 82 | email = EmailMultiAlternativesRetrying( 83 | "Multi Subject", 84 | "Content", 85 | "from@example.com", 86 | ["to@example.com"], 87 | ) 88 | email.attach_alternative("

Content

", "text/html") 89 | email.send( 90 | retry_options={ 91 | "retry_exceptions": (SomeException,), 92 | # Overriding defaults to avoid the test being slow. 93 | "sleeptime": 0.02, 94 | "jitter": 0.01, 95 | } 96 | ) 97 | sent = mail.outbox[-1] 98 | # sanity check 99 | assert sent.subject == "Multi Subject" 100 | -------------------------------------------------------------------------------- /kuma/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import requests_mock 3 | from django.contrib.auth.models import Group 4 | from django.core.cache import caches 5 | from django.urls import set_urlconf 6 | from django.utils import timezone 7 | from django.utils.translation import activate 8 | 9 | from kuma.users.models import UserProfile 10 | 11 | 12 | @pytest.fixture(autouse=True) 13 | def clear_cache(): 14 | """ 15 | Before every pytest test function starts, it clears the cache. 16 | """ 17 | caches["default"].clear() 18 | 19 | 20 | @pytest.fixture(autouse=True) 21 | def set_default_language(): 22 | activate("en-US") 23 | 24 | 25 | @pytest.fixture(autouse=True) 26 | def reset_urlconf(): 27 | """ 28 | Reset the default urlconf used by "reverse" to the one provided 29 | by settings.ROOT_URLCONF. 30 | 31 | Django resets the default urlconf back to settings.ROOT_URLCONF at 32 | the beginning of each request, but if the incoming request has a 33 | "urlconf" attribute, the default urlconf is changed to its value for 34 | the remainder of the request, so that all subsequent "reverse" calls 35 | use that value (unless they explicitly specify a different one). The 36 | problem occurs when a test is run that uses the "request.urlconf" 37 | mechanism, setting the default urlconf to something other than 38 | settings.ROOT_URLCONF, and then subsequent tests make "reverse" calls 39 | that fail because they're expecting a default urlconf of 40 | settings.ROOT_URLCONF (i.e., they're not explicitly providing a 41 | urlconf value to the "reverse" call). 42 | """ 43 | set_urlconf(None) 44 | yield 45 | set_urlconf(None) 46 | 47 | 48 | @pytest.fixture 49 | def beta_testers_group(db): 50 | return Group.objects.create(name="Beta Testers") 51 | 52 | 53 | @pytest.fixture 54 | def wiki_user(db, django_user_model): 55 | """A test user.""" 56 | return django_user_model.objects.create( 57 | username="wiki_user", 58 | email="wiki_user@example.com", 59 | date_joined=timezone.now(), 60 | ) 61 | 62 | 63 | @pytest.fixture 64 | def user_client(client, wiki_user): 65 | """A test client with wiki_user logged in.""" 66 | wiki_user.set_password("password") 67 | wiki_user.save() 68 | client.login(username=wiki_user.username, password="password") 69 | return client 70 | 71 | 72 | @pytest.fixture 73 | def subscriber_client(client, wiki_user): 74 | """A test client with wiki_user logged in and a paying subscriber.""" 75 | UserProfile.objects.create(user=wiki_user, is_subscriber=True) 76 | wiki_user.set_password("password") 77 | wiki_user.save() 78 | client.login(username=wiki_user.username, password="password") 79 | return client 80 | 81 | 82 | @pytest.fixture 83 | def stripe_user(wiki_user): 84 | wiki_user.stripe_customer_id = "fakeCustomerID123" 85 | wiki_user.save() 86 | return wiki_user 87 | 88 | 89 | @pytest.fixture 90 | def stripe_user_client(client, stripe_user): 91 | """A test client with wiki_user logged in and with a stripe_customer_id.""" 92 | stripe_user.set_password("password") 93 | stripe_user.save() 94 | client.login(username=stripe_user.username, password="password") 95 | return client 96 | 97 | 98 | @pytest.fixture 99 | def mock_requests(): 100 | with requests_mock.Mocker() as mocker: 101 | yield mocker 102 | -------------------------------------------------------------------------------- /kuma/users/tasks.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from celery import task 4 | from django.contrib.auth import get_user_model 5 | 6 | from kuma.users.auth import KumaOIDCAuthenticationBackend 7 | from kuma.users.models import AccountEvent, UserProfile 8 | from kuma.users.utils import get_valid_subscription_type_or_none 9 | 10 | 11 | @task 12 | def process_event_delete_user(event_id): 13 | event = AccountEvent.objects.get(id=event_id) 14 | try: 15 | user = get_user_model().objects.get(username=event.fxa_uid) 16 | except get_user_model().DoesNotExist: 17 | return 18 | 19 | user.delete() 20 | 21 | event.status = AccountEvent.PROCESSED 22 | event.save() 23 | 24 | 25 | @task 26 | def process_event_subscription_state_change(event_id): 27 | event = AccountEvent.objects.get(id=event_id) 28 | try: 29 | user = get_user_model().objects.get(username=event.fxa_uid) 30 | profile = UserProfile.objects.get(user=user) 31 | except get_user_model().DoesNotExist: 32 | return 33 | 34 | payload = json.loads(event.payload) 35 | 36 | last_event = AccountEvent.objects.filter( 37 | fxa_uid=event.fxa_uid, 38 | status=AccountEvent.EventStatus.PROCESSED, 39 | event_type=AccountEvent.EventType.SUBSCRIPTION_CHANGED, 40 | ).first() 41 | 42 | if last_event: 43 | last_event_payload = json.loads(last_event.payload) 44 | if last_event_payload["changeTime"] >= payload["changeTime"]: 45 | event.status = AccountEvent.EventStatus.IGNORED 46 | event.save() 47 | return 48 | 49 | subscription_type = get_valid_subscription_type_or_none( 50 | payload.get("capabilities", []) 51 | ) 52 | if not user.is_staff: 53 | if payload["isActive"]: 54 | profile.subscription_type = subscription_type 55 | profile.is_subscriber = True 56 | else: 57 | profile.subscription_type = "" 58 | profile.is_subscriber = False 59 | profile.save() 60 | 61 | event.status = AccountEvent.EventStatus.PROCESSED 62 | event.save() 63 | 64 | 65 | @task 66 | def process_event_password_change(event_id): 67 | event = AccountEvent.objects.get(id=event_id) 68 | event.status = AccountEvent.EventStatus.PROCESSED 69 | event.save() 70 | 71 | 72 | @task 73 | def process_event_profile_change(event_id): 74 | event = AccountEvent.objects.get(id=event_id) 75 | try: 76 | user = get_user_model().objects.get(username=event.fxa_uid) 77 | profile = UserProfile.objects.get(user=user) 78 | except get_user_model().DoesNotExist: 79 | return 80 | 81 | refresh_token = profile.fxa_refresh_token 82 | 83 | if not refresh_token: 84 | event.status = AccountEvent.IGNORED 85 | event.save() 86 | return 87 | 88 | fxa = KumaOIDCAuthenticationBackend() 89 | token_info = fxa.get_token( 90 | { 91 | "client_id": fxa.OIDC_RP_CLIENT_ID, 92 | "client_secret": fxa.OIDC_RP_CLIENT_SECRET, 93 | "grant_type": "refresh_token", 94 | "refresh_token": refresh_token, 95 | "ttl": 60 * 5, 96 | } 97 | ) 98 | access_token = token_info.get("access_token") 99 | user_info = fxa.get_userinfo(access_token, None, None) 100 | fxa.update_user(user, user_info) 101 | 102 | event.status = AccountEvent.EventStatus.PROCESSED 103 | event.save() 104 | -------------------------------------------------------------------------------- /kuma/documenturls/models.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from django.db import models 3 | from django.db.models.signals import pre_save 4 | from django.dispatch import receiver 5 | from redo import retrying 6 | 7 | 8 | def download_url(url, retry_options=None): 9 | retry_options = retry_options or { 10 | "retry_exceptions": (requests.exceptions.Timeout,), 11 | "sleeptime": 2, 12 | "attempts": 5, 13 | } 14 | with retrying(requests.get, **retry_options) as retrying_get: 15 | response = retrying_get(url, allow_redirects=False) 16 | response.raise_for_status() 17 | 18 | return response 19 | 20 | 21 | class DocumentURL(models.Model): 22 | """There are documents we look up for things like bookmarks and notes. 23 | These are not legacy Wiki documents but rather remote URLs. 24 | """ 25 | 26 | # E.g. /en-us/docs/web/javascript (note that it's lowercased!) 27 | # (the longest URI in all our of content as of mid-2021 is 121 characters) 28 | uri = models.CharField(max_length=250, unique=True, verbose_name="URI") 29 | # E.g. https://developer.allizom.org/en-us/docs/web/javascript/index.json 30 | absolute_url = models.URLField(verbose_name="Absolute URL") 31 | # If it's applicable, it's a download of the `index.json` for that URL. 32 | metadata = models.JSONField(null=True) 33 | # If the URI for some reason becomes invalid, rather than deleting it 34 | # or storing a boolean, note *when* it became invalid. 35 | invalid = models.DateTimeField(null=True) 36 | created = models.DateTimeField(auto_now_add=True) 37 | modified = models.DateTimeField(auto_now=True) 38 | 39 | class Meta: 40 | verbose_name = "Document URL" 41 | 42 | def __str__(self): 43 | return self.uri 44 | 45 | @classmethod 46 | def normalize_uri(cls, uri): 47 | return uri.lower().strip() 48 | 49 | 50 | @receiver(pre_save, sender=DocumentURL) 51 | def assert_lowercase_uri(sender, instance, **kwargs): 52 | # Because it's so important that the `uri` is lowercased, this makes 53 | # absolutely sure it's always so. 54 | # Ideally, the client should check this at upsert, but if it's missed, 55 | # this last resort will make sure it doesn't get in in 56 | instance.uri = DocumentURL.normalize_uri(instance.uri) 57 | 58 | 59 | class DocumentURLCheck(models.Model): 60 | document_url = models.ForeignKey( 61 | DocumentURL, on_delete=models.CASCADE, verbose_name="Document URL" 62 | ) 63 | http_error = models.IntegerField(verbose_name="HTTP error") 64 | headers = models.JSONField(default=dict) 65 | created = models.DateTimeField(auto_now_add=True) 66 | 67 | class Meta: 68 | verbose_name = "Document URL Check" 69 | 70 | def __str__(self): 71 | return f"{self.http_error} on {self.document_url.absolute_url}" 72 | 73 | @classmethod 74 | def check_uri(cls, uri, cleanup_old=False, retry_options=None): 75 | document_url = DocumentURL.objects.get(uri=DocumentURL.normalize_uri(uri)) 76 | 77 | response = download_url(document_url.absolute_url, retry_options=retry_options) 78 | 79 | headers = dict(response.headers) 80 | checked = cls.objects.create( 81 | document_url=document_url, 82 | http_error=response.status_code, 83 | headers=headers, 84 | ) 85 | if cleanup_old: 86 | cls.objects.filter(document_url=document_url).exclude( 87 | id=checked.id 88 | ).delete() 89 | return checked 90 | -------------------------------------------------------------------------------- /kuma/api/v1/views.py: -------------------------------------------------------------------------------- 1 | from typing import Literal, Optional, Union 2 | 3 | from django.conf import settings 4 | from django.middleware.csrf import get_token 5 | from ninja import Router, Schema 6 | 7 | from kuma.api.v1.auth import profile_auth 8 | from kuma.api.v1.forms import AccountSettingsForm 9 | from kuma.api.v1.plus.notifications import Ok 10 | from kuma.users.models import UserProfile 11 | 12 | from .api import api 13 | 14 | settings_router = Router(auth=profile_auth, tags=["settings"]) 15 | 16 | 17 | class AnonResponse(Schema): 18 | geo: Optional[dict] 19 | 20 | 21 | class AuthResponse(AnonResponse): 22 | username: str 23 | is_authenticated: bool = True 24 | email: str 25 | is_staff: Optional[bool] 26 | is_superuser: Optional[bool] 27 | avatar_url: Optional[str] 28 | is_subscriber: Optional[bool] 29 | subscription_type: Optional[str] 30 | 31 | 32 | @api.get( 33 | "/whoami", 34 | auth=None, 35 | exclude_none=True, 36 | response=Union[AuthResponse, AnonResponse], 37 | url_name="whoami", 38 | ) 39 | def whoami(request): 40 | """ 41 | Return a JSON object representing the current user, either 42 | authenticated or anonymous. 43 | """ 44 | data = {} 45 | user = request.user 46 | cloudfront_country_header = "HTTP_CLOUDFRONT_VIEWER_COUNTRY_NAME" 47 | cloudfront_country_value = request.META.get(cloudfront_country_header) 48 | if cloudfront_country_value: 49 | data["geo"] = {"country": cloudfront_country_value} 50 | 51 | if not user.is_authenticated: 52 | return data 53 | 54 | data["username"] = user.username 55 | data["is_authenticated"] = True 56 | data["email"] = user.email 57 | 58 | if user.is_staff: 59 | data["is_staff"] = True 60 | if user.is_superuser: 61 | data["is_superuser"] = True 62 | 63 | profile = UserProfile.objects.filter(user=user).first() 64 | if profile: 65 | data["avatar_url"] = profile.avatar 66 | data["is_subscriber"] = profile.is_subscriber 67 | data["subscription_type"] = profile.subscription_type 68 | return data 69 | 70 | 71 | @settings_router.delete("/", url_name="settings") 72 | def delete_user(request): 73 | request.user.delete() 74 | return {"deleted": True} 75 | 76 | 77 | @settings_router.get("/", url_name="settings") 78 | def account_settings(request): 79 | user_profile: UserProfile = request.auth 80 | return { 81 | "csrfmiddlewaretoken": get_token(request), 82 | "locale": user_profile.locale if user_profile.pk else None, 83 | } 84 | 85 | 86 | class FormErrors(Schema): 87 | ok: Literal[False] = False 88 | errors: dict[str, list[dict[str, str]]] 89 | 90 | 91 | @settings_router.post("/", response={200: Ok, 400: FormErrors}, url_name="settings") 92 | def save_settings(request): 93 | user_profile: UserProfile = request.auth 94 | form = AccountSettingsForm(request.POST) 95 | if not form.is_valid(): 96 | return 400, {"errors": form.errors.get_json_data()} 97 | 98 | set_locale = None 99 | if form.cleaned_data.get("locale"): 100 | user_profile.locale = form.cleaned_data["locale"] 101 | user_profile.save() 102 | 103 | response = api.create_response(request, Ok.from_orm(True)) 104 | response.set_cookie( 105 | key=settings.LANGUAGE_COOKIE_NAME, 106 | value=set_locale, 107 | max_age=settings.LANGUAGE_COOKIE_AGE, 108 | path=settings.LANGUAGE_COOKIE_PATH, 109 | domain=settings.LANGUAGE_COOKIE_DOMAIN, 110 | secure=settings.LANGUAGE_COOKIE_SECURE, 111 | ) 112 | return response 113 | 114 | return True 115 | -------------------------------------------------------------------------------- /kuma/users/tests/test_tasks.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from kuma.users.models import AccountEvent, UserProfile 4 | from kuma.users.tasks import process_event_subscription_state_change 5 | 6 | 7 | @pytest.mark.django_db 8 | def test_process_event_subscription_state_change(wiki_user): 9 | 10 | profile = UserProfile.objects.create(user=wiki_user) 11 | assert profile.subscription_type == "" 12 | 13 | created_event = AccountEvent.objects.create( 14 | event_type=AccountEvent.EventType.SUBSCRIPTION_CHANGED, 15 | status=AccountEvent.EventStatus.PENDING, 16 | fxa_uid=wiki_user.username, 17 | id=1, 18 | payload='{"capabilities": ["mdn_plus_5y"], "isActive": true}', 19 | ) 20 | 21 | process_event_subscription_state_change(created_event.id) 22 | profile.refresh_from_db() 23 | assert profile.subscription_type == "mdn_plus_5y" 24 | 25 | 26 | @pytest.mark.django_db 27 | def test_empty_subscription_inactive_change(wiki_user): 28 | 29 | profile = UserProfile.objects.create(user=wiki_user) 30 | assert profile.subscription_type == "" 31 | 32 | created_event = AccountEvent.objects.create( 33 | event_type=AccountEvent.EventType.SUBSCRIPTION_CHANGED, 34 | status=AccountEvent.EventStatus.PENDING, 35 | fxa_uid=wiki_user.username, 36 | id=1, 37 | payload='{"capabilities": [], "isActive": false}', 38 | ) 39 | 40 | process_event_subscription_state_change(created_event.id) 41 | profile.refresh_from_db() 42 | assert profile.subscription_type == "" 43 | assert not profile.is_subscriber 44 | 45 | 46 | @pytest.mark.django_db 47 | def test_valid_subscription_inactive_change(wiki_user): 48 | 49 | profile = UserProfile.objects.create(user=wiki_user) 50 | assert profile.subscription_type == "" 51 | 52 | created_event = AccountEvent.objects.create( 53 | event_type=AccountEvent.EventType.SUBSCRIPTION_CHANGED, 54 | status=AccountEvent.EventStatus.PENDING, 55 | fxa_uid=wiki_user.username, 56 | id=1, 57 | payload='{"capabilities": ["mdn_plus_5m"], "isActive": false}', 58 | ) 59 | 60 | process_event_subscription_state_change(created_event.id) 61 | profile.refresh_from_db() 62 | assert profile.subscription_type == "" 63 | assert not profile.is_subscriber 64 | 65 | 66 | @pytest.mark.django_db 67 | def test_invalid_subscription_change(wiki_user): 68 | 69 | profile = UserProfile.objects.create(user=wiki_user) 70 | assert profile.subscription_type == "" 71 | 72 | created_event = AccountEvent.objects.create( 73 | event_type=AccountEvent.EventType.SUBSCRIPTION_CHANGED, 74 | status=AccountEvent.EventStatus.PENDING, 75 | fxa_uid=wiki_user.username, 76 | id=1, 77 | payload='{"capabilities": ["invalid_subscription"], "isActive": true}', 78 | ) 79 | 80 | process_event_subscription_state_change(created_event.id) 81 | profile.refresh_from_db() 82 | assert profile.subscription_type == "" 83 | 84 | 85 | @pytest.mark.django_db 86 | def test_multiple_valid_subscription_change_takes_first_in_array(wiki_user): 87 | 88 | profile = UserProfile.objects.create(user=wiki_user) 89 | assert profile.subscription_type == "" 90 | 91 | created_event = AccountEvent.objects.create( 92 | event_type=AccountEvent.EventType.SUBSCRIPTION_CHANGED, 93 | status=AccountEvent.EventStatus.PENDING, 94 | fxa_uid=wiki_user.username, 95 | id=1, 96 | payload='{"capabilities": ["mdn_plus", "mdn_plus_5y", "mdn_plus_5m"], "isActive": true}', 97 | ) 98 | 99 | process_event_subscription_state_change(created_event.id) 100 | profile.refresh_from_db() 101 | # Ensure only first (lexicographical) valid is persisted 102 | assert profile.subscription_type == "mdn_plus_5m" 103 | -------------------------------------------------------------------------------- /kuma/users/checks.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import urljoin, urlparse 2 | 3 | from django.conf import settings 4 | from django.core.checks import Error, Warning 5 | 6 | from kuma.core.utils import requests_retry_session 7 | 8 | MISSING_OIDC_RP_CLIENT_ID_ERROR = "kuma.users.E001" 9 | MISSING_OIDC_RP_CLIENT_SECRET_ERROR = "kuma.users.E002" 10 | MISSING_OIDC_CONFIGURATION_ERROR = "kuma.users.E003" 11 | 12 | 13 | def oidc_config_check(app_configs, **kwargs): 14 | errors = [] 15 | 16 | for id, key in ( 17 | (MISSING_OIDC_RP_CLIENT_ID_ERROR, "OIDC_RP_CLIENT_ID"), 18 | (MISSING_OIDC_RP_CLIENT_SECRET_ERROR, "OIDC_RP_CLIENT_SECRET"), 19 | ): 20 | if not getattr(settings, key, None): 21 | class_ = Warning if settings.DEBUG else Error 22 | errors.append( 23 | class_( 24 | f"{key} environment variable is not set or is empty", 25 | id=id, 26 | ) 27 | ) 28 | 29 | if settings.OIDC_CONFIGURATION_CHECK: 30 | errors.extend(_get_oidc_configuration_errors(MISSING_OIDC_CONFIGURATION_ERROR)) 31 | 32 | return errors 33 | 34 | 35 | def _get_oidc_configuration_errors(id): 36 | errors = [] 37 | 38 | configuration_url = settings.OIDC_CONFIGURATION_URL 39 | parsed = urlparse(configuration_url) 40 | if not parsed.path or parsed.path == "/": 41 | default_path = "/.well-known/openid-configuration" 42 | parsed._replace(path=default_path) 43 | configuration_url = urljoin(configuration_url, default_path) 44 | response = requests_retry_session().get(configuration_url) 45 | response.raise_for_status() 46 | openid_configuration = response.json() 47 | 48 | for key, setting_key in ( 49 | ("userinfo_endpoint", "OIDC_OP_USER_ENDPOINT"), 50 | ("authorization_endpoint", "OIDC_OP_AUTHORIZATION_ENDPOINT"), 51 | ("token_endpoint", "OIDC_OP_TOKEN_ENDPOINT"), 52 | ): 53 | setting_value = getattr(settings, setting_key, None) 54 | if key not in openid_configuration and setting_value: 55 | errors.append( 56 | Warning( 57 | f"{setting_key} is set but {key!r} is not exposed in {configuration_url}", 58 | id=id, 59 | ) 60 | ) 61 | continue 62 | config_value = openid_configuration[key] 63 | if setting_value and config_value != setting_value: 64 | errors.append( 65 | Error( 66 | f"{setting_key}'s value is different from that on {configuration_url}" 67 | f" ({setting_value!r} != {config_value!r}", 68 | id=id, 69 | ) 70 | ) 71 | 72 | # ##TODO 14/02/22 - The additional profile:subscriptions scope is currently missing from the supportes scopes in oidc config 73 | # settings.OIDC_RP_SCOPES can have less but not more that what's supported 74 | # scopes_requested = set(settings.OIDC_RP_SCOPES.split()) 75 | # scopes_supported = set(openid_configuration["scopes_supported"]) 76 | # if scopes_supported - scopes_requested: 77 | # errors.append( 78 | # Error( 79 | # f"Invalid settings.OIDC_RP_SCOPES ({settings.OIDC_RP_SCOPES!r}). " 80 | # f"Requested: {scopes_requested}, Supported: {scopes_supported}", 81 | # id=id, 82 | # ) 83 | # ) 84 | 85 | if settings.OIDC_RP_SIGN_ALGO not in set( 86 | openid_configuration["id_token_signing_alg_values_supported"] 87 | ): 88 | errors.append( 89 | Error( 90 | f"Invalid settings.OIDC_RP_SIGN_ALGO. " 91 | f"{settings.OIDC_RP_SIGN_ALGO!r} not in " 92 | f'{openid_configuration["id_token_signing_alg_values_supported"]}', 93 | id=id, 94 | ) 95 | ) 96 | 97 | return errors 98 | -------------------------------------------------------------------------------- /kuma/attachments/views.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import json 3 | from pathlib import Path 4 | 5 | from django.conf import settings 6 | from django.http import HttpResponseNotFound 7 | from django.shortcuts import redirect 8 | 9 | from .utils import ( 10 | convert_to_http_date, 11 | full_attachment_url, 12 | full_mindtouch_attachment_url, 13 | ) 14 | 15 | # In the olden days of Kuma, we used to have people upload files into the Wiki. 16 | # These are called Attachments. 17 | # Later we realized it's bad to host these files on disk so we uploaded them 18 | # all into S3 and in the Django view, we simply redirected to the public 19 | # S3 URL. E.g. 20 | # GET /files/16014/LeagueMonoVariable.ttf 21 | # ==> 301 https://media.prod.mdn.mozit.cloud/attachments/2018/06/05/16014/96e6cc293bd18f32a17371500d01301b/LeagueMonoVariable.ttf 22 | # 23 | # Now, you can't upload files any more but we have these absolute URLs 24 | # peppered throughout the translated-content since we moved to Yari. Cleaning 25 | # all of that up is considered too hard so it's best to just keep serving the 26 | # redirects. 27 | # But in an effort to seriously simplify and be able to clean up a much of 28 | # MySQL data, we decided to scrape all the legacy file attachments mentioned 29 | # through out all mdn/content and mdn/translated-content. We then looked up 30 | # each URL's final destination URL. All of these we dumped into the 31 | # file called 'redirects.json' which is stored here on disk in this folder. 32 | # Now the view function just needs to reference that mapping. If something's 33 | # not in the mapping it's either a fumbly typo or it's simple a file attachment 34 | # URL not used anywhere in the content or translated-content. 35 | 36 | REDIRECTS_FILE = Path(__file__).parent / "redirects.json" 37 | assert REDIRECTS_FILE.exists(), REDIRECTS_FILE 38 | with open(REDIRECTS_FILE) as f: 39 | _redirects = json.load(f) 40 | assert _redirects 41 | 42 | # The mindtouch_redirects.json file is for URLs like /@api/deki/files/5695/=alpha.png 43 | # and the json file points to the final destination on S3. 44 | 45 | MINDTOUCH_REDIRECTS_FILE = Path(__file__).parent / "mindtouch_redirects.json" 46 | assert MINDTOUCH_REDIRECTS_FILE.exists(), MINDTOUCH_REDIRECTS_FILE 47 | with open(MINDTOUCH_REDIRECTS_FILE) as f: 48 | _mindtouch_redirects = json.load(f) 49 | assert _mindtouch_redirects 50 | 51 | 52 | def raw_file(request, attachment_id, filename): 53 | """ 54 | Serve up an attachment's file. 55 | """ 56 | if attachment_id not in _redirects: 57 | return HttpResponseNotFound(f"{attachment_id} not a known redirect") 58 | 59 | if settings.DOMAIN in request.get_host(): 60 | file_url = full_attachment_url(attachment_id, filename) 61 | return redirect(file_url, permanent=True) 62 | 63 | return _redirect_final_path(_redirects[attachment_id]) 64 | 65 | 66 | def mindtouch_file_redirect(request, file_id, filename): 67 | """Redirect an old MindTouch file URL to a new kuma file URL.""" 68 | if file_id not in _mindtouch_redirects: 69 | return HttpResponseNotFound(f"{file_id} not a known redirect") 70 | 71 | if settings.DOMAIN in request.get_host(): 72 | file_url = full_mindtouch_attachment_url(file_id, filename) 73 | return redirect(file_url, permanent=True) 74 | 75 | return _redirect_final_path(_mindtouch_redirects[file_id]) 76 | 77 | 78 | def _redirect_final_path(final_path): 79 | assert settings.ATTACHMENTS_AWS_S3_CUSTOM_DOMAIN 80 | redirect_url = "https://" if settings.ATTACHMENTS_AWS_S3_SECURE_URLS else "http://" 81 | redirect_url += settings.ATTACHMENTS_AWS_S3_CUSTOM_DOMAIN 82 | redirect_url += final_path 83 | response = redirect(redirect_url) 84 | 85 | olden = datetime.datetime.now() - datetime.timedelta(days=365) 86 | response["Last-Modified"] = convert_to_http_date(olden) 87 | response["X-Frame-Options"] = f"ALLOW-FROM {settings.DOMAIN}" 88 | return response 89 | -------------------------------------------------------------------------------- /docs/tests.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | The Kuma test suite 3 | ====================== 4 | 5 | Kuma has a fairly comprehensive Python test suite. Changes should not break 6 | tests. Only change a test if there is a good reason to change the expected 7 | behavior. New code should come with tests. 8 | 9 | Commands should be run inside the development environment, after ``make bash``. 10 | 11 | 12 | Running the test suite 13 | ====================== 14 | If you followed the steps in :doc:`the installation docs `, 15 | then all you should need to do to run the test suite is:: 16 | 17 | make test 18 | 19 | The default options for running the test are in ``pytest.ini``. This is a 20 | good set of defaults. 21 | 22 | If you ever need to change the defaults, you can do so at the command 23 | line by running what the Make task does behind the scenes:: 24 | 25 | py.test kuma 26 | 27 | Some helpful command line arguments to py.test (won't work on ``make test``): 28 | 29 | ``--pdb``: 30 | Drop into pdb on test failure. 31 | 32 | ``--create-db``: 33 | Create a new test database. 34 | 35 | ``--showlocals``: 36 | Shows local variables in tracebacks on errors. 37 | 38 | ``--exitfirst``: 39 | Exits on the first failure. 40 | 41 | See ``py.test --help`` for more arguments. 42 | 43 | Running subsets of tests and specific tests 44 | ------------------------------------------- 45 | There are a bunch of ways to specify a subset of tests to run: 46 | 47 | * only tests marked with the 'spam' marker:: 48 | 49 | py.test -m spam 50 | 51 | * all the tests but those marked with the 'spam' marker:: 52 | 53 | py.test -m "not spam" 54 | 55 | * all the tests but the ones in ``kuma/core``:: 56 | 57 | py.test --ignore kuma/core 58 | 59 | * all the tests that have "foobar" in their names:: 60 | 61 | py.test -k foobar 62 | 63 | * all the tests that don't have "foobar" in their names:: 64 | 65 | py.test -k "not foobar" 66 | 67 | * tests in a certain directory:: 68 | 69 | py.test kuma/wiki/ 70 | 71 | * specific test:: 72 | 73 | py.test kuma/wiki/tests/test_views.py::RedirectTests::test_redirects_only_internal 74 | 75 | See http://pytest.org/latest/usage.html for more examples. 76 | 77 | Showing test coverage 78 | --------------------- 79 | While running the tests you can record which part of the code base is covered 80 | by test cases. To show the results at the end of the test run use this command:: 81 | 82 | make coveragetest 83 | 84 | To generate an HTML coverage report, use:: 85 | 86 | make coveragetesthtml 87 | 88 | The test database 89 | ----------------- 90 | The test suite will create a new database named ``test_%s`` where ``%s`` is 91 | whatever value you have for ``settings.DATABASES['default']['NAME']``. Make 92 | sure the user has ``ALL`` on the test database as well. 93 | 94 | 95 | Markers 96 | ======= 97 | See:: 98 | 99 | py.test --markers 100 | 101 | 102 | for the list of available markers. 103 | 104 | To add a marker, add it to the ``pytest.ini`` file. 105 | 106 | To use a marker, add a decorator to the class or function. Examples:: 107 | 108 | import pytest 109 | 110 | @pytest.mark.spam 111 | class SpamTests(TestCase): 112 | ... 113 | 114 | class OtherSpamTests(TestCase): 115 | @pytest.mark.spam 116 | def test_something(self): 117 | ... 118 | 119 | 120 | Adding tests 121 | ============ 122 | Code should be written so that it can be tested, and then there should be tests for 123 | it. 124 | 125 | When adding code to an app, tests should be added in that app that cover the 126 | new functionality. All apps have a ``tests`` module where tests should go. They 127 | will be discovered automatically by the test runner as long as the look like a 128 | test. 129 | 130 | Changing tests 131 | ============== 132 | Unless the current behavior, and thus the test that verifies that behavior is 133 | correct, is demonstrably wrong, don't change tests. Tests may be refactored as 134 | long as it's clear that the result is the same. 135 | 136 | 137 | Removing tests 138 | ============== 139 | On those rare, wonderful occasions when we get to remove code, we should remove 140 | the tests for it, as well. 141 | 142 | If we liberate some functionality into a new package, the tests for that 143 | functionality should move to that package, too. 144 | -------------------------------------------------------------------------------- /kuma/api/v1/tests/test_views.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from django.contrib.auth.models import User 3 | 4 | from kuma.core.tests import assert_no_cache_header 5 | from kuma.core.urlresolvers import reverse 6 | 7 | 8 | @pytest.mark.parametrize("http_method", ["put", "post", "delete", "options", "head"]) 9 | def test_whoami_disallowed_methods(client, http_method): 10 | """HTTP methods other than GET are not allowed.""" 11 | url = reverse("api-v1:whoami") 12 | response = getattr(client, http_method)(url) 13 | assert response.status_code == 405 14 | 15 | 16 | @pytest.mark.django_db 17 | def test_whoami_anonymous(client): 18 | """Test response for anonymous users.""" 19 | url = reverse("api-v1:whoami") 20 | response = client.get(url) 21 | assert response.status_code == 200 22 | assert response["content-type"] == "application/json; charset=utf-8" 23 | assert response.json() == {} 24 | assert_no_cache_header(response) 25 | 26 | 27 | @pytest.mark.django_db 28 | def test_whoami_anonymous_cloudfront_geo(client): 29 | """Test response for anonymous users.""" 30 | url = reverse("api-v1:whoami") 31 | response = client.get(url, HTTP_CLOUDFRONT_VIEWER_COUNTRY_NAME="US of A") 32 | assert response.status_code == 200 33 | assert response["content-type"] == "application/json; charset=utf-8" 34 | assert response.json()["geo"] == {"country": "US of A"} 35 | 36 | 37 | @pytest.mark.django_db 38 | @pytest.mark.parametrize( 39 | "is_staff,is_superuser", 40 | [(False, False), (True, True)], 41 | ids=("muggle", "wizard"), 42 | ) 43 | def test_whoami( 44 | user_client, 45 | wiki_user, 46 | is_staff, 47 | is_superuser, 48 | ): 49 | """Test responses for logged-in users.""" 50 | wiki_user.is_staff = is_staff 51 | wiki_user.is_superuser = is_superuser 52 | wiki_user.is_staff = is_staff 53 | wiki_user.save() 54 | url = reverse("api-v1:whoami") 55 | response = user_client.get(url) 56 | assert response.status_code == 200 57 | assert response["content-type"] == "application/json; charset=utf-8" 58 | expect = { 59 | "username": wiki_user.username, 60 | "is_authenticated": True, 61 | "email": "wiki_user@example.com", 62 | } 63 | if is_staff: 64 | expect["is_staff"] = True 65 | if is_superuser: 66 | expect["is_superuser"] = True 67 | 68 | assert response.json() == expect 69 | assert_no_cache_header(response) 70 | 71 | 72 | @pytest.mark.django_db 73 | def test_account_settings_auth(client): 74 | url = reverse("api-v1:settings") 75 | response = client.get(url) 76 | assert response.status_code == 401 77 | response = client.delete(url) 78 | assert response.status_code == 401 79 | response = client.post(url, {}) 80 | assert response.status_code == 401 81 | 82 | 83 | def test_account_settings_delete(user_client, wiki_user): 84 | username = wiki_user.username 85 | response = user_client.delete(reverse("api-v1:settings")) 86 | assert response.status_code == 200 87 | assert not User.objects.filter(username=username).exists() 88 | 89 | 90 | def test_get_and_set_settings_happy_path(user_client): 91 | url = reverse("api-v1:settings") 92 | response = user_client.get(url) 93 | assert response.status_code == 200 94 | assert_no_cache_header(response) 95 | assert response.json()["locale"] is None 96 | 97 | response = user_client.post(url, {"locale": "zh-CN"}) 98 | assert response.status_code == 200 99 | 100 | response = user_client.post(url, {"locale": "fr"}) 101 | assert response.status_code == 200 102 | 103 | response = user_client.get(url) 104 | assert response.status_code == 200 105 | assert response.json()["locale"] == "fr" 106 | 107 | # You can also omit certain things and things won't be set 108 | response = user_client.post(url, {}) 109 | assert response.status_code == 200 110 | response = user_client.get(url) 111 | assert response.status_code == 200 112 | assert response.json()["locale"] == "fr" 113 | 114 | 115 | def test_set_settings_validation_errors(user_client): 116 | url = reverse("api-v1:settings") 117 | response = user_client.post(url, {"locale": "never heard of"}) 118 | assert response.status_code == 400 119 | assert response.json()["errors"]["locale"][0]["code"] == "invalid_choice" 120 | assert response.json()["errors"]["locale"][0]["message"] 121 | -------------------------------------------------------------------------------- /kuma/notifications/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2021-11-05 15:19 2 | 3 | import django.db.models.deletion 4 | from django.conf import settings 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name="CompatibilityData", 19 | fields=[ 20 | ( 21 | "id", 22 | models.BigAutoField( 23 | auto_created=True, 24 | primary_key=True, 25 | serialize=False, 26 | verbose_name="ID", 27 | ), 28 | ), 29 | ("bcd", models.JSONField()), 30 | ("created", models.DateTimeField(auto_now_add=True)), 31 | ], 32 | ), 33 | migrations.CreateModel( 34 | name="NotificationData", 35 | fields=[ 36 | ( 37 | "id", 38 | models.BigAutoField( 39 | auto_created=True, 40 | primary_key=True, 41 | serialize=False, 42 | verbose_name="ID", 43 | ), 44 | ), 45 | ("title", models.CharField(max_length=256)), 46 | ("text", models.CharField(max_length=256)), 47 | ("created", models.DateTimeField(auto_now_add=True)), 48 | ("modified", models.DateTimeField(auto_now=True)), 49 | ], 50 | ), 51 | migrations.CreateModel( 52 | name="Watch", 53 | fields=[ 54 | ( 55 | "id", 56 | models.BigAutoField( 57 | auto_created=True, 58 | primary_key=True, 59 | serialize=False, 60 | verbose_name="ID", 61 | ), 62 | ), 63 | ("slug", models.SlugField()), 64 | ("path", models.CharField(max_length=4096)), 65 | ], 66 | ), 67 | migrations.CreateModel( 68 | name="UserWatch", 69 | fields=[ 70 | ( 71 | "id", 72 | models.BigAutoField( 73 | auto_created=True, 74 | primary_key=True, 75 | serialize=False, 76 | verbose_name="ID", 77 | ), 78 | ), 79 | ( 80 | "user", 81 | models.ForeignKey( 82 | on_delete=django.db.models.deletion.CASCADE, 83 | to=settings.AUTH_USER_MODEL, 84 | ), 85 | ), 86 | ( 87 | "watch", 88 | models.ForeignKey( 89 | on_delete=django.db.models.deletion.CASCADE, 90 | to="notifications.watch", 91 | ), 92 | ), 93 | ], 94 | options={"db_table": "notifications_watch_users"}, 95 | ), 96 | migrations.CreateModel( 97 | name="Notification", 98 | fields=[ 99 | ( 100 | "id", 101 | models.BigAutoField( 102 | auto_created=True, 103 | primary_key=True, 104 | serialize=False, 105 | verbose_name="ID", 106 | ), 107 | ), 108 | ("read", models.BooleanField(default=False)), 109 | ( 110 | "notification", 111 | models.ForeignKey( 112 | on_delete=django.db.models.deletion.CASCADE, 113 | to="notifications.notificationdata", 114 | ), 115 | ), 116 | ( 117 | "user", 118 | models.ForeignKey( 119 | on_delete=django.db.models.deletion.CASCADE, 120 | to=settings.AUTH_USER_MODEL, 121 | ), 122 | ), 123 | ], 124 | ), 125 | ] 126 | -------------------------------------------------------------------------------- /kuma/api/v1/tests/test_search.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | import pytest 4 | from elasticmock import FakeElasticsearch 5 | 6 | from kuma.core.urlresolvers import reverse 7 | 8 | 9 | @pytest.mark.django_db 10 | def test_search_validation_problems(user_client, settings): 11 | url = reverse("api.v1.search") 12 | 13 | # locale invalid 14 | response = user_client.get(url, {"q": "x", "locale": "xxx"}) 15 | assert response.status_code == 400 16 | assert response.json()["errors"]["locale"][0]["code"] == "invalid_choice" 17 | # all non-200 OK responses should NOT have a Cache-Control header set. 18 | assert "cache-control" not in response 19 | 20 | # 'q' exceeds max allowed characters 21 | response = user_client.get(url, {"q": "x" * (settings.ES_Q_MAXLENGTH + 1)}) 22 | assert response.status_code == 400 23 | assert response.json()["errors"]["q"][0]["code"] == "max_length" 24 | 25 | # 'q' is empty or missing 26 | response = user_client.get(url, {"q": ""}) 27 | assert response.status_code == 400 28 | assert response.json()["errors"]["q"][0]["code"] == "required" 29 | response = user_client.get(url) 30 | assert response.status_code == 400 31 | assert response.json()["errors"]["q"][0]["code"] == "required" 32 | 33 | # 'page' is not a valid number 34 | response = user_client.get(url, {"q": "x", "page": "x"}) 35 | assert response.status_code == 400 36 | assert response.json()["errors"]["page"][0]["code"] == "invalid" 37 | response = user_client.get(url, {"q": "x", "page": "-1"}) 38 | assert response.status_code == 400 39 | assert response.json()["errors"]["page"][0]["code"] == "min_value" 40 | 41 | # 'sort' not a valid value 42 | response = user_client.get(url, {"q": "x", "sort": "neverheardof"}) 43 | assert response.status_code == 400 44 | assert response.json()["errors"]["sort"][0]["code"] == "invalid_choice" 45 | 46 | # 'slug_prefix' has to be anything but empty 47 | response = user_client.get(url, {"q": "x", "slug_prefix": ""}) 48 | assert response.status_code == 400 49 | assert response.json()["errors"]["slug_prefix"][0]["code"] == "invalid_choice" 50 | 51 | 52 | class FindEverythingFakeElasticsearch(FakeElasticsearch): 53 | def search(self, *args, **kwargs): 54 | # This trick is what makes the mock so basic. It basically removes 55 | # any search query so that it just returns EVERYTHING that's been indexed. 56 | kwargs.pop("body", None) 57 | result = super().search(*args, **kwargs) 58 | # Due to a bug in ElasticMock, instead of returning an object for the 59 | # `response.hits.total`, it returns just an integer. We'll need to fix that. 60 | if isinstance(result["hits"]["total"], int): 61 | result["hits"]["total"] = { 62 | "value": result["hits"]["total"], 63 | "relation": "eq", 64 | } 65 | return result 66 | 67 | 68 | @pytest.fixture 69 | def mock_elasticsearch(): 70 | fake_elasticsearch = FindEverythingFakeElasticsearch() 71 | with patch("elasticsearch_dsl.search.get_connection") as get_connection: 72 | get_connection.return_value = fake_elasticsearch 73 | yield fake_elasticsearch 74 | 75 | 76 | def test_search_basic_match(user_client, settings, mock_elasticsearch): 77 | mock_elasticsearch.index( 78 | settings.SEARCH_INDEX_NAME, 79 | { 80 | "id": "/en-us/docs/Foo", 81 | "title": "Foo Title", 82 | "summary": "Foo summary", 83 | "locale": "en-us", 84 | "slug": "Foo", 85 | "popularity": 0, 86 | }, 87 | id="/en-us/docs/Foo", 88 | ) 89 | url = reverse("api.v1.search") 90 | response = user_client.get(url, {"q": "foo bar"}) 91 | assert response.status_code == 200 92 | assert "public" in response["Cache-Control"] 93 | assert "max-age=" in response["Cache-Control"] 94 | assert "max-age=0" not in response["Cache-Control"] 95 | 96 | assert response["content-type"] == "application/json" 97 | assert response["Access-Control-Allow-Origin"] == "*" 98 | data = response.json() 99 | assert data["metadata"]["page"] == 1 100 | assert data["metadata"]["size"] 101 | assert data["metadata"]["took_ms"] 102 | assert data["metadata"]["total"]["value"] == 1 103 | assert data["metadata"]["total"]["relation"] == "eq" 104 | assert data["suggestions"] == [] 105 | assert data["documents"] == [ 106 | { 107 | "highlight": {"body": [], "title": []}, 108 | "locale": "en-us", 109 | "mdn_url": "/en-us/docs/Foo", 110 | "popularity": 0, 111 | "score": 1.0, 112 | "slug": "Foo", 113 | "title": "Foo Title", 114 | "summary": "Foo summary", 115 | } 116 | ] 117 | -------------------------------------------------------------------------------- /kuma/attachments/tests/test_views.py: -------------------------------------------------------------------------------- 1 | from kuma.core.urlresolvers import reverse 2 | 3 | from ..utils import full_attachment_url, full_mindtouch_attachment_url 4 | 5 | 6 | def test_mindtouch_redirect(client, settings, sample_mindtouch_attachment_redirect): 7 | 8 | settings.DOMAIN = "testserver" 9 | settings.ATTACHMENT_HOST = "demos" 10 | settings.ALLOWED_HOSTS.append("demos") 11 | 12 | filename = sample_mindtouch_attachment_redirect["url"].split("/")[-1] 13 | id = sample_mindtouch_attachment_redirect["id"] 14 | mindtouch_url = reverse( 15 | "attachments.mindtouch_file_redirect", 16 | args=(), 17 | kwargs={ 18 | "file_id": id, 19 | "filename": filename, 20 | }, 21 | ) 22 | response = client.get(mindtouch_url, HTTP_HOST=settings.ATTACHMENT_HOST) 23 | assert response.status_code == 302 24 | # Figure out the external scheme + host for our attachments bucket 25 | endpoint_url = settings.ATTACHMENTS_AWS_S3_ENDPOINT_URL 26 | custom_proto = "https" if settings.ATTACHMENTS_AWS_S3_SECURE_URLS else "http" 27 | custom_url = f"{custom_proto}://{settings.ATTACHMENTS_AWS_S3_CUSTOM_DOMAIN}" 28 | bucket_url = ( 29 | custom_url if settings.ATTACHMENTS_AWS_S3_CUSTOM_DOMAIN else endpoint_url 30 | ) 31 | 32 | # Verify we're redirecting to the intended bucket or custom frontend 33 | assert response["location"].startswith(bucket_url) 34 | 35 | assert response["x-frame-options"] == f"ALLOW-FROM {settings.DOMAIN}" 36 | assert response["Last-Modified"] 37 | 38 | 39 | def test_mindtouch_redirect_requires_attachment_host( 40 | client, settings, sample_mindtouch_attachment_redirect 41 | ): 42 | 43 | settings.DOMAIN = "testserver" 44 | settings.ATTACHMENT_HOST = "demos" 45 | settings.ALLOWED_HOSTS.append("demos") 46 | 47 | filename = sample_mindtouch_attachment_redirect["url"].split("/")[-1] 48 | id = sample_mindtouch_attachment_redirect["id"] 49 | mindtouch_url = reverse( 50 | "attachments.mindtouch_file_redirect", 51 | args=(), 52 | kwargs={ 53 | "file_id": id, 54 | "filename": filename, 55 | }, 56 | ) 57 | # Note! Not using the correct `HTTP_HOST=settings.ATTACHMENT_HOST` 58 | response = client.get(mindtouch_url) 59 | assert response.status_code == 301 60 | url = full_mindtouch_attachment_url(id, filename) 61 | assert response["Location"] == url 62 | 63 | 64 | def test_mindtouch_not_found(client, settings): 65 | 66 | settings.DOMAIN = "testserver" 67 | settings.ATTACHMENT_HOST = "demos" 68 | settings.ALLOWED_HOSTS.append("demos") 69 | 70 | mindtouch_url = reverse( 71 | "attachments.mindtouch_file_redirect", 72 | args=(), 73 | kwargs={ 74 | "file_id": 12345678, 75 | "filename": "anything.png", 76 | }, 77 | ) 78 | response = client.get(mindtouch_url) 79 | assert response.status_code == 404 80 | 81 | 82 | def test_raw_file_requires_attachment_host( 83 | client, settings, sample_attachment_redirect 84 | ): 85 | settings.DOMAIN = "testserver" 86 | settings.ATTACHMENT_HOST = "demos" 87 | settings.ALLOWED_HOSTS.append("demos") 88 | 89 | filename = sample_attachment_redirect["url"].split("/")[-1] 90 | url = full_attachment_url(sample_attachment_redirect["id"], filename) 91 | # Force the HOST header to look like something other than "demos". 92 | response = client.get(url, HTTP_HOST="testserver") 93 | assert response.status_code == 301 94 | assert response["Location"] == url 95 | 96 | response = client.get(url, HTTP_HOST=settings.ATTACHMENT_HOST) 97 | # Figure out the external scheme + host for our attachments bucket 98 | endpoint_url = settings.ATTACHMENTS_AWS_S3_ENDPOINT_URL 99 | custom_proto = "https" if settings.ATTACHMENTS_AWS_S3_SECURE_URLS else "http" 100 | custom_url = f"{custom_proto}://{settings.ATTACHMENTS_AWS_S3_CUSTOM_DOMAIN}" 101 | bucket_url = ( 102 | custom_url if settings.ATTACHMENTS_AWS_S3_CUSTOM_DOMAIN else endpoint_url 103 | ) 104 | 105 | # Verify we're redirecting to the intended bucket or custom frontend 106 | assert response.status_code == 302 107 | assert response["location"].startswith(bucket_url) 108 | 109 | assert response["x-frame-options"] == f"ALLOW-FROM {settings.DOMAIN}" 110 | assert response["Last-Modified"] 111 | 112 | 113 | def test_raw_file_not_found(client, settings): 114 | settings.DOMAIN = "testserver" 115 | settings.ATTACHMENT_HOST = "demos" 116 | settings.ALLOWED_HOSTS.append("demos") 117 | 118 | # Any ID we definitely know is not in the 'redirects.json' file 119 | url = full_attachment_url(12345678, "foo.png") 120 | response = client.get(url, HTTP_HOST=settings.ATTACHMENT_HOST) 121 | assert response.status_code == 404 122 | --------------------------------------------------------------------------------