├── .github ├── CODEOWNERS ├── dependabot.yml ├── workflows │ ├── commitlint.yml │ ├── self-assign-issue.yml │ ├── add-depr-ticket-to-depr-board.yml │ ├── pypi-publish.yml │ ├── add-remove-label-on-comment.yml │ ├── upgrade-python-requirements.yml │ └── ci.yml ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── edx_django_utils ├── db │ ├── tests │ │ ├── __init__.py │ │ ├── test_queryset_utils.py │ │ └── test_read_replica.py │ ├── __init__.py │ ├── queryset_utils.py │ └── read_replica.py ├── admin │ ├── tests │ │ ├── __init__.py │ │ ├── models.py │ │ └── test_mixins.py │ └── mixins.py ├── cache │ ├── tests │ │ ├── __init__.py │ │ └── test_middleware.py │ ├── __init__.py │ ├── middleware.py │ └── README.rst ├── ip │ ├── internal │ │ ├── __init__.py │ │ └── tests │ │ │ └── __init__.py │ └── README.rst ├── logging │ ├── tests │ │ ├── __init__.py │ │ ├── test_logging.py │ │ └── test_log_sensitive.py │ ├── internal │ │ ├── __init__.py │ │ └── filters.py │ ├── __init__.py │ └── README.rst ├── security │ ├── csp │ │ ├── __init__.py │ │ ├── tests │ │ │ └── test_middleware.py │ │ └── middleware.py │ └── README.rst ├── user │ ├── tests │ │ ├── __init__.py │ │ └── test_user.py │ ├── management │ │ ├── __init__.py │ │ ├── commands │ │ │ ├── __init__.py │ │ │ └── manage_group.py │ │ └── tests │ │ │ └── __init__.py │ ├── README.rst │ └── __init__.py ├── monitoring │ ├── internal │ │ ├── __init__.py │ │ ├── code_owner │ │ │ ├── __init__.py │ │ │ └── middleware.py │ │ ├── transactions.py │ │ └── utils.py │ ├── tests │ │ ├── __init__.py │ │ ├── code_owner │ │ │ ├── __init__.py │ │ │ ├── mock_views.py │ │ │ └── test_utils.py │ │ ├── test_utils.py │ │ └── test_custom_monitoring.py │ ├── docs │ │ ├── how_tos │ │ │ ├── search_new_relic.rst │ │ │ ├── update_monitoring_for_squad_or_theme_changes.rst │ │ │ └── add_code_owner_custom_attribute_to_an_ida.rst │ │ └── decisions │ │ │ ├── 0001-monitoring-by-code-owner.rst │ │ │ ├── 0004-code-owner-theme-and-squad.rst │ │ │ ├── 0002-custom-monitoring-language.rst │ │ │ └── 0003-code-owner-for-celery-tasks.rst │ ├── __init__.py │ ├── middleware.py │ ├── README.rst │ └── utils.py ├── __init__.py ├── wsgi.py ├── apps.py ├── plugins │ ├── registry.py │ ├── __init__.py │ ├── plugin_apps.py │ ├── utils.py │ ├── plugin_manager.py │ ├── constants.py │ ├── docs │ │ ├── how_tos │ │ │ └── how_to_enable_plugins_for_an_ida.rst │ │ └── decisions │ │ │ └── 0001-plugin-contexts.rst │ ├── plugin_settings.py │ ├── plugin_urls.py │ ├── plugin_signals.py │ ├── README.rst │ ├── pluggable_override.py │ └── plugin_contexts.py ├── urls.py └── tests │ └── test_pluggable_override.py ├── docs ├── readme.rst ├── changelog.rst ├── plugins │ ├── readme.rst │ ├── decisions │ │ └── 0001-plugin-contexts.rst │ └── how_tos │ │ ├── how_to_create_a_plugin_app.rst │ │ └── how_to_enable_plugins_for_an_ida.rst ├── monitoring │ ├── how_tos │ │ ├── search_new_relic.rst │ │ ├── using_custom_attributes.rst │ │ ├── add_code_owner_custom_attribute_to_an_ida.rst │ │ └── update_monitoring_for_squad_or_theme_changes.rst │ ├── decisions │ │ ├── 0001-monitoring-by-code-owner.rst │ │ ├── 0002-custom-monitoring-language.rst │ │ ├── 0003-code-owner-for-celery-tasks.rst │ │ └── 0004-code-owner-theme-and-squad.rst │ └── README.rst ├── _static │ └── theme_overrides.css ├── getting_started.rst ├── decisions │ ├── 0001-purpose-of-this-repo.rst │ ├── 0002-extract-plugins-infrastructure-from-edx-platform.rst │ ├── 0003-logging-filters-for-user-and-ip.rst │ ├── 0005-user-and-group-management-commands.rst │ ├── 0004-public-api-and-app-organization.rst │ └── 0006-content-security-policy-middleware.rst ├── testing.rst └── index.rst ├── setup.cfg ├── tests └── test_models.py ├── requirements ├── pip.in ├── pip-tools.in ├── ci.in ├── pip.txt ├── quality.in ├── doc.in ├── test.in ├── dev.in ├── base.in ├── private.readme ├── pip-tools.txt ├── ci.txt ├── constraints.txt ├── base.txt ├── test.txt ├── quality.txt └── doc.txt ├── .coveragerc ├── openedx.yaml ├── pylintrc_tweaks ├── codecov.yml ├── MANIFEST.in ├── test_utils └── __init__.py ├── .readthedocs.yaml ├── .tx └── config ├── catalog-info.yaml ├── manage.py ├── .gitignore ├── tox.ini ├── test_settings.py ├── Makefile └── setup.py /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @edx/arch-bom 2 | -------------------------------------------------------------------------------- /edx_django_utils/db/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edx_django_utils/admin/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edx_django_utils/cache/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edx_django_utils/ip/internal/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edx_django_utils/logging/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edx_django_utils/security/csp/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edx_django_utils/user/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /edx_django_utils/ip/internal/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edx_django_utils/logging/internal/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/internal/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edx_django_utils/user/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGELOG.rst 2 | -------------------------------------------------------------------------------- /edx_django_utils/user/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/tests/code_owner/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/plugins/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../edx_django_utils/plugins/README.rst 2 | -------------------------------------------------------------------------------- /edx_django_utils/user/management/tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Tests for Management Commands.""" 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [isort] 2 | indent=' ' 3 | line_length=120 4 | multi_line_output=3 5 | 6 | [wheel] 7 | universal = 1 8 | -------------------------------------------------------------------------------- /tests/test_models.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Tests for the `edx-django-utils` models module. 4 | """ 5 | -------------------------------------------------------------------------------- /docs/monitoring/how_tos/search_new_relic.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../edx_django_utils/monitoring/docs/how_tos/search_new_relic.rst 2 | -------------------------------------------------------------------------------- /docs/plugins/decisions/0001-plugin-contexts.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../edx_django_utils/plugins/docs/decisions/0001-plugin-contexts.rst 2 | -------------------------------------------------------------------------------- /requirements/pip.in: -------------------------------------------------------------------------------- 1 | # Core dependencies for installing other packages 2 | 3 | -c constraints.txt 4 | 5 | pip 6 | setuptools 7 | wheel 8 | -------------------------------------------------------------------------------- /docs/monitoring/how_tos/using_custom_attributes.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../edx_django_utils/monitoring/docs/how_tos/using_custom_attributes.rst 2 | -------------------------------------------------------------------------------- /docs/plugins/how_tos/how_to_create_a_plugin_app.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app.rst 2 | -------------------------------------------------------------------------------- /docs/plugins/how_tos/how_to_enable_plugins_for_an_ida.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../edx_django_utils/plugins/docs/how_tos/how_to_enable_plugins_for_an_ida.rst 2 | -------------------------------------------------------------------------------- /docs/monitoring/decisions/0001-monitoring-by-code-owner.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../edx_django_utils/monitoring/docs/decisions/0001-monitoring-by-code-owner.rst 2 | -------------------------------------------------------------------------------- /docs/monitoring/decisions/0002-custom-monitoring-language.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../edx_django_utils/monitoring/docs/decisions/0002-custom-monitoring-language.rst 2 | -------------------------------------------------------------------------------- /docs/monitoring/decisions/0003-code-owner-for-celery-tasks.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../edx_django_utils/monitoring/docs/decisions/0003-code-owner-for-celery-tasks.rst 2 | -------------------------------------------------------------------------------- /docs/monitoring/decisions/0004-code-owner-theme-and-squad.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../edx_django_utils/monitoring/docs/decisions/0004-code-owner-theme-and-squad.rst 2 | -------------------------------------------------------------------------------- /docs/monitoring/how_tos/add_code_owner_custom_attribute_to_an_ida.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../edx_django_utils/monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst 2 | -------------------------------------------------------------------------------- /docs/monitoring/how_tos/update_monitoring_for_squad_or_theme_changes.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../edx_django_utils/monitoring/docs/how_tos/update_monitoring_for_squad_or_theme_changes.rst 2 | -------------------------------------------------------------------------------- /edx_django_utils/cache/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Cache utilities public api 3 | 4 | See README.rst for details. 5 | """ 6 | 7 | from .utils import DEFAULT_REQUEST_CACHE, RequestCache, TieredCache, get_cache_key 8 | -------------------------------------------------------------------------------- /edx_django_utils/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | EdX utilities for Django Application development.. 3 | """ 4 | 5 | __version__ = "5.7.0" 6 | 7 | default_app_config = ( 8 | "edx_django_utils.apps.EdxDjangoUtilsConfig" 9 | ) 10 | -------------------------------------------------------------------------------- /requirements/pip-tools.in: -------------------------------------------------------------------------------- 1 | # Just the dependencies to run pip-tools, mainly for the "upgrade" make target 2 | 3 | -c constraints.txt 4 | pip-tools # Contains pip-compile, used to generate pip requirements files 5 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/internal/code_owner/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This directory should only be used internally. 3 | 4 | Its public API is exposed in the top-level monitoring __init__.py. 5 | See its README.rst for details. 6 | """ 7 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | data_file = .coverage 4 | source=edx_django_utils 5 | omit = 6 | test_settings 7 | */migrations/* 8 | */admin.py 9 | */static/* 10 | */templates/* 11 | */plugins/* 12 | -------------------------------------------------------------------------------- /edx_django_utils/logging/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Logging utilities public api 3 | 4 | See README.rst for details. 5 | """ 6 | from .internal.filters import RemoteIpFilter, UserIdFilter 7 | from .internal.log_sensitive import encrypt_for_log 8 | -------------------------------------------------------------------------------- /edx_django_utils/wsgi.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from django.core.wsgi import get_wsgi_application 5 | 6 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'edx_django_utils.settings') 7 | 8 | application = get_wsgi_application() -------------------------------------------------------------------------------- /openedx.yaml: -------------------------------------------------------------------------------- 1 | # openedx.yaml 2 | 3 | --- 4 | owner: edx/arch-review 5 | nick: edx-django-utils 6 | tags: 7 | - tools 8 | oeps: 9 | oep-2: true 10 | oep-7: true 11 | oep-18: true 12 | 13 | tags: 14 | - library 15 | -------------------------------------------------------------------------------- /requirements/ci.in: -------------------------------------------------------------------------------- 1 | # Requirements for running tests in Travis 2 | 3 | -c constraints.txt 4 | 5 | tox # Virtualenv management for tests 6 | tox-battery # Makes tox aware of requirements file changes 7 | -------------------------------------------------------------------------------- /pylintrc_tweaks: -------------------------------------------------------------------------------- 1 | # pylintrc tweaks for use with edx_lint. 2 | [MASTER] 3 | ignore = migrations 4 | load-plugins = edx_lint.pylint,pylint_django,pylint_celery 5 | 6 | [MESSAGES CONTROL] 7 | disable+= 8 | consider-using-with, 9 | consider-using-f-string -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Adding new check for github-actions 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "weekly" 8 | reviewers: 9 | - "openedx/arbi-bom" 10 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | enabled: yes 6 | threshold: 0.1 7 | target: auto 8 | patch: 9 | default: 10 | enabled: yes 11 | target: 74% 12 | 13 | comment: false 14 | -------------------------------------------------------------------------------- /.github/workflows/commitlint.yml: -------------------------------------------------------------------------------- 1 | # Run commitlint on the commit messages in a pull request. 2 | 3 | name: Lint Commit Messages 4 | 5 | on: 6 | - pull_request 7 | 8 | jobs: 9 | commitlint: 10 | uses: openedx/.github/.github/workflows/commitlint.yml@master 11 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS 2 | include CHANGELOG.rst 3 | include CONTRIBUTING.rst 4 | include LICENSE.txt 5 | include README.rst 6 | include requirements/base.in 7 | recursive-include edx_django_utils *.html *.png *.gif *js *.css *jpg *jpeg *svg *py 8 | include requirements/constraints.txt 9 | -------------------------------------------------------------------------------- /edx_django_utils/admin/tests/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class GenericModel(models.Model): 5 | name = models.CharField(max_length=100) 6 | description = models.TextField() 7 | created = models.DateField() 8 | 9 | class Meta: 10 | ordering = ('name',) 11 | -------------------------------------------------------------------------------- /edx_django_utils/apps.py: -------------------------------------------------------------------------------- 1 | """ 2 | edx_django_utils Django application initialization. 3 | """ 4 | 5 | from django.apps import AppConfig 6 | 7 | 8 | class EdxDjangoUtilsConfig(AppConfig): 9 | """ 10 | Configuration for the edx_django_utils Django application. 11 | """ 12 | 13 | name = 'edx_django_utils' 14 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/tests/code_owner/mock_views.py: -------------------------------------------------------------------------------- 1 | """ 2 | Mock views with a different module to enable testing of mapping 3 | code_owner to modules. Trying to mock __module__ on a view was 4 | getting too complex. 5 | """ 6 | from django.views.generic import View 7 | 8 | 9 | class MockViewTest(View): 10 | """ 11 | Mock view for use in testing. 12 | """ 13 | -------------------------------------------------------------------------------- /test_utils/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test utilities. 3 | 4 | Since py.test discourages putting __init__.py into test directory (i.e. making tests a package) 5 | one cannot import from anywhere under tests folder. However, some utility classes/methods might be useful 6 | in multiple test modules (i.e. factoryboy factories, base test classes). So this package is the place to put them. 7 | """ 8 | -------------------------------------------------------------------------------- /docs/_static/theme_overrides.css: -------------------------------------------------------------------------------- 1 | /* override table width restrictions */ 2 | .wy-table-responsive table td, .wy-table-responsive table th { 3 | /* !important prevents the common CSS stylesheets from 4 | overriding this as on RTD they are loaded after this stylesheet */ 5 | white-space: normal !important; 6 | } 7 | 8 | .wy-table-responsive { 9 | overflow: visible !important; 10 | } 11 | -------------------------------------------------------------------------------- /edx_django_utils/user/README.rst: -------------------------------------------------------------------------------- 1 | Django User and Group Utilities 2 | =============================== 3 | 4 | User and group code useful to multiple django services will live here. 5 | 6 | To use the user and group management commands, add ``edx_django_utils.user`` to ``INSTALLED_APPS`` of the django service. 7 | 8 | .. _User and Group Management Commands: /docs/decisions/0005-user-and-group-management-commands.rst 9 | -------------------------------------------------------------------------------- /requirements/pip.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.8 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | wheel==0.41.2 8 | # via -r requirements/pip.in 9 | 10 | # The following packages are considered to be unsafe in a requirements file: 11 | pip==23.2.1 12 | # via -r requirements/pip.in 13 | setuptools==68.2.0 14 | # via -r requirements/pip.in 15 | -------------------------------------------------------------------------------- /requirements/quality.in: -------------------------------------------------------------------------------- 1 | # Requirements for code quality checks 2 | 3 | -c constraints.txt 4 | -r test.txt # Core and testing dependencies for this package 5 | 6 | edx-lint # edX pylint rules and plugins 7 | isort # to standardize order of imports 8 | pycodestyle # PEP 8 compliance validation 9 | pydocstyle # PEP 257 compliance validation 10 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | * edx-django-utils version: 2 | * Django version: 3 | * Python version: 4 | * Operating System: 5 | 6 | ### Description 7 | 8 | Describe what you were trying to get done. 9 | Tell us what happened, what went wrong, and what you expected to happen. 10 | 11 | ### What I Did 12 | 13 | ``` 14 | Paste the command(s) you ran and the output. 15 | If there was a crash, please include the traceback here. 16 | ``` 17 | -------------------------------------------------------------------------------- /requirements/doc.in: -------------------------------------------------------------------------------- 1 | # Requirements for documentation validation 2 | 3 | -c constraints.txt 4 | -r test.txt # Core and testing dependencies for this package 5 | 6 | doc8 # reStructuredText style checker 7 | sphinx-book-theme # Common theme for all Open edX projects 8 | readme_renderer # Validates README.rst for usage on PyPI 9 | Sphinx # Documentation builder 10 | twine -------------------------------------------------------------------------------- /.github/workflows/self-assign-issue.yml: -------------------------------------------------------------------------------- 1 | # This workflow runs when a comment is made on the ticket 2 | # If the comment starts with "assign me" it assigns the author to the 3 | # ticket (case insensitive) 4 | 5 | name: Assign comment author to ticket if they say "assign me" 6 | on: 7 | issue_comment: 8 | types: [created] 9 | 10 | jobs: 11 | self_assign_by_comment: 12 | uses: openedx/.github/.github/workflows/self-assign-issue.yml@master 13 | -------------------------------------------------------------------------------- /requirements/test.in: -------------------------------------------------------------------------------- 1 | # Requirements for test runs. 2 | 3 | -c constraints.txt 4 | -r base.txt # Core dependencies for this package 5 | 6 | ddt # Run a test case multiple times with different input 7 | mock # Backport of unittest.mock, available in Python 3.3 8 | pytest-cov # pytest extension for code coverage statistics 9 | pytest-django # pytest extension for better Django support 10 | -------------------------------------------------------------------------------- /edx_django_utils/db/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utilities for working effectively with databases in django. 3 | 4 | read_replica: Tools for making queries from the read-replica. 5 | queryset_utils: Utils to use with Django QuerySets. 6 | """ 7 | 8 | from .queryset_utils import chunked_queryset 9 | from .read_replica import ( 10 | ReadReplicaRouter, 11 | read_queries_only, 12 | read_replica_or_default, 13 | use_read_replica_if_available, 14 | write_queries 15 | ) 16 | -------------------------------------------------------------------------------- /docs/getting_started.rst: -------------------------------------------------------------------------------- 1 | Getting Started 2 | =============== 3 | 4 | If you have not already done so, create or activate a `virtualenv`_. Unless otherwise stated, assume all terminal code 5 | below is executed within the virtualenv. 6 | 7 | .. _virtualenv: https://virtualenvwrapper.readthedocs.org/en/latest/ 8 | 9 | 10 | Install dependencies 11 | -------------------- 12 | Dependencies can be installed via the command below. 13 | 14 | .. code-block:: bash 15 | 16 | $ make requirements 17 | -------------------------------------------------------------------------------- /edx_django_utils/security/README.rst: -------------------------------------------------------------------------------- 1 | Security Utils 2 | ############## 3 | 4 | Common security utilities. 5 | 6 | Content-Security-Policy middleware 7 | ********************************** 8 | 9 | Add the middleware ``'edx_django_utils.security.csp.middleware.content_security_policy_middleware'`` near the beginning of your ``MIDDLEWARE`` list in order to add ``Content-Security-Policy`` and ``Content-Security-Policy-Report-Only`` headers. See ``csp/middleware.py`` for configuration and details. 10 | -------------------------------------------------------------------------------- /edx_django_utils/plugins/registry.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code to create Registry of django app plugins 3 | 4 | Please remember to expose any new public methods in the `__init__.py` file. 5 | """ 6 | from .plugin_manager import PluginManager 7 | 8 | 9 | class DjangoAppRegistry(PluginManager): 10 | """ 11 | DjangoAppRegistry is a registry of django app plugins. 12 | """ 13 | 14 | 15 | def get_plugin_app_configs(project_type): 16 | return DjangoAppRegistry.get_available_plugins(project_type).values() 17 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/docs/how_tos/search_new_relic.rst: -------------------------------------------------------------------------------- 1 | Searching New Relic 2 | =================== 3 | 4 | The search script `new_relic_search.py`_ is generally useful for searching NRQL (New Relic Query Language) and text widgets in New Relic. It searches the NRQL in New Relic alert policies, and in NRQL and text widgets in New Relic dashboards. Use ``--help`` for more details. 5 | 6 | .. _new_relic_search.py: https://github.com/openedx/edx-django-utils/blob/master/edx_django_utils/monitoring/scripts/new_relic_search.py 7 | -------------------------------------------------------------------------------- /requirements/dev.in: -------------------------------------------------------------------------------- 1 | # Additional requirements for development of this application 2 | 3 | -c constraints.txt 4 | -r pip-tools.txt # pip-tools and its dependencies, for managing requirements files 5 | -r quality.txt # Core and quality check dependencies 6 | -r ci.txt # dependencies needed to setup testing environment in CI 7 | 8 | diff-cover # Changeset diff test coverage 9 | edx-i18n-tools # For i18n_tool dummy 10 | python-dateutil # Date utility for local script 11 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Build documentation in the docs/ directory with Sphinx 9 | sphinx: 10 | configuration: docs/conf.py 11 | # TODO: Fix autodoc warnings and set this to true. See https://github.com/openedx/edx-django-utils/issues/304 12 | fail_on_warning: false 13 | 14 | python: 15 | version: 3.8 16 | install: 17 | - requirements: requirements/doc.txt 18 | -------------------------------------------------------------------------------- /.tx/config: -------------------------------------------------------------------------------- 1 | [main] 2 | host = https://www.transifex.com 3 | 4 | [o:open-edx:p:edx-platform:r:edx-django-utils] 5 | file_filter = edx_django_utils/conf/locale//LC_MESSAGES/django.po 6 | source_file = edx_django_utils/conf/locale/en/LC_MESSAGES/django.po 7 | source_lang = en 8 | type = PO 9 | 10 | [o:open-edx:p:edx-platform:r:edx-django-utils-js] 11 | file_filter = edx_django_utils/conf/locale//LC_MESSAGES/djangojs.po 12 | source_file = edx_django_utils/conf/locale/en/LC_MESSAGES/djangojs.po 13 | source_lang = en 14 | type = PO 15 | 16 | -------------------------------------------------------------------------------- /edx_django_utils/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Plugins infrastructure 3 | 4 | See README.rst for details. 5 | """ 6 | 7 | from .constants import PluginContexts, PluginSettings, PluginSignals, PluginURLs 8 | from .pluggable_override import pluggable_override 9 | from .plugin_apps import get_plugin_apps 10 | from .plugin_contexts import get_plugins_view_context 11 | from .plugin_manager import PluginError, PluginManager 12 | from .plugin_settings import add_plugins 13 | from .plugin_signals import connect_plugin_receivers 14 | from .plugin_urls import get_plugin_url_patterns 15 | from .registry import get_plugin_app_configs 16 | -------------------------------------------------------------------------------- /edx_django_utils/ip/README.rst: -------------------------------------------------------------------------------- 1 | IP Address Utils 2 | ################ 3 | 4 | Utilities for safely and correctly determining the IP address(es) of a request. 5 | 6 | See ``edx_django_utils/ip/__init__.py`` for documentation and a list of everything included in the public API. 7 | 8 | Usage 9 | ***** 10 | 11 | The short version: 12 | 13 | 1. Configure ``CLOSEST_CLIENT_IP_FROM_HEADERS`` 14 | 2. Make sure ``init_client_ips`` is called as early as possible in your middleware stack 15 | 3. Call ``get_safest_client_ip`` whenever you want to know the caller's IP address 16 | 17 | For details, see ``__init__.py`` module docstring. 18 | -------------------------------------------------------------------------------- /catalog-info.yaml: -------------------------------------------------------------------------------- 1 | # This file records information about this repo. Its use is described in OEP-55: 2 | # https://open-edx-proposals.readthedocs.io/en/latest/processes/oep-0055-proc-project-maintainers.html 3 | 4 | apiVersion: backstage.io/v1alpha1 5 | kind: Component 6 | metadata: 7 | name: 'edx-django-utils' 8 | description: "edX utilities for Django Application development" 9 | links: 10 | - url: "https://edx-django-utils.readthedocs.org" 11 | title: "Full Documentation" 12 | icon: "Web" 13 | annotations: 14 | openedx.org/arch-interest-groups: "" 15 | spec: 16 | owner: group:arch-bom 17 | type: 'library' 18 | lifecycle: 'production' 19 | -------------------------------------------------------------------------------- /edx_django_utils/user/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | User and group utilities useful to multiple django services will live here. 3 | """ 4 | 5 | import random 6 | import string 7 | 8 | 9 | def generate_password(length=12, chars=string.ascii_letters + string.digits): 10 | """Generate a valid random password""" 11 | if length < 8: 12 | raise ValueError("password must be at least 8 characters") 13 | 14 | choice = random.SystemRandom().choice 15 | 16 | password = '' 17 | password += choice(string.digits) 18 | password += choice(string.ascii_letters) 19 | password += ''.join([choice(chars) for _i in range(length - 2)]) 20 | return password 21 | -------------------------------------------------------------------------------- /requirements/base.in: -------------------------------------------------------------------------------- 1 | # Core requirements for using this application 2 | 3 | -c constraints.txt 4 | 5 | Django # Web application framework 6 | django-waffle # Allows for feature toggles in Django. 7 | newrelic # New Relic agent for performance monitoring 8 | psutil # Library for retrieving information on running processes and system utilization 9 | stevedore # Used by plugins to make importing easier 10 | django-crum # Used by logging filters 11 | PyNaCl # User-friendly cryptography (wrapper and bindings for libsodium) 12 | click # CLI tools 13 | -------------------------------------------------------------------------------- /edx_django_utils/logging/README.rst: -------------------------------------------------------------------------------- 1 | Logging Utils 2 | ============= 3 | 4 | Several utilities for assisting with logging. 5 | 6 | See ``__init__.py`` for a list of everything included in the public API. 7 | 8 | Logging filters 9 | --------------- 10 | 11 | - ``RemoteIpFilter``: A logging filter that adds the remote IP to the logging context 12 | - ``UserIdFilter``: A logging filter that adds userid to the logging context 13 | 14 | Logging of sensitive information 15 | -------------------------------- 16 | 17 | ``encrypt_for_log`` allows encrypting a string in a way appropriate for logging. See module docstring for more information. 18 | 19 | This package also exposes a CLI command ``log-sensitive`` for key generation and decryption. 20 | -------------------------------------------------------------------------------- /.github/workflows/add-depr-ticket-to-depr-board.yml: -------------------------------------------------------------------------------- 1 | # Run the workflow that adds new tickets that are either: 2 | # - labelled "DEPR" 3 | # - title starts with "[DEPR]" 4 | # - body starts with "Proposal Date" (this is the first template field) 5 | # to the org-wide DEPR project board 6 | 7 | name: Add newly created DEPR issues to the DEPR project board 8 | 9 | on: 10 | issues: 11 | types: [opened] 12 | 13 | jobs: 14 | routeissue: 15 | uses: openedx/.github/.github/workflows/add-depr-ticket-to-depr-board.yml@master 16 | secrets: 17 | GITHUB_APP_ID: ${{ secrets.GRAPHQL_AUTH_APP_ID }} 18 | GITHUB_APP_PRIVATE_KEY: ${{ secrets.GRAPHQL_AUTH_APP_PEM }} 19 | SLACK_BOT_TOKEN: ${{ secrets.SLACK_ISSUE_BOT_TOKEN }} 20 | -------------------------------------------------------------------------------- /requirements/private.readme: -------------------------------------------------------------------------------- 1 | # If there are any Python packages you want to keep in your virtualenv beyond 2 | # those listed in the official requirements files, create a "private.in" file 3 | # and list them there. Generate the corresponding "private.txt" file pinning 4 | # all of their indirect dependencies to specific versions as follows: 5 | 6 | # pip-compile private.in 7 | 8 | # This allows you to use "pip-sync" without removing these packages: 9 | 10 | # pip-sync requirements/*.txt 11 | 12 | # "private.in" and "private.txt" aren't checked into git to avoid merge 13 | # conflicts, and the presence of this file allows "private.*" to be 14 | # included in scripted pip-sync usage without requiring that those files be 15 | # created first. 16 | -------------------------------------------------------------------------------- /requirements/pip-tools.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.8 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | build==1.0.3 8 | # via pip-tools 9 | click==8.1.7 10 | # via pip-tools 11 | importlib-metadata==6.8.0 12 | # via build 13 | packaging==23.1 14 | # via build 15 | pip-tools==7.3.0 16 | # via -r requirements/pip-tools.in 17 | pyproject-hooks==1.0.0 18 | # via build 19 | tomli==2.0.1 20 | # via 21 | # build 22 | # pip-tools 23 | # pyproject-hooks 24 | wheel==0.41.2 25 | # via pip-tools 26 | zipp==3.16.2 27 | # via importlib-metadata 28 | 29 | # The following packages are considered to be unsafe in a requirements file: 30 | # pip 31 | # setuptools 32 | -------------------------------------------------------------------------------- /.github/workflows/pypi-publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish package to PyPi 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | 10 | push: 11 | runs-on: ubuntu-20.04 12 | 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v1 16 | - name: setup python 17 | uses: actions/setup-python@v2 18 | with: 19 | python-version: 3.8 20 | 21 | - name: Install pip 22 | run: pip install -r requirements/pip.txt 23 | 24 | - name: Build package 25 | run: python setup.py sdist bdist_wheel 26 | 27 | - name: Publish to PyPi 28 | uses: pypa/gh-action-pypi-publish@master 29 | with: 30 | user: __token__ 31 | password: ${{ secrets.PYPI_UPLOAD_TOKEN }} 32 | -------------------------------------------------------------------------------- /.github/workflows/add-remove-label-on-comment.yml: -------------------------------------------------------------------------------- 1 | # This workflow runs when a comment is made on the ticket 2 | # If the comment starts with "label: " it tries to apply 3 | # the label indicated in rest of comment. 4 | # If the comment starts with "remove label: ", it tries 5 | # to remove the indicated label. 6 | # Note: Labels are allowed to have spaces and this script does 7 | # not parse spaces (as often a space is legitimate), so the command 8 | # "label: really long lots of words label" will apply the 9 | # label "really long lots of words label" 10 | 11 | name: Allows for the adding and removing of labels via comment 12 | 13 | on: 14 | issue_comment: 15 | types: [created] 16 | 17 | jobs: 18 | add_remove_labels: 19 | uses: openedx/.github/.github/workflows/add-remove-label-on-comment.yml@master 20 | 21 | -------------------------------------------------------------------------------- /requirements/ci.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.8 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | distlib==0.3.7 8 | # via virtualenv 9 | filelock==3.12.3 10 | # via 11 | # tox 12 | # virtualenv 13 | packaging==23.1 14 | # via tox 15 | platformdirs==3.10.0 16 | # via virtualenv 17 | pluggy==1.3.0 18 | # via tox 19 | py==1.11.0 20 | # via tox 21 | six==1.16.0 22 | # via tox 23 | tomli==2.0.1 24 | # via tox 25 | tox==3.28.0 26 | # via 27 | # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt 28 | # -r requirements/ci.in 29 | # tox-battery 30 | tox-battery==0.6.2 31 | # via -r requirements/ci.in 32 | typing-extensions==4.7.1 33 | # via filelock 34 | virtualenv==20.24.5 35 | # via tox 36 | -------------------------------------------------------------------------------- /docs/decisions/0001-purpose-of-this-repo.rst: -------------------------------------------------------------------------------- 1 | Purpose of this Repo 2 | ==================== 3 | 4 | Status 5 | ------ 6 | 7 | Accepted 8 | 9 | Context 10 | ------- 11 | 12 | When creating some caching utilities in Ecommerce, we wanted the ability to share the utilities across IDAs, without the overhead of creating a unique repository for every cross-cutting concern. 13 | 14 | 15 | Decision 16 | -------- 17 | 18 | This repository was created as a home for django utilities that might be useful to OpenEdx. 19 | 20 | .. note:: Some utilities may warrant their own repository. A judgement call needs to be made as to whether code properly belongs here or not. Please review with the Architecture Team if you have any questions. 21 | 22 | 23 | Consequences 24 | ------------ 25 | 26 | Certain tools that were created in edx-platform(or other IDAs) will be moved to this shared library. 27 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for utilities in monitoring. 3 | """ 4 | 5 | from unittest.mock import patch 6 | 7 | from edx_django_utils.monitoring import background_task 8 | 9 | 10 | @patch('edx_django_utils.monitoring.internal.utils.newrelic') 11 | def test_background_task_wrapper(wrapped_nr): 12 | # We are verifying that this returns the correct decorated function 13 | # in the two cases we care about. 14 | returned_func = background_task() 15 | 16 | assert returned_func == wrapped_nr.agent.background_task() 17 | 18 | 19 | @patch('edx_django_utils.monitoring.internal.utils.newrelic', None) 20 | def test_background_task_wrapper_no_new_relic(): 21 | # Test that the decorator behaves as a no-op when newrelic is not set. 22 | returned_func = background_task() 23 | wrapped_value = returned_func('a') 24 | 25 | assert wrapped_value == 'a' 26 | -------------------------------------------------------------------------------- /edx_django_utils/urls.py: -------------------------------------------------------------------------------- 1 | """django_forum URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/4.0/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.contrib import admin 17 | from django.urls import path 18 | # include 19 | 20 | urlpatterns = [ 21 | path('admin/', admin.site.urls), 22 | # path('', include('posts.urls')), 23 | ] -------------------------------------------------------------------------------- /edx_django_utils/plugins/plugin_apps.py: -------------------------------------------------------------------------------- 1 | """ 2 | Methods to get plugin apps 3 | 4 | Please remember to expose any new public methods in the `__init__.py` file. 5 | """ 6 | from logging import getLogger 7 | 8 | from . import constants, registry 9 | 10 | log = getLogger(__name__) 11 | 12 | 13 | def get_plugin_apps(project_type): 14 | """ 15 | Returns a list of all registered Plugin Apps, expected to be added to 16 | the INSTALLED_APPS list for the given project_type. 17 | """ 18 | plugin_apps = [ 19 | "{module_name}.{class_name}".format( 20 | module_name=app_config.__module__, class_name=app_config.__name__, 21 | ) 22 | for app_config in registry.get_plugin_app_configs(project_type) 23 | if getattr(app_config, constants.PLUGIN_APP_CLASS_ATTRIBUTE_NAME, None) 24 | is not None 25 | ] 26 | log.debug("Plugin Apps: Found %s", plugin_apps) 27 | return plugin_apps 28 | -------------------------------------------------------------------------------- /edx_django_utils/logging/internal/filters.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django-based logging filters 3 | """ 4 | 5 | from logging import Filter 6 | 7 | from crum import get_current_request, get_current_user 8 | 9 | 10 | class RemoteIpFilter(Filter): 11 | """ 12 | A logging filter that adds the remote IP to the logging context 13 | """ 14 | def filter(self, record): 15 | request = get_current_request() 16 | if request and 'REMOTE_ADDR' in request.META: 17 | record.remoteip = request.META['REMOTE_ADDR'] 18 | else: 19 | record.remoteip = None 20 | return True 21 | 22 | 23 | class UserIdFilter(Filter): 24 | """ 25 | A logging filter that adds userid to the logging context 26 | """ 27 | def filter(self, record): 28 | user = get_current_user() 29 | if user and user.pk: 30 | record.userid = user.pk 31 | else: 32 | record.userid = None 33 | return True 34 | -------------------------------------------------------------------------------- /docs/testing.rst: -------------------------------------------------------------------------------- 1 | .. _chapter-testing: 2 | 3 | Testing 4 | ======= 5 | 6 | edx-django-utils has an assortment of test cases and code quality 7 | checks to catch potential problems during development. To run them all in the 8 | version of Python you chose for your virtualenv: 9 | 10 | .. code-block:: bash 11 | 12 | $ make validate 13 | 14 | To run just the unit tests: 15 | 16 | .. code-block:: bash 17 | 18 | $ make test 19 | 20 | To run just the unit tests and check diff coverage 21 | 22 | .. code-block:: bash 23 | 24 | $ make diff_cover 25 | 26 | To run just the code quality checks: 27 | 28 | .. code-block:: bash 29 | 30 | $ make quality 31 | 32 | To run the unit tests under every supported Python version and the code 33 | quality checks: 34 | 35 | .. code-block:: bash 36 | 37 | $ make test-all 38 | 39 | To generate and open an HTML report of how much of the code is covered by 40 | test cases: 41 | 42 | .. code-block:: bash 43 | 44 | $ make coverage 45 | -------------------------------------------------------------------------------- /.github/workflows/upgrade-python-requirements.yml: -------------------------------------------------------------------------------- 1 | name: Upgrade Requirements 2 | 3 | on: 4 | schedule: 5 | - cron: "45 1 * * 1" 6 | workflow_dispatch: 7 | inputs: 8 | branch: 9 | description: 'Target branch to create requirements PR against' 10 | required: true 11 | default: 'master' 12 | jobs: 13 | call-upgrade-python-requirements-workflow: 14 | with: 15 | branch: ${{ github.event.inputs.branch || 'master' }} 16 | team_reviewers: "arbi-bom" 17 | email_address: arbi-bom@edx.org 18 | send_success_notification: false 19 | secrets: 20 | requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} 21 | requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }} 22 | edx_smtp_username: ${{ secrets.EDX_SMTP_USERNAME }} 23 | edx_smtp_password: ${{ secrets.EDX_SMTP_PASSWORD }} 24 | uses: openedx/.github/.github/workflows/upgrade-python-requirements.yml@master 25 | -------------------------------------------------------------------------------- /requirements/constraints.txt: -------------------------------------------------------------------------------- 1 | # Version constraints for pip-installation. 2 | # 3 | # This file doesn't install any packages. It specifies version constraints 4 | # that will be applied if a package is needed. 5 | # 6 | # When pinning something here, please provide an explanation of why. Ideally, 7 | # link to other information that will help people in the future to remove the 8 | # pin when possible. Writing an issue against the offending project and 9 | # linking to it here is good. 10 | 11 | # Common constraints for edx repos 12 | -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt 13 | 14 | # diff-cover latest requires (pluggy>=0.13.1,<0.14.0) 15 | # which conflicts with pytest(pluggy>=0.12,<2.0.0) and tox(pluggy>0.12) both of these fetch pluggy==1.0.0 16 | diff-cover<6.2.2 17 | 18 | 19 | # greater version failing docs build 20 | sphinx==4.2.0 21 | 22 | # version 1.0.0 requires docutils >0.19 but sphinx@4.2.0 needs docutils<0.18 23 | doc8<1.0.0 24 | -------------------------------------------------------------------------------- /docs/decisions/0002-extract-plugins-infrastructure-from-edx-platform.rst: -------------------------------------------------------------------------------- 1 | Extract plugins Infrastructure from edx-platform 2 | ================================================ 3 | 4 | Status 5 | ------ 6 | 7 | Accepted 8 | 9 | Context 10 | ------- 11 | 12 | We have experimented with adding plugins to edx-platform and are now ready to move this functionality to other IDAs. However, the django app plugins infrastructure is currently located in edx-platform and thus not easily usable by IDAs. 13 | 14 | This Django App Plugin functionality allows for Django-framework code to be 15 | encapsulated within each Django app, rather than having a monolith Project that 16 | is aware of the details of its Django apps. 17 | 18 | Decision 19 | -------- 20 | 21 | Move plugin infrastructure to edx-django-utils. 22 | 23 | .. note:: Currently, we've decided this plugin enablement code doesn't warrant its own repository. 24 | 25 | Consequences 26 | ------------ 27 | 28 | Fix all links to this code in edx-platform. 29 | -------------------------------------------------------------------------------- /edx_django_utils/user/tests/test_user.py: -------------------------------------------------------------------------------- 1 | """Test user functions""" 2 | 3 | import pytest 4 | from django.test import TestCase 5 | 6 | from edx_django_utils.user import generate_password 7 | 8 | 9 | class GeneratePasswordTest(TestCase): 10 | """Tests formation of randomly generated passwords.""" 11 | 12 | def test_default_args(self): 13 | password = generate_password() 14 | assert 12 == len(password) 15 | assert any(c.isdigit for c in password) 16 | assert any(c.isalpha for c in password) 17 | 18 | def test_length(self): 19 | length = 25 20 | assert length == len(generate_password(length=length)) 21 | 22 | def test_chars(self): 23 | char = '!' 24 | password = generate_password(length=12, chars=(char,)) 25 | 26 | assert any(c.isdigit for c in password) 27 | assert any(c.isalpha for c in password) 28 | assert (char * 10) == password[2:] 29 | 30 | def test_min_length(self): 31 | with pytest.raises(ValueError): 32 | generate_password(length=7) 33 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Metrics utilities public api 3 | 4 | See README.rst for details. 5 | """ 6 | from .internal.code_owner.middleware import CodeOwnerMonitoringMiddleware 7 | from .internal.code_owner.utils import ( 8 | get_code_owner_from_module, 9 | set_code_owner_attribute, 10 | set_code_owner_attribute_from_module 11 | ) 12 | from .internal.middleware import ( 13 | CachedCustomMonitoringMiddleware, 14 | CookieMonitoringMiddleware, 15 | DeploymentMonitoringMiddleware, 16 | MonitoringMemoryMiddleware 17 | ) 18 | from .internal.transactions import ( 19 | function_trace, 20 | get_current_transaction, 21 | ignore_transaction, 22 | set_monitoring_transaction_name 23 | ) 24 | from .internal.utils import ( 25 | accumulate, 26 | background_task, 27 | increment, 28 | record_exception, 29 | set_custom_attribute, 30 | set_custom_attributes_for_course_key 31 | ) 32 | # "set_custom_metric*" methods are deprecated 33 | from .utils import set_custom_metric, set_custom_metrics_for_course_key 34 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Django administration utility. 4 | """ 5 | 6 | import os 7 | import sys 8 | 9 | PWD = os.path.abspath(os.path.dirname(__file__)) 10 | 11 | if __name__ == '__main__': 12 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'test_settings') 13 | sys.path.append(PWD) 14 | try: 15 | from django.core.management import execute_from_command_line 16 | except ImportError: 17 | # The above import may fail for some other reason. Ensure that the 18 | # issue is really that Django is missing to avoid masking other 19 | # exceptions on Python 2. 20 | try: 21 | import django # pylint: disable=unused-import 22 | except ImportError as error: 23 | raise ImportError( 24 | "Couldn't import Django. Are you sure it's installed and " 25 | "available on your PYTHONPATH environment variable? Did you " 26 | "forget to activate a virtual environment?" 27 | ) from error 28 | raise 29 | execute_from_command_line(sys.argv) 30 | -------------------------------------------------------------------------------- /requirements/base.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.8 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | asgiref==3.7.2 8 | # via django 9 | cffi==1.15.1 10 | # via pynacl 11 | click==8.1.7 12 | # via -r requirements/base.in 13 | django==3.2.21 14 | # via 15 | # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt 16 | # -r requirements/base.in 17 | # django-crum 18 | # django-waffle 19 | django-crum==0.7.9 20 | # via -r requirements/base.in 21 | django-waffle==4.0.0 22 | # via -r requirements/base.in 23 | newrelic==9.0.0 24 | # via -r requirements/base.in 25 | pbr==5.11.1 26 | # via stevedore 27 | psutil==5.9.5 28 | # via -r requirements/base.in 29 | pycparser==2.21 30 | # via cffi 31 | pynacl==1.5.0 32 | # via -r requirements/base.in 33 | pytz==2023.3.post1 34 | # via django 35 | sqlparse==0.4.4 36 | # via django 37 | stevedore==5.1.0 38 | # via -r requirements/base.in 39 | typing-extensions==4.7.1 40 | # via asgiref 41 | -------------------------------------------------------------------------------- /docs/decisions/0003-logging-filters-for-user-and-ip.rst: -------------------------------------------------------------------------------- 1 | Logging filters for user and IP 2 | ================================================ 3 | 4 | Status 5 | ------ 6 | 7 | Accepted 8 | 9 | Context 10 | ------- 11 | 12 | As part of the Security Working Group's work on SEG-34, we recognized that the `logging filters`_ for 13 | LMS users and remote IP addresses were not reusable by other IDAs from inside LMS. 14 | 15 | .. _logging filters: https://github.com/openedx/edx-platform/blob/11e4cab6220c8c503787142f48a352410191de0a/openedx/core/djangoapps/util/log_utils.py#L16 16 | 17 | Decision 18 | -------- 19 | 20 | We decided to move the LMS users and remote IP addresses to this library, so these filters may be re-used by any edx component. Of particular use, we can update the logging settings in the IDA cookie-cutter repo to reference these filters in the standard logging context that is created from the repo for new IDAs. 21 | 22 | Consequences 23 | ------------ 24 | 25 | We will need to: 26 | * Update the IDA cookie-cutter once these are available. 27 | * Remove these classes from edx-platform. 28 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Python CI 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: 8 | - '**' 9 | 10 | 11 | jobs: 12 | run_tests: 13 | name: tests 14 | runs-on: ${{ matrix.os }} 15 | strategy: 16 | matrix: 17 | os: [ubuntu-20.04] 18 | python-version: ['3.8'] 19 | toxenv: [docs, quality, django32, django42] 20 | 21 | steps: 22 | - uses: actions/checkout@v1 23 | - name: setup python 24 | uses: actions/setup-python@v2 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | 28 | - name: Install pip 29 | run: pip install -r requirements/pip.txt 30 | 31 | - name: Install Dependencies 32 | run: pip install -r requirements/ci.txt 33 | 34 | - name: Run Tests 35 | env: 36 | TOXENV: ${{ matrix.toxenv }} 37 | run: tox 38 | 39 | - name: Run coverage 40 | if: matrix.python-version == '3.8' && matrix.toxenv == 'django42' 41 | uses: codecov/codecov-action@v3 42 | with: 43 | flags: unittests 44 | fail_ci_if_error: true 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | __pycache__ 3 | .pytest_cache 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Packages 9 | *.egg 10 | *.egg-info 11 | dist 12 | build 13 | eggs 14 | parts 15 | bin 16 | var 17 | sdist 18 | develop-eggs 19 | .installed.cfg 20 | lib 21 | lib64 22 | 23 | # Installer logs 24 | pip-log.txt 25 | 26 | # Unit test / coverage reports 27 | .cache/ 28 | .coverage 29 | .coverage.* 30 | .pytest_cache 31 | .tox 32 | coverage.xml 33 | htmlcov/ 34 | 35 | # Translations 36 | *.mo 37 | 38 | # IDEs and text editors 39 | *~ 40 | *.swp 41 | .idea/ 42 | .project 43 | .pycharm_helpers/ 44 | .pydevproject 45 | 46 | # The Silver Searcher 47 | .agignore 48 | 49 | # OS X artifacts 50 | *.DS_Store 51 | 52 | # Logging 53 | log/ 54 | logs/ 55 | chromedriver.log 56 | ghostdriver.log 57 | 58 | # Complexity 59 | output/*.html 60 | output/*/index.html 61 | 62 | # Sphinx 63 | docs/_build 64 | docs/modules.rst 65 | docs/edx_django_utils.rst 66 | docs/edx_django_utils.*.rst 67 | 68 | # Private requirements 69 | requirements/private.in 70 | requirements/private.txt 71 | 72 | # tox environment temporary artifacts 73 | tests/__init__.py 74 | 75 | # Development task artifacts 76 | default.db 77 | 78 | # Floobits pairing 79 | .floo 80 | .flooignore 81 | -------------------------------------------------------------------------------- /edx_django_utils/admin/mixins.py: -------------------------------------------------------------------------------- 1 | """ 2 | Mixins for use in Django Admin classes. 3 | """ 4 | 5 | 6 | class ReadOnlyAdminMixin: 7 | """ 8 | Disables all editing capabilities for the admin's model. 9 | An example usage: a Django proxy model which provides data to perform some other action. 10 | """ 11 | def __init__(self, *args, **kwargs): 12 | super().__init__(*args, **kwargs) 13 | self.list_display_links = None 14 | self.readonly_fields = [f.name for f in self.model._meta.get_fields()] 15 | 16 | def get_actions(self, request): 17 | actions = super().get_actions(request) 18 | if 'delete_selected' in actions: 19 | del actions["delete_selected"] # pragma: no cover 20 | return actions 21 | 22 | def has_add_permission(self, request): 23 | return False 24 | 25 | def has_delete_permission(self, request, obj=None): # pylint: disable=unused-argument 26 | return False 27 | 28 | def save_model(self, request, obj, form, change): 29 | pass # pragma: no cover 30 | 31 | def delete_model(self, request, obj): 32 | pass # pragma: no cover 33 | 34 | def save_related(self, request, form, formsets, change): 35 | pass # pragma: no cover 36 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | **Description:** 2 | 3 | Describe in a couple of sentence what this PR adds 4 | 5 | **JIRA:** 6 | 7 | [XXX-XXXX](https://openedx.atlassian.net/browse/XXX-XXXX) 8 | 9 | **Dependencies:** 10 | 11 | List dependencies on other outstanding PRs, issues, etc. 12 | 13 | **Merge deadline:** 14 | 15 | List merge deadline (if any) 16 | 17 | **Installation instructions:** 18 | 19 | List any non-trivial installation 20 | instructions. 21 | 22 | **Testing instructions:** 23 | 24 | 1. Open page A 25 | 2. Do thing B 26 | 3. Expect C to happen 27 | 4. If D happened instead - check failed. 28 | 29 | **Reviewers:** 30 | - [ ] tag reviewer 31 | 32 | **Merge checklist:** 33 | - [ ] All reviewers approved 34 | - [ ] CI build is green 35 | - [ ] Version bumped 36 | - [ ] Changelog record added 37 | - [ ] Documentation updated (not only docstrings) 38 | - [ ] Commits are squashed 39 | 40 | **Post merge:** 41 | - [ ] Create a tag 42 | - [ ] Check new version is pushed to PyPi after tag-triggered build is 43 | finished. 44 | - [ ] Delete working branch (if not needed anymore) 45 | 46 | **Author concerns:** 47 | 48 | List any concerns about this PR - inelegant 49 | solutions, hacks, quick-and-dirty implementations, concerns about 50 | migrations, etc. 51 | -------------------------------------------------------------------------------- /edx_django_utils/plugins/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | utils to help with imports 3 | 4 | Please remember to expose any new public methods in the `__init__.py` file. 5 | """ 6 | from importlib import import_module as system_import_module 7 | 8 | from django.utils.module_loading import import_string 9 | 10 | 11 | def import_module(module_path): 12 | """ 13 | Import and returns the module at the specific path. 14 | 15 | Args: 16 | module_path is the full path to the module, including the package name. 17 | """ 18 | return system_import_module(module_path) 19 | 20 | 21 | def get_module_path(app_config, plugin_config, plugin_cls): 22 | return "{package_path}.{module_path}".format( 23 | package_path=app_config.name, 24 | module_path=plugin_config.get( 25 | plugin_cls.RELATIVE_PATH, plugin_cls.DEFAULT_RELATIVE_PATH 26 | ), 27 | ) 28 | 29 | 30 | def import_attr(attr_path): 31 | """ 32 | Import and returns a module's attribute at the specific path. 33 | 34 | Args: 35 | attr_path should be of the form: 36 | {full_module_path}.attr_name 37 | """ 38 | return import_string(attr_path) 39 | 40 | 41 | def import_attr_in_module(imported_module, attr_name): 42 | """ 43 | Import and returns the attribute with name attr_name 44 | in the given module. 45 | """ 46 | return getattr(imported_module, attr_name) 47 | -------------------------------------------------------------------------------- /docs/decisions/0005-user-and-group-management-commands.rst: -------------------------------------------------------------------------------- 1 | User and Group Management Commands 2 | ================================== 3 | 4 | Context 5 | ------- 6 | 7 | Open edX administrators ocassionally need to grant permissions to staff users for certain django services (e.g. ecommerce, registrar). Right now, this is done manually via django admin of the django service. This method of granting permissions is not endorsed, since it is difficult to review, audit, and track changes to user access over time. 8 | 9 | edx-platform, however, defines ``manage_user`` and ``manage_group`` management commands, which allow users to be managed via an external system (such as one, for example, that defines permissions declaratively in version-controlled YAML). These user and group management commands are supposedly generic enough to be used via a common library/app (such as edx-django-utils) so that this users and group management scheme can be brought to other IDAs. 10 | 11 | Decision 12 | -------- 13 | 14 | User and group management commands should be moved to edx-django-utils from edx-platform so that the commands are available to other services. 15 | 16 | The original idea for this decision came from an `edx.org private discussion on app permissions`_. 17 | 18 | .. _`edx.org private discussion on app permissions`: https://github.com/openedx/app-permissions/blob/master/docs/known-issues.md#it-only-works-on-edxapp 19 | 20 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/docs/decisions/0001-monitoring-by-code-owner.rst: -------------------------------------------------------------------------------- 1 | Monitoring by Code Owner 2 | ************************ 3 | 4 | Status 5 | ====== 6 | 7 | Accepted 8 | 9 | Context 10 | ======= 11 | 12 | We originally implemented the "code_owner" custom attribute in edx-platform for split-ownership of the LMS. See the original `ADR in edx-platform for monitoring by code owner`_. 13 | 14 | Owners wanted to be able to see transactions that they owned, in any IDA. 15 | 16 | .. _ADR in edx-platform for monitoring by code owner: https://github.com/openedx/edx-platform/blob/59e0f6efcf2a297806918f8e0020255c1f59ea5f/lms/djangoapps/monitoring/docs/decisions/0001-monitoring-by-code-owner.rst 17 | 18 | Decision 19 | ======== 20 | 21 | We will move the "code_owner" custom attribute code to these shared monitoring utilities so it is available for all IDAs. 22 | 23 | The ability to add a catch-all configuration if there are no other matches will also be added in follow-up work. 24 | 25 | Consequences 26 | ============ 27 | 28 | IDA owners will be able to add middleware and a Django Setting to have the same "code_owner" attribute available across all IDAs that are owned. 29 | 30 | At this time, in the case of an IDA with split-ownership, maintenance of the Django Setting is still manual. In other words, new paths with new owners will needed to be added to the setting. Otherwise, the catch-all (if configured) will be marked as the code owner. 31 | -------------------------------------------------------------------------------- /docs/monitoring/README.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | This README explains the purpose of this directory in the form of a decision document. 4 | 5 | Context 6 | ------- 7 | 8 | At this time, we don't have a standard way to pull in rst docs from outside of the main docs directory. See https://github.com/sphinx-doc/sphinx/issues/701 for details on this issue. 9 | 10 | Decision 11 | -------- 12 | 13 | Copy monitoring/docs files to the main docs directory, and use an include statement as a quick temporary solution to publishing docs with the following benefits: 14 | 15 | * Original docs stay close to the code. 16 | * Any old github references to the old locations would still be accurate. 17 | * We may be able to implement a more automated solution in the future that doesn't break the new Readthedoc locations. 18 | 19 | Consequences 20 | ------------ 21 | 22 | New docs need to be copied to be included in the index.rst, which is only one minor additional step. 23 | 24 | Rejected Alternatives 25 | --------------------- 26 | 27 | The following alternatives were temporarily rejected for the sake of expediency. The decision could be updated in more time were invested on this issue more globally. 28 | 29 | #. Move the monitoring/docs folder under the main docs folder. This would break existing references to github docs. Someone could choose to make this change in the future. 30 | #. Create sphinx plugin to automatically copy docs. This is a potentially good idea, but I am not investing in it at this time. 31 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/docs/decisions/0004-code-owner-theme-and-squad.rst: -------------------------------------------------------------------------------- 1 | Code Owner Theme and Squad 2 | ========================== 3 | 4 | Status 5 | ------ 6 | 7 | Accepted 8 | 9 | Context 10 | ------- 11 | 12 | As detailed in the `Monitoring by Code Owner ADR`_, we added a ``code_owner`` custom attribute for monitoring by code owner. The value for this attribute had the format 'theme-squad'. 13 | 14 | The problems with this configuration is that for theme name changes, or when squads transfer themes, any monitoring referencing the full name would also need to be updated. 15 | 16 | .. _Monitoring by Code Owner ADR: https://github.com/openedx/edx-platform/blob/master/lms/djangoapps/monitoring/docs/decisions/0001-monitoring-by-code-owner.rst 17 | 18 | Decision 19 | -------- 20 | 21 | We will add a ``code_owner_squad`` custom attribute. Monitoring will now be able to refer to ``code_owner_squad`` with a value of 'squad', and will be unaffected by theme name changes. 22 | 23 | Additionally, we are adding ``code_owner_theme`` for similar convenience if there is a need for theme-based monitoring. 24 | 25 | We will leave the original ``code_owner`` custom attribute for backward compatability, and for cases where theme and squad are not used. 26 | 27 | Consequences 28 | ------------ 29 | 30 | * Theme name changes may now result in a one time fix for those that were relying on ``code_owner``. Monitoring can switch from ``code_owner`` to ``code_owner_squad``, rather than a change for every theme update in the future. 31 | -------------------------------------------------------------------------------- /edx_django_utils/cache/middleware.py: -------------------------------------------------------------------------------- 1 | """ 2 | Caching utility middleware. 3 | """ 4 | from django.utils.deprecation import MiddlewareMixin 5 | 6 | from . import RequestCache, TieredCache 7 | 8 | 9 | class RequestCacheMiddleware(MiddlewareMixin): 10 | """ 11 | Middleware to clear the request cache as appropriate for new requests. 12 | """ 13 | def process_request(self, request): 14 | """ 15 | Clears the request cache before processing the request. 16 | """ 17 | RequestCache.clear_all_namespaces() 18 | 19 | def process_response(self, request, response): 20 | """ 21 | Clear the request cache after processing a response. 22 | """ 23 | # This is to just to do some basic cleanup, and isn't 24 | # necessary for correctness; the next request will have the 25 | # cache cleared during process_request anyhow. 26 | # 27 | # Note that clearing in process_exception would actually cause 28 | # problems for other middleware that want to read the cache in 29 | # their process_response. 30 | RequestCache.clear_all_namespaces() 31 | return response 32 | 33 | 34 | class TieredCacheMiddleware(MiddlewareMixin): 35 | """ 36 | Middleware to store whether or not to force django cache misses. 37 | """ 38 | def process_request(self, request): 39 | """ 40 | Stores whether or not FORCE_CACHE_MISS_PARAM was supplied in the 41 | request. 42 | """ 43 | TieredCache._get_and_set_force_cache_miss(request) # pylint: disable=protected-access 44 | -------------------------------------------------------------------------------- /docs/decisions/0004-public-api-and-app-organization.rst: -------------------------------------------------------------------------------- 1 | Public API and App Organization 2 | =============================== 3 | 4 | Status 5 | ------ 6 | 7 | Accepted 8 | 9 | Context 10 | ------- 11 | 12 | The original apps in the library (e.g. ``cache`` and ``monitoring``) exposed a public API via ``__init__.py``. 13 | 14 | There are several problems with the original organization of the app code: 15 | 16 | * It was easy to forget to add new code to the public API, or ignore this requirement. For example, the Middleware didn't follow the same process. 17 | * It was easy for a user of the library to mistakenly use code from a different module in the app, rather than through the public API. 18 | 19 | Decision 20 | -------- 21 | 22 | All implementation code should be moved to an ``internal`` module. 23 | 24 | using monitoring as an example, the public API would be exposed as follows in ``edx_django_utils/monitoring/__init__.py``:: 25 | 26 | from .internal.somemodule import ... 27 | 28 | The benefits of this setup include: 29 | 30 | * A clear designation of what is part of the public API. 31 | * The ability to refactor the implementation without changing the API. 32 | * A clear reminder to developers adding new code that it needs to be exposed if it is public. 33 | * A clear reminder to developers using the library not to use code from the internal implementation. 34 | 35 | Consequences 36 | ------------ 37 | 38 | Whenever a new class or function is added to the edx_django_utils public API, it should be implemented in the Django app's ``internal`` module and explicitly imported in its ``__init__.py`` module. 39 | 40 | Additionally, some existing apps will need to be refactored. 41 | -------------------------------------------------------------------------------- /edx_django_utils/admin/tests/test_mixins.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for admin mixins. 3 | """ 4 | import pytest 5 | from django.contrib.admin import ModelAdmin 6 | from django.contrib.admin.sites import AdminSite 7 | 8 | from edx_django_utils.admin.mixins import ReadOnlyAdminMixin 9 | from edx_django_utils.admin.tests.models import GenericModel 10 | 11 | 12 | class ReadOnlyAdmin(ReadOnlyAdminMixin, ModelAdmin): 13 | """ 14 | Test admin interface which adds the mixin to be tested. 15 | """ 16 | pass 17 | 18 | 19 | @pytest.fixture 20 | def site(): 21 | return AdminSite() 22 | 23 | 24 | @pytest.fixture 25 | def a_request(): 26 | class MockRequest: 27 | GET = [] 28 | return MockRequest() 29 | 30 | 31 | @pytest.fixture 32 | def read_only_admin(site): 33 | return ReadOnlyAdmin(GenericModel, site) 34 | 35 | 36 | class TestReadOnlyAdminMixin: 37 | def test_create(self, read_only_admin, a_request): 38 | assert not read_only_admin.has_add_permission(a_request) 39 | 40 | def test_delete(self, read_only_admin, a_request): 41 | assert not read_only_admin.has_delete_permission(a_request) 42 | 43 | def test_delete_action(self, read_only_admin, a_request): 44 | actions = read_only_admin.get_actions(a_request) 45 | assert 'delete_selected' not in actions 46 | 47 | def test_list_display_links(self, read_only_admin): 48 | assert read_only_admin.list_display_links is None 49 | 50 | def test_read_only_fields(self, read_only_admin): 51 | # The GenericModel has three fields plus an Django-added id field - ensure *all* are marked as read-only. 52 | assert len(read_only_admin.readonly_fields) == 4 53 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/docs/how_tos/update_monitoring_for_squad_or_theme_changes.rst: -------------------------------------------------------------------------------- 1 | Update Monitoring for Squad or Theme Changes 2 | ============================================ 3 | 4 | .. contents:: 5 | :local: 6 | :depth: 2 7 | 8 | Understanding code owner custom attributes 9 | ------------------------------------------ 10 | 11 | If you first need some background on the ``code_owner_squad`` and ``code_owner_theme`` custom attributes, see :doc:`add_code_owner_custom_attribute_to_an_ida`. 12 | 13 | Expand and contract name changes 14 | -------------------------------- 15 | 16 | NRQL (New Relic Query Language) statements that use the ``code_owner_squad`` or ``code_owner_theme`` (or ``code_owner``) custom attributes may be found in alert conditions or dashboards. 17 | 18 | To change a squad or theme name, you should *expand* the NRQL before the change, and *contract* the NRQL after the change. 19 | 20 | .. note:: 21 | 22 | For edx.org, it is useful to wait a month before implementing the contract phase of the monitoring update. 23 | 24 | Example expand phase NRQL:: 25 | 26 | code_owner_squad IN ('old-squad-name', 'new-squad-name') 27 | code_owner_theme IN ('old-theme-name', 'new-theme-name') 28 | 29 | Example contract phase NRQL:: 30 | 31 | code_owner_squad = 'new-squad-name' 32 | code_owner_theme = 'new-theme-name' 33 | 34 | To find the relevant NRQL to update, see `Searching New Relic NRQL`_. 35 | 36 | Searching New Relic NRQL 37 | ------------------------ 38 | 39 | See :doc:`search_new_relic` for general information about the ``new_relic_search.py`` script. 40 | 41 | This script can be especially useful for helping with the expand/contract phase when changing squad or theme names. For example, you could use the following:: 42 | 43 | new_relic_search.py --regex old-squad-name 44 | new_relic_search.py --regex new-squad-name 45 | -------------------------------------------------------------------------------- /edx_django_utils/plugins/plugin_manager.py: -------------------------------------------------------------------------------- 1 | """ 2 | Adds support for first class plugins that can be added to an IDA. 3 | 4 | Please remember to expose any new public methods in the `__init__.py` file. 5 | """ 6 | 7 | import functools 8 | from collections import OrderedDict 9 | 10 | from stevedore.extension import ExtensionManager 11 | 12 | 13 | class PluginError(Exception): 14 | """ 15 | Base Exception for when an error was found regarding plugins. 16 | """ 17 | 18 | 19 | class PluginManager: 20 | """ 21 | Base class that manages plugins for an IDA. 22 | """ 23 | 24 | @classmethod 25 | @functools.lru_cache(maxsize=None) 26 | def get_available_plugins(cls, namespace=None): 27 | """ 28 | Returns a dict of all the plugins that have been made available. 29 | """ 30 | # Note: we're creating the extension manager lazily to ensure that the Python path 31 | # has been correctly set up. Trying to create this statically will fail, unfortunately. 32 | plugins = OrderedDict() 33 | # pylint: disable=no-member 34 | extension_manager = ExtensionManager( 35 | namespace=namespace or cls.NAMESPACE 36 | ) 37 | for plugin_name in extension_manager.names(): 38 | plugins[plugin_name] = extension_manager[plugin_name].plugin 39 | return plugins 40 | 41 | @classmethod 42 | def get_plugin(cls, name, namespace=None): 43 | """ 44 | Returns the plugin with the given name. 45 | """ 46 | plugins = cls.get_available_plugins(namespace) 47 | if name not in plugins: 48 | raise PluginError( 49 | "No such plugin {name} for entry point {namespace}".format( 50 | name=name, 51 | namespace=namespace or cls.NAMESPACE, # pylint: disable=no-member 52 | ) 53 | ) 54 | return plugins[name] 55 | -------------------------------------------------------------------------------- /edx_django_utils/db/queryset_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utils related to QuerySets. 3 | """ 4 | 5 | 6 | def chunked_queryset(queryset, chunk_size=2000): 7 | """ 8 | Slice a queryset into chunks. 9 | 10 | The function slices a queryset into smaller QuerySets containing chunk_size objects and then yields them. It is 11 | used to avoid memory error when processing huge querysets, and also to avoid database errors due to the 12 | database pulling the whole table at once. Additionally, without using a chunked queryset, concurrent database 13 | modification while processing a large table might repeat or skip some entries. 14 | 15 | Warning: It throws away your sorting and sort queryset based on `pk`. Only recommended for large QuerySets where 16 | order does not matter. 17 | (e.g: Can be used in management commands to back-fill data based on Queryset having millions of objects.) 18 | 19 | Source: https://www.djangosnippets.org/snippets/10599/ 20 | 21 | Example Usage: 22 | queryset = User.objects.all() 23 | for chunked_queryset in chunked_queryset(queryset): 24 | print(chunked_queryset.count()) 25 | 26 | Argument: 27 | chunk_size (int): Size of desired batch. 28 | 29 | Return: 30 | QuerySet: Iterator with sliced Queryset. 31 | """ 32 | start_pk = 0 33 | queryset = queryset.order_by('pk') 34 | 35 | while True: 36 | # No entry left 37 | if not queryset.filter(pk__gt=start_pk).exists(): 38 | return 39 | 40 | try: 41 | # Fetch chunk_size entries if possible 42 | end_pk = queryset.filter(pk__gt=start_pk).values_list('pk', flat=True)[chunk_size - 1] 43 | 44 | # Fetch rest entries if less than chunk_size left 45 | except IndexError: 46 | end_pk = queryset.values_list('pk', flat=True).last() 47 | 48 | yield queryset.filter(pk__gt=start_pk).filter(pk__lte=end_pk) 49 | 50 | start_pk = end_pk 51 | -------------------------------------------------------------------------------- /edx_django_utils/logging/tests/test_logging.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for logging. 3 | """ 4 | 5 | from unittest.mock import MagicMock, patch 6 | 7 | from django.test import TestCase 8 | 9 | from edx_django_utils.logging import RemoteIpFilter, UserIdFilter 10 | 11 | 12 | class MockRecord: 13 | """ 14 | Mocks a logging construct to receive data to be interpolated. 15 | """ 16 | def __init__(self): 17 | self.userid = None 18 | self.remoteip = None 19 | 20 | 21 | class TestLoggingFilters(TestCase): 22 | """ 23 | Test the logging filters for users and IP addresses 24 | """ 25 | 26 | @patch('edx_django_utils.logging.internal.filters.get_current_user') 27 | def test_userid_filter(self, mock_get_user): 28 | mock_user = MagicMock() 29 | mock_user.pk = '1234' 30 | mock_get_user.return_value = mock_user 31 | 32 | user_filter = UserIdFilter() 33 | test_record = MockRecord() 34 | user_filter.filter(test_record) 35 | 36 | self.assertEqual(test_record.userid, '1234') 37 | 38 | def test_userid_filter_no_user(self): 39 | user_filter = UserIdFilter() 40 | test_record = MockRecord() 41 | user_filter.filter(test_record) 42 | 43 | self.assertEqual(test_record.userid, None) 44 | 45 | @patch('edx_django_utils.logging.internal.filters.get_current_request') 46 | def test_remoteip_filter(self, mock_get_request): 47 | mock_request = MagicMock() 48 | mock_request.META = {'REMOTE_ADDR': '192.168.1.1'} 49 | mock_get_request.return_value = mock_request 50 | 51 | ip_filter = RemoteIpFilter() 52 | test_record = MockRecord() 53 | ip_filter.filter(test_record) 54 | 55 | self.assertEqual(test_record.remoteip, '192.168.1.1') 56 | 57 | def test_remoteip_filter_no_request(self): 58 | ip_filter = RemoteIpFilter() 59 | test_record = MockRecord() 60 | ip_filter.filter(test_record) 61 | 62 | self.assertEqual(test_record.remoteip, None) 63 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.8 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | asgiref==3.7.2 8 | # via 9 | # -r requirements/base.txt 10 | # django 11 | cffi==1.15.1 12 | # via 13 | # -r requirements/base.txt 14 | # pynacl 15 | click==8.1.7 16 | # via -r requirements/base.txt 17 | coverage[toml]==7.3.1 18 | # via pytest-cov 19 | ddt==1.6.0 20 | # via -r requirements/test.in 21 | # via 22 | # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt 23 | # -r requirements/base.txt 24 | # django-crum 25 | # django-waffle 26 | django-crum==0.7.9 27 | # via -r requirements/base.txt 28 | django-waffle==4.0.0 29 | # via -r requirements/base.txt 30 | exceptiongroup==1.1.3 31 | # via pytest 32 | iniconfig==2.0.0 33 | # via pytest 34 | mock==5.1.0 35 | # via -r requirements/test.in 36 | newrelic==9.0.0 37 | # via -r requirements/base.txt 38 | packaging==23.1 39 | # via pytest 40 | pbr==5.11.1 41 | # via 42 | # -r requirements/base.txt 43 | # stevedore 44 | pluggy==1.3.0 45 | # via pytest 46 | psutil==5.9.5 47 | # via -r requirements/base.txt 48 | pycparser==2.21 49 | # via 50 | # -r requirements/base.txt 51 | # cffi 52 | pynacl==1.5.0 53 | # via -r requirements/base.txt 54 | pytest==7.4.2 55 | # via 56 | # pytest-cov 57 | # pytest-django 58 | pytest-cov==4.1.0 59 | # via -r requirements/test.in 60 | pytest-django==4.5.2 61 | # via -r requirements/test.in 62 | pytz==2023.3.post1 63 | # via 64 | # -r requirements/base.txt 65 | # django 66 | sqlparse==0.4.4 67 | # via 68 | # -r requirements/base.txt 69 | # django 70 | stevedore==5.1.0 71 | # via -r requirements/base.txt 72 | tomli==2.0.1 73 | # via 74 | # coverage 75 | # pytest 76 | typing-extensions==4.7.1 77 | # via 78 | # -r requirements/base.txt 79 | # asgiref 80 | -------------------------------------------------------------------------------- /edx_django_utils/tests/test_pluggable_override.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for utilities. 3 | """ 4 | 5 | from django.test import override_settings 6 | 7 | from edx_django_utils.plugins import pluggable_override 8 | 9 | 10 | @pluggable_override('OVERRIDE_TRANSFORM') 11 | def transform(x): 12 | return x + 10 13 | 14 | 15 | def decrement(prev_fn, x): 16 | if x >= 10: 17 | return x - 1 18 | else: 19 | return prev_fn(x) - 1 20 | 21 | 22 | def double(prev_fn, x): 23 | if x >= 11: 24 | return x * 2 25 | else: 26 | return prev_fn(x) * 2 27 | 28 | 29 | def test_no_override(): 30 | """Test that the original function is called when an override is not specified.""" 31 | assert transform(10) == 20 32 | 33 | 34 | @override_settings(OVERRIDE_TRANSFORM="{}.decrement".format(__name__)) 35 | def test_override(): 36 | """Test that the overriding function is called.""" 37 | assert transform(10) == 9 38 | 39 | 40 | @override_settings(OVERRIDE_TRANSFORM="{}.decrement".format(__name__)) 41 | def test_call_original_function(): 42 | """Test that the overriding function calls the base one.""" 43 | assert transform(9) == 18 44 | 45 | 46 | @override_settings(OVERRIDE_TRANSFORM="{0}.decrement,{0}.double".format(__name__).split(',')) 47 | def test_multiple_overrides_call_last_function(): 48 | """Test that the newest (last) overriding function is called when multiple overrides are specified.""" 49 | assert transform(11) == 22 50 | 51 | 52 | @override_settings(OVERRIDE_TRANSFORM="{0}.decrement,{0}.double".format(__name__).split(',')) 53 | def test_multiple_overrides_fallback_to_previous_function(): 54 | """Test that the last overriding function can call the previous one from the chain.""" 55 | assert transform(10) == 18 56 | 57 | 58 | @override_settings(OVERRIDE_TRANSFORM="{0}.decrement,{0}.double".format(__name__).split(',')) 59 | def test_multiple_overrides_fallback_to_base_function(): 60 | """Test that the overriding functions can eventually call the base one.""" 61 | assert transform(9) == 36 62 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/docs/decisions/0002-custom-monitoring-language.rst: -------------------------------------------------------------------------------- 1 | Custom Monitoring Language 2 | ========================== 3 | 4 | Status 5 | ------ 6 | 7 | Accepted 8 | 9 | Context 10 | ------- 11 | 12 | Although the monitoring library is meant to encapsulate monitoring functionality, our primary implementation works with New Relic. New Relic has the concept of "events" (with "attributes") and "metrics". However, this library was original written using the term "metric" when referring to New Relic "attributes", and not for New Relic "metrics". 13 | 14 | Decision 15 | -------- 16 | 17 | When referring to New Relic event attributes, we will use the term "attribute", and not the word "metric". We will continue to use the term "custom" for attributes that we define, rather than those that come standard from New Relic. Note: New Relic uses the terms "attribute" and "parameter" interchangeably, but the term "attribute" is more popular in the UI. 18 | 19 | Additionally, we will use the term "custom monitoring" as a more general term for any of our custom additions for monitoring purposes, whether this refers to "custom events", "custom attributes", or "custom metrics". 20 | 21 | Consequences 22 | ------------ 23 | 24 | * Switching from "metric" to "attribute" enables us to save the term "metric" for when and if we encapsulate methods that actually write New Relic metrics. 25 | * Because many methods will need to be renamed, we will deprecate the older methods and classes to maintain backward-compatibility until we retire all uses of the old names. 26 | 27 | Resources 28 | --------- 29 | 30 | * `New Relic custom events`_ 31 | 32 | * `New Relic custom attributes`_ 33 | 34 | * `New Relic custom metrics`_ 35 | 36 | .. _New Relic custom events: https://docs.newrelic.com/docs/insights/event-data-sources/custom-events 37 | .. _New Relic custom attributes: https://docs.newrelic.com/docs/insights/event-data-sources/custom-events/new-relic-apm-report-custom-attributes 38 | .. _New Relic custom metrics: https://docs.newrelic.com/docs/agents/manage-apm-agents/agent-data/collect-custom-metrics 39 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py38-django{32, 42} 3 | 4 | [doc8] 5 | ignore = D000, D001 6 | 7 | [pycodestyle] 8 | exclude = .git,.tox,migrations 9 | ignore = E121,E123,E126,E133,E226,E241,E242,E704,W503,W504,W505,E501 10 | 11 | [pydocstyle] 12 | ignore = D101,D200,D203,D212,D215,D404,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414 13 | match-dir = (?!migrations) 14 | 15 | [pytest] 16 | DJANGO_SETTINGS_MODULE = test_settings 17 | addopts = --cov edx_django_utils --cov-report term-missing --cov-report xml 18 | norecursedirs = .* docs requirements 19 | 20 | [testenv] 21 | deps = 22 | django32: Django>=3.2,<4.0 23 | django42: Django>=4.2,<4.3 24 | -r{toxinidir}/requirements/test.txt 25 | commands = 26 | python -Wd -m pytest {posargs} 27 | 28 | [testenv:docs] 29 | setenv = 30 | DJANGO_SETTINGS_MODULE = test_settings 31 | PYTHONPATH = {toxinidir} 32 | whitelist_externals = 33 | make 34 | rm 35 | deps = 36 | -r{toxinidir}/requirements/doc.txt 37 | commands = 38 | doc8 --ignore-path docs/_build README.rst docs 39 | rm -f docs/edx_django_utils.rst 40 | rm -f docs/modules.rst 41 | make -C docs clean 42 | make -C docs html 43 | python setup.py bdist_wheel 44 | twine check dist/* 45 | 46 | [testenv:quality] 47 | setenv = 48 | DJANGO_SETTINGS_MODULE = test_settings 49 | PYTHONPATH = {toxinidir} 50 | whitelist_externals = 51 | make 52 | rm 53 | touch 54 | deps = 55 | -r{toxinidir}/requirements/quality.txt 56 | commands = 57 | touch tests/__init__.py 58 | pylint edx_django_utils tests test_utils manage.py setup.py 59 | rm tests/__init__.py 60 | pycodestyle edx_django_utils tests manage.py setup.py 61 | pydocstyle edx_django_utils tests manage.py setup.py 62 | isort --check-only --diff tests test_utils edx_django_utils manage.py setup.py test_settings.py 63 | make selfcheck 64 | 65 | [testenv:isort] 66 | whitelist_externals = 67 | make 68 | deps = 69 | -r{toxinidir}/requirements/quality.txt 70 | commands = 71 | isort tests test_utils edx_django_utils manage.py setup.py test_settings.py 72 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. edx-django-utils documentation master file, created by 2 | sphinx-quickstart on Mon Jul 23 18:17:54 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | edx-django-utils 7 | =============================== 8 | EdX utilities for Django Application development. 9 | 10 | .. toctree:: 11 | :maxdepth: 1 12 | :caption: General Contents 13 | 14 | README 15 | getting_started 16 | testing 17 | Package Reference 18 | changelog 19 | 20 | .. toctree:: 21 | :maxdepth: 1 22 | :caption: Decisions 23 | 24 | decisions/0001-purpose-of-this-repo 25 | decisions/0002-extract-plugins-infrastructure-from-edx-platform 26 | decisions/0003-logging-filters-for-user-and-ip 27 | decisions/0004-public-api-and-app-organization 28 | decisions/0005-user-and-group-management-commands 29 | 30 | .. toctree:: 31 | :maxdepth: 1 32 | :caption: Monitoring Decisions 33 | 34 | monitoring/decisions/0001-monitoring-by-code-owner 35 | monitoring/decisions/0002-custom-monitoring-language 36 | monitoring/decisions/0003-code-owner-for-celery-tasks 37 | monitoring/decisions/0004-code-owner-theme-and-squad 38 | 39 | .. toctree:: 40 | :maxdepth: 1 41 | :caption: Monitoring How-Tos 42 | 43 | monitoring/how_tos/add_code_owner_custom_attribute_to_an_ida 44 | monitoring/how_tos/using_custom_attributes 45 | monitoring/how_tos/search_new_relic_nrql 46 | monitoring/how_tos/update_monitoring_for_squad_or_theme_changes 47 | 48 | .. toctree:: 49 | :maxdepth: 1 50 | :caption: Plugins README 51 | 52 | plugins/readme 53 | 54 | .. toctree:: 55 | :maxdepth: 1 56 | :caption: Plugins Decisions 57 | 58 | plugins/decisions/0001-plugin-contexts 59 | 60 | .. toctree:: 61 | :maxdepth: 1 62 | :caption: Plugins How-Tos 63 | 64 | plugins/how_tos/how_to_enable_plugins_for_an_ida 65 | plugins/how_tos/how_to_create_a_plugin_app 66 | 67 | 68 | Indices and tables 69 | ================== 70 | 71 | * :ref:`genindex` 72 | * :ref:`modindex` 73 | * :ref:`search` 74 | -------------------------------------------------------------------------------- /edx_django_utils/plugins/constants.py: -------------------------------------------------------------------------------- 1 | """ 2 | Constants/Enums used during interfacing with plugin system 3 | """ 4 | 5 | # Name of the class attribute to put in the AppConfig class of the Plugin App. 6 | PLUGIN_APP_CLASS_ATTRIBUTE_NAME = "plugin_app" 7 | 8 | 9 | # Name of the function that belongs in the plugin Django app's settings file. 10 | # The function should be defined as: 11 | # def plugin_settings(settings): 12 | # # enter code that should be injected into the given settings module. 13 | PLUGIN_APP_SETTINGS_FUNC_NAME = "plugin_settings" 14 | 15 | 16 | class PluginSettings(): 17 | """ 18 | The PluginSettings enum defines dictionary field names (and defaults) 19 | that can be specified by a Plugin App in order to configure the settings 20 | that are injected into the project. 21 | """ 22 | 23 | CONFIG = "settings_config" 24 | RELATIVE_PATH = "relative_path" 25 | DEFAULT_RELATIVE_PATH = "settings" 26 | 27 | 28 | class PluginURLs(): 29 | """ 30 | The PluginURLs enum defines dictionary field names (and defaults) that can 31 | be specified by a Plugin App in order to configure the URLs that are 32 | injected into the project. 33 | """ 34 | 35 | CONFIG = "url_config" 36 | RELATIVE_PATH = "relative_path" 37 | DEFAULT_RELATIVE_PATH = "urls" 38 | 39 | 40 | class PluginSignals(): 41 | """ 42 | The PluginSignals enum defines dictionary field names (and defaults) 43 | that can be specified by a Plugin App in order to configure the signals 44 | that it receives. 45 | """ 46 | 47 | CONFIG = "signals_config" 48 | 49 | RECEIVERS = "receivers" 50 | DISPATCH_UID = "dispatch_uid" 51 | RECEIVER_FUNC_NAME = "receiver_func_name" 52 | SENDER_PATH = "sender_path" 53 | SIGNAL_PATH = "signal_path" 54 | 55 | RELATIVE_PATH = "relative_path" 56 | DEFAULT_RELATIVE_PATH = "signals" 57 | 58 | 59 | class PluginContexts(): 60 | """ 61 | The PluginContexts enum defines dictionary field names (and defaults) 62 | that can be specified by a Plugin App in order to configure the 63 | additional views it would like to add context into. 64 | """ 65 | 66 | CONFIG = "view_context_config" 67 | -------------------------------------------------------------------------------- /edx_django_utils/plugins/docs/how_tos/how_to_enable_plugins_for_an_ida.rst: -------------------------------------------------------------------------------- 1 | How to enable plugins for an IDA 2 | ================================ 3 | 4 | .. note:: Enabling plugins only needs to be done once per IDA. If you are creating a plugin app for an IDA that already supports plugins, see :doc:`how to create a plugin app `. 5 | 6 | If you are unsure if your IDA supports plugins, you can look for ``INSTALLED_APPS.extend(get_plugin_apps(...))`` in your settings file. 7 | 8 | Overview 9 | -------- 10 | 11 | Plugins are meant to be automatically discovered and installed by an Independently Deployable Application (IDA). In order for an IDA to recognize and install plugins, a one time setup is required in each IDA. This how-to guide is for this one-time preparation of an IDA. 12 | 13 | Django Projects 14 | --------------- 15 | 16 | In order to enable this functionality in a Django project, the project needs to 17 | update: 18 | 19 | 1. its settings to extend its INSTALLED_APPS to include the Plugin Apps 20 | :: 21 | 22 | INSTALLED_APPS.extend(get_plugin_apps(...)) 23 | 24 | 2. its settings to add all Plugin Settings 25 | :: 26 | 27 | add_plugins(__name__, ...) 28 | 29 | 3. its urls to add all Plugin URLs 30 | :: 31 | 32 | urlpatterns.extend(get_plugin_url_patterns(...)) 33 | 34 | 4. its setup to register PluginsConfig (for connecting Plugin Signals) 35 | :: 36 | 37 | from setuptools import setup 38 | setup( 39 | ... 40 | entry_points={ 41 | "lms.djangoapp": [ 42 | "plugins = openedx.core.djangoapps.plugins.apps:PluginsConfig", 43 | ], 44 | "cms.djangoapp": [ 45 | "plugins = openedx.core.djangoapps.plugins.apps:PluginsConfig", 46 | ], 47 | } 48 | ) 49 | 50 | .. note:: For a plugin that will appear in a single IDA, you could create constants like `ProjectType found in edx-platform`_. If the plugin is for many IDAs, we need to add a capability to this library with a global constant. 51 | 52 | .. _ProjectType found in edx-platform: https://github.com/openedx/edx-platform/blob/dbe40dae1a8b50fea0954e85f76ebf244129186e/openedx/core/djangoapps/plugins/constants.py#L14-L22 53 | -------------------------------------------------------------------------------- /edx_django_utils/plugins/plugin_settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Functions that deal with settings related to plugins 3 | 4 | Please remember to expose any new public methods in the `__init__.py` file. 5 | """ 6 | from logging import getLogger 7 | 8 | from . import constants, registry, utils 9 | 10 | log = getLogger(__name__) 11 | 12 | 13 | def add_plugins(settings_path, project_type, settings_type): 14 | """ 15 | Updates the module at the given ``settings_path`` with all Plugin 16 | Settings appropriate for the given project_type and settings_type. 17 | """ 18 | settings_module = utils.import_module(settings_path) 19 | for plugin_settings in _iter_plugins(project_type, settings_type): 20 | settings_func = getattr( 21 | plugin_settings, constants.PLUGIN_APP_SETTINGS_FUNC_NAME 22 | ) 23 | settings_func(settings_module) 24 | 25 | 26 | def _iter_plugins(project_type, settings_type): 27 | """ 28 | Yields Plugin Settings modules that are registered for the given 29 | project_type and settings_type. 30 | """ 31 | for app_config in registry.get_plugin_app_configs(project_type): 32 | settings_config = _get_config(app_config, project_type, settings_type) 33 | if settings_config is None: 34 | log.debug( 35 | "Plugin Apps [Settings]: Did NOT find %s for %s and %s", 36 | app_config.name, 37 | project_type, 38 | settings_type, 39 | ) 40 | continue 41 | 42 | plugin_settings_path = utils.get_module_path( 43 | app_config, settings_config, constants.PluginSettings 44 | ) 45 | 46 | log.debug( 47 | "Plugin Apps [Settings]: Found %s for %s and %s", 48 | app_config.name, 49 | project_type, 50 | settings_type, 51 | ) 52 | yield utils.import_module(plugin_settings_path) 53 | 54 | 55 | def _get_config(app_config, project_type, settings_type): 56 | plugin_config = getattr(app_config, constants.PLUGIN_APP_CLASS_ATTRIBUTE_NAME, {}) 57 | settings_config = plugin_config.get(constants.PluginSettings.CONFIG, {}) 58 | project_type_settings = settings_config.get(project_type, {}) 59 | return project_type_settings.get(settings_type) 60 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/docs/decisions/0003-code-owner-for-celery-tasks.rst: -------------------------------------------------------------------------------- 1 | Code Owner for Celery Tasks 2 | =========================== 3 | 4 | Status 5 | ------ 6 | 7 | Accepted 8 | 9 | Context 10 | ------- 11 | 12 | As detailed in the `Monitoring by Code Owner ADR`_, we were able to add a ``code_owner`` custom attribute to web transactions using a special middleware. Since middleware is not run for celery tasks (non-web transactions), this solution cannot be used. 13 | 14 | .. _Monitoring by Code Owner ADR: https://github.com/openedx/edx-platform/blob/master/lms/djangoapps/monitoring/docs/decisions/0001-monitoring-by-code-owner.rst 15 | 16 | Decision 17 | -------- 18 | 19 | We implemented a ``@set_code_owner_attribute`` decorator that would add the ``code_owner`` custom attribute for a celery task, and added the decorator to all the celery tasks. See the `celery section of the code_owner how-to`_ for usage details. 20 | 21 | .. _celery section of the code_owner how-to: https://github.com/openedx/edx-django-utils/blob/6ed6de25d487314faa01ed72afd190db95afd1e8/edx_django_utils/monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst#handling-celery-tasks 22 | 23 | Consequences 24 | ------------ 25 | 26 | * We added the new decorator as needed for all existing tasks. 27 | * New celery tasks will need to add the same decorator. See "Rejected Alternatives" for a potential alternative. 28 | 29 | (Rejected) Alternatives 30 | ----------------------- 31 | 32 | An untested potential alternative to the ``@set_code_owner_attribute`` decorator is to try celery's `task_prerun signal`_ in an IDA, which would also ensure all future celery tasks are automatically handled. Although this is a potentially superior solution, it was missed at the time of implementation. We will not be switching to this solution at this time given we have a working solution and other priorities, but it is a potentially viable solution if the need arises. 33 | 34 | Additionally, if this alternative solution were implemented, it would be best to not add celery as a dependency to this library, and to document a new edx-platform implementation. It is unlikely that the solution will be needed outside of edx-platform. 35 | 36 | .. _task_prerun signal: https://docs.celeryproject.org/en/stable/userguide/signals.html#task-prerun 37 | -------------------------------------------------------------------------------- /edx_django_utils/db/tests/test_queryset_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests of edx_django_utils.db.queryset_utils. 3 | """ 4 | from ddt import data, ddt, unpack 5 | from django.contrib import auth 6 | from django.test import TestCase 7 | 8 | from edx_django_utils.db.queryset_utils import chunked_queryset 9 | 10 | User = auth.get_user_model() 11 | 12 | 13 | @ddt 14 | class TestQuerysetUtils(TestCase): 15 | """ 16 | Tests of edx_django_utils.db.queryset_utils. 17 | """ 18 | @unpack 19 | @data( 20 | (30, 10, [10, 10, 10]), 21 | (31, 10, [10, 10, 10, 1]), 22 | (10, 10, [10]), 23 | (7, 10, [7]), 24 | (0, 10, [0]), 25 | ) 26 | def test_chunked_queryset(self, query_size, chunk_size, expected_batches): 27 | User.objects.all().delete() 28 | 29 | # create objects size of query_size 30 | for number in range(query_size): 31 | User.objects.create(username="username_{number}".format(number=number)) 32 | 33 | queryset = User.objects.all() 34 | 35 | self.assertEqual(queryset.count(), query_size) 36 | for (batch_num, chunked_query) in enumerate(chunked_queryset(queryset, chunk_size)): 37 | self.assertEqual(chunked_query.count(), expected_batches[batch_num]) 38 | 39 | def test_concurrent_update(self): 40 | """ 41 | Test concurrent database modification wouldn't skip records. 42 | """ 43 | User.objects.all().delete() 44 | 45 | # Create 14 objects. 46 | for number in range(14): 47 | User.objects.create(username="username_{number}".format(number=number)) 48 | 49 | queryset = User.objects.all() 50 | 51 | # Now create chunks of size 10. 52 | chunked_query = chunked_queryset(queryset, chunk_size=10) 53 | 54 | # As there a total 14 objects and chunk size is 10, Assert first chunk should contain 10 objects. 55 | first_chunk = next(chunked_query) 56 | self.assertEqual(first_chunk.count(), 10) 57 | 58 | # Lets create a new object while iterating over the chunked_queryset. 59 | User.objects.create(username="one-more-user") 60 | 61 | # As now there are total 15 objects, the second chunk should contain 5 objects instead of 4. 62 | # that implies concurrent database modification won't skip records in this process. 63 | second_chunk = next(chunked_query) 64 | self.assertEqual(second_chunk.count(), 5) 65 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/middleware.py: -------------------------------------------------------------------------------- 1 | """ 2 | Deprecated Middleware for backward-compatibility. 3 | 4 | IMPORTANT: No new classes should be added to this file. 5 | TODO: Remove this file once these classes are no longer used. 6 | 7 | """ 8 | import warnings 9 | 10 | from .internal.middleware import CachedCustomMonitoringMiddleware as InternalCachedCustomMonitoringMiddleware 11 | from .internal.middleware import MonitoringMemoryMiddleware as InternalMonitoringMemoryMiddleware 12 | from .internal.utils import set_custom_attribute 13 | 14 | 15 | class CachedCustomMonitoringMiddleware(InternalCachedCustomMonitoringMiddleware): 16 | """ 17 | Deprecated class for handling middleware. Class has been moved to public API. 18 | """ 19 | def __init__(self, *args, **kwargs): 20 | super().__init__(*args, **kwargs) 21 | msg = "Use 'edx_django_utils.monitoring.CachedCustomMonitoringMiddleware' in place of " \ 22 | "'edx_django_utils.monitoring.middleware.CachedCustomMonitoringMiddleware'." 23 | warnings.warn(msg, DeprecationWarning) 24 | set_custom_attribute('deprecated_monitoring_middleware', 'CachedCustomMonitoringMiddleware') 25 | 26 | 27 | class MonitoringCustomMetricsMiddleware(InternalCachedCustomMonitoringMiddleware): 28 | """ 29 | Deprecated class for handling middleware. Class has been renamed to CachedCustomMonitoringMiddleware. 30 | """ 31 | def __init__(self, *args, **kwargs): 32 | super().__init__(*args, **kwargs) 33 | msg = "Use 'edx_django_utils.monitoring.CachedCustomMonitoringMiddleware' in place of " \ 34 | "'edx_django_utils.monitoring.middleware.MonitoringCustomMetricsMiddleware'." 35 | warnings.warn(msg, DeprecationWarning) 36 | set_custom_attribute('deprecated_monitoring_middleware', 'MonitoringCustomMetricsMiddleware') 37 | 38 | 39 | class MonitoringMemoryMiddleware(InternalMonitoringMemoryMiddleware): 40 | """ 41 | Deprecated class for handling middleware. Class has been moved to public API. 42 | """ 43 | def __init__(self, *args, **kwargs): 44 | super().__init__(*args, **kwargs) 45 | msg = "Use 'edx_django_utils.monitoring.MonitoringMemoryMiddleware' in place of " \ 46 | "'edx_django_utils.monitoring.middleware.MonitoringMemoryMiddleware'." 47 | warnings.warn(msg, DeprecationWarning) 48 | set_custom_attribute('deprecated_monitoring_middleware', 'MonitoringMemoryMiddleware') 49 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/README.rst: -------------------------------------------------------------------------------- 1 | Monitoring Utils 2 | ================ 3 | 4 | This is our primary abstraction from 3rd party monitoring libraries such as newrelic.agent. It includes middleware and utility methods for adding custom attributes and for better monitoring memory consumption. 5 | 6 | See ``__init__.py`` for a list of everything included in the public API. 7 | 8 | If, for some reason, you need low level access to the newrelic agent, please extend this library to implement the feature that you want. Applications should never include ``import newrelic.agent`` directly. 9 | 10 | Using Custom Attributes 11 | ----------------------- 12 | 13 | For help writing and using custom attributes, see docs/how_tos/using_custom_attributes.rst. 14 | 15 | Setting up Middleware 16 | --------------------- 17 | 18 | Here is how you add the middleware: 19 | 20 | .. code-block:: 21 | 22 | MIDDLEWARE = ( 23 | 'edx_django_utils.cache.middleware.RequestCacheMiddleware', 24 | 25 | # Add monitoring middleware immediately after RequestCacheMiddleware 26 | 'edx_django_utils.monitoring.DeploymentMonitoringMiddleware', 27 | 'edx_django_utils.monitoring.CookieMonitoringMiddleware', 28 | 'edx_django_utils.monitoring.CodeOwnerMonitoringMiddleware', 29 | 'edx_django_utils.monitoring.CachedCustomMonitoringMiddleware', 30 | 'edx_django_utils.monitoring.MonitoringMemoryMiddleware', 31 | ) 32 | 33 | Cached Custom Monitoring Middleware 34 | ----------------------------------- 35 | 36 | The middleware ``CachedCustomMonitoringMiddleware`` is required to allow certain utility methods, like ``accumulate`` and ``increment``, to work appropriately. 37 | 38 | Code Owner Custom Attribute 39 | --------------------------- 40 | 41 | See docstring for ``CodeOwnerMonitoringMiddleware`` for configuring the ``code_owner`` custom attribute for your IDA. 42 | 43 | Cookie Monitoring Middleware 44 | ---------------------------- 45 | 46 | See docstring for configuring ``CookieMonitoringMiddleware`` to monitor cookie header size. 47 | 48 | Also see ``monitoring/scripts/process_cookie_monitoring_logs.py`` for processing log messages. 49 | 50 | Deployment Monitoring Middleware 51 | -------------------------------- 52 | 53 | Simply add ``DeploymentMonitoringMiddleware`` to monitor the python and django version of each request. See docstring for details. 54 | 55 | Monitoring Memory Usage 56 | ----------------------- 57 | 58 | In addition to adding the MonitoringMemoryMiddleware, you will need to enable a waffle switch ``edx_django_utils.monitoring.enable_memory_middleware`` to enable the additional monitoring. 59 | -------------------------------------------------------------------------------- /edx_django_utils/plugins/plugin_urls.py: -------------------------------------------------------------------------------- 1 | """ 2 | Urls utility functions 3 | 4 | Please remember to expose any new public methods in the `__init__.py` file. 5 | """ 6 | from logging import getLogger 7 | 8 | from django.urls import include, re_path 9 | 10 | from . import constants, registry, utils 11 | 12 | log = getLogger(__name__) 13 | 14 | 15 | def _get_url(url_module_path, url_config): 16 | """ 17 | function constructs the appropriate URL 18 | """ 19 | namespace = url_config[constants.PluginURLs.NAMESPACE] 20 | app_name = url_config.get(constants.PluginURLs.APP_NAME) 21 | regex = url_config.get(constants.PluginURLs.REGEX, r"") 22 | 23 | if namespace: 24 | return re_path(regex, include((url_module_path, app_name), namespace=namespace)) 25 | else: 26 | return re_path(regex, include(url_module_path)) 27 | 28 | 29 | def get_plugin_url_patterns(project_type): 30 | """ 31 | Returns a list of all registered Plugin URLs, expected to be added to 32 | the URL patterns for the given project_type. 33 | """ 34 | return [ 35 | _get_url(url_module_path, url_config) 36 | for url_module_path, url_config in _iter_plugins(project_type) 37 | ] 38 | 39 | 40 | def _iter_plugins(project_type): 41 | """ 42 | Yields the module path and configuration for Plugin URLs registered for 43 | the given project_type. 44 | """ 45 | for app_config in registry.get_plugin_app_configs(project_type): 46 | url_config = _get_config(app_config, project_type) 47 | if url_config is None: 48 | log.debug( 49 | "Plugin Apps [URLs]: Did NOT find %s for %s", 50 | app_config.name, 51 | project_type, 52 | ) 53 | continue 54 | 55 | urls_module_path = utils.get_module_path( 56 | app_config, url_config, constants.PluginURLs 57 | ) 58 | url_config[constants.PluginURLs.NAMESPACE] = url_config.get( 59 | constants.PluginURLs.NAMESPACE, app_config.name 60 | ) 61 | url_config[constants.PluginURLs.APP_NAME] = app_config.name 62 | log.debug( 63 | "Plugin Apps [URLs]: Found %s with namespace %s for %s", 64 | app_config.name, 65 | url_config[constants.PluginURLs.NAMESPACE], 66 | project_type, 67 | ) 68 | yield urls_module_path, url_config 69 | 70 | 71 | def _get_config(app_config, project_type): 72 | plugin_config = getattr(app_config, constants.PLUGIN_APP_CLASS_ATTRIBUTE_NAME, {}) 73 | url_config = plugin_config.get(constants.PluginURLs.CONFIG, {}) 74 | return url_config.get(project_type) 75 | -------------------------------------------------------------------------------- /test_settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | These settings are here to use during tests, because django requires them. 3 | 4 | In a real-world use case, apps in this project are installed into other 5 | Django applications, so these settings will not be used. 6 | """ 7 | 8 | from os.path import abspath, dirname, join 9 | 10 | 11 | def root(*args): 12 | """ 13 | Get the absolute path of the given path relative to the project root. 14 | """ 15 | return join(abspath(dirname(__file__)), *args) 16 | 17 | 18 | DATABASES = { 19 | "default": { 20 | "ENGINE": "django.db.backends.sqlite3", 21 | "NAME": "default.db", 22 | "USER": "", 23 | "PASSWORD": "", 24 | "HOST": "", 25 | "PORT": "", 26 | }, 27 | "read_replica": { 28 | "ENGINE": "django.db.backends.sqlite3", 29 | "NAME": "read_replica.db", 30 | "USER": "", 31 | "PASSWORD": "", 32 | "HOST": "", 33 | "PORT": "", 34 | }, 35 | } 36 | 37 | DEBUG = False 38 | 39 | ALLOWED_HOSTS = ['*'] 40 | 41 | INSTALLED_APPS = ( 42 | "django.contrib.auth", 43 | "django.contrib.contenttypes", 44 | "waffle", 45 | "edx_django_utils", 46 | "edx_django_utils.admin.tests", 47 | "edx_django_utils.user", 48 | 'django.contrib.admin', 49 | 'django.contrib.messages', 50 | ) 51 | 52 | LOCALE_PATHS = [root("edx_django_utils", "conf", "locale")] 53 | 54 | ROOT_URLCONF = "edx_django_utils.urls" 55 | 56 | TEMPLATES = [ 57 | { 58 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 59 | 'DIRS': ['templates'], 60 | 'APP_DIRS': True, 61 | 'OPTIONS': { 62 | 'context_processors': [ 63 | 'django.template.context_processors.debug', 64 | 'django.template.context_processors.request', 65 | 'django.contrib.auth.context_processors.auth', 66 | 'django.contrib.messages.context_processors.messages', 67 | ], 68 | }, 69 | }, 70 | ] 71 | 72 | WSGI_APPLICATION = 'edx_django_utils.wsgi.application' 73 | 74 | 75 | 76 | SECRET_KEY = "insecure-secret-key" 77 | 78 | MIDDLEWARE = [ 79 | 'corsheaders.middleware.CorsMiddleware', 80 | 'django.middleware.security.SecurityMiddleware', 81 | "whitenoise.middleware.WhiteNoiseMiddleware", 82 | 'django.contrib.sessions.middleware.SessionMiddleware', 83 | 'django.middleware.common.CommonMiddleware', 84 | 'django.middleware.csrf.CsrfViewMiddleware', 85 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 86 | 'django.contrib.messages.middleware.MessageMiddleware', 87 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 88 | ] -------------------------------------------------------------------------------- /edx_django_utils/monitoring/internal/transactions.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is a collection of transaction-related monitoring utilities. 3 | 4 | Usage: 5 | 6 | from edx_django_utils.monitoring import ignore_transaction 7 | ... 8 | # called from inside a view you want ignored 9 | ignore_transaction() 10 | 11 | Please remember to expose any new methods in the `__init__.py` file. 12 | """ 13 | 14 | from contextlib import contextmanager 15 | 16 | try: 17 | import newrelic.agent 18 | except ImportError: 19 | newrelic = None # pylint: disable=invalid-name 20 | 21 | 22 | def set_monitoring_transaction_name(name, group=None, priority=None): 23 | """ 24 | Sets the transaction name for monitoring. 25 | 26 | This is not cached, and only support reporting to New Relic. 27 | 28 | """ 29 | if newrelic: # pragma: no cover 30 | newrelic.agent.set_transaction_name(name, group, priority) 31 | 32 | 33 | def ignore_transaction(): 34 | """ 35 | Ignore the transaction in monitoring 36 | 37 | This allows us to ignore code paths that are unhelpful to include, such as 38 | `/health/` checks. 39 | """ 40 | if newrelic: # pragma: no cover 41 | newrelic.agent.ignore_transaction() 42 | 43 | 44 | @contextmanager 45 | def function_trace(function_name): 46 | """ 47 | Wraps a chunk of code that we want to appear as a separate, explicit, 48 | segment in our monitoring tools. 49 | """ 50 | # Not covering this because if we mock it, we're not really testing anything 51 | # anyway. If something did break, it should show up in tests for apps that 52 | # use this code with newrelic enabled, on whatever version of newrelic they 53 | # run. 54 | if newrelic: # pragma: no cover 55 | if newrelic.version_info[0] >= 5: 56 | with newrelic.agent.FunctionTrace(function_name): 57 | yield 58 | else: 59 | nr_transaction = newrelic.agent.current_transaction() 60 | with newrelic.agent.FunctionTrace(nr_transaction, function_name): 61 | yield 62 | else: 63 | yield 64 | 65 | 66 | class MonitoringTransaction(): 67 | """ 68 | Represents a monitoring transaction (likely the current transaction). 69 | """ 70 | def __init__(self, transaction): 71 | self.transaction = transaction 72 | 73 | @property 74 | def name(self): 75 | """ 76 | The name of the transaction. 77 | 78 | For NewRelic, the name may look like: 79 | openedx.core.djangoapps.contentserver.middleware:StaticContentServer 80 | 81 | """ 82 | if self.transaction and hasattr(self.transaction, 'name'): 83 | return self.transaction.name 84 | return None 85 | 86 | 87 | def get_current_transaction(): 88 | """ 89 | Returns the current transaction. 90 | """ 91 | current_transaction = None 92 | if newrelic: 93 | current_transaction = newrelic.agent.current_transaction() 94 | 95 | return MonitoringTransaction(current_transaction) 96 | -------------------------------------------------------------------------------- /edx_django_utils/plugins/plugin_signals.py: -------------------------------------------------------------------------------- 1 | """ 2 | Allows plugins to work with django signals 3 | 4 | Please remember to expose any new public methods in the `__init__.py` file. 5 | """ 6 | from logging import getLogger 7 | 8 | from . import constants, registry, utils 9 | 10 | log = getLogger(__name__) 11 | 12 | 13 | def connect_plugin_receivers(project_type): 14 | """ 15 | connects receivers to right signal 16 | """ 17 | for signals_module, signals_config in _iter_plugins(project_type): 18 | for signal, receiver_func, receiver_config in _iter_receivers( 19 | signals_module, signals_config 20 | ): 21 | signal.connect( 22 | receiver_func, 23 | sender=_get_sender(receiver_config), 24 | dispatch_uid=_get_dispatch_uuid(receiver_config, receiver_func), 25 | ) 26 | 27 | 28 | def _iter_receivers(signals_module, signals_config): 29 | """ 30 | Generator for ___ TODO 31 | """ 32 | for receiver_config in signals_config.get(constants.PluginSignals.RECEIVERS, []): 33 | receiver_func = utils.import_attr_in_module( 34 | signals_module, receiver_config[constants.PluginSignals.RECEIVER_FUNC_NAME], 35 | ) 36 | signal = utils.import_attr(receiver_config[constants.PluginSignals.SIGNAL_PATH]) 37 | yield signal, receiver_func, receiver_config 38 | 39 | 40 | def _iter_plugins(project_type): 41 | """ 42 | Generator for ___ TODO 43 | """ 44 | for app_config in registry.get_plugin_app_configs(project_type): 45 | signals_config = _get_config(app_config, project_type) 46 | if signals_config is None: 47 | log.debug( 48 | "Plugin Apps [Signals]: Did NOT find %s for %s", 49 | app_config.name, 50 | project_type, 51 | ) 52 | continue 53 | 54 | signals_module_path = utils.get_module_path( 55 | app_config, signals_config, constants.PluginSignals 56 | ) 57 | signals_module = utils.import_module(signals_module_path) 58 | 59 | log.debug( 60 | "Plugin Apps [Signals]: Found %s with %d receiver(s) for %s", 61 | app_config.name, 62 | len(signals_config.get(constants.PluginSignals.RECEIVERS, [])), 63 | project_type, 64 | ) 65 | yield signals_module, signals_config 66 | 67 | 68 | def _get_config(app_config, project_type): 69 | plugin_config = getattr(app_config, constants.PLUGIN_APP_CLASS_ATTRIBUTE_NAME, {}) 70 | signals_config = plugin_config.get(constants.PluginSignals.CONFIG, {}) 71 | return signals_config.get(project_type) 72 | 73 | 74 | def _get_sender(receiver_config): 75 | sender_path = receiver_config.get(constants.PluginSignals.SENDER_PATH) 76 | if sender_path: 77 | return utils.import_attr(sender_path) 78 | return None 79 | 80 | 81 | def _get_dispatch_uuid(receiver_config, receiver_func): 82 | return receiver_config.get(constants.PluginSignals.DISPATCH_UID) or f"{receiver_func.__module__}.{receiver_func.__name__}" 83 | -------------------------------------------------------------------------------- /edx_django_utils/plugins/README.rst: -------------------------------------------------------------------------------- 1 | Django App Plugins 2 | ================== 3 | 4 | .. contents:: 5 | :local: 6 | :depth: 2 7 | 8 | Overview 9 | -------- 10 | 11 | This provides functionality to enable improved plugin support for Django apps. 12 | 13 | It can be used to extend any service with code that is not core to the service, 14 | but will be discoverable by that service for an individual deployment. 15 | 16 | Once a Django project is enhanced with this functionality, any participating 17 | Django app (a.k.a. Plugin App) that is PIP-installed on the system is 18 | automatically included in the Django project's INSTALLED_APPS list. In addition, 19 | the participating Django app's URLs and Settings are automatically recognized by 20 | the Django project. Furthermore, the Plugin Signals feature allows Plugin Apps 21 | to shift their dependencies on Django Signal Senders from code-time to runtime. 22 | 23 | While Django+Python already support dynamic installation of components/apps, 24 | they do not have out-of-the-box support for plugin apps that auto-install 25 | into a containing Django project. 26 | 27 | Enable Plugins in an IDA 28 | ------------------------ 29 | 30 | See :doc:`instructions to enable plugins for an IDA `. 31 | 32 | Creating a Plugin App 33 | --------------------- 34 | 35 | See :doc:`how to create a plugin app `. 36 | 37 | .. note:: Do not use this plugin framework for required apps. Instead, explicitly add your required app to the ``INSTALLED_APPS`` of the IDA. 38 | 39 | Design Principles 40 | ----------------- 41 | 42 | This Django App Plugin functionality allows for Django-framework code to be 43 | encapsulated within each Django app, rather than having a monolith Project that 44 | is aware of the details of its Django apps. It is motivated by the following 45 | design principles: 46 | 47 | * Single Responsibility Principle, which says "a class or module should have 48 | one, and only one, reason to change." When code related to a single Django app 49 | changes, there's no reason for its containing project to also change. The 50 | encapsulation and modularity resulting from code being co-located with its 51 | owning Django app helps prevent "God objects" that have too much responsibility 52 | and knowledge of the details. 53 | 54 | * Open Closed Principle, which says "software entities should be open for 55 | extension, but closed for modification." IDAs are extensible via 56 | installation of Django apps. Having automatic Django App Plugin support allows 57 | for this extensibility without modification to an IDA. Currently, this is only 58 | set up in the edx platform. Going forward, we expect this capability to be widely 59 | used by other IDAs to enable enhancement without the need to modify core IDA code. 60 | 61 | * Dependency Inversion Principle, which says "high level modules should not 62 | depend upon low level modules." The high-level module here is the Django 63 | project, while the participating Django app is the low-level module. For 64 | long-term maintenance of a system, dependencies should go from low-level 65 | modules/details to higher level ones. 66 | -------------------------------------------------------------------------------- /edx_django_utils/logging/tests/test_log_sensitive.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for util.logging 3 | """ 4 | 5 | import re 6 | import subprocess 7 | 8 | import pytest 9 | 10 | from edx_django_utils.logging.internal.log_sensitive import decrypt_log_message, encrypt_for_log, generate_reader_keys 11 | 12 | 13 | def test_encryption_no_key(): 14 | to_log = encrypt_for_log("Testing testing 1234", None) 15 | assert to_log == '[encryption failed, no key]' 16 | 17 | 18 | def test_encryption_round_trip(): 19 | reader_keys = generate_reader_keys() 20 | reader_public_64 = reader_keys['public'] 21 | reader_private_64 = reader_keys['private'] 22 | 23 | to_log = encrypt_for_log("Testing testing 1234", reader_public_64) 24 | re_base64 = r'[a-zA-Z0-9/+=]' 25 | assert re.fullmatch(f'\\[encrypted: {re_base64}+\\|{re_base64}+\\]', to_log) 26 | 27 | to_decrypt = to_log.partition('[encrypted: ')[2].rstrip(']') 28 | 29 | decrypted = decrypt_log_message(to_decrypt, reader_private_64) 30 | assert decrypted == "Testing testing 1234" 31 | 32 | # Also check that decryption still works if someone accidentally 33 | # copies in the trailing framing "]" character, just as a 34 | # nice-to-have. (base64 module should handle this already, since 35 | # it stops reading at the first invalid base64 character.) 36 | decrypted_again = decrypt_log_message(to_decrypt + ']', reader_private_64) 37 | assert decrypted_again == "Testing testing 1234" 38 | 39 | 40 | def test_full_cli(tmp_path): 41 | def do_call(args, stdin=None): 42 | return subprocess.run( 43 | ['log-sensitive', *args], check=True, 44 | input=(stdin.encode() if stdin else None), capture_output=True, 45 | ).stdout.decode() 46 | 47 | # Generate keys and save the private key to file 48 | genkeys_out = do_call(['gen-keys']) 49 | gen_pub_64 = re.search(r'YOUR_DEBUG_PUBLIC_KEY[^"]*"([^"]*)"', genkeys_out).group(1) 50 | gen_priv_64 = re.search(r'"([^"]*)" \(private\)', genkeys_out).group(1) 51 | priv_key_file = tmp_path / 'log_sensitive_private.key' 52 | with open(priv_key_file, 'w') as priv_f: 53 | priv_f.write(gen_priv_64) 54 | 55 | sample_plaintext = "The Magic Words are Squeamish Ossifrage" 56 | 57 | encrypted_raw = do_call( 58 | ['encrypt', '--public-key', gen_pub_64, '--message-file', '-'], 59 | stdin=sample_plaintext 60 | ) 61 | 62 | # Check that there's a useful error message if the [encrypted: ...] wrapper is left in 63 | with pytest.raises(subprocess.CalledProcessError) as e: 64 | decrypted = do_call( 65 | ['decrypt', '--private-key-file', priv_key_file, '--message-file', '-'], 66 | stdin=encrypted_raw 67 | ) 68 | assert 'Only include the Base64' in e.value.stderr.decode() 69 | 70 | # Get rid of [encrypted: ...] wrapper/delimiter 71 | encrypted = encrypted_raw.split(' ')[1].rstrip(']') 72 | decrypted = do_call( 73 | ['decrypt', '--private-key-file', priv_key_file, '--message-file', '-'], 74 | stdin=encrypted 75 | ) 76 | 77 | # Decrypted output comes out of CLI with an extra \n, so get rid of that first 78 | assert decrypted.strip('\n') == sample_plaintext 79 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Deprecated monitoring helpers for backward-compatibility. 3 | 4 | IMPORTANT: No new code should be added to this file. 5 | TODO: Remove this file once this code are no longer used. 6 | 7 | """ 8 | import warnings 9 | 10 | from .internal.utils import accumulate as internal_accumulate 11 | from .internal.utils import increment as internal_increment 12 | from .internal.utils import set_custom_attribute as internal_set_custom_attribute 13 | from .internal.utils import set_custom_attributes_for_course_key as internal_set_custom_attributes_for_course_key 14 | 15 | 16 | def accumulate(name, value): 17 | """ 18 | Deprecated method. Use public API instead. 19 | """ 20 | msg = "Use 'monitoring.accumulate' in place of 'monitoring.utils.accumulate'." 21 | warnings.warn(msg, DeprecationWarning) 22 | internal_set_custom_attribute('deprecated_monitoring_utils', f'accumulate[{name}]') 23 | internal_accumulate(name, value) 24 | 25 | 26 | def increment(name): 27 | """ 28 | Deprecated method. Use public API instead. 29 | """ 30 | msg = "Use 'monitoring.increment' in place of 'monitoring.utils.increment'." 31 | warnings.warn(msg, DeprecationWarning) 32 | internal_set_custom_attribute('deprecated_monitoring_utils', f'increment[{name}]') 33 | internal_increment(name) 34 | 35 | 36 | def set_custom_attribute(key, value): 37 | """ 38 | Deprecated method. Use public API instead. 39 | """ 40 | msg = "Use 'monitoring.set_custom_attribute' in place of 'monitoring.utils.set_custom_attribute'." 41 | warnings.warn(msg, DeprecationWarning) 42 | internal_set_custom_attribute('deprecated_monitoring_utils', f'set_custom_attribute[{key}]') 43 | internal_set_custom_attribute(key, value) 44 | 45 | 46 | def set_custom_attributes_for_course_key(course_key): 47 | """ 48 | Deprecated method. Use public API instead. 49 | """ 50 | msg = "Use 'monitoring.set_custom_attributes_for_course_key' in place of " \ 51 | "'monitoring.utils.set_custom_attributes_for_course_key'." 52 | warnings.warn(msg, DeprecationWarning) 53 | internal_set_custom_attribute( 54 | 'deprecated_monitoring_utils', 55 | 'set_custom_attributes_for_course_key[{}]'.format(str(course_key)) 56 | ) 57 | internal_set_custom_attributes_for_course_key(course_key) 58 | 59 | 60 | def set_custom_metric(key, value): # pragma: no cover 61 | """ 62 | Deprecated method to set monitoring custom attribute. 63 | """ 64 | msg = "Use 'set_custom_attribute' in place of 'set_custom_metric'." 65 | warnings.warn(msg, DeprecationWarning) 66 | internal_set_custom_attribute('deprecated_monitoring_utils', f'set_custom_metric[{key}]') 67 | internal_set_custom_attribute(key, value) 68 | 69 | 70 | def set_custom_metrics_for_course_key(course_key): # pragma: no cover 71 | """ 72 | Deprecated method to set monitoring custom attributes for course key. 73 | """ 74 | msg = "Use 'set_custom_attributes_for_course_key' in place of 'set_custom_metrics_for_course_key'." 75 | warnings.warn(msg, DeprecationWarning) 76 | internal_set_custom_attribute( 77 | 'deprecated_monitoring_utils', 78 | 'set_custom_metrics_for_course_key[{}]'.format(str(course_key)) 79 | ) 80 | internal_set_custom_attributes_for_course_key(course_key) 81 | -------------------------------------------------------------------------------- /edx_django_utils/plugins/pluggable_override.py: -------------------------------------------------------------------------------- 1 | """ 2 | Allows overriding existing functions and methods with alternative implementations. 3 | """ 4 | 5 | import functools 6 | from importlib import import_module 7 | 8 | from django.conf import settings 9 | 10 | 11 | def pluggable_override(override): 12 | """ 13 | This decorator allows overriding any function or method by pointing to an alternative implementation 14 | with `override` param. 15 | :param override: path to the alternative function 16 | 17 | Example usage: 18 | 19 | 1. Add this decorator to an existing function `OVERRIDE_TRANSFORM` is the variable name in settings that can be 20 | used for overriding this function. Remember to add the `OVERRIDE_` prefix to the name to have the consistent 21 | namespace for the overrides. 22 | >>> @pluggable_override('OVERRIDE_TRANSFORM') 23 | ... def transform(value): 24 | ... return value + 10 25 | 26 | 2. Prepare an alternative implementation. It will have the same set of arguments as the original function, with the 27 | `prev_fn` added at the beginning. 28 | >>> def decrement(prev_fn, value): 29 | ... if value >= 10: 30 | ... return value - 1 # Return the decremented value. 31 | ... else: 32 | ... return prev_fn(value) - 1 # Call the original `transform` method before decrementing and returning. 33 | 34 | 3. Specify the path in settings (e.g. in `envs/private.py`): 35 | >>> OVERRIDE_TRANSFORM = 'transform_plugin.decrement' 36 | 37 | You can also chain overrides: 38 | >>> OVERRIDE_TRANSFORM = [ 39 | ... 'transform_plugin.decrement', 40 | ... 'transform_plugin.increment', 41 | ... ] 42 | 43 | Another example: 44 | 45 | 1. We want to limit access to a Django view (e.g. `common.djangoapps.student.views.dashboard.student_dashboard`) 46 | to allow only logged in users. To do this add `OVERRIDE_DASHBOARD` to the original function: 47 | >>> @pluggable_override('OVERRIDE_DASHBOARD') 48 | ... def student_dashboard(request): 49 | ... ... # The rest of the implementation is not relevant in this case. 50 | 51 | 2. Prepare an alternative implementation (e.g. in `envs/private.py` to make this example simpler): 52 | >>> from django.contrib.auth.decorators import login_required 53 | ... 54 | ... def dashboard(prev_fn, request): 55 | ... return login_required(prev_fn)(request) 56 | ... 57 | ... OVERRIDE_DASHBOARD = 'lms.envs.private.dashboard' 58 | """ 59 | def decorator(f): 60 | @functools.wraps(f) 61 | def wrapper(*args, **kwargs): 62 | prev_fn = f 63 | 64 | override_functions = getattr(settings, override, ()) 65 | 66 | if isinstance(override_functions, str): 67 | override_functions = [override_functions] 68 | 69 | for impl in override_functions: 70 | module, function = impl.rsplit('.', 1) 71 | mod = import_module(module) 72 | func = getattr(mod, function) 73 | 74 | prev_fn = functools.partial(func, prev_fn) 75 | # Call the last specified function. It can call the previous one, which can call the previous one, etc. 76 | # (until it reaches the base implementation). It can also return without calling `prev_fn`. 77 | return prev_fn(*args, **kwargs) 78 | return wrapper 79 | return decorator 80 | -------------------------------------------------------------------------------- /edx_django_utils/cache/tests/test_middleware.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for the RequestCacheMiddleware. 3 | """ 4 | from unittest.mock import MagicMock, Mock 5 | 6 | from django.test import RequestFactory, TestCase 7 | 8 | from edx_django_utils.cache import middleware 9 | from edx_django_utils.cache.utils import FORCE_CACHE_MISS_PARAM, SHOULD_FORCE_CACHE_MISS_KEY, RequestCache 10 | 11 | TEST_KEY = "clobert" 12 | EXPECTED_VALUE = "bertclob" 13 | TEST_NAMESPACE = "test_namespace" 14 | 15 | 16 | class TestRequestCacheMiddleware(TestCase): # pylint: disable=missing-class-docstring 17 | 18 | def setUp(self): 19 | super().setUp() 20 | self.mock_response = Mock() 21 | self.middleware = middleware.RequestCacheMiddleware(self.mock_response) 22 | self.request = RequestFactory().get('/') 23 | 24 | self.request_cache = RequestCache() 25 | self.other_request_cache = RequestCache(TEST_NAMESPACE) 26 | self._dirty_request_cache() 27 | 28 | def test_process_request(self): 29 | self.middleware.process_request(self.request) 30 | 31 | self._check_request_caches_cleared() 32 | 33 | def test_process_response(self): 34 | response = self.middleware.process_response(self.request, EXPECTED_VALUE) 35 | 36 | self.assertEqual(response, EXPECTED_VALUE) 37 | self._check_request_caches_cleared() 38 | 39 | def _check_request_caches_cleared(self): 40 | """ Checks that all request caches were cleared. """ 41 | self.assertFalse(self.request_cache.get_cached_response(TEST_KEY).is_found) 42 | self.assertFalse(self.other_request_cache.get_cached_response(TEST_KEY).is_found) 43 | 44 | def _dirty_request_cache(self): 45 | """ Dirties the request caches to ensure the middleware is clearing it. """ 46 | self.request_cache.set(TEST_KEY, EXPECTED_VALUE) 47 | self.other_request_cache.set(TEST_KEY, EXPECTED_VALUE) 48 | 49 | 50 | class TestTieredCacheMiddleware(TestCase): # pylint: disable=missing-class-docstring 51 | 52 | def setUp(self): 53 | super().setUp() 54 | self.mock_response = Mock() 55 | self.middleware = middleware.TieredCacheMiddleware(self.mock_response) 56 | self.request = RequestFactory().get('/') 57 | self.request.user = self._mock_user(is_staff=True) 58 | 59 | self.request_cache = RequestCache() 60 | self.request_cache.clear_all_namespaces() 61 | 62 | def test_process_request(self): 63 | self.middleware.process_request(self.request) 64 | 65 | self.assertFalse(self.request_cache.get_cached_response(SHOULD_FORCE_CACHE_MISS_KEY).value) 66 | 67 | def test_process_request_force_cache_miss(self): 68 | request = RequestFactory().get(f'/?{FORCE_CACHE_MISS_PARAM}=tRuE') 69 | request.user = self._mock_user(is_staff=True) 70 | 71 | self.middleware.process_request(request) 72 | 73 | self.assertTrue(self.request_cache.get_cached_response(SHOULD_FORCE_CACHE_MISS_KEY).value) 74 | 75 | def test_process_request_force_cache_miss_non_staff(self): 76 | request = RequestFactory().get(f'/?{FORCE_CACHE_MISS_PARAM}=tRuE') 77 | request.user = self._mock_user(is_staff=False) 78 | 79 | self.middleware.process_request(request) 80 | 81 | self.assertFalse(self.request_cache.get_cached_response(SHOULD_FORCE_CACHE_MISS_KEY).value) 82 | 83 | def _mock_user(self, is_staff=True): 84 | mock_user = MagicMock() 85 | mock_user.is_active = True 86 | mock_user.is_staff = is_staff 87 | return mock_user 88 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean compile_translations coverage diff_cover docs dummy_translations \ 2 | extract_translations fake_translations help pull_translations push_translations \ 3 | quality requirements selfcheck test test-all upgrade validate 4 | 5 | .DEFAULT_GOAL := help 6 | 7 | define BROWSER_PYSCRIPT 8 | import os, webbrowser, sys 9 | try: 10 | from urllib import pathname2url 11 | except: 12 | from urllib.request import pathname2url 13 | 14 | webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) 15 | endef 16 | export BROWSER_PYSCRIPT 17 | BROWSER := python -c "$$BROWSER_PYSCRIPT" 18 | 19 | help: ## display this help message 20 | @echo "Please use \`make ' where is one of" 21 | @perl -nle'print $& if m{^[a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m %-25s\033[0m %s\n", $$1, $$2}' 22 | 23 | clean: ## remove generated byte code, coverage reports, and build artifacts 24 | find . -name '__pycache__' -exec rm -rf {} + 25 | find . -name '*.pyc' -exec rm -f {} + 26 | find . -name '*.pyo' -exec rm -f {} + 27 | find . -name '*~' -exec rm -f {} + 28 | coverage erase 29 | rm -fr build/ 30 | rm -fr dist/ 31 | rm -fr *.egg-info 32 | 33 | coverage: clean ## generate and view HTML coverage report 34 | pytest --cov-report html 35 | $(BROWSER) htmlcov/index.html 36 | 37 | docs: ## generate Sphinx HTML documentation, including API docs 38 | tox -e docs 39 | $(BROWSER) docs/_build/html/index.html 40 | 41 | upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade 42 | upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in 43 | pip install -qr requirements/pip-tools.txt 44 | # Make sure to compile files after any other files they include! 45 | pip-compile --allow-unsafe --rebuild --upgrade -o requirements/pip.txt requirements/pip.in 46 | pip-compile --no-emit-trusted-host --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in 47 | pip install -qr requirements/pip.txt 48 | pip install -qr requirements/pip-tools.txt 49 | pip-compile --no-emit-trusted-host --upgrade -o requirements/base.txt requirements/base.in 50 | pip-compile --no-emit-trusted-host --upgrade -o requirements/test.txt requirements/test.in 51 | pip-compile --no-emit-trusted-host --upgrade -o requirements/doc.txt requirements/doc.in 52 | pip-compile --no-emit-trusted-host --upgrade -o requirements/quality.txt requirements/quality.in 53 | pip-compile --no-emit-trusted-host --upgrade -o requirements/ci.txt requirements/ci.in 54 | pip-compile --no-emit-trusted-host --upgrade -o requirements/dev.txt requirements/dev.in 55 | # Let tox control the Django version for tests 56 | sed '/^[dD]jango==/d' requirements/test.txt > requirements/test.tmp 57 | mv requirements/test.tmp requirements/test.txt 58 | 59 | isort: ## fixes isort issues found during quality check 60 | tox -e isort 61 | 62 | quality: ## check coding style with pycodestyle and pylint 63 | tox -e quality 64 | 65 | requirements: ## install development environment requirements 66 | pip install -r requirements/pip.txt 67 | pip install -qr requirements/pip-tools.txt 68 | pip-sync requirements/dev.txt requirements/private.* 69 | pip install . # CLI entry points 70 | 71 | test: clean ## run tests in the current virtualenv 72 | pytest 73 | 74 | diff_cover: test ## find diff lines that need test coverage 75 | diff-cover coverage.xml 76 | 77 | test-all: ## run tests on every supported Python/Django combination 78 | tox -e quality 79 | tox 80 | 81 | validate: quality test ## run tests and quality checks 82 | 83 | selfcheck: ## check that the Makefile is well-formed 84 | @echo "The Makefile is well-formed." 85 | -------------------------------------------------------------------------------- /edx_django_utils/user/management/commands/manage_group.py: -------------------------------------------------------------------------------- 1 | from django.apps import apps 2 | from django.contrib.auth.models import Group, Permission 3 | from django.contrib.contenttypes.models import ContentType 4 | from django.core.exceptions import ValidationError 5 | from django.core.management.base import BaseCommand, CommandError 6 | from django.db import transaction 7 | from django.utils.translation import gettext as _ 8 | 9 | class Command(BaseCommand): 10 | help = "Creates the specified group, if it does not exist, and sets its permissions." 11 | 12 | def add_arguments(self, parser): 13 | parser.add_argument('group_name') 14 | parser.add_argument('--remove', dest='is_remove', action='store_true') 15 | parser.add_argument('-p', '--permissions', nargs='*', default=[]) 16 | 17 | def _handle_remove(self, group_name): 18 | try: 19 | Group.objects.get(name=group_name).delete() 20 | self.stderr.write(_('Removed group: "{}"').format(group_name)) 21 | except Group.DoesNotExist: 22 | self.stderr.write(_('Did not find a group with name "{}" - skipping.').format(group_name)) 23 | 24 | @transaction.atomic 25 | def handle(self, group_name, is_remove, permissions=None, **options): 26 | if is_remove: 27 | self._handle_remove(group_name) 28 | return 29 | 30 | group, created = Group.objects.get_or_create(name=group_name) 31 | 32 | if created: 33 | try: 34 | group.full_clean() 35 | except ValidationError as exc: 36 | raise CommandError( 37 | _( 38 | 'Invalid group name: "{group_name}". {messages}' 39 | ).format( 40 | group_name=group_name, 41 | messages=exc.messages[0] 42 | ) 43 | ) from None 44 | self.stderr.write(_('Created new group: "{}"').format(group_name)) 45 | else: 46 | self.stderr.write(_('Found existing group: "{}"').format(group_name)) 47 | 48 | new_permissions = self._resolve_permissions(permissions or set()) 49 | 50 | group.permissions.set(new_permissions) 51 | group.save() 52 | 53 | def _resolve_permissions(self, permissions): 54 | new_permissions = set() 55 | for permission in permissions: 56 | try: 57 | app_label, model_name, codename = permission.split(':') 58 | except ValueError: 59 | raise CommandError(_( 60 | 'Invalid permission option: "{}". Please specify permissions ' 61 | 'using the format: app_label:model_name:permission_codename.' 62 | ).format(permission)) from None 63 | try: 64 | model_class = apps.get_model(app_label, model_name) 65 | except LookupError as exc: 66 | raise CommandError(str(exc)) from None 67 | 68 | content_type = ContentType.objects.get_for_model(model_class, for_concrete_model=False) 69 | try: 70 | new_permission = Permission.objects.get( 71 | content_type=content_type, 72 | codename=codename, 73 | ) 74 | except Permission.DoesNotExist: 75 | raise CommandError( 76 | _( 77 | 'Invalid permission codename: "{codename}". No such permission exists ' 78 | 'for the model {module}.{model_name}.' 79 | ).format( 80 | codename=codename, 81 | module=model_class.__module__, 82 | model_name=model_class.__name__, 83 | ) 84 | ) from None 85 | new_permissions.add(new_permission) 86 | return new_permissions 87 | -------------------------------------------------------------------------------- /edx_django_utils/db/tests/test_read_replica.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests of edx_django_utils.db.read_replica. 3 | """ 4 | 5 | from importlib import reload 6 | 7 | from django.conf import settings 8 | from django.contrib import auth 9 | from django.test import TestCase, override_settings 10 | 11 | from .. import read_replica 12 | 13 | WRITER_ONLY_DATABASES = settings.DATABASES.copy() 14 | del WRITER_ONLY_DATABASES["read_replica"] 15 | 16 | READ_REPLICA_DATABASES = settings.DATABASES 17 | DATABASE_ROUTERS = settings.DATABASE_ROUTERS + [ 18 | "edx_django_utils.db.read_replica.ReadReplicaRouter" 19 | ] 20 | User = auth.get_user_model() 21 | 22 | 23 | class TestReadReplica(TestCase): 24 | """ 25 | Tests of edx_django_utils.db.read_replica. 26 | """ 27 | 28 | databases = ["default", "read_replica"] 29 | 30 | def setUp(self): 31 | super().setUp() 32 | self.addCleanup(reload, read_replica) 33 | 34 | def test_read_inside_write_error(self): 35 | with self.assertRaises(AssertionError): 36 | with read_replica.write_queries(): 37 | with read_replica.read_queries_only(): 38 | pass # pragma: no cover 39 | 40 | def test_write_inside_read_error(self): 41 | with self.assertRaises(AssertionError): 42 | with read_replica.read_queries_only(): 43 | with read_replica.write_queries(): 44 | pass # pragma: no cover 45 | 46 | def test_write_inside_write_ok(self): 47 | with read_replica.write_queries(): 48 | with read_replica.write_queries(): 49 | pass 50 | 51 | def test_read_inside_read_ok(self): 52 | with read_replica.read_queries_only(): 53 | with read_replica.read_queries_only(): 54 | pass 55 | 56 | def test_read_replica_name_from_default(self): 57 | reload(read_replica) 58 | assert read_replica.READ_REPLICA_NAME == "read_replica" 59 | 60 | @override_settings(EDX_READ_REPLICA_DB_NAME="new_read_replica") 61 | def test_read_replica_name_from_settings(self): 62 | reload(read_replica) 63 | assert read_replica.READ_REPLICA_NAME == "new_read_replica" 64 | 65 | @override_settings(DATABASES=WRITER_ONLY_DATABASES) 66 | def test_writer_name_from_default(self): 67 | reload(read_replica) 68 | assert read_replica.WRITER_NAME == "default" 69 | 70 | @override_settings(EDX_WRITER_DB_NAME="new_default") 71 | def test_writer_name_from_settings(self): 72 | reload(read_replica) 73 | assert read_replica.WRITER_NAME == "new_default" 74 | 75 | @override_settings(DATABASES=WRITER_ONLY_DATABASES) 76 | def test_read_replica_fallback(self): 77 | reload(read_replica) 78 | assert read_replica.READ_REPLICA_OR_DEFAULT == read_replica.WRITER_NAME 79 | 80 | @override_settings(DATABASES=READ_REPLICA_DATABASES) 81 | def test_read_replica_exists(self): 82 | reload(read_replica) 83 | assert read_replica.READ_REPLICA_OR_DEFAULT == read_replica.READ_REPLICA_NAME 84 | 85 | @override_settings( 86 | DATABASES=READ_REPLICA_DATABASES, DATABASE_ROUTERS=DATABASE_ROUTERS 87 | ) 88 | def test_read_only_queries_from_database(self): 89 | reload(read_replica) 90 | User(username="test_user").save(using=read_replica.WRITER_NAME) 91 | 92 | assert User.objects.all().count() == 1 93 | with read_replica.read_queries_only(): 94 | assert User.objects.all().count() == 0 95 | 96 | @override_settings( 97 | DATABASES=READ_REPLICA_DATABASES, DATABASE_ROUTERS=DATABASE_ROUTERS 98 | ) 99 | def test_router_writes(self): 100 | reload(read_replica) 101 | User(username="test_user").save() 102 | 103 | assert User.objects.all().count() == 1 104 | with read_replica.read_queries_only(): 105 | assert User.objects.all().count() == 0 106 | -------------------------------------------------------------------------------- /requirements/quality.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.8 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | asgiref==3.7.2 8 | # via 9 | # -r requirements/test.txt 10 | # django 11 | astroid==2.15.6 12 | # via 13 | # pylint 14 | # pylint-celery 15 | cffi==1.15.1 16 | # via 17 | # -r requirements/test.txt 18 | # pynacl 19 | click==8.1.7 20 | # via 21 | # -r requirements/test.txt 22 | # click-log 23 | # code-annotations 24 | # edx-lint 25 | click-log==0.4.0 26 | # via edx-lint 27 | code-annotations==1.5.0 28 | # via edx-lint 29 | coverage[toml]==7.3.1 30 | # via 31 | # -r requirements/test.txt 32 | # pytest-cov 33 | ddt==1.6.0 34 | # via -r requirements/test.txt 35 | dill==0.3.7 36 | # via pylint 37 | django==3.2.21 38 | # via 39 | # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt 40 | # -r requirements/test.txt 41 | # django-crum 42 | # django-waffle 43 | django-crum==0.7.9 44 | # via -r requirements/test.txt 45 | django-waffle==4.0.0 46 | # via -r requirements/test.txt 47 | edx-lint==5.3.4 48 | # via -r requirements/quality.in 49 | exceptiongroup==1.1.3 50 | # via 51 | # -r requirements/test.txt 52 | # pytest 53 | iniconfig==2.0.0 54 | # via 55 | # -r requirements/test.txt 56 | # pytest 57 | isort==5.12.0 58 | # via 59 | # -r requirements/quality.in 60 | # pylint 61 | jinja2==3.1.2 62 | # via code-annotations 63 | lazy-object-proxy==1.9.0 64 | # via astroid 65 | markupsafe==2.1.3 66 | # via jinja2 67 | mccabe==0.7.0 68 | # via pylint 69 | mock==5.1.0 70 | # via -r requirements/test.txt 71 | newrelic==9.0.0 72 | # via -r requirements/test.txt 73 | packaging==23.1 74 | # via 75 | # -r requirements/test.txt 76 | # pytest 77 | pbr==5.11.1 78 | # via 79 | # -r requirements/test.txt 80 | # stevedore 81 | platformdirs==3.10.0 82 | # via pylint 83 | pluggy==1.3.0 84 | # via 85 | # -r requirements/test.txt 86 | # pytest 87 | psutil==5.9.5 88 | # via -r requirements/test.txt 89 | pycodestyle==2.11.0 90 | # via -r requirements/quality.in 91 | pycparser==2.21 92 | # via 93 | # -r requirements/test.txt 94 | # cffi 95 | pydocstyle==6.3.0 96 | # via -r requirements/quality.in 97 | pylint==2.17.5 98 | # via 99 | # edx-lint 100 | # pylint-celery 101 | # pylint-django 102 | # pylint-plugin-utils 103 | pylint-celery==0.3 104 | # via edx-lint 105 | pylint-django==2.5.3 106 | # via edx-lint 107 | pylint-plugin-utils==0.8.2 108 | # via 109 | # pylint-celery 110 | # pylint-django 111 | pynacl==1.5.0 112 | # via -r requirements/test.txt 113 | pytest==7.4.2 114 | # via 115 | # -r requirements/test.txt 116 | # pytest-cov 117 | # pytest-django 118 | pytest-cov==4.1.0 119 | # via -r requirements/test.txt 120 | pytest-django==4.5.2 121 | # via -r requirements/test.txt 122 | python-slugify==8.0.1 123 | # via code-annotations 124 | pytz==2023.3.post1 125 | # via 126 | # -r requirements/test.txt 127 | # django 128 | pyyaml==6.0.1 129 | # via code-annotations 130 | six==1.16.0 131 | # via edx-lint 132 | snowballstemmer==2.2.0 133 | # via pydocstyle 134 | sqlparse==0.4.4 135 | # via 136 | # -r requirements/test.txt 137 | # django 138 | stevedore==5.1.0 139 | # via 140 | # -r requirements/test.txt 141 | # code-annotations 142 | text-unidecode==1.3 143 | # via python-slugify 144 | tomli==2.0.1 145 | # via 146 | # -r requirements/test.txt 147 | # coverage 148 | # pylint 149 | # pytest 150 | tomlkit==0.12.1 151 | # via pylint 152 | typing-extensions==4.7.1 153 | # via 154 | # -r requirements/test.txt 155 | # asgiref 156 | # astroid 157 | # pylint 158 | wrapt==1.15.0 159 | # via astroid 160 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst: -------------------------------------------------------------------------------- 1 | Add Code_Owner Custom Attributes to an IDA 2 | ========================================== 3 | 4 | .. contents:: 5 | :local: 6 | :depth: 2 7 | 8 | What are the code owner custom attributes? 9 | ------------------------------------------ 10 | 11 | The code owner custom attributes can be used to create custom dashboards and alerts for monitoring the things that you own. It was originally introduced for the LMS, as is described in this `ADR on monitoring by code owner`_. 12 | 13 | The code owner custom attributes consist of: 14 | 15 | * code_owner: The owner name. When themes and squads are used, this will be the theme and squad names joined by a hyphen. 16 | * code_owner_theme: The theme name of the owner. 17 | * code_owner_squad: The squad name of the owner. Use this to avoid issues when theme name changes. 18 | 19 | You can now easily add this same attribute to any IDA so that your dashboards and alerts can work across multiple IDAs at once. 20 | 21 | If you want to know about custom attributes in general, see :doc:`using_custom_attributes`. 22 | 23 | .. _ADR on monitoring by code owner: https://github.com/openedx/edx-platform/blob/master/lms/djangoapps/monitoring/docs/decisions/0001-monitoring-by-code-owner.rst 24 | 25 | Setting up the Middleware 26 | ------------------------- 27 | 28 | You simply need to add ``edx_django_utils.monitoring.CodeOwnerMonitoringMiddleware`` as described in the README to make this functionality available. Then it is ready to be configured. 29 | 30 | Handling celery tasks 31 | --------------------- 32 | 33 | Celery tasks require use of a special decorator to set the ``code_owner`` custom attributes because no middleware will be run. 34 | 35 | Here is an example:: 36 | 37 | @task() 38 | @set_code_owner_attribute 39 | def example_task(): 40 | ... 41 | 42 | If the task is not compatible with additional decorators, you can use the following alternative:: 43 | 44 | @task() 45 | def example_task(): 46 | set_code_owner_attribute_from_module(__name__) 47 | ... 48 | 49 | An untested potential alternative to the decorator is documented in the `Code Owner for Celery Tasks ADR`_, should we run into maintenance issues using the decorator. 50 | 51 | .. _Code Owner for Celery Tasks ADR: https://github.com/openedx/edx-django-utils/blob/master/edx_django_utils/monitoring/docs/decisions/0003-code-owner-for-celery-tasks.rst 52 | Configuring your app settings 53 | ----------------------------- 54 | 55 | Once the Middleware is made available, simply set the Django Settings ``CODE_OWNER_MAPPINGS`` and ``CODE_OWNER_THEMES`` appropriately. 56 | 57 | The following example shows how you can include an optional config for a catch-all using ``'*'``. Although you might expect this example to use Python, it is intentionally illustrated in YAML because the catch-all requires special care in YAML. 58 | 59 | :: 60 | 61 | # YAML format of example CODE_OWNER_MAPPINGS 62 | CODE_OWNER_MAPPINGS: 63 | theme-x-team-red: 64 | - xblock_django 65 | - openedx.core.djangoapps.xblock 66 | theme-x-team-blue: 67 | - '*' # IMPORTANT: you must surround * with quotes in yml 68 | 69 | # YAML format of example CODE_OWNER_THEMES 70 | CODE_OWNER_THEMES: 71 | theme-x: 72 | - theme-x-team-red 73 | - theme-x-team-blue 74 | 75 | How to find and fix code_owner mappings 76 | --------------------------------------- 77 | 78 | If you are missing the ``code_owner`` custom attributes on a particular Transaction or Error, or if ``code_owner`` is matching the catch-all, but you want to add a more specific mapping, you can use the other `code_owner supporting attributes`_ to determine what the appropriate mappings should be. 79 | 80 | .. _code_owner supporting attributes: https://github.com/openedx/edx-django-utils/blob/c022565/edx_django_utils/monitoring/internal/code_owner/middleware.py#L30-L34 81 | 82 | Updating New Relic monitoring 83 | ----------------------------- 84 | 85 | To update monitoring in the event of a squad or theme name change, see :doc:`update_monitoring_for_squad_or_theme_changes`. 86 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/internal/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Defines several custom monitoring helpers, some of which work with 3 | the CachedCustomMonitoringMiddleware. 4 | 5 | Usage: 6 | 7 | from edx_django_utils.monitoring import accumulate 8 | ... 9 | accumulate('xb_user_state.get_many.num_items', 4) 10 | 11 | There is no need to do anything else. The custom attributes are automatically 12 | cleared before the next request. 13 | 14 | We try to keep track of our custom monitoring at: 15 | https://openedx.atlassian.net/wiki/spaces/PERF/pages/54362736/Custom+Attributes+in+New+Relic 16 | 17 | At this time, the custom monitoring will only be reported to New Relic. 18 | 19 | """ 20 | from .middleware import CachedCustomMonitoringMiddleware 21 | 22 | try: 23 | import newrelic.agent 24 | except ImportError: # pragma: no cover 25 | newrelic = None # pylint: disable=invalid-name 26 | 27 | 28 | def accumulate(name, value): 29 | """ 30 | Accumulate monitoring custom attribute for the current request. 31 | 32 | The named attribute is accumulated by a numerical amount using the sum. All 33 | attributes are queued up in the request_cache for this request. At the end of 34 | the request, the monitoring_utils middleware will batch report all 35 | queued accumulated attributes to the monitoring tool (e.g. New Relic). 36 | 37 | Arguments: 38 | name (str): The attribute name. It should be period-delimited, and 39 | increase in specificity from left to right. For example: 40 | 'xb_user_state.get_many.num_items'. 41 | value (number): The amount to accumulate into the named attribute. When 42 | accumulate() is called multiple times for a given attribute name 43 | during a request, the sum of the values for each call is reported 44 | for that attribute. For attributes which don't make sense to accumulate, 45 | use ``set_custom_attribute`` instead. 46 | 47 | """ 48 | CachedCustomMonitoringMiddleware.accumulate_attribute(name, value) 49 | 50 | 51 | def increment(name): 52 | """ 53 | Increment a monitoring custom attribute representing a counter. 54 | 55 | Here we simply accumulate a new custom attribute with a value of 1, and the 56 | middleware should automatically aggregate this attribute. 57 | """ 58 | accumulate(name, 1) 59 | 60 | 61 | def set_custom_attributes_for_course_key(course_key): 62 | """ 63 | Set monitoring custom attributes related to a course key. 64 | 65 | This is not cached, and only support reporting to New Relic Insights. 66 | 67 | """ 68 | if newrelic: # pragma: no cover 69 | newrelic.agent.add_custom_parameter('course_id', str(course_key)) 70 | newrelic.agent.add_custom_parameter('org', str(course_key.org)) 71 | 72 | 73 | def set_custom_attribute(key, value): 74 | """ 75 | Set monitoring custom attribute. 76 | 77 | This is not cached, and only support reporting to New Relic Insights. 78 | 79 | """ 80 | if newrelic: # pragma: no cover 81 | # note: parameter is new relic's older name for attributes 82 | newrelic.agent.add_custom_parameter(key, value) 83 | 84 | 85 | def record_exception(): 86 | """ 87 | Records a caught exception to the monitoring system. 88 | 89 | Note: By default, only unhandled exceptions are monitored. This function 90 | can be called to record exceptions as monitored errors, even if you handle 91 | the exception gracefully from a user perspective. 92 | 93 | For more details, see: 94 | https://docs.newrelic.com/docs/agents/python-agent/python-agent-api/recordexception-python-agent-api 95 | 96 | """ 97 | if newrelic: # pragma: no cover 98 | newrelic.agent.record_exception() 99 | 100 | 101 | def background_task(*args, **kwargs): 102 | """ 103 | Handles monitoring for background tasks that are not passed in through the web server like 104 | celery and event consuming tasks. 105 | 106 | For more details, see: 107 | https://docs.newrelic.com/docs/apm/agents/python-agent/supported-features/monitor-non-web-scripts-worker-processes-tasks-functions 108 | 109 | """ 110 | def noop_decorator(func): 111 | return func 112 | 113 | if newrelic: # pragma: no cover 114 | return newrelic.agent.background_task(*args, **kwargs) 115 | else: 116 | return noop_decorator 117 | -------------------------------------------------------------------------------- /edx_django_utils/plugins/plugin_contexts.py: -------------------------------------------------------------------------------- 1 | """ 2 | Various functions to get view contexts 3 | 4 | Please remember to expose any new public methods in the `__init__.py` file. 5 | """ 6 | import functools 7 | from importlib import import_module 8 | from logging import getLogger 9 | 10 | from . import constants, registry 11 | 12 | log = getLogger(__name__) 13 | 14 | 15 | def get_plugins_view_context(project_type, view_name, existing_context=None): 16 | """ 17 | Returns a dict of additional view context. Will check if any plugin apps 18 | have that view in their view_context_config, and if so will call their 19 | selected function to get their context dicts. 20 | 21 | Params: 22 | project_type: a string that determines which project the view is being called in. See the 23 | ProjectType enum in 24 | https://github.com/openedx/edx-platform/blob/2dc79bcab42dafed2c122eb808cdd5604327c890/openedx/core/ 25 | djangoapps/plugins/constants.py#L14 26 | for valid options for edx-platform. 27 | view_name: a string that determines which view needs the additional context. These are globally unique and 28 | noted in the api.py in the view's app. 29 | existing_context: a dictionary which includes all of the data that the page was going to render with prior 30 | to the addition of each plugin's context. This is what will be passed to plugins so they may choose 31 | what data to add to the view. 32 | """ 33 | aggregate_context = {"plugins": {}} 34 | 35 | if existing_context is None: 36 | existing_context = {} 37 | 38 | context_functions = _get_cached_context_functions_for_view(project_type, view_name) 39 | 40 | for (context_function, plugin_name) in context_functions: 41 | try: 42 | plugin_context = context_function(existing_context) 43 | except Exception as exc: # pylint: disable=broad-except 44 | # We're catching this because we don't want the core to blow up when a 45 | # plugin is broken. This exception will probably need some sort of 46 | # monitoring hooked up to it to make sure that these errors don't go 47 | # unseen. 48 | log.exception("Failed to call plugin context function. Error: %s", exc) 49 | continue 50 | 51 | aggregate_context["plugins"][plugin_name] = plugin_context 52 | 53 | return aggregate_context 54 | 55 | 56 | @functools.lru_cache(maxsize=None) 57 | def _get_cached_context_functions_for_view(project_type, view_name): 58 | """ 59 | Returns a list of tuples where the first item is the context function 60 | and the second item is the name of the plugin it's being called from. 61 | 62 | NOTE: These will be functions will be cached (in RAM not memcache) on this unique 63 | combination. If we enable many new views to use this system, we may notice an 64 | increase in memory usage as the entirety of these functions will be held in memory. 65 | """ 66 | context_functions = [] 67 | for app_config in registry.get_plugin_app_configs(project_type): 68 | context_function_path = _get_context_function_path( 69 | app_config, project_type, view_name 70 | ) 71 | if context_function_path: 72 | module_path, _, name = context_function_path.rpartition(".") 73 | try: 74 | module = import_module(module_path) 75 | except ImportError: 76 | log.exception( 77 | "Failed to import %s plugin when creating %s context", 78 | module_path, 79 | view_name, 80 | ) 81 | continue 82 | context_function = getattr(module, name, None) 83 | if context_function: 84 | plugin_name, _, _ = module_path.partition(".") 85 | context_functions.append((context_function, plugin_name)) 86 | else: 87 | log.warning( 88 | "Failed to retrieve %s function from %s plugin when creating %s context", 89 | name, 90 | module_path, 91 | view_name, 92 | ) 93 | return context_functions 94 | 95 | 96 | def _get_context_function_path(app_config, project_type, view_name): 97 | plugin_config = getattr(app_config, constants.PLUGIN_APP_CLASS_ATTRIBUTE_NAME, {}) 98 | context_config = plugin_config.get(constants.PluginContexts.CONFIG, {}) 99 | project_type_settings = context_config.get(project_type, {}) 100 | return project_type_settings.get(view_name) 101 | -------------------------------------------------------------------------------- /edx_django_utils/plugins/docs/decisions/0001-plugin-contexts.rst: -------------------------------------------------------------------------------- 1 | Plugin Contexts 2 | =============== 3 | 4 | Status 5 | ------ 6 | 7 | Accepted 8 | 9 | Context 10 | ------- 11 | 12 | .. note:: This decision was originally written while this plugins directory still lived in edx-platform. So the language of this decision reflects the origins of this file in edx-platform. Assume the words below also apply to any IDA that has this plugin system setup. 13 | 14 | edx-platform contains a plugin system (https://github.com/openedx/edx-platform/tree/master/openedx/core/djangoapps/plugins) which allows new Django apps to be installed inside the LMS and Studio without requiring the LMS/Studio to know about them. This is what enables us to move to a small and extensible core. While we had the ability to add settings, URLs, and signal handlers in our plugins, there wasn't any way for a plugin to affect the commonly used pages that the core was delivering. Thus a plugin couldn't change any details on the dashboard, courseware, or any other rendered page that the platform delivered. 15 | 16 | Decisions 17 | --------- 18 | 19 | We have added the ability to add page context additions to the plugin system. This means that a plugin will be able to add context to any view where it is enabled. To support this we have decided: 20 | 21 | 1. Plugins will define a callable function that the LMS and/or studio can import and call, which will return additional context to be added. 22 | 2. Every page that a plugin wants to add context to, must add a line to add the plugin contexts directly before the render. 23 | 3. Plugin context will live in a dictionary called "plugins" that will be passed into the context the templates receive. The structure will look like: 24 | 25 | .. code-block:: 26 | 27 | { 28 | ..existing context values.. 29 | "plugins": { 30 | "my_new_plugin": {... my_new_plugins's values ...}, 31 | "my_other_plugin": {... my_other_plugin's values ...}, 32 | } 33 | } 34 | 35 | 4. Each view will have a constant name that will be defined within it's app's API.py which will be used by plugins. These must be globally unique. These will also be recorded in the rendering app's README.rst file. 36 | 5. Plugin apps have the option to either use the view name strings directly or import the constants from the rendering app's api.py if the plugin is part of the edx-platform repo. 37 | 6. For now, in order to use these new context data items, we must use theming alongside this to keep the new context out of the core. This may be iterated on in the future. 38 | 39 | Implementation 40 | -------------- 41 | 42 | In the plugin app 43 | ~~~~~~~~~~~~~~~~~ 44 | 45 | Config 46 | ++++++ 47 | 48 | Inside of the AppConfig of your new plugin app, add a "view_context_config" item like below. 49 | 50 | * The format will be ``{"globally_unique_view_name": "function_inside_plugin_app"}`` 51 | * The function name & path don't need to be named anything specific, so long as they work 52 | * These functions will be called on **every** render of that view, so keep them efficient or memoize them if they aren't user specific. 53 | 54 | .. code-block:: 55 | 56 | class MyAppConfig(AppConfig): 57 | name = "my_app" 58 | 59 | plugin_app = { 60 | "view_context_config": { 61 | "lms.djangoapp": { 62 | "course_dashboard": "my_app.context_api.get_dashboard_context" 63 | } 64 | } 65 | } 66 | 67 | Function 68 | ++++++++ 69 | 70 | The function that will be called by the plugin system should accept a single parameter which will be the previously existing context. It should then return a dictionary which consists of items which will be added to the context 71 | 72 | Example: 73 | .. code-block:: 74 | 75 | def my_context_function(existing_context, *args, **kwargs): 76 | additional_context = {"some_plugin_value": 10} 77 | if existing_context.get("some_core_value"): 78 | additional_context.append({"some_other_plugin_value": True}) 79 | return additional_context 80 | 81 | 82 | In the core (LMS / Studio) 83 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 84 | The view you wish to add context to should have the following pieces enabled: 85 | 86 | * A constant defined inside the app's for the globally unique view name. 87 | * The view must call lines similar to the below right before the render so that the plugin has the full context. 88 | .. code-block:: 89 | 90 | context_from_plugins = get_plugins_view_context( 91 | 'lms.djangoapp', 92 | current_app.api.THIS_VIEW_NAME, 93 | context 94 | ) 95 | context.update(context_from_plugins) 96 | -------------------------------------------------------------------------------- /edx_django_utils/cache/README.rst: -------------------------------------------------------------------------------- 1 | Cache Utils 2 | =========== 3 | 4 | Cache utilities that implement `OEP-0022: Caching in Django`_. 5 | 6 | .. _`OEP-0022: Caching in Django`: https://github.com/openedx/open-edx-proposals/blob/master/oeps/oep-0022-bp-django-caches.rst 7 | 8 | get_cache_key 9 | ------------- 10 | 11 | A function for easily creating cache keys. See its docstring for details. 12 | 13 | RequestCache 14 | ------------ 15 | 16 | A thread-local for storing request scoped cache values. 17 | 18 | An optional namespace can be used with the RequestCache, or you can use the `DEFAULT_REQUEST_CACHE`. 19 | 20 | RequestCacheMiddleware 21 | ---------------------- 22 | 23 | You must include 'edx_django_utils.cache.middleware.RequestCacheMiddleware' when using the RequestCache to ensure it is emptied between requests. This should be added before most middleware, in case any other middleware wants to use the request cache. 24 | 25 | Note: This middleware may just be a safety net, but safe is good. 26 | 27 | TieredCache 28 | ----------- 29 | 30 | The first tier is the default request cache that is tied to the life of a given request. The second tier is the Django cache -- e.g. the "default" entry in settings.CACHES, typically backed by memcached. 31 | 32 | Some baseline rules: 33 | 34 | 1. Treat it as a global namespace, like any other cache. The per-request local cache is only going to live for the lifetime of one request, but the backing cache is going to be something like Memcached, where key collision is possible. 35 | 36 | 2. Timeouts are ignored for the purposes of the in-memory request cache, but do apply to the Django cache. One consequence of this is that sending an explicit timeout of 0 in `set_all_tiers` will cause that item to only be cached across the duration of the request and will not cause a write to the remote cache. 37 | 38 | Sample Usage (cache hit):: 39 | 40 | x_cached_response = TieredCache.get_cached_response(key) 41 | if x_cached_response.is_found: 42 | return x_cached_response.value 43 | # calculate x, set in cache, and return value. 44 | 45 | Sample Usage (cache miss):: 46 | 47 | x_cached_response = TieredCache.get_cached_response(key) 48 | if not x_cached_response.is_found: 49 | # calculate x, set in cache, and return value. 50 | return x_cached_response.value 51 | 52 | Warning when storing bools 53 | ^^^^^^^^^^^^^^^^^^^^^^^^^^ 54 | 55 | **Warning**: When storing a bool in a TieredCache that uses Memcached, `Memcached will return an int`_. However, the RequestCache will return a bool. Therefore, the first time a bool is set the TieredCache will return a bool and in later requests the TieredCache will return an int. 56 | 57 | Where possible, you can ensure a consistent return value by storing ``int(my_bool)`` rather than ``my_bool``. 58 | 59 | Additionally, when checking the value, do the following check that works for ints:: 60 | 61 | # do this. 62 | if my_bool_cached_response.is_found: 63 | if my_bool_cached_response.value: 64 | ... 65 | 66 | Do **not** explictly test against ``True`` or ``False``:: 67 | 68 | # do NOT do this. 69 | if my_bool_cached_response.is_found: 70 | if my_bool_cached_response.value is True: 71 | ... 72 | 73 | .. _Memcached will return an int: https://stackoverflow.com/questions/8169001/why-is-bool-a-subclass-of-int 74 | 75 | TieredCacheMiddleware 76 | --------------------- 77 | 78 | You must include 'edx_django_utils.cache.middleware.TieredCacheMiddleware' when using the TieredCache if you want to enable the `Force Django Cache Miss`_ functionality. 79 | 80 | This middleware should come after the required RequestCacheMiddleware, which the TieredCache needs because it uses RequestCache internally. Additionally, since this functionality checks for staff permissions, it should come after any authentication middleware. Here is an example:: 81 | 82 | MIDDLEWARE = ( 83 | 'edx_django_utils.cache.middleware.RequestCacheMiddleware', 84 | 'django.contrib.sessions.middleware.SessionMiddleware', 85 | ... 86 | # TieredCacheMiddleware middleware must come after these. 87 | 'edx_django_utils.cache.middleware.TieredCacheMiddleware', 88 | ) 89 | 90 | Force Django Cache Miss 91 | ^^^^^^^^^^^^^^^^^^^^^^^ 92 | 93 | To force recompute a value stored in the django cache, add the query parameter 'force_cache_miss'. This will force a CACHE_MISS. 94 | 95 | This requires staff permissions. 96 | 97 | Example:: 98 | 99 | http://clobert.com/api/v1/resource?force_cache_miss=true 100 | 101 | 102 | CachedResponse 103 | -------------- 104 | 105 | A CachedResponse includes the cache miss/hit status (is_found) and the value stored in the cache (for cache hits). 106 | 107 | The purpose of the CachedResponse is to avoid a common bug with the default Django cache interface where a cache hit that is Falsey (e.g. None) is misinterpreted as a cache miss. 108 | 109 | An example of the Bug:: 110 | 111 | # DON'T DO THIS! 112 | cache_value = cache.get(key) 113 | if cache_value: 114 | # calculated value is None, set None in cache, and return value. 115 | # BUG: None will be treated as a cache miss every time. 116 | return cache_value 117 | 118 | Future Ideas 119 | ------------ 120 | 121 | * See `ARCH-240`_ for a discussion of additional cache utilities that could be made available. 122 | 123 | .. _ARCH-240: https://openedx.atlassian.net/browse/ARCH-240 124 | -------------------------------------------------------------------------------- /edx_django_utils/db/read_replica.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools for making queries read from the read-replica database, rather than from the writer database. 3 | 4 | The read-replica can be used for 5 | a. long-running queries that aren't time sensitive 6 | b. reads of rows that are frequently written, but where reads can be eventually consistent 7 | 8 | Settings: 9 | EDX_READ_REPLICA_DB_NAME: The name of the read-replica in the DATABASES django setting. 10 | Defaults to "read_replica". 11 | EDX_WRITER_DB_NAME: The name of the writer in the DATABASES django setting. 12 | Defaults to "default". 13 | """ 14 | import threading 15 | from contextlib import contextmanager 16 | 17 | from django.conf import settings 18 | 19 | READ_REPLICA_NAME = getattr(settings, "EDX_READ_REPLICA_DB_NAME", "read_replica") 20 | WRITER_NAME = getattr(settings, "EDX_WRITER_DB_NAME", "default") 21 | 22 | READ_REPLICA_OR_DEFAULT = ( 23 | READ_REPLICA_NAME if READ_REPLICA_NAME in settings.DATABASES else WRITER_NAME 24 | ) 25 | 26 | 27 | def use_read_replica_if_available(queryset): 28 | """ 29 | If there is a database called 'read_replica', 30 | use that database for the queryset / manager. 31 | 32 | Example usage: 33 | queryset = use_read_replica_if_available(SomeModel.objects.filter(...)) 34 | 35 | Arguments: 36 | queryset (QuerySet) 37 | 38 | Returns: QuerySet 39 | """ 40 | return queryset.using(READ_REPLICA_OR_DEFAULT) 41 | 42 | 43 | def read_replica_or_default(): 44 | """ 45 | If there is a database called READ_REPLICA_DB, 46 | return READ_REPLICA_DB, otherwise return WRITER_NAME. 47 | 48 | This function is similiar to `use_read_replica_if_available`, 49 | but is be more syntactically convenient for method call chaining. 50 | Also, it always falls back to WRITER_NAME, 51 | no matter what the queryset was using before. 52 | 53 | Example usage: 54 | queryset = SomeModel.objects.filter(...).using(read_replica_or_default()) 55 | 56 | Returns: str 57 | """ 58 | return READ_REPLICA_OR_DEFAULT 59 | 60 | 61 | @contextmanager 62 | def read_queries_only(): 63 | """ 64 | A context manager that sets all reads inside it to be from the read-replica. 65 | 66 | It is an error to call this from inside a write_queries context. 67 | 68 | The ReadReplicaRouter must be used for this decorator to affect queries. 69 | """ 70 | old_db_name = _storage.db_name 71 | assert ( 72 | old_db_name is None or old_db_name == READ_REPLICA_NAME 73 | ), "Can't use read_queries_only inside a write_queries contextmanager" 74 | _storage.db_name = READ_REPLICA_NAME 75 | try: 76 | yield 77 | finally: 78 | _storage.db_name = old_db_name 79 | 80 | 81 | @contextmanager 82 | def write_queries(): 83 | """ 84 | A context manager that sets all reads inside it to be from the writer. 85 | Use this to annotate code that has both reads and writes, where the writes depend 86 | on the values read. This will allow all of that code to exist within a transaction. 87 | 88 | Using this contextmanager will prevent any contained call from using `read_queries_only` 89 | in order to read from the read-replica. 90 | 91 | It is an error to call this from inside a read_queries_only context. 92 | 93 | The ReadReplicaRouter must be used for this decorator to affect queries. 94 | """ 95 | old_db_name = _storage.db_name 96 | assert ( 97 | old_db_name is None or old_db_name == WRITER_NAME 98 | ), "Can't use write_queries inside a read_only_queries contextmanager" 99 | 100 | _storage.db_name = WRITER_NAME 101 | try: 102 | yield 103 | finally: 104 | _storage.db_name = old_db_name 105 | 106 | 107 | class _ReadReplicaRouterStorage(threading.local): 108 | def __init__(self): 109 | super().__init__() 110 | self.db_name = None 111 | 112 | 113 | _storage = _ReadReplicaRouterStorage() 114 | 115 | 116 | class ReadReplicaRouter: 117 | """ 118 | A database router that by default, reads from the writer database, 119 | but can be overridden with a context manager to route all reads 120 | to the read-replica. 121 | 122 | See https://docs.djangoproject.com/en/2.2/topics/db/multi-db/#automatic-database-routing 123 | """ 124 | 125 | def db_for_read(self, model, **hints): # pylint: disable=unused-argument 126 | """ 127 | Reads go the active reader name 128 | """ 129 | return ( 130 | _storage.db_name if _storage.db_name in settings.DATABASES else WRITER_NAME 131 | ) 132 | 133 | def db_for_write(self, model, **hints): # pylint: disable=unused-argument 134 | """ 135 | Writes always go to the writer. 136 | """ 137 | return WRITER_NAME 138 | 139 | def allow_relation( 140 | self, obj1, obj2, **hints 141 | ): # pylint: disable=unused-argument, protected-access 142 | """ 143 | Relations between objects are allowed if both objects are 144 | in either the read-replica or the writer. 145 | """ 146 | db_list = (READ_REPLICA_NAME, WRITER_NAME) 147 | if obj1._state.db in db_list and obj2._state.db in db_list: 148 | return True 149 | return None 150 | 151 | def allow_migrate( 152 | self, db, app_label, model_name=None, **hints 153 | ): # pylint: disable=unused-argument 154 | """ 155 | All non-auth models end up in this pool. 156 | """ 157 | return True 158 | -------------------------------------------------------------------------------- /edx_django_utils/security/csp/tests/test_middleware.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for Content-Security-Policy middleware. 3 | """ 4 | 5 | from unittest import TestCase 6 | from unittest.mock import Mock, patch 7 | 8 | import ddt 9 | import pytest 10 | from django.core.exceptions import MiddlewareNotUsed 11 | from django.test import override_settings 12 | 13 | import edx_django_utils.security.csp.middleware as csp 14 | 15 | 16 | @ddt.ddt 17 | class TestLoadHeaders(TestCase): 18 | """Test loading of headers from settings.""" 19 | 20 | @ddt.unpack 21 | @ddt.data( 22 | # Empty settings 23 | [{}, {}], 24 | # The reporting URL and endpoint names alone don't do anything 25 | [{"CSP_STATIC_REPORTING_URI": "http://localhost"}, {}], 26 | [{"CSP_STATIC_REPORTING_NAME": "default"}, {}], 27 | [ 28 | { 29 | "CSP_STATIC_REPORTING_URI": "http://localhost", 30 | "CSP_STATIC_REPORTING_NAME": "default" 31 | }, 32 | {}, 33 | ], 34 | # Just the enforcement header 35 | [ 36 | {"CSP_STATIC_ENFORCE": "default-src https:"}, 37 | {'Content-Security-Policy': "default-src https:"}, 38 | ], 39 | # Just the reporting header 40 | [ 41 | {"CSP_STATIC_REPORT_ONLY": "default-src 'none'"}, 42 | {'Content-Security-Policy-Report-Only': "default-src 'none'"}, 43 | ], 44 | # Reporting URL is automatically appended to headers 45 | [ 46 | { 47 | "CSP_STATIC_ENFORCE": "default-src https:", 48 | "CSP_STATIC_REPORT_ONLY": "default-src 'none'", 49 | "CSP_STATIC_REPORTING_URI": "http://localhost", 50 | }, 51 | { 52 | 'Content-Security-Policy': "default-src https:; report-uri http://localhost", 53 | 'Content-Security-Policy-Report-Only': "default-src 'none'; report-uri http://localhost", 54 | }, 55 | ], 56 | # ...and when an endpoint name is supplied, 57 | # Reporting-Endpoints is added and a report-to directive is 58 | # included. 59 | [ 60 | { 61 | "CSP_STATIC_ENFORCE": "default-src https:", 62 | "CSP_STATIC_REPORT_ONLY": "default-src 'none'", 63 | "CSP_STATIC_REPORTING_URI": "http://localhost", 64 | "CSP_STATIC_REPORTING_NAME": "default", 65 | }, 66 | { 67 | 'Reporting-Endpoints': 'default="http://localhost"', 68 | 'Content-Security-Policy': "default-src https:; report-uri http://localhost; report-to default", 69 | 'Content-Security-Policy-Report-Only': ( 70 | "default-src 'none'; report-uri http://localhost; report-to default" 71 | ), 72 | }, 73 | ], 74 | # Adding a reporting endpoint name without a URL doesn't change anything. 75 | [ 76 | { 77 | "CSP_STATIC_REPORT_ONLY": "default-src 'none'", 78 | "CSP_STATIC_REPORTING_NAME": "default", 79 | }, 80 | {'Content-Security-Policy-Report-Only': "default-src 'none'"}, 81 | ], 82 | # Any newlines and trailing semicolon are stripped. 83 | [ 84 | { 85 | "CSP_STATIC_REPORT_ONLY": "default-src 'self'; \n \t frame-src 'none'; \n ", 86 | "CSP_STATIC_REPORTING_URI": "http://localhost", 87 | }, 88 | { 89 | 'Content-Security-Policy-Report-Only': ( 90 | "default-src 'self'; frame-src 'none'; " 91 | "report-uri http://localhost" 92 | ), 93 | }, 94 | ], 95 | ) 96 | def test_load_headers(self, settings, headers): 97 | with override_settings(**settings): 98 | assert csp._load_headers() == headers # pylint: disable=protected-access 99 | 100 | 101 | @ddt.ddt 102 | class TestHeaderManipulation(TestCase): 103 | """Test _append_headers""" 104 | 105 | @ddt.unpack 106 | @ddt.data( 107 | [{}, {}, {}], 108 | [ 109 | {'existing': 'aaa', 'multi': '111'}, 110 | {'multi': '222', 'new': 'xxx'}, 111 | {'existing': 'aaa', 'multi': '111, 222', 'new': 'xxx'}, 112 | ], 113 | ) 114 | def test_append_headers(self, response_headers, more_headers, expected): 115 | csp._append_headers(response_headers, more_headers) # pylint: disable=protected-access 116 | assert response_headers == expected 117 | 118 | 119 | @ddt.ddt 120 | class TestCSPMiddleware(TestCase): 121 | """Test the actual middleware.""" 122 | 123 | def setUp(self): 124 | super().setUp() 125 | self.fake_response = Mock() 126 | self.fake_response.headers = {'Existing': 'something'} 127 | 128 | def test_make_middleware_unused(self): 129 | with pytest.raises(MiddlewareNotUsed): 130 | csp.content_security_policy_middleware(lambda _: self.fake_response) 131 | 132 | @override_settings(CSP_STATIC_ENFORCE="default-src: https:") 133 | def test_make_middleware_configured(self): 134 | handler = csp.content_security_policy_middleware(lambda _: self.fake_response) 135 | 136 | assert handler(Mock()) is self.fake_response 137 | 138 | # Headers have been mutated in place (if flag enabled) 139 | assert self.fake_response.headers == { 140 | 'Existing': 'something', 141 | 'Content-Security-Policy': 'default-src: https:', 142 | } 143 | -------------------------------------------------------------------------------- /requirements/doc.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.8 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | accessible-pygments==0.0.4 8 | # via pydata-sphinx-theme 9 | alabaster==0.7.13 10 | # via sphinx 11 | asgiref==3.7.2 12 | # via 13 | # -r requirements/test.txt 14 | # django 15 | babel==2.12.1 16 | # via 17 | # pydata-sphinx-theme 18 | # sphinx 19 | beautifulsoup4==4.12.2 20 | # via pydata-sphinx-theme 21 | certifi==2023.7.22 22 | # via requests 23 | cffi==1.15.1 24 | # via 25 | # -r requirements/test.txt 26 | # cryptography 27 | # pynacl 28 | charset-normalizer==3.2.0 29 | # via requests 30 | click==8.1.7 31 | # via -r requirements/test.txt 32 | coverage[toml]==7.3.1 33 | # via 34 | # -r requirements/test.txt 35 | # pytest-cov 36 | cryptography==41.0.3 37 | # via secretstorage 38 | ddt==1.6.0 39 | # via -r requirements/test.txt 40 | django==3.2.21 41 | # via 42 | # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt 43 | # -r requirements/test.txt 44 | # django-crum 45 | # django-waffle 46 | django-crum==0.7.9 47 | # via -r requirements/test.txt 48 | django-waffle==4.0.0 49 | # via -r requirements/test.txt 50 | doc8==0.11.2 51 | # via 52 | # -c requirements/constraints.txt 53 | # -r requirements/doc.in 54 | docutils==0.17.1 55 | # via 56 | # doc8 57 | # pydata-sphinx-theme 58 | # readme-renderer 59 | # restructuredtext-lint 60 | # sphinx 61 | exceptiongroup==1.1.3 62 | # via 63 | # -r requirements/test.txt 64 | # pytest 65 | idna==3.4 66 | # via requests 67 | imagesize==1.4.1 68 | # via sphinx 69 | importlib-metadata==6.8.0 70 | # via 71 | # keyring 72 | # twine 73 | importlib-resources==6.0.1 74 | # via keyring 75 | iniconfig==2.0.0 76 | # via 77 | # -r requirements/test.txt 78 | # pytest 79 | jaraco-classes==3.3.0 80 | # via keyring 81 | jeepney==0.8.0 82 | # via 83 | # keyring 84 | # secretstorage 85 | jinja2==3.1.2 86 | # via sphinx 87 | keyring==24.2.0 88 | # via twine 89 | markdown-it-py==3.0.0 90 | # via rich 91 | markupsafe==2.1.3 92 | # via jinja2 93 | mdurl==0.1.2 94 | # via markdown-it-py 95 | mock==5.1.0 96 | # via -r requirements/test.txt 97 | more-itertools==10.1.0 98 | # via jaraco-classes 99 | newrelic==9.0.0 100 | # via -r requirements/test.txt 101 | nh3==0.2.14 102 | # via readme-renderer 103 | packaging==23.1 104 | # via 105 | # -r requirements/test.txt 106 | # pydata-sphinx-theme 107 | # pytest 108 | # sphinx 109 | pbr==5.11.1 110 | # via 111 | # -r requirements/test.txt 112 | # stevedore 113 | pkginfo==1.9.6 114 | # via twine 115 | pluggy==1.3.0 116 | # via 117 | # -r requirements/test.txt 118 | # pytest 119 | psutil==5.9.5 120 | # via -r requirements/test.txt 121 | pycparser==2.21 122 | # via 123 | # -r requirements/test.txt 124 | # cffi 125 | pydata-sphinx-theme==0.13.3 126 | # via sphinx-book-theme 127 | pygments==2.16.1 128 | # via 129 | # accessible-pygments 130 | # doc8 131 | # pydata-sphinx-theme 132 | # readme-renderer 133 | # rich 134 | # sphinx 135 | pynacl==1.5.0 136 | # via -r requirements/test.txt 137 | pytest==7.4.2 138 | # via 139 | # -r requirements/test.txt 140 | # pytest-cov 141 | # pytest-django 142 | pytest-cov==4.1.0 143 | # via -r requirements/test.txt 144 | pytest-django==4.5.2 145 | # via -r requirements/test.txt 146 | pytz==2023.3.post1 147 | # via 148 | # -r requirements/test.txt 149 | # babel 150 | # django 151 | readme-renderer==42.0 152 | # via 153 | # -r requirements/doc.in 154 | # twine 155 | requests==2.31.0 156 | # via 157 | # requests-toolbelt 158 | # sphinx 159 | # twine 160 | requests-toolbelt==1.0.0 161 | # via twine 162 | restructuredtext-lint==1.4.0 163 | # via doc8 164 | rfc3986==2.0.0 165 | # via twine 166 | rich==13.5.2 167 | # via twine 168 | secretstorage==3.3.3 169 | # via keyring 170 | snowballstemmer==2.2.0 171 | # via sphinx 172 | soupsieve==2.5 173 | # via beautifulsoup4 174 | sphinx==4.2.0 175 | # via 176 | # -c requirements/constraints.txt 177 | # -r requirements/doc.in 178 | # pydata-sphinx-theme 179 | # sphinx-book-theme 180 | sphinx-book-theme==1.0.1 181 | # via -r requirements/doc.in 182 | sphinxcontrib-applehelp==1.0.4 183 | # via sphinx 184 | sphinxcontrib-devhelp==1.0.2 185 | # via sphinx 186 | sphinxcontrib-htmlhelp==2.0.1 187 | # via sphinx 188 | sphinxcontrib-jsmath==1.0.1 189 | # via sphinx 190 | sphinxcontrib-qthelp==1.0.3 191 | # via sphinx 192 | sphinxcontrib-serializinghtml==1.1.5 193 | # via sphinx 194 | sqlparse==0.4.4 195 | # via 196 | # -r requirements/test.txt 197 | # django 198 | stevedore==5.1.0 199 | # via 200 | # -r requirements/test.txt 201 | # doc8 202 | tomli==2.0.1 203 | # via 204 | # -r requirements/test.txt 205 | # coverage 206 | # pytest 207 | twine==4.0.2 208 | # via -r requirements/doc.in 209 | typing-extensions==4.7.1 210 | # via 211 | # -r requirements/test.txt 212 | # asgiref 213 | # pydata-sphinx-theme 214 | # rich 215 | urllib3==2.0.4 216 | # via 217 | # requests 218 | # twine 219 | zipp==3.16.2 220 | # via 221 | # importlib-metadata 222 | # importlib-resources 223 | 224 | # The following packages are considered to be unsafe in a requirements file: 225 | # setuptools 226 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/internal/code_owner/middleware.py: -------------------------------------------------------------------------------- 1 | """ 2 | Middleware for code_owner custom attribute 3 | """ 4 | import logging 5 | 6 | from django.urls import resolve 7 | from django.urls.exceptions import Resolver404 8 | 9 | from ..transactions import get_current_transaction 10 | from ..utils import set_custom_attribute 11 | from .utils import ( 12 | _get_catch_all_code_owner, 13 | get_code_owner_from_module, 14 | is_code_owner_mappings_configured, 15 | set_code_owner_custom_attributes 16 | ) 17 | 18 | log = logging.getLogger(__name__) 19 | 20 | 21 | class CodeOwnerMonitoringMiddleware: 22 | """ 23 | Django middleware object to set custom attributes for the owner of each view. 24 | 25 | For instructions on usage, see: 26 | https://github.com/openedx/edx-django-utils/blob/master/edx_django_utils/monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst 27 | 28 | Custom attributes set: 29 | - code_owner: The owning team mapped to the current view. 30 | - code_owner_module: The module found from the request or current transaction. 31 | - code_owner_path_error: The error mapping by path, if code_owner isn't found in other ways. 32 | - code_owner_transaction_error: The error mapping by transaction, if code_owner isn't found in other ways. 33 | - code_owner_transaction_name: The current transaction name used to try to map to code_owner. 34 | This can be used to find missing mappings. 35 | 36 | """ 37 | def __init__(self, get_response): 38 | self.get_response = get_response 39 | 40 | def __call__(self, request): 41 | response = self.get_response(request) 42 | self._set_code_owner_attribute(request) 43 | return response 44 | 45 | def process_exception(self, request, exception): # pylint: disable=W0613 46 | self._set_code_owner_attribute(request) 47 | 48 | def _set_code_owner_attribute(self, request): 49 | """ 50 | Sets the code_owner custom attribute for the request. 51 | """ 52 | code_owner = None 53 | module = self._get_module_from_request(request) 54 | if module: 55 | code_owner = get_code_owner_from_module(module) 56 | if not code_owner: 57 | code_owner = _get_catch_all_code_owner() 58 | 59 | if code_owner: 60 | set_code_owner_custom_attributes(code_owner) 61 | 62 | def _get_module_from_request(self, request): 63 | """ 64 | Get the module from the request path or the current transaction. 65 | 66 | Side-effects: 67 | Sets code_owner_module custom attribute, used to determine code_owner. 68 | If module was not found, may set code_owner_path_error and/or 69 | code_owner_transaction_error custom attributes if applicable. 70 | 71 | Returns: 72 | str: module name or None if not found 73 | 74 | """ 75 | if not is_code_owner_mappings_configured(): 76 | return None 77 | 78 | module, path_error = self._get_module_from_request_path(request) 79 | if module: 80 | set_custom_attribute('code_owner_module', module) 81 | return module 82 | 83 | module, transaction_error = self._get_module_from_current_transaction() 84 | if module: 85 | set_custom_attribute('code_owner_module', module) 86 | return module 87 | 88 | # monitor errors if module was not found 89 | if path_error: 90 | set_custom_attribute('code_owner_path_error', path_error) 91 | if transaction_error: 92 | set_custom_attribute('code_owner_transaction_error', transaction_error) 93 | return None 94 | 95 | def _get_module_from_request_path(self, request): 96 | """ 97 | Uses the request path to get the view_func module. 98 | 99 | Returns: 100 | (str, str): (module, error_message), where at least one of these should be None 101 | 102 | """ 103 | try: 104 | view_func, _, _ = resolve(request.path) 105 | module = view_func.__module__ 106 | return module, None 107 | # TODO: Replace ImportError with ModuleNotFoundError when Python 3.5 support is dropped. 108 | except (ImportError, Resolver404) as e: 109 | return None, str(e) 110 | except Exception as e: # pylint: disable=broad-except; #pragma: no cover 111 | # will remove broad exceptions after ensuring all proper cases are covered 112 | set_custom_attribute('deprecated_broad_except__get_module_from_request_path', e.__class__) 113 | return None, str(e) 114 | 115 | def _get_module_from_current_transaction(self): 116 | """ 117 | Uses the current transaction to get the module. 118 | 119 | Side-effects: 120 | Sets code_owner_transaction_name custom attribute, used to determine code_owner 121 | 122 | Returns: 123 | (str, str): (module, error_message), where at least one of these should be None 124 | 125 | """ 126 | try: 127 | # Example: openedx.core.djangoapps.contentserver.middleware:StaticContentServer 128 | transaction_name = get_current_transaction().name 129 | if not transaction_name: 130 | return None, 'No current transaction name found.' 131 | module = transaction_name.split(':')[0] 132 | set_custom_attribute('code_owner_transaction_name', transaction_name) 133 | return module, None 134 | except Exception as e: # pylint: disable=broad-except 135 | # will remove broad exceptions after ensuring all proper cases are covered 136 | set_custom_attribute('deprecated_broad_except___get_module_from_current_transaction', e.__class__) 137 | return None, str(e) 138 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Note: pylint disable of useless-suppression is required because the errors 3 | # are different for python 2.7 and python 3.6. 4 | # pylint: disable=C0111,useless-suppression 5 | """ 6 | Package metadata for edx-django-utils. 7 | """ 8 | 9 | import os 10 | import re 11 | import sys 12 | 13 | from setuptools import setup 14 | 15 | 16 | def get_version(*file_paths): 17 | """ 18 | Extract the version string from the file at the given relative path fragments. 19 | """ 20 | filename = os.path.join(os.path.dirname(__file__), *file_paths) 21 | version_file = open(filename, encoding="utf8").read() 22 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", 23 | version_file, re.M) 24 | if version_match: 25 | return version_match.group(1) 26 | raise RuntimeError('Unable to find version string.') 27 | 28 | 29 | def load_requirements(*requirements_paths): 30 | """ 31 | Load all requirements from the specified requirements files. 32 | 33 | Requirements will include any constraints from files specified 34 | with -c in the requirements files. 35 | Returns a list of requirement strings. 36 | """ 37 | # UPDATED VIA SEMGREP - if you need to remove/modify this method remove this line and add a comment specifying why. 38 | 39 | requirements = {} 40 | constraint_files = set() 41 | 42 | # groups "my-package-name<=x.y.z,..." into ("my-package-name", "<=x.y.z,...") 43 | requirement_line_regex = re.compile(r"([a-zA-Z0-9-_.]+)([<>=][^#\s]+)?") 44 | 45 | def add_version_constraint_or_raise(current_line, current_requirements, add_if_not_present): 46 | regex_match = requirement_line_regex.match(current_line) 47 | if regex_match: 48 | package = regex_match.group(1) 49 | version_constraints = regex_match.group(2) 50 | existing_version_constraints = current_requirements.get(package, None) 51 | # it's fine to add constraints to an unconstrained package, but raise an error if there are already 52 | # constraints in place 53 | if existing_version_constraints and existing_version_constraints != version_constraints: 54 | raise BaseException(f'Multiple constraint definitions found for {package}:' 55 | f' "{existing_version_constraints}" and "{version_constraints}".' 56 | f'Combine constraints into one location with {package}' 57 | f'{existing_version_constraints},{version_constraints}.') 58 | if add_if_not_present or package in current_requirements: 59 | current_requirements[package] = version_constraints 60 | 61 | # process .in files and store the path to any constraint files that are pulled in 62 | for path in requirements_paths: 63 | with open(path) as reqs: 64 | for line in reqs: 65 | if is_requirement(line): 66 | add_version_constraint_or_raise(line, requirements, True) 67 | if line and line.startswith('-c') and not line.startswith('-c http'): 68 | constraint_files.add(os.path.dirname(path) + '/' + line.split('#')[0].replace('-c', '').strip()) 69 | 70 | # process constraint files and add any new constraints found to existing requirements 71 | for constraint_file in constraint_files: 72 | with open(constraint_file) as reader: 73 | for line in reader: 74 | if is_requirement(line): 75 | add_version_constraint_or_raise(line, requirements, False) 76 | 77 | # process back into list of pkg><=constraints strings 78 | constrained_requirements = [f'{pkg}{version or ""}' for (pkg, version) in sorted(requirements.items())] 79 | return constrained_requirements 80 | 81 | 82 | def is_requirement(line): 83 | """ 84 | Return True if the requirement line is a package requirement. 85 | 86 | Returns: 87 | bool: True if the line is not blank, a comment, 88 | a URL, or an included file 89 | """ 90 | # UPDATED VIA SEMGREP - if you need to remove/modify this method remove this line and add a comment specifying why 91 | 92 | return line and line.strip() and not line.startswith(('-r', '#', '-e', 'git+', '-c')) 93 | 94 | 95 | VERSION = get_version('edx_django_utils', '__init__.py') 96 | 97 | if sys.argv[-1] == 'tag': 98 | print("Tagging the version on github:") 99 | os.system("git tag -a %s -m 'version %s'" % (VERSION, VERSION)) 100 | os.system("git push --tags") 101 | sys.exit() 102 | 103 | README = open(os.path.join(os.path.dirname(__file__), 'README.rst'), encoding="utf8").read() 104 | CHANGELOG = open(os.path.join(os.path.dirname(__file__), 'CHANGELOG.rst'), encoding="utf8").read() 105 | 106 | setup( 107 | name='edx-django-utils', 108 | version=VERSION, 109 | description="""EdX utilities for Django Application development.""", 110 | long_description=README + '\n\n' + CHANGELOG, 111 | long_description_content_type="text/x-rst", 112 | author='edX', 113 | author_email='oscm@edx.org', 114 | url='https://github.com/openedx/edx-django-utils', 115 | packages=[ 116 | 'edx_django_utils', 117 | ], 118 | include_package_data=True, 119 | install_requires=load_requirements('requirements/base.in'), 120 | entry_points={ 121 | 'console_scripts': [ 122 | 'log-sensitive = edx_django_utils.logging.internal.log_sensitive:cli', 123 | ], 124 | }, 125 | zip_safe=False, 126 | keywords='Django edx', 127 | classifiers=[ 128 | 'Development Status :: 3 - Alpha', 129 | 'Framework :: Django', 130 | 'Framework :: Django :: 3.2', 131 | 'Framework :: Django :: 4.2', 132 | 'Intended Audience :: Developers', 133 | 'License :: OSI Approved :: Apache Software License', 134 | 'Natural Language :: English', 135 | 'Programming Language :: Python :: 3', 136 | 'Programming Language :: Python :: 3.8', 137 | ], 138 | ) 139 | -------------------------------------------------------------------------------- /edx_django_utils/security/csp/middleware.py: -------------------------------------------------------------------------------- 1 | """ 2 | Middleware to add Content-Security-Policy and related headers. 3 | 4 | The ``content_security_policy_middleware`` middleware function can add 5 | ``Content-Security-Policy``, ``Content-Security-Policy-Report-Only``, and 6 | ``Reporting-Endpoints`` HTTP response headers. This functionality is 7 | configured by Django settings. 8 | 9 | This middleware only supports static values for the headers. That is, it does not 10 | support the ``strict-dynamic`` directive, which requires coordination of nonces or 11 | hashes between the response header and response body. 12 | 13 | Additionally, this middleware can be extended to support other security headers 14 | as per the application's requirements. For example, it can be modified to include 15 | headers like `X-Content-Type-Options`, `X-Frame-Options`, `X-XSS-Protection`, etc. 16 | """ 17 | import re 18 | 19 | from django.conf import settings 20 | from django.core.exceptions import MiddlewareNotUsed 21 | 22 | 23 | def _load_headers() -> dict: 24 | """ 25 | Return a dict of headers to append to every response, based on settings. 26 | """ 27 | # .. setting_name: CSP_STATIC_ENFORCE 28 | # .. setting_default: None 29 | # .. setting_description: Content-Security-Policy header to attach to all responses. 30 | # This should include everything but the ``report-to`` or ``report-uri`` clauses; those 31 | # will be appended automatically according to the ``CSP_STATIC_REPORTING_NAME`` and 32 | # ``CSP_STATIC_REPORTING_URI`` settings. Newlines are permitted and will be replaced with spaces. 33 | # A trailing `;` is also permitted. 34 | # .. setting_warning: Setting the CSP header to too strict a value can cause your pages to 35 | # break. It is strongly recommended that deployers start by using ``CSP_STATIC_REPORT_ONLY`` (along 36 | # with the reporting settings) and only move or copy the policies into ``CSP_STATIC_ENFORCE`` after 37 | # confirming that the received CSP reports only represent false positives. (The report-only 38 | # and enforcement headers may be used at the same time.) 39 | enforce_policies = getattr(settings, 'CSP_STATIC_ENFORCE', None) 40 | 41 | # .. setting_name: CSP_STATIC_REPORT_ONLY 42 | # .. setting_default: None 43 | # .. setting_description: Content-Security-Policy-Report-Only header to attach to 44 | # all responses. See ``CSP_STATIC_ENFORCE`` for details. 45 | report_policies = getattr(settings, 'CSP_STATIC_REPORT_ONLY', None) 46 | 47 | # .. setting_name: CSP_STATIC_REPORTING_URI 48 | # .. setting_default: None 49 | # .. setting_description: URL of reporting server. This will be used for both Level 2 and 50 | # Level 3 reports. If there are any semicolons or commas in the URL, they must be URL-encoded. 51 | # Level 3 reporting is only enabled if ``CSP_STATIC_REPORTING_NAME`` is also set. 52 | reporting_uri = getattr(settings, 'CSP_STATIC_REPORTING_URI', None) 53 | 54 | # .. setting_name: CSP_STATIC_REPORTING_NAME 55 | # .. setting_default: None 56 | # .. setting_description: Used for CSP Level 3 reporting. This sets the name to use in the 57 | # report-to CSP field and the Reporting-Endpoints header. If omitted, then Level 3 CSP 58 | # reporting will not be enabled. If present, this must be a string starting with an ASCII 59 | # letter and can contain ASCII letters, numbers, hyphen, underscore, and some other characters. 60 | # See https://www.rfc-editor.org/rfc/rfc8941.html#section-3.3.4 for full grammar. 61 | reporting_endpoint_name = getattr(settings, 'CSP_STATIC_REPORTING_NAME', None) 62 | 63 | if not enforce_policies and not report_policies: 64 | return {} 65 | 66 | headers = {} 67 | 68 | reporting_suffix = '' 69 | if reporting_uri: 70 | reporting_suffix = f"; report-uri {reporting_uri}" 71 | if reporting_endpoint_name: 72 | headers['Reporting-Endpoints'] = f'{reporting_endpoint_name}="{reporting_uri}"' 73 | reporting_suffix += f"; report-to {reporting_endpoint_name}" 74 | 75 | def clean_header(value): 76 | # Collapse any internal whitespace that contains a newline. This allows 77 | # writing the setting value as a multi-line string, which is useful for 78 | # CSP -- the values can be quite long. 79 | value = re.sub("\\s*\n\\s*", " ", value).strip() 80 | # Remove any trailing semicolon, which we allow (for convenience). 81 | # The CSP spec does not allow trailing semicolons or empty directives. 82 | value = re.sub("[;\\s]+$", "", value) 83 | return value 84 | 85 | if enforce_policies: 86 | headers['Content-Security-Policy'] = clean_header(enforce_policies) + reporting_suffix 87 | 88 | if report_policies: 89 | headers['Content-Security-Policy-Report-Only'] = clean_header(report_policies) + reporting_suffix 90 | 91 | # Add other security headers here as per the application's requirements 92 | 93 | return headers 94 | 95 | 96 | def _append_headers(response_headers, more_headers): 97 | """ 98 | Append to the response headers. If a header already exists, assume it is 99 | permitted to be multi-valued (comma-separated), and update the existing value. 100 | 101 | Arguments: 102 | response_headers: response.headers (or any dict-like object), to be modified 103 | more_headers: Dict of header names to values 104 | """ 105 | for k, v in more_headers.items(): 106 | if existing := response_headers.get(k): 107 | response_headers[k] = f"{existing}, {v}" 108 | else: 109 | response_headers[k] = v 110 | 111 | 112 | def content_security_policy_middleware(get_response): 113 | """ 114 | Middleware that adds Content-Security-Policy and related headers, if enabled. 115 | 116 | This should be reasonably high up in the middleware chain, since it should 117 | apply to all responses. 118 | """ 119 | # Constant across all requests, since they're based on static settings. 120 | csp_headers = _load_headers() 121 | if not csp_headers: 122 | raise MiddlewareNotUsed() # tell Django to skip this middleware 123 | 124 | def middleware_handler(request): 125 | response = get_response(request) 126 | # Reporting-Endpoints, CSP, and CSP-RO can all be multi-valued 127 | # (comma-separated) headers, though the CSP spec says "SHOULD NOT" 128 | # for the latter two. 129 | _append_headers(response.headers, csp_headers) 130 | return response 131 | 132 | return middleware_handler 133 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/tests/code_owner/test_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for the code_owner monitoring middleware 3 | """ 4 | import timeit 5 | from unittest import TestCase 6 | from unittest.mock import call, patch 7 | 8 | import ddt 9 | from django.test import override_settings 10 | 11 | from edx_django_utils.monitoring import ( 12 | get_code_owner_from_module, 13 | set_code_owner_attribute, 14 | set_code_owner_attribute_from_module 15 | ) 16 | from edx_django_utils.monitoring.internal.code_owner.utils import clear_cached_mappings 17 | 18 | 19 | @set_code_owner_attribute 20 | def decorated_function(pass_through): 21 | """ 22 | For testing the set_code_owner_attribute decorator. 23 | """ 24 | return pass_through 25 | 26 | 27 | @ddt.ddt 28 | class MonitoringUtilsTests(TestCase): 29 | """ 30 | Tests for the code_owner monitoring utility functions 31 | """ 32 | def setUp(self): 33 | super().setUp() 34 | clear_cached_mappings() 35 | 36 | @override_settings(CODE_OWNER_MAPPINGS={ 37 | 'team-red': [ 38 | 'openedx.core.djangoapps.xblock', 39 | 'lms.djangoapps.grades', 40 | ], 41 | 'team-blue': [ 42 | 'common.djangoapps.xblock_django', 43 | ], 44 | }) 45 | @ddt.data( 46 | ('xbl', None), 47 | ('xblock_2', None), 48 | ('xblock', 'team-red'), 49 | ('openedx.core.djangoapps', None), 50 | ('openedx.core.djangoapps.xblock', 'team-red'), 51 | ('openedx.core.djangoapps.xblock.views', 'team-red'), 52 | ('grades', 'team-red'), 53 | ('lms.djangoapps.grades', 'team-red'), 54 | ('xblock_django', 'team-blue'), 55 | ('common.djangoapps.xblock_django', 'team-blue'), 56 | ) 57 | @ddt.unpack 58 | def test_code_owner_mapping_hits_and_misses(self, module, expected_owner): 59 | actual_owner = get_code_owner_from_module(module) 60 | self.assertEqual(expected_owner, actual_owner) 61 | 62 | @override_settings(CODE_OWNER_MAPPINGS=['invalid_setting_as_list']) 63 | @patch('edx_django_utils.monitoring.internal.code_owner.utils.log') 64 | def test_code_owner_mapping_with_invalid_dict(self, mock_logger): 65 | with self.assertRaises(TypeError): 66 | get_code_owner_from_module('xblock') 67 | 68 | mock_logger.exception.assert_called_with( 69 | 'Error processing CODE_OWNER_MAPPINGS. list indices must be integers or slices, not str', 70 | ) 71 | 72 | def test_code_owner_mapping_with_no_settings(self): 73 | self.assertIsNone(get_code_owner_from_module('xblock')) 74 | 75 | def test_code_owner_mapping_with_no_module(self): 76 | self.assertIsNone(get_code_owner_from_module(None)) 77 | 78 | def test_mapping_performance(self): 79 | code_owner_mappings = { 80 | 'team-red': [] 81 | } 82 | # create a long list of mappings that are nearly identical 83 | for n in range(1, 200): 84 | path = f'openedx.core.djangoapps.{n}' 85 | code_owner_mappings['team-red'].append(path) 86 | with override_settings(CODE_OWNER_MAPPINGS=code_owner_mappings): 87 | call_iterations = 100 88 | time = timeit.timeit( 89 | # test a module name that matches nearly to the end, but doesn't actually match 90 | lambda: get_code_owner_from_module('openedx.core.djangoapps.XXX.views'), number=call_iterations 91 | ) 92 | average_time = time / call_iterations 93 | self.assertLess(average_time, 0.0005, f'Mapping takes {average_time}s which is too slow.') 94 | 95 | @override_settings( 96 | CODE_OWNER_MAPPINGS={'team-red': ['edx_django_utils.monitoring.tests.code_owner.test_utils']}, 97 | CODE_OWNER_THEMES={'team': ['team-red']}, 98 | ) 99 | @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') 100 | def test_set_code_owner_attribute_success(self, mock_set_custom_attribute): 101 | self.assertEqual(decorated_function('test'), 'test') 102 | self._assert_set_custom_attribute( 103 | mock_set_custom_attribute, code_owner='team-red', module=__name__, check_theme_and_squad=True 104 | ) 105 | 106 | @override_settings( 107 | CODE_OWNER_MAPPINGS={'team-red': ['edx_django_utils.monitoring.tests.code_owner.test_utils']}, 108 | CODE_OWNER_THEMES='invalid-setting', 109 | ) 110 | def test_set_code_owner_attribute_with_invalid_setting(self): 111 | with self.assertRaises(TypeError): 112 | decorated_function('test') 113 | 114 | @override_settings(CODE_OWNER_MAPPINGS={ 115 | 'team-red': ['*'] 116 | }) 117 | @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') 118 | def test_set_code_owner_attribute_catch_all(self, mock_set_custom_attribute): 119 | self.assertEqual(decorated_function('test'), 'test') 120 | self._assert_set_custom_attribute(mock_set_custom_attribute, code_owner='team-red', module=__name__) 121 | 122 | @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') 123 | def test_set_code_owner_attribute_no_mappings(self, mock_set_custom_attribute): 124 | self.assertEqual(decorated_function('test'), 'test') 125 | self._assert_set_custom_attribute(mock_set_custom_attribute, code_owner=None, module=__name__) 126 | 127 | @override_settings(CODE_OWNER_MAPPINGS={ 128 | 'team-red': ['edx_django_utils.monitoring.tests.code_owner.test_utils'] 129 | }) 130 | @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') 131 | def test_set_code_owner_attribute_from_module_success(self, mock_set_custom_attribute): 132 | set_code_owner_attribute_from_module(__name__) 133 | self._assert_set_custom_attribute(mock_set_custom_attribute, code_owner='team-red', module=__name__) 134 | 135 | def _assert_set_custom_attribute(self, mock_set_custom_attribute, code_owner, module, check_theme_and_squad=False): 136 | """ 137 | Helper to assert that the proper set_custom_metric calls were made. 138 | """ 139 | call_list = [] 140 | if code_owner: 141 | call_list.append(call('code_owner', code_owner)) 142 | if check_theme_and_squad: 143 | call_list.append(call('code_owner_theme', code_owner.split('-')[0])) 144 | call_list.append(call('code_owner_squad', code_owner.split('-')[1])) 145 | call_list.append(call('code_owner_module', module)) 146 | mock_set_custom_attribute.assert_has_calls(call_list, any_order=True) 147 | -------------------------------------------------------------------------------- /docs/decisions/0006-content-security-policy-middleware.rst: -------------------------------------------------------------------------------- 1 | 6. Content-Security-Policy middleware 2 | ##################################### 3 | 4 | Status 5 | ****** 6 | 7 | Provisional (2023-04-07) 8 | 9 | Context 10 | ******* 11 | 12 | 2U's Security Working Group is interested in adding the ``Content-Security-Policy`` response header (CSP) to as many edx.org domains as possible, in order to provide a layer of security on top of existing XSS mitigations and security measures. Some other deployers are also likely to be interested in using CSP. (There are also two additional headers ``Content-Security-Policy-Report-Only`` (CSP-RO) and ``Reporting-Endpoints``; together these will be referred to as the "CSP headers".) 13 | 14 | ``Content-Security-Policy`` is a powerful HTTP response header that can: 15 | 16 | - Block XSS, exfiltration, UI redress, and various other kinds of attacks 17 | - Gather reports of violations, allowing deployers to discover vulnerabilities even while CSP prevents them from being exploited 18 | - Enforce the use of a business process for adding new external scripts to the site 19 | 20 | At the most basic level, CSP allows the server to tell the browser to refuse to load any Javascript that isn't on an allowlist. More advanced deployments can restrict other kinds of resources (including service workers, child frames, images, and XHR connections). A static set of headers can only support an allowlist that is based on domain names and paths, but features such as ``strict-dynamic`` and ``nonce`` allow the server to vouch for scripts individually. This is more secure but requires more integration with application code. 21 | 22 | If fully deployed, there is a reduction of risk to learners, partners, and the site's reputation. However, for full effect, these security headers must be returned on all responses, including non-HTML resources and error pages. CSP headers must also be tailored not only to each IDA and MFE, but to each *deployment*, as different deployers will load images and scripts from different domains. 23 | 24 | CSP policies are often very long (1 kB in one example) and are built from a number of directives that each contain an allowlist or configuration. They cannot be loosened by the addition of later headers, but only tightened. This means that the server must return the entire policy in one shot, or alternatively emit some directives as separate headers. The use of ``strict-dynamic`` can shrink the header and make the policy more flexible, but requires integration between body and headers: The headers must either contain hashes of the scripts, or nonces that are also referenced in the HTML. 25 | 26 | Microfrontends are generally served from static file hosting such as Amazon S3 and cannot take advantage of ``strict-dynamic``; they are limited to source allowlists set by the fileserver or a CDN. In contrast, Django-based IDAs could use ``strict-dynamic`` (via nonces and hashes) because of the ability to customize header and body contents for each response. 27 | 28 | As of April 2023, most MFEs and other IDAs include inline Javascript, which severely limits the protection that CSP can provide. However, an initial deployment of CSP that locks down script source domains to an allowlist and reports on the locations of remaining inline scripts would be a good start towards cleaning up these inline scripts. In other words, the goal here is to get *started* on CSP. 29 | 30 | Decision 31 | ******** 32 | 33 | We will add a CSP middleware to edx-django-utils that can attach CSP headers based on Django settings. These will be static values, but with the option for simultaneous use of different CSP and CSP-RO headers, allowing safe rollout of increased restrictions. 34 | 35 | We will enable this middleware for all Django-based IDAs. 36 | 37 | Consequences 38 | ************ 39 | 40 | Django-based IDAs will for now be restricted to using static headers, and will not support ``strict-dynamic``. This is equivalent in configurability to having a CDN attach headers, and is good enough for a first pass, but will almost certainly need to be replaced with a more integrated system later as IDAs are adjusted to become amenable to ``strict-dynamic``. 41 | 42 | Rejected Alternatives 43 | ********************* 44 | 45 | Configure at the CDN level 46 | ========================== 47 | 48 | Since the initial version of this is equivalent to CDN headers in flexibility, we could just require that deployers use a CDN (or their origin web server) to attach these headers. This would meet 2U's needs in the short term. However, this would be a move in the wrong direction if we want to support more advanced, easier-to-use configurations in the future such as ``strict-dynamic``. It would also leave each deployer to build their own header-attaching solution. 49 | 50 | Attach arbitrary headers 51 | ======================== 52 | 53 | Since the CSP headers are staticly configured, it could be sufficient to make (or install from PyPI) a middleware that just attaches HTTP response headers. However, none were readily available that also appeared suitable. 54 | 55 | Additionally, the CSP policies are quite long, and it is preferable to be able to write them in multiline strings with free choice of line breaks and indentation. YAML mostly supports this, but only if *exactly* the right multiline string syntax is used (only 1 of the 9 multiline string forms is appropriate) so this would be error-prone and fragile. 56 | 57 | Having custom code not only allows collapsing whitespace runs but also allows trimming a trailing semicolon. CSP headers do not permit trailing delimiters or empty directives, but as seen in many programming languages, allowing trailing delimiters allows for better maintainability. CSP-specific code can remove the final delimiter and make mistakes less likely. 58 | 59 | django-csp package 60 | ================== 61 | 62 | https://github.com/mozilla/django-csp would technically work, and it’s nice that it breaks apart the CSP into pieces that can be separately controlled and even modified by views and middleware. This would immediately allow progress towards ``strict-dynamic`` CSP and even allow view-specific allow-lists. However, it doesn’t allow both CSP and CSP-RO at the same time. This is not a deal-breaker, but it does make deployment harder and somewhat reduces security, as it means that any time a directive is to be tightened the header must be taken out of enforcement mode for some observation period. (It is also not clear whether this library is maintained at this point.) 63 | 64 | If this library is at some point changed to support simultaneous use of CSP and CSP-RO, or if another library is developed that has equivalent properties, then it would be appropriate to migrate to using that. 65 | 66 | References 67 | ********** 68 | 69 | - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy - Overview of CSP and its options 70 | - https://www.w3.org/TR/CSP3/ - Spec that defines CSP 71 | - https://w3c.github.io/reporting/ - Spec that defines the ``Reporting-Endpoints`` header 72 | -------------------------------------------------------------------------------- /edx_django_utils/monitoring/tests/test_custom_monitoring.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for CachedCustomMonitoringMiddleware and associated utilities. 3 | 4 | Note: See test_middleware.py for the rest of the middleware tests. 5 | """ 6 | from unittest.mock import Mock, call, patch 7 | 8 | import ddt 9 | from django.test import TestCase 10 | 11 | from edx_django_utils.cache import RequestCache 12 | from edx_django_utils.monitoring import ( 13 | CachedCustomMonitoringMiddleware, 14 | accumulate, 15 | get_current_transaction, 16 | increment, 17 | record_exception 18 | ) 19 | 20 | from ..middleware import CachedCustomMonitoringMiddleware as DeprecatedCachedCustomMonitoringMiddleware 21 | from ..middleware import MonitoringCustomMetricsMiddleware as DeprecatedMonitoringCustomMetricsMiddleware 22 | from ..utils import accumulate as deprecated_accumulate 23 | from ..utils import increment as deprecated_increment 24 | from ..utils import set_custom_attribute as deprecated_set_custom_attribute 25 | from ..utils import set_custom_attributes_for_course_key as deprecated_set_custom_attributes_for_course_key 26 | 27 | 28 | @ddt.ddt 29 | class TestCustomMonitoringMiddleware(TestCase): 30 | """ 31 | Test the monitoring_utils middleware and helpers 32 | """ 33 | def setUp(self): 34 | super().setUp() 35 | self.mock_response = Mock() 36 | RequestCache.clear_all_namespaces() 37 | 38 | @patch('newrelic.agent') 39 | @ddt.data( 40 | (CachedCustomMonitoringMiddleware, False, 'process_response'), 41 | (CachedCustomMonitoringMiddleware, False, 'process_exception'), 42 | (DeprecatedCachedCustomMonitoringMiddleware, True, 'process_response'), 43 | (DeprecatedMonitoringCustomMetricsMiddleware, True, 'process_response'), 44 | ) 45 | @ddt.unpack 46 | def test_accumulate_and_increment( 47 | self, cached_monitoring_middleware_class, is_deprecated, middleware_method_name, mock_newrelic_agent 48 | ): 49 | """ 50 | Test normal usage of collecting custom attributes and reporting to New Relic 51 | """ 52 | accumulate('hello', 10) 53 | accumulate('world', 10) 54 | accumulate('world', 10) 55 | increment('foo') 56 | increment('foo') 57 | 58 | # based on the attribute data above, we expect the following calls to newrelic: 59 | nr_agent_calls_expected = [ 60 | call('hello', 10), 61 | call('world', 20), 62 | call('foo', 2), 63 | ] 64 | 65 | # fake a response to trigger attributes reporting 66 | middleware_method = getattr(cached_monitoring_middleware_class(self.mock_response), middleware_method_name) 67 | middleware_method( 68 | 'fake request', 69 | 'fake response', 70 | ) 71 | 72 | # Assert call counts to newrelic.agent.add_custom_parameter() 73 | expected_call_count = len(nr_agent_calls_expected) 74 | if is_deprecated: 75 | expected_call_count += 1 76 | measured_call_count = mock_newrelic_agent.add_custom_parameter.call_count 77 | self.assertEqual(expected_call_count, measured_call_count) 78 | 79 | # Assert call args to newrelic.agent.add_custom_parameter(). Due to 80 | # the nature of python dicts, call order is undefined. 81 | mock_newrelic_agent.add_custom_parameter.assert_has_calls(nr_agent_calls_expected, any_order=True) 82 | 83 | @patch('newrelic.agent') 84 | @ddt.data( 85 | (CachedCustomMonitoringMiddleware, False), 86 | (DeprecatedCachedCustomMonitoringMiddleware, True), 87 | (DeprecatedMonitoringCustomMetricsMiddleware, True), 88 | ) 89 | @ddt.unpack 90 | def test_accumulate_with_illegal_value( 91 | self, cached_monitoring_middleware_class, is_deprecated, mock_newrelic_agent 92 | ): 93 | """ 94 | Test monitoring accumulate with illegal value that can't be added. 95 | """ 96 | accumulate('hello', None) 97 | accumulate('hello', 10) 98 | 99 | # based on the metric data above, we expect the following calls to newrelic: 100 | nr_agent_calls_expected = [ 101 | call('hello', None), 102 | call('error_adding_accumulated_metric', 'name=hello, new_value=10, cached_value=None'), 103 | ] 104 | 105 | self.mock_response = Mock() 106 | # fake a response to trigger metrics reporting 107 | cached_monitoring_middleware_class(self.mock_response).process_response( 108 | 'fake request', 109 | 'fake response', 110 | ) 111 | 112 | # Assert call counts to newrelic.agent.add_custom_parameter() 113 | expected_call_count = len(nr_agent_calls_expected) 114 | if is_deprecated: 115 | expected_call_count += 1 116 | measured_call_count = mock_newrelic_agent.add_custom_parameter.call_count 117 | self.assertEqual(expected_call_count, measured_call_count) 118 | 119 | # Assert call args to newrelic.agent.add_custom_parameter(). 120 | mock_newrelic_agent.add_custom_parameter.assert_has_calls(nr_agent_calls_expected, any_order=True) 121 | 122 | @patch('newrelic.agent') 123 | def test_get_current_transaction(self, mock_newrelic_agent): 124 | mock_newrelic_agent.current_transaction().name = 'fake-transaction' 125 | current_transaction = get_current_transaction() 126 | self.assertEqual(current_transaction.name, 'fake-transaction') 127 | 128 | def test_get_current_transaction_without_newrelic(self): 129 | current_transaction = get_current_transaction() 130 | self.assertEqual(current_transaction.name, None) 131 | 132 | @patch('edx_django_utils.monitoring.utils.internal_accumulate') 133 | def test_deprecated_accumulate(self, mock_accumulate): 134 | deprecated_accumulate('key', 1) 135 | mock_accumulate.assert_called_with('key', 1) 136 | 137 | @patch('edx_django_utils.monitoring.utils.internal_increment') 138 | def test_deprecated_increment(self, mock_increment): 139 | deprecated_increment('key') 140 | mock_increment.assert_called_with('key') 141 | 142 | @patch('edx_django_utils.monitoring.utils.internal_set_custom_attribute') 143 | def test_deprecated_set_custom_attribute(self, mock_set_custom_attribute): 144 | deprecated_set_custom_attribute('key', True) 145 | mock_set_custom_attribute.assert_called_with('key', True) 146 | 147 | @patch('edx_django_utils.monitoring.utils.internal_set_custom_attributes_for_course_key') 148 | def test_deprecated_set_custom_attributes_for_course_key(self, mock_set_custom_attributes_for_course_key): 149 | deprecated_set_custom_attributes_for_course_key('key') 150 | mock_set_custom_attributes_for_course_key.assert_called_with('key') 151 | 152 | @patch('newrelic.agent.record_exception') 153 | def test_record_exception(self, mock_record_exception): 154 | record_exception() 155 | mock_record_exception.assert_called_once() 156 | --------------------------------------------------------------------------------