├── .coveragerc ├── .dockerignore ├── .github ├── docker-compose-ci.yml └── workflows │ ├── add-depr-ticket-to-depr-board.yml │ ├── add-remove-label-on-comment.yml │ ├── ci.yml │ ├── commitlint.yml │ ├── mysql8-migrations.yml │ ├── push-docker-images.yml │ ├── self-assign-issue.yml │ └── upgrade-python-requirements.yml ├── .gitignore ├── .readthedocs.yaml ├── .test_env ├── Dockerfile ├── LICENSE.txt ├── Makefile ├── README.rst ├── analytics_data_api ├── __init__.py ├── constants │ ├── __init__.py │ ├── country.py │ ├── engagement_events.py │ ├── enrollment_modes.py │ ├── genders.py │ └── learner.py ├── docker_gunicorn_configuration.py ├── fixtures │ ├── problem_response_answer_distribution.json │ └── problem_response_answer_distribution_analytics_v1.json ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ ├── generate_data.py │ │ ├── generate_fake_course_data.py │ │ ├── generate_stage_course_data.py │ │ ├── set_api_key.py │ │ └── tests │ │ ├── test_generate_fake_course_data.py │ │ └── test_generate_stage_course_data.py ├── middleware.py ├── models.py ├── renderers.py ├── tests │ ├── __init__.py │ ├── test_middleware.py │ ├── test_renderers.py │ ├── test_throttles.py │ └── test_utils.py ├── throttles.py ├── urls.py ├── utils.py ├── v0 │ ├── __init__.py │ ├── apps.py │ ├── exceptions.py │ ├── models.py │ ├── serializers.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_models.py │ │ ├── test_serializers.py │ │ ├── test_urls.py │ │ ├── utils.py │ │ └── views │ │ │ ├── __init__.py │ │ │ ├── test_course_summaries.py │ │ │ ├── test_courses.py │ │ │ ├── test_enterprise_learner_engagements.py │ │ │ ├── test_problems.py │ │ │ ├── test_programs.py │ │ │ ├── test_utils.py │ │ │ └── test_videos.py │ ├── urls │ │ ├── __init__.py │ │ ├── course_summaries.py │ │ ├── courses.py │ │ ├── learners.py │ │ ├── problems.py │ │ ├── programs.py │ │ └── videos.py │ └── views │ │ ├── __init__.py │ │ ├── course_summaries.py │ │ ├── courses.py │ │ ├── learners.py │ │ ├── problems.py │ │ ├── programs.py │ │ ├── utils.py │ │ └── videos.py └── v1 │ ├── __init__.py │ └── urls.py ├── analyticsdataserver ├── __init__.py ├── clients.py ├── router.py ├── settings │ ├── __init__.py │ ├── base.py │ ├── devstack.py │ ├── local.py │ ├── local_mysql.py │ ├── logger.py │ ├── production.py │ └── test.py ├── tests │ ├── test_clients.py │ ├── test_router.py │ ├── test_utils.py │ ├── test_views.py │ └── utils.py ├── urls.py ├── utils.py ├── views.py └── wsgi.py ├── docker-compose.yml ├── docs ├── api │ ├── Makefile │ ├── __init__.py │ ├── requirements.txt │ └── source │ │ ├── __init__.py │ │ ├── authentication.rst │ │ ├── change_log.rst │ │ ├── conf.py │ │ ├── courses.rst │ │ ├── endpoints.rst │ │ ├── images │ │ ├── api_test.png │ │ ├── api_test_expand.png │ │ └── api_test_response.png │ │ ├── index.rst │ │ ├── links.rst │ │ ├── overview.rst │ │ ├── problems.rst │ │ ├── read_me.rst │ │ ├── setup.rst │ │ └── videos.rst ├── apiary │ └── apiary.apib └── decisions │ ├── 01-pipeline-choice.rst │ └── 02-no-pii.rst ├── manage.py ├── openedx.yaml ├── pylintrc ├── pytest.ini ├── requirements.txt ├── requirements ├── base.in ├── base.txt ├── ci.in ├── ci.txt ├── constraints.txt ├── dev.in ├── dev.txt ├── django.txt ├── doc.in ├── doc.txt ├── pip.in ├── pip.txt ├── pip_tools.in ├── pip_tools.txt ├── production.in ├── production.txt ├── test.in ├── test.txt ├── tox.in └── tox.txt ├── scripts └── post-pip-compile.sh ├── setup.cfg ├── static ├── css │ └── edx-swagger.css └── images │ ├── favicon.ico │ └── header-logo.png └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = analyticsdataserver/settings* 3 | *wsgi.py 4 | 5 | source = analyticsdataserver, analytics_data_api 6 | branch = True 7 | 8 | [report] 9 | # Regexes for lines to exclude from consideration 10 | exclude_lines = 11 | # Have to re-enable the standard pragma 12 | pragma: no cover 13 | 14 | raise NotImplementedError 15 | 16 | [html] 17 | directory = ${COVERAGE_DIR}/html/ 18 | 19 | [xml] 20 | output = ${COVERAGE_DIR}/coverage.xml 21 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | Dockerfile 2 | -------------------------------------------------------------------------------- /.github/docker-compose-ci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | 3 | services: 4 | 5 | analytics_api: 6 | image: edxops/analytics-api-dev:latest 7 | container_name: analytics_api_testing 8 | volumes: 9 | - ..:/edx/app/analytics_api/analytics_api 10 | command: tail -f /dev/null 11 | environment: 12 | ELASTICSEARCH_LEARNERS_HOST: "http://es:9223" 13 | depends_on: 14 | - "es" 15 | 16 | es: 17 | image: docker.elastic.co/elasticsearch/elasticsearch:7.10.1 18 | container_name: es 19 | environment: 20 | - node.name=es 21 | - cluster.name=docker-cluster 22 | - cluster.initial_master_nodes=es 23 | - bootstrap.memory_lock=true 24 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 25 | - http.port=9223 26 | ulimits: 27 | memlock: 28 | soft: -1 29 | hard: -1 30 | volumes: 31 | - data01:/usr/share/elasticsearch/data 32 | ports: 33 | - "9223:9223" 34 | 35 | volumes: 36 | data01: 37 | driver: local 38 | -------------------------------------------------------------------------------- /.github/workflows/add-depr-ticket-to-depr-board.yml: -------------------------------------------------------------------------------- 1 | # Run the workflow that adds new tickets that are either: 2 | # - labelled "DEPR" 3 | # - title starts with "[DEPR]" 4 | # - body starts with "Proposal Date" (this is the first template field) 5 | # to the org-wide DEPR project board 6 | 7 | name: Add newly created DEPR issues to the DEPR project board 8 | 9 | on: 10 | issues: 11 | types: [opened] 12 | 13 | jobs: 14 | routeissue: 15 | uses: openedx/.github/.github/workflows/add-depr-ticket-to-depr-board.yml@master 16 | secrets: 17 | GITHUB_APP_ID: ${{ secrets.GRAPHQL_AUTH_APP_ID }} 18 | GITHUB_APP_PRIVATE_KEY: ${{ secrets.GRAPHQL_AUTH_APP_PEM }} 19 | SLACK_BOT_TOKEN: ${{ secrets.SLACK_ISSUE_BOT_TOKEN }} 20 | -------------------------------------------------------------------------------- /.github/workflows/add-remove-label-on-comment.yml: -------------------------------------------------------------------------------- 1 | # This workflow runs when a comment is made on the ticket 2 | # If the comment starts with "label: " it tries to apply 3 | # the label indicated in rest of comment. 4 | # If the comment starts with "remove label: ", it tries 5 | # to remove the indicated label. 6 | # Note: Labels are allowed to have spaces and this script does 7 | # not parse spaces (as often a space is legitimate), so the command 8 | # "label: really long lots of words label" will apply the 9 | # label "really long lots of words label" 10 | 11 | name: Allows for the adding and removing of labels via comment 12 | 13 | on: 14 | issue_comment: 15 | types: [created] 16 | 17 | jobs: 18 | add_remove_labels: 19 | uses: openedx/.github/.github/workflows/add-remove-label-on-comment.yml@master 20 | 21 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Python CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - '**' 10 | 11 | jobs: 12 | run_tests: 13 | name: Tests 14 | runs-on: ${{ matrix.os }} 15 | strategy: 16 | matrix: 17 | os: 18 | - ubuntu-20.04 19 | python-version: 20 | - 3.8 21 | targets: [ 'quality','main.test','docs' ] 22 | 23 | steps: 24 | - uses: actions/checkout@v3 25 | - name: setup python 26 | uses: actions/setup-python@v3 27 | with: 28 | python-version: ${{ matrix.python-version }} 29 | 30 | - name: Start Container 31 | run: docker-compose -f .github/docker-compose-ci.yml up -d 32 | 33 | - name: Install dependencies 34 | run: | 35 | pip install -r requirements/pip.txt 36 | pip install -r requirements/ci.txt 37 | 38 | - name: Run Tests 39 | run: docker exec -t analytics_api_testing bash -c "cd /edx/app/analytics_api/analytics_api/ 40 | && export TOXENV=django42 && make test.requirements tox.requirements ${{ matrix.targets }}" 41 | 42 | - name: Run Coverage 43 | if: matrix.python-version == '3.8' && matrix.targets=='main.test' 44 | uses: codecov/codecov-action@v1 45 | with: 46 | fail_ci_if_error: true 47 | -------------------------------------------------------------------------------- /.github/workflows/commitlint.yml: -------------------------------------------------------------------------------- 1 | # Run commitlint on the commit messages in a pull request. 2 | 3 | name: Lint Commit Messages 4 | 5 | on: 6 | - pull_request 7 | 8 | jobs: 9 | commitlint: 10 | uses: openedx/.github/.github/workflows/commitlint.yml@master 11 | -------------------------------------------------------------------------------- /.github/workflows/mysql8-migrations.yml: -------------------------------------------------------------------------------- 1 | name: Migrations check on mysql8 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | push: 7 | branches: 8 | - master 9 | 10 | jobs: 11 | check_migrations: 12 | name: check migrations 13 | runs-on: ${{ matrix.os }} 14 | strategy: 15 | matrix: 16 | os: [ ubuntu-20.04 ] 17 | python-version: [ 3.8 ] 18 | 19 | steps: 20 | - name: Checkout repo 21 | uses: actions/checkout@v2 22 | 23 | - name: Setup Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v2 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | 28 | - name: Install system Packages 29 | run: | 30 | sudo apt-get update 31 | sudo apt-get install -y libxmlsec1-dev 32 | - name: Get pip cache dir 33 | id: pip-cache-dir 34 | run: | 35 | echo "::set-output name=dir::$(pip cache dir)" 36 | - name: Cache pip dependencies 37 | id: cache-dependencies 38 | uses: actions/cache@v2 39 | with: 40 | path: ${{ steps.pip-cache-dir.outputs.dir }} 41 | key: ${{ runner.os }}-pip-${{ hashFiles('requirements/pip_tools.txt') }} 42 | restore-keys: ${{ runner.os }}-pip- 43 | 44 | - name: Ubuntu and sql Versions 45 | run: | 46 | lsb_release -a 47 | mysql -V 48 | - name: Install Python dependencies 49 | run: | 50 | pip install -r requirements/production.txt 51 | pip uninstall -y mysqlclient 52 | pip install --no-binary mysqlclient mysqlclient 53 | pip uninstall -y xmlsec 54 | pip install --no-binary xmlsec xmlsec==1.3.13 55 | - name: Initiate Services 56 | run: | 57 | sudo /etc/init.d/mysql start 58 | - name: Reset mysql password 59 | run: | 60 | cat <=2.2.0 requires pkg-config (https://github.com/PyMySQL/mysqlclient/issues/620) 6 | 7 | RUN apt update && \ 8 | DEBIAN_FRONTEND=noninteractive apt-get install -qy \ 9 | curl \ 10 | vim \ 11 | language-pack-en \ 12 | build-essential \ 13 | python3.8-dev \ 14 | python3-virtualenv \ 15 | python3.8-distutils \ 16 | libmysqlclient-dev \ 17 | pkg-config \ 18 | libssl-dev && \ 19 | rm -rf /var/lib/apt/lists/* 20 | 21 | # Use UTF-8. 22 | RUN locale-gen en_US.UTF-8 23 | ENV LANG en_US.UTF-8 24 | ENV LANGUAGE en_US:en 25 | ENV LC_ALL en_US.UTF-8 26 | 27 | ARG COMMON_APP_DIR="/edx/app" 28 | ARG ANALYTICS_API_SERVICE_NAME="analytics_api" 29 | ENV ANALYTICS_API_HOME "${COMMON_APP_DIR}/${ANALYTICS_API_SERVICE_NAME}" 30 | ARG ANALYTICS_API_APP_DIR="${COMMON_APP_DIR}/${ANALYTICS_API_SERVICE_NAME}" 31 | ARG ANALYTICS_API_VENV_DIR="${COMMON_APP_DIR}/${ANALYTICS_API_SERVICE_NAME}/venvs/${ANALYTICS_API_SERVICE_NAME}" 32 | ARG ANALYTICS_API_CODE_DIR="${ANALYTICS_API_APP_DIR}/${ANALYTICS_API_SERVICE_NAME}" 33 | 34 | ENV ANALYTICS_API_CODE_DIR="${ANALYTICS_API_CODE_DIR}" 35 | ENV PATH "${ANALYTICS_API_VENV_DIR}/bin:$PATH" 36 | ENV COMMON_CFG_DIR "/edx/etc" 37 | ENV ANALYTICS_API_CFG "/edx/etc/${ANALYTICS_API_SERVICE_NAME}.yml" 38 | 39 | # Working directory will be root of repo. 40 | WORKDIR ${ANALYTICS_API_CODE_DIR} 41 | 42 | RUN virtualenv -p python3.8 --always-copy ${ANALYTICS_API_VENV_DIR} 43 | 44 | # Expose canonical Analytics port 45 | EXPOSE 19001 46 | 47 | FROM base as prod 48 | 49 | ENV DJANGO_SETTINGS_MODULE "analyticsdataserver.settings.production" 50 | 51 | COPY requirements/production.txt ${ANALYTICS_API_CODE_DIR}/requirements/production.txt 52 | 53 | RUN pip install -r ${ANALYTICS_API_CODE_DIR}/requirements/production.txt 54 | 55 | # Copy over rest of code. 56 | # We do this AFTER requirements so that the requirements cache isn't busted 57 | # every time any bit of code is changed. 58 | 59 | COPY . . 60 | 61 | # exec /edx/app/analytics_api/venvs/analytics_api/bin/gunicorn -c /edx/app/analytics_api/analytics_api_gunicorn.py analyticsdataserver.wsgi:application 62 | 63 | CMD ["gunicorn" , "-b", "0.0.0.0:8100", "--pythonpath", "/edx/app/analytics_api/analytics_api","analyticsdataserver.wsgi:application"] 64 | 65 | FROM base as dev 66 | 67 | ENV DJANGO_SETTINGS_MODULE "analyticsdataserver.settings.devstack" 68 | 69 | COPY requirements/dev.txt ${ANALYTICS_API_CODE_DIR}/requirements/dev.txt 70 | 71 | RUN pip install -r ${ANALYTICS_API_CODE_DIR}/requirements/dev.txt 72 | 73 | # Copy over rest of code. 74 | # We do this AFTER requirements so that the requirements cache isn't busted 75 | # every time any bit of code is changed. 76 | COPY . . 77 | 78 | # Devstack related step for backwards compatibility 79 | RUN touch /edx/app/${ANALYTICS_API_SERVICE_NAME}/${ANALYTICS_API_SERVICE_NAME}_env 80 | 81 | CMD while true; do python ./manage.py runserver 0.0.0.0:8110; sleep 2; done 82 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ROOT = $(shell echo "$$PWD") 2 | COVERAGE_DIR = $(ROOT)/build/coverage 3 | DATABASES = default analytics analytics_v1 4 | .DEFAULT_GOAL := help 5 | 6 | TOX='' 7 | 8 | ifdef TOXENV 9 | TOX := tox -- #to isolate each tox environment if TOXENV is defined 10 | endif 11 | 12 | help: ## display this help message 13 | @echo "Please use \`make ' where is one of" 14 | @perl -nle'print $& if m{^[\.a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m %-25s\033[0m %s\n", $$1, $$2}' 15 | 16 | .PHONY: requirements develop clean diff.report view.diff.report quality static docs 17 | 18 | requirements: ## install base requirements 19 | pip3 install -q -r requirements/base.txt 20 | 21 | production-requirements: ## install production requirements 22 | pip3 install -r requirements.txt 23 | 24 | test.requirements: requirements ## install base and test requirements 25 | pip3 install -q -r requirements/test.txt 26 | 27 | tox.requirements: ## install tox requirements 28 | pip3 install -q -r requirements/tox.txt 29 | 30 | develop: test.requirements ## install test and dev requirements 31 | pip3 install -q -r requirements/dev.txt 32 | 33 | upgrade: 34 | pip3 install -q -r requirements/pip_tools.txt 35 | pip-compile --upgrade --allow-unsafe -o requirements/pip.txt requirements/pip.in 36 | pip-compile --upgrade -o requirements/pip_tools.txt requirements/pip_tools.in 37 | pip install -qr requirements/pip.txt 38 | pip install -qr requirements/pip_tools.txt 39 | pip-compile --upgrade -o requirements/base.txt requirements/base.in 40 | pip-compile --upgrade -o requirements/doc.txt requirements/doc.in 41 | pip-compile --upgrade -o requirements/dev.txt requirements/dev.in 42 | pip-compile --upgrade -o requirements/production.txt requirements/production.in 43 | pip-compile --upgrade -o requirements/test.txt requirements/test.in 44 | pip-compile --upgrade -o requirements/tox.txt requirements/tox.in 45 | pip-compile --upgrade -o requirements/ci.txt requirements/ci.in 46 | scripts/post-pip-compile.sh \ 47 | requirements/pip_tools.txt \ 48 | requirements/base.txt \ 49 | requirements/doc.txt \ 50 | requirements/dev.txt \ 51 | requirements/production.txt \ 52 | requirements/test.txt \ 53 | requirements/tox.txt \ 54 | requirements/ci.txt 55 | ## Let tox control the Django version for tests 56 | grep -e "^django==" requirements/base.txt > requirements/django.txt 57 | sed '/^[dD]jango==/d' requirements/test.txt > requirements/test.tmp 58 | mv requirements/test.tmp requirements/test.txt 59 | 60 | 61 | clean: 62 | $(TOX)coverage erase 63 | find . -name '*.pyc' -delete 64 | 65 | main.test: clean 66 | export COVERAGE_DIR=$(COVERAGE_DIR) && \ 67 | $(TOX)pytest --cov-report html --cov-report xml 68 | 69 | test: main.test 70 | 71 | diff.report: test.requirements ## Show the diff in quality and coverage 72 | diff-cover $(COVERAGE_DIR)/coverage.xml --html-report $(COVERAGE_DIR)/diff_cover.html 73 | diff-quality --violations=pycodestyle --html-report $(COVERAGE_DIR)/diff_quality_pycodestyle.html 74 | diff-quality --violations=pylint --html-report $(COVERAGE_DIR)/diff_quality_pylint.html 75 | 76 | view.diff.report: ## Show the diff in quality and coverage using xdg 77 | xdg-open file:///$(COVERAGE_DIR)/diff_cover.html 78 | xdg-open file:///$(COVERAGE_DIR)/diff_quality_pycodestyle.html 79 | xdg-open file:///$(COVERAGE_DIR)/diff_quality_pylint.html 80 | 81 | run_check_isort: 82 | $(TOX)isort --check-only --recursive --diff analytics_data_api/ analyticsdataserver/ 83 | 84 | run_pycodestyle: 85 | $(TOX)pycodestyle --config=.pycodestyle analytics_data_api analyticsdataserver 86 | 87 | run_pylint: 88 | $(TOX)pylint -j 0 --rcfile=pylintrc analytics_data_api analyticsdataserver 89 | 90 | run_isort: 91 | $(TOX)isort --recursive analytics_data_api/ analyticsdataserver/ 92 | 93 | quality: run_pylint run_check_isort run_pycodestyle ## run_pylint, run_check_isort, run_pycodestyle (Installs tox requirements.) 94 | 95 | validate: test.requirements test quality ## Runs make test and make quality. (Installs test requirements.) 96 | 97 | static: ## Runs collectstatic 98 | python manage.py collectstatic --noinput 99 | 100 | migrate: ## Runs django migrations with syncdb and default database 101 | ./manage.py migrate --noinput --run-syncdb --database=default 102 | 103 | migrate-all: ## Runs migrations on all databases 104 | $(foreach db_name,$(DATABASES),./manage.py migrate --noinput --run-syncdb --database=$(db_name);) 105 | 106 | loaddata: migrate-all ## Runs migrations and generates fake data 107 | python manage.py loaddata problem_response_answer_distribution --database=analytics 108 | python manage.py loaddata problem_response_answer_distribution_analytics_v1 --database=analytics_v1 109 | python manage.py generate_fake_course_data --database=analytics 110 | python manage.py generate_fake_course_data --database=analytics_v1 111 | 112 | demo: requirements clean loaddata ## Runs make clean, requirements, and loaddata, sets api key to edx 113 | python manage.py set_api_key edx edx 114 | 115 | # Target used by edx-analytics-dashboard during its testing. 116 | github_ci: test.requirements clean migrate-all ## Used by CI for testing 117 | python manage.py set_api_key edx edx 118 | python manage.py loaddata problem_response_answer_distribution --database=analytics 119 | python manage.py loaddata problem_response_answer_distribution_analytics_v1 --database=analytics_v1 120 | python manage.py generate_fake_course_data --database=analytics --num-weeks=2 --no-videos --course-id "edX/DemoX/Demo_Course" 121 | python manage.py generate_fake_course_data --database=analytics_v1 --num-weeks=2 --no-videos --course-id "edX/DemoX/Demo_Course" 122 | 123 | docs: tox.requirements 124 | tox -e docs 125 | -------------------------------------------------------------------------------- /analytics_data_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/analytics_data_api/__init__.py -------------------------------------------------------------------------------- /analytics_data_api/constants/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/analytics_data_api/constants/__init__.py -------------------------------------------------------------------------------- /analytics_data_api/constants/country.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file holds constants and helper functions related to countries. All codes are assumed to be valid ISO 3166 country 3 | codes. 4 | """ 5 | 6 | 7 | from collections import namedtuple 8 | 9 | from django_countries import countries 10 | 11 | Country = namedtuple('Country', 'name, alpha2, alpha3, numeric') 12 | 13 | UNKNOWN_COUNTRY_CODE = 'UNKNOWN' 14 | UNKNOWN_COUNTRY = Country(UNKNOWN_COUNTRY_CODE, None, None, None) 15 | 16 | 17 | def _get_country_property(code, property_name): 18 | return str(getattr(countries, property_name)(code)) 19 | 20 | 21 | def get_country(code): 22 | if not code: 23 | return UNKNOWN_COUNTRY 24 | 25 | name = _get_country_property(code, 'name') 26 | if not name: 27 | return UNKNOWN_COUNTRY 28 | 29 | args = [] 30 | properties = ['alpha2', 'alpha3', 'numeric'] 31 | for property_name in properties: 32 | args.append(_get_country_property(code, property_name)) 33 | 34 | return Country(name, *args) 35 | -------------------------------------------------------------------------------- /analytics_data_api/constants/engagement_events.py: -------------------------------------------------------------------------------- 1 | ATTEMPTED = 'attempted' 2 | ATTEMPTS_PER_COMPLETED = 'attempts_per_completed' 3 | COMPLETED = 'completed' 4 | CONTRIBUTED = 'contributed' 5 | VIEWED = 'viewed' 6 | 7 | DISCUSSION = 'discussion' 8 | PROBLEM = 'problem' 9 | VIDEO = 'video' 10 | PROBLEMS = 'problems' 11 | VIDEOS = 'videos' 12 | 13 | INDIVIDUAL_EVENTS = [ 14 | 'problem_attempts_per_completed', 15 | 'problem_attempted', 16 | 'problem_completed', 17 | 'discussion_contributed', 18 | 'video_viewed' 19 | ] 20 | 21 | EVENTS = [ 22 | 'problem_attempts_per_completed', 23 | 'problems_attempted', 24 | 'problems_completed', 25 | 'discussion_contributions', 26 | 'videos_viewed' 27 | ] 28 | -------------------------------------------------------------------------------- /analytics_data_api/constants/enrollment_modes.py: -------------------------------------------------------------------------------- 1 | AUDIT = 'audit' 2 | CREDIT = 'credit' 3 | HONOR = 'honor' 4 | PROFESSIONAL = 'professional' 5 | PROFESSIONAL_NO_ID = 'no-id-professional' 6 | VERIFIED = 'verified' 7 | MASTERS = 'masters' 8 | 9 | ALL = [AUDIT, CREDIT, HONOR, PROFESSIONAL, PROFESSIONAL_NO_ID, VERIFIED, MASTERS] 10 | -------------------------------------------------------------------------------- /analytics_data_api/constants/genders.py: -------------------------------------------------------------------------------- 1 | FEMALE = 'female' 2 | MALE = 'male' 3 | OTHER = 'other' 4 | UNKNOWN = 'unknown' 5 | ALL = [FEMALE, MALE, OTHER, UNKNOWN] 6 | -------------------------------------------------------------------------------- /analytics_data_api/constants/learner.py: -------------------------------------------------------------------------------- 1 | SEGMENTS = ["highly_engaged", "disengaging", "struggling", "inactive", "unenrolled"] 2 | UUID_REGEX_PATTERN = r'[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?4[0-9a-fA-F]{3}-?[89abAB][0-9a-fA-F]{3}-?[0-9a-fA-F]{12}' 3 | -------------------------------------------------------------------------------- /analytics_data_api/docker_gunicorn_configuration.py: -------------------------------------------------------------------------------- 1 | """ 2 | gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html 3 | This file is created and updated by ansible, edit at your peril 4 | """ 5 | 6 | timeout = 300 7 | bind = "127.0.0.1:8100" 8 | workers = 2 9 | worker_class = "gevent" 10 | 11 | 12 | def pre_request(worker, req): 13 | worker.log.info("%s %s" % (req.method, req.path)) 14 | -------------------------------------------------------------------------------- /analytics_data_api/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/analytics_data_api/management/__init__.py -------------------------------------------------------------------------------- /analytics_data_api/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/analytics_data_api/management/commands/__init__.py -------------------------------------------------------------------------------- /analytics_data_api/management/commands/generate_fake_course_data.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=line-too-long,invalid-name 2 | 3 | import datetime 4 | import logging 5 | 6 | from django.core.management.base import BaseCommand 7 | from django.utils import timezone 8 | 9 | from analytics_data_api.management.commands.generate_data import ( 10 | fake_video_ids_fallback, 11 | fetch_videos_from_course_blocks, 12 | generate_all_video_data, 13 | generate_daily_data, 14 | generate_learner_engagement_data, 15 | generate_learner_engagement_range_data, 16 | generate_program_data, 17 | generate_tags_distribution_data, 18 | generate_weekly_data, 19 | ) 20 | 21 | logging.basicConfig(level=logging.INFO) 22 | logger = logging.getLogger(__name__) 23 | 24 | 25 | class Command(BaseCommand): 26 | help = 'Generate fake data' 27 | 28 | def add_arguments(self, parser): 29 | parser.add_argument( 30 | '--num-weeks', 31 | action='store', 32 | type=int, 33 | dest='num_weeks', 34 | help='Number of weeks worth of data to generate.', 35 | ) 36 | parser.add_argument( 37 | '--course-id', 38 | action='store', 39 | dest='course_id', 40 | default='course-v1:edX+DemoX+Demo_Course', 41 | help='Course ID for which to generate fake data', 42 | ) 43 | parser.add_argument( 44 | '--username', 45 | action='store', 46 | dest='username', 47 | default='ed_xavier', 48 | help='Username for which to generate fake data', 49 | ) 50 | parser.add_argument( 51 | '--no-videos', 52 | action='store_false', 53 | dest='videos', 54 | default=True, 55 | help='Disables pulling video ids from the LMS server to generate fake video data and instead uses fake ids.' 56 | ) 57 | parser.add_argument( 58 | '--database', 59 | action='store', 60 | dest='database', 61 | default='default', 62 | help='Database in which to generate fake date', 63 | ) 64 | 65 | def handle(self, *args, **options): 66 | course_id = options['course_id'] 67 | username = options['username'] 68 | videos = options['videos'] 69 | database = options['database'] 70 | if videos: 71 | video_ids = fetch_videos_from_course_blocks(course_id) 72 | if not video_ids: 73 | logger.warning("Falling back to fake video id due to Course Blocks API failure...") 74 | video_ids = fake_video_ids_fallback() 75 | else: 76 | logger.info("Option to generate videos with ids pulled from the LMS is disabled, using fake video ids...") 77 | video_ids = fake_video_ids_fallback() 78 | start_date = timezone.now() - datetime.timedelta(weeks=10) 79 | 80 | num_weeks = options['num_weeks'] 81 | if num_weeks: 82 | end_date = start_date + datetime.timedelta(weeks=num_weeks) 83 | else: 84 | end_date = timezone.now().replace(microsecond=0) 85 | 86 | logger.info("Generating data for %s in database %s", course_id, database) 87 | 88 | generate_weekly_data(course_id, start_date, end_date, database) 89 | generate_daily_data(course_id, start_date, end_date, database) 90 | generate_program_data([course_id], 'Demo Program', 'Demo_Program', database) 91 | generate_all_video_data(course_id, video_ids, database) 92 | generate_learner_engagement_data(course_id, username, start_date, end_date, database) 93 | generate_learner_engagement_range_data(course_id, start_date.date(), end_date.date(), database) 94 | generate_tags_distribution_data(course_id, database) 95 | -------------------------------------------------------------------------------- /analytics_data_api/management/commands/generate_stage_course_data.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=line-too-long,invalid-name 2 | 3 | import datetime 4 | import logging 5 | 6 | from django.core.management.base import BaseCommand 7 | 8 | from analytics_data_api.management.commands.generate_data import ( 9 | fake_video_ids_fallback, 10 | generate_all_video_data, 11 | generate_daily_data, 12 | generate_learner_engagement_data, 13 | generate_learner_engagement_range_data, 14 | generate_program_data, 15 | generate_tags_distribution_data, 16 | generate_weekly_data, 17 | ) 18 | from analytics_data_api.v0 import models 19 | 20 | logging.basicConfig(level=logging.INFO) 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | COURSE_IDS = ['course-v1:edX+DemoX+Demo_Course', 'course-v1:edX+Insights+Stage', 'course-v1:edX+Insights+Fake+Data'] 25 | PROGRAM_COURSE_IDS = ['course-v1:edX+Insights+Stage', 'course-v1:edX+Insights+Fake+Data'] 26 | 27 | 28 | class Command(BaseCommand): 29 | help = 'Generate fake data for stage environment' 30 | 31 | def add_arguments(self, parser): 32 | parser.add_argument( 33 | '--database', 34 | action='store', 35 | dest='database', 36 | default='default', 37 | help='Database in which to generate fake date', 38 | ) 39 | 40 | def get_start_date(self, database): 41 | enrollments = models.CourseEnrollmentDaily.objects.using(database) 42 | # default to four weeks of data if no data exists 43 | start_date = datetime.datetime.now() - datetime.timedelta(weeks=4) 44 | if enrollments: 45 | latest_date = enrollments.latest().date 46 | # want to return latest date + some time difference to avoid overlap 47 | start_date = datetime.datetime.combine( 48 | latest_date + datetime.timedelta(days=1), datetime.datetime.min.time() 49 | ) 50 | return start_date 51 | 52 | def handle(self, *args, **options): 53 | usernames = ['ed_xavier', 'xander_x', 'alice_bob'] 54 | program_title = 'edX Insights' 55 | database = options['database'] 56 | 57 | logger.info("Option to generate videos with ids pulled from the LMS is disabled, using fake video ids...") 58 | video_ids = fake_video_ids_fallback() 59 | 60 | start_date = self.get_start_date(database) 61 | end_date = datetime.datetime.now() 62 | 63 | for course_id in COURSE_IDS: 64 | logger.info("Generating data for %s in database %s", course_id, database) 65 | generate_weekly_data(course_id, start_date, end_date, database, delete_data=False) 66 | generate_daily_data( 67 | course_id, start_date, end_date, database, 68 | delete_data=False, add_birth_year=False, use_current_cumulative=True 69 | ) 70 | generate_all_video_data(course_id, video_ids, database) 71 | for username in usernames: 72 | generate_learner_engagement_data(course_id, username, start_date, end_date, database) 73 | generate_learner_engagement_range_data(course_id, start_date.date(), end_date.date(), database) 74 | generate_tags_distribution_data(course_id, database) 75 | 76 | generate_program_data(PROGRAM_COURSE_IDS, program_title, '01n3fb1531o8470b832209243z7y421a', database) 77 | -------------------------------------------------------------------------------- /analytics_data_api/management/commands/set_api_key.py: -------------------------------------------------------------------------------- 1 | """A command to set the API key for a user using when using TokenAuthentication.""" 2 | 3 | from django.contrib.auth import get_user_model 4 | from django.core.management.base import BaseCommand, CommandError 5 | 6 | from analytics_data_api.utils import delete_user_auth_token, set_user_auth_token 7 | 8 | User = get_user_model() 9 | 10 | 11 | class Command(BaseCommand): 12 | """A command to set the API key for a user using when using TokenAuthentication.""" 13 | 14 | help = 'Set the API key for the specified user.' 15 | 16 | def add_arguments(self, parser): 17 | parser.add_argument('username', nargs='?') 18 | parser.add_argument('api_key', nargs='?') 19 | parser.add_argument( 20 | '--delete-key', 21 | action='store_true', 22 | default=False, 23 | help="Delete API key for user", 24 | ) 25 | 26 | def handle(self, *args, **options): 27 | if options['username'] is None: 28 | raise CommandError("You must supply a username.") 29 | 30 | username = options['username'] 31 | 32 | if options['delete_key']: 33 | delete_user_auth_token(username) 34 | print(f'Removed API key for user: <{username}>') 35 | else: 36 | if options['api_key'] is None: 37 | raise CommandError("You must supply both a username and key.") 38 | 39 | # pylint: disable=no-member 40 | user, _ = User.objects.get_or_create(username=username) 41 | 42 | try: 43 | key = options['api_key'] 44 | set_user_auth_token(user, key) 45 | except AttributeError: 46 | print("The key %s is in use by another user. Please select another key." % key) 47 | -------------------------------------------------------------------------------- /analytics_data_api/management/commands/tests/test_generate_fake_course_data.py: -------------------------------------------------------------------------------- 1 | from django.core.management import call_command 2 | from django.test import TestCase 3 | 4 | from analytics_data_api.tests.test_utils import set_databases 5 | from analytics_data_api.v0 import models 6 | 7 | 8 | @set_databases 9 | class GenerateFakeCourseDataTests(TestCase): 10 | def testNormalRun(self): 11 | num_weeks = 2 12 | course_id = "edX/DemoX/Demo_Course" 13 | 14 | call_command( 15 | 'generate_fake_course_data', 16 | f"--num-weeks={num_weeks}", 17 | "--no-videos", 18 | "--course-id", course_id, 19 | "--database", 'analytics', 20 | ) 21 | 22 | for model in [models.CourseEnrollmentDaily, 23 | models.CourseEnrollmentModeDaily, 24 | models.CourseEnrollmentByGender, 25 | models.CourseEnrollmentByEducation, 26 | models.CourseEnrollmentByBirthYear, 27 | models.CourseEnrollmentByCountry, 28 | models.CourseMetaSummaryEnrollment, 29 | models.CourseProgramMetadata]: 30 | self.assertTrue(model.objects.filter(course_id=course_id, ).exists()) 31 | self.assertEqual(model.objects.filter(course_id=course_id).count(), model.objects.all().count()) 32 | -------------------------------------------------------------------------------- /analytics_data_api/management/commands/tests/test_generate_stage_course_data.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from django.core.management import call_command 4 | from django.test import TestCase 5 | from freezegun import freeze_time 6 | 7 | from analytics_data_api.management.commands.generate_stage_course_data import COURSE_IDS, PROGRAM_COURSE_IDS 8 | from analytics_data_api.tests.test_utils import set_databases 9 | from analytics_data_api.v0 import models 10 | 11 | 12 | @set_databases 13 | class GenerateStageCourseDataTests(TestCase): 14 | 15 | def test_run_with_no_preexisting_data(self): 16 | """ 17 | Test that data is generated for the past four weeks 18 | """ 19 | 20 | call_command( 21 | 'generate_stage_course_data', 22 | f"--database", 'analytics', 23 | ) 24 | 25 | for model in [models.CourseEnrollmentDaily, 26 | models.CourseEnrollmentModeDaily, 27 | models.CourseEnrollmentByGender, 28 | models.CourseEnrollmentByEducation, 29 | models.CourseEnrollmentByCountry, 30 | models.CourseMetaSummaryEnrollment]: 31 | for course_id in COURSE_IDS: 32 | self.assertTrue(model.objects.filter(course_id=course_id).exists()) 33 | 34 | if model != models.CourseMetaSummaryEnrollment: 35 | # course meta summary generates a wider date range of data 36 | earliest_date = model.objects.filter(course_id=course_id).earliest('date').date 37 | self.assertEqual(earliest_date, datetime.date.today() - datetime.timedelta(weeks=4)) 38 | 39 | # check that Insights courses are in program table, while demo course is not 40 | for course_id in COURSE_IDS: 41 | if course_id in PROGRAM_COURSE_IDS: 42 | self.assertTrue( 43 | models.CourseProgramMetadata.objects.filter(course_id=course_id).exists() 44 | ) 45 | else: 46 | self.assertFalse( 47 | models.CourseProgramMetadata.objects.filter(course_id=course_id).exists() 48 | ) 49 | 50 | self.assertFalse(models.CourseEnrollmentByBirthYear.objects.exists()) 51 | 52 | def test_run_with_preexisting_data(self): 53 | """ 54 | Test that data is generated for the time between preexisting data and now, 55 | e.g. if data was added a week ago, should only add a weeks worth of data. 56 | No previous data should be deleted. 57 | """ 58 | 59 | start_date = datetime.date.today() - datetime.timedelta(weeks=6) 60 | end_date = datetime.date.today() 61 | 62 | # call command first time to generate data 63 | with freeze_time(datetime.date.today() - datetime.timedelta(weeks=2)): 64 | call_command( 65 | 'generate_stage_course_data', 66 | f"--database", 'analytics', 67 | ) 68 | test_enrollment = models.CourseEnrollmentDaily.objects\ 69 | .filter(course_id='course-v1:edX+DemoX+Demo_Course').earliest('date') 70 | 71 | # go ahead two weeks to generate data 72 | call_command( 73 | 'generate_stage_course_data', 74 | f"--database", 'analytics', 75 | ) 76 | 77 | for model in [models.CourseEnrollmentDaily, 78 | models.CourseEnrollmentModeDaily, 79 | models.CourseEnrollmentByGender, 80 | models.CourseEnrollmentByEducation, 81 | models.CourseEnrollmentByCountry, 82 | models.CourseMetaSummaryEnrollment]: 83 | for course_id in COURSE_IDS: 84 | self.assertTrue(model.objects.filter(course_id=course_id).exists()) 85 | 86 | if model != models.CourseMetaSummaryEnrollment: 87 | # course meta summary generates a wider date range of data 88 | earliest_date = model.objects.filter(course_id=course_id).earliest('date').date 89 | self.assertEqual(earliest_date, start_date) 90 | 91 | # assert that old data wasn't deleted 92 | self.assertEqual(test_enrollment.count, models.CourseEnrollmentDaily.objects.get(id=test_enrollment.id).count) 93 | 94 | # assert that date range is covered 95 | check_date = start_date 96 | time_delta = datetime.timedelta(days=1) 97 | 98 | while check_date <= end_date: 99 | enrollment_objs = models.CourseEnrollmentDaily.objects.filter( 100 | course_id='course-v1:edX+DemoX+Demo_Course', date=check_date 101 | ) 102 | self.assertEqual(len(enrollment_objs), 1) 103 | check_date += time_delta 104 | -------------------------------------------------------------------------------- /analytics_data_api/middleware.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from threading import local 3 | 4 | from django.conf import settings 5 | from django.http.response import JsonResponse 6 | from django.utils.deprecation import MiddlewareMixin 7 | from rest_framework import status 8 | 9 | from analytics_data_api.v0.exceptions import ( 10 | CannotCreateReportDownloadLinkError, 11 | CourseKeyMalformedError, 12 | CourseNotSpecifiedError, 13 | ReportFileNotFoundError, 14 | ) 15 | 16 | thread_data = local() 17 | 18 | 19 | class RequestVersionMiddleware: 20 | """ 21 | Add a database hint, analyticsapi_database, in the form of an attribute in thread-local storage. 22 | This is used by the AnalyticsAPIRouter to switch databases between the v0 and v1 views. 23 | """ 24 | def __init__(self, get_response): 25 | self.get_response = get_response 26 | 27 | def __call__(self, request): 28 | if 'api/v1' in request.path: 29 | thread_data.analyticsapi_database = getattr(settings, 'ANALYTICS_DATABASE_V1') 30 | else: 31 | thread_data.analyticsapi_database = getattr(settings, 'ANALYTICS_DATABASE', 'default') 32 | response = self.get_response(request) 33 | return response 34 | 35 | 36 | class BaseProcessErrorMiddleware(MiddlewareMixin, metaclass=abc.ABCMeta): 37 | """ 38 | Base error. 39 | """ 40 | 41 | @abc.abstractproperty 42 | def error(self): 43 | """ Error class to catch. """ 44 | 45 | @abc.abstractproperty 46 | def error_code(self): 47 | """ Error code to return. """ 48 | 49 | @abc.abstractproperty 50 | def status_code(self): 51 | """ HTTP status code to return. """ 52 | 53 | def process_exception(self, _request, exception): 54 | if isinstance(exception, self.error): 55 | return JsonResponse({ 56 | "error_code": self.error_code, 57 | "developer_message": str(exception) 58 | }, status=self.status_code) 59 | return None 60 | 61 | 62 | class CourseNotSpecifiedErrorMiddleware(BaseProcessErrorMiddleware): 63 | """ 64 | Raise 400 course not specified. 65 | """ 66 | 67 | @property 68 | def error(self): 69 | return CourseNotSpecifiedError 70 | 71 | @property 72 | def error_code(self): 73 | return 'course_not_specified' 74 | 75 | @property 76 | def status_code(self): 77 | return status.HTTP_400_BAD_REQUEST 78 | 79 | 80 | class CourseKeyMalformedErrorMiddleware(BaseProcessErrorMiddleware): 81 | """ 82 | Raise 400 if course key is malformed. 83 | """ 84 | 85 | @property 86 | def error(self): 87 | return CourseKeyMalformedError 88 | 89 | @property 90 | def error_code(self): 91 | return 'course_key_malformed' 92 | 93 | @property 94 | def status_code(self): 95 | return status.HTTP_400_BAD_REQUEST 96 | 97 | 98 | class ReportFileNotFoundErrorMiddleware(BaseProcessErrorMiddleware): 99 | """ 100 | Raise 404 if the report file isn't present 101 | """ 102 | 103 | @property 104 | def error(self): 105 | return ReportFileNotFoundError 106 | 107 | @property 108 | def error_code(self): 109 | return 'report_file_not_found' 110 | 111 | @property 112 | def status_code(self): 113 | return status.HTTP_404_NOT_FOUND 114 | 115 | 116 | class CannotCreateDownloadLinkErrorMiddleware(BaseProcessErrorMiddleware): 117 | """ 118 | Raise 501 if the filesystem doesn't support creating download links 119 | """ 120 | 121 | @property 122 | def error(self): 123 | return CannotCreateReportDownloadLinkError 124 | 125 | @property 126 | def error_code(self): 127 | return 'cannot_create_report_download_link' 128 | 129 | @property 130 | def status_code(self): 131 | return status.HTTP_501_NOT_IMPLEMENTED 132 | -------------------------------------------------------------------------------- /analytics_data_api/models.py: -------------------------------------------------------------------------------- 1 | # This file should be empty. Place your models in the package that corresponds to an API version (e.g. v0, v1). This 2 | # file only exists to make Django happy. 3 | -------------------------------------------------------------------------------- /analytics_data_api/renderers.py: -------------------------------------------------------------------------------- 1 | """ 2 | Custom REST framework renderers common to all versions of the API. 3 | """ 4 | 5 | 6 | from ordered_set import OrderedSet 7 | from rest_framework_csv.renderers import CSVRenderer 8 | 9 | 10 | class ResultsOnlyRendererMixin: 11 | """ 12 | Render data using just the results array. 13 | 14 | Use with PaginatedHeadersMixin to preserve the pagination links in the response header. 15 | """ 16 | results_field = 'results' 17 | 18 | def render(self, data, *args, **kwargs): 19 | """ 20 | Replace the rendered data with just what is in the results_field. 21 | """ 22 | if not isinstance(data, list): 23 | data = data.get(self.results_field, []) 24 | return super().render(data, *args, **kwargs) 25 | 26 | 27 | class DynamicFieldsCsvRenderer(CSVRenderer): 28 | """ 29 | Allows the `fields` query parameter to determine which fields should be 30 | returned in the response, and in what order. 31 | 32 | Note that if no header is provided, and the fields_param query string 33 | parameter is not found in the request, the fields are rendered in 34 | alphabetical order. 35 | """ 36 | # Name of the query string parameter to check for the fields list 37 | # Set to None to ensure that any request fields will not override 38 | fields_param = 'fields' 39 | 40 | # Seperator character(s) to split the fields parameter list 41 | fields_sep = ',' 42 | 43 | # Set to None to flatten lists into one heading per value. 44 | # Otherwise, concatenate lists delimiting with the given string. 45 | concatenate_lists_sep = ', ' 46 | 47 | def flatten_list(self, l): 48 | if self.concatenate_lists_sep is None: 49 | return super().flatten_list(l) 50 | return {'': self.concatenate_lists_sep.join(l)} 51 | 52 | def get_header(self, data, renderer_context): 53 | """Return the list of header fields, determined by class settings and context.""" 54 | 55 | # Start with the previously-set list of header fields 56 | header = renderer_context.get('header', self.header) 57 | 58 | # If no previous set, then determine the candidates from the data 59 | if header is None: 60 | header = set() 61 | data = self.flatten_data(data) 62 | for item in data: 63 | header.update(list(item.keys())) 64 | 65 | # Alphabetize header fields by default, since 66 | # flatten_data() makes field order indeterminate. 67 | header = sorted(header) 68 | 69 | # If configured to, examine the query parameters for the requsted header fields 70 | request = renderer_context.get('request') 71 | if request is not None and self.fields_param is not None: 72 | 73 | request_fields = request.query_params.get(self.fields_param) 74 | if request_fields is not None: 75 | 76 | requested = OrderedSet() 77 | for request_field in request_fields.split(self.fields_sep): 78 | 79 | # Only fields in the original candidate header set are valid 80 | if request_field in header: 81 | requested.update((request_field,)) 82 | 83 | header = requested 84 | 85 | return header 86 | 87 | def render(self, data, media_type=None, renderer_context=None, writer_opts=None): 88 | """Override the default "get headers" behaviour, then render the data.""" 89 | renderer_context = renderer_context or {} 90 | self.header = self.get_header(data, renderer_context) 91 | self.labels = renderer_context.get('labels') 92 | return super().render(data, media_type, renderer_context, writer_opts) 93 | 94 | 95 | class PaginatedCsvRenderer(ResultsOnlyRendererMixin, DynamicFieldsCsvRenderer): 96 | """ 97 | Render results-only CSV data with dynamically-determined fields. 98 | """ 99 | media_type = 'text/csv' 100 | -------------------------------------------------------------------------------- /analytics_data_api/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/analytics_data_api/tests/__init__.py -------------------------------------------------------------------------------- /analytics_data_api/tests/test_middleware.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.test import override_settings 3 | 4 | from analytics_data_api.middleware import thread_data 5 | from analytics_data_api.tests.test_utils import set_databases 6 | from analytics_data_api.v0.tests.views import CourseSamples 7 | from analyticsdataserver.tests.utils import TestCaseWithAuthentication 8 | 9 | 10 | @set_databases 11 | class RequestVersionMiddleware(TestCaseWithAuthentication): 12 | def test_request_version_middleware_v1(self): 13 | self.authenticated_get('/api/v1/courses/{}/activity'.format( 14 | CourseSamples.course_ids[0])) 15 | 16 | self.assertEqual(thread_data.analyticsapi_database, getattr(settings, 'ANALYTICS_DATABASE_V1')) 17 | 18 | @override_settings(ANALYTICS_DATABASE_V1=None) 19 | def test_request_version_middleware_v1_no_setting(self): 20 | self.authenticated_get('/api/v1/courses/{}/activity'.format( 21 | CourseSamples.course_ids[0])) 22 | 23 | self.assertEqual(thread_data.analyticsapi_database, getattr(settings, 'ANALYTICS_DATABASE_V1')) 24 | 25 | def test_request_version_middleware_v0(self): 26 | self.authenticated_get('/api/v0/courses/{}/activity'.format( 27 | CourseSamples.course_ids[0])) 28 | 29 | self.assertEqual("analytics", getattr(thread_data, 'analyticsapi_database')) 30 | -------------------------------------------------------------------------------- /analytics_data_api/tests/test_renderers.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for the custom REST framework renderers. 3 | """ 4 | 5 | from unittest.mock import MagicMock, PropertyMock 6 | 7 | from django.test import TestCase 8 | 9 | from analytics_data_api.renderers import PaginatedCsvRenderer 10 | 11 | 12 | class PaginatedCsvRendererTests(TestCase): 13 | 14 | def setUp(self): 15 | super().setUp() 16 | self.renderer = PaginatedCsvRenderer() 17 | self.data = {'results': [ 18 | { 19 | 'string': 'ab,c', 20 | 'list': ['a', 'b', 'c'], 21 | 'dict': {'a': 1, 'b': 2, 'c': 3}, 22 | }, { 23 | 'string': 'def', 24 | 'string2': 'ghi', 25 | 'list': ['d', 'e', 'f', 'g'], 26 | 'dict': {'d': 4, 'b': 5, 'c': 6}, 27 | }, 28 | ]} 29 | self.context = {} 30 | 31 | def set_request(self, params=None): 32 | request = MagicMock() 33 | mock_params = PropertyMock(return_value=params) 34 | type(request).query_params = mock_params 35 | self.context['request'] = request 36 | 37 | def test_csv_media_type(self): 38 | self.assertEqual(self.renderer.media_type, 'text/csv') 39 | 40 | def test_render(self): 41 | rendered_data = self.renderer.render(self.data, renderer_context=self.context) 42 | self.assertEqual(rendered_data, 43 | b'dict.a,dict.b,dict.c,dict.d,list,string,string2\r\n' 44 | b'1,2,3,,"a, b, c","ab,c",\r\n' 45 | b',5,6,4,"d, e, f, g",def,ghi\r\n') 46 | 47 | def test_render_fields(self): 48 | self.set_request(dict(fields='string2,invalid,dict.b,list,dict.a,string')) 49 | rendered_data = self.renderer.render(self.data, renderer_context=self.context) 50 | self.assertEqual(rendered_data, 51 | b'string2,dict.b,list,dict.a,string\r\n' 52 | b',2,"a, b, c",1,"ab,c"\r\n' 53 | b'ghi,5,"d, e, f, g",,def\r\n') 54 | 55 | def test_render_flatten_lists(self): 56 | self.renderer.concatenate_lists_sep = None 57 | rendered_data = self.renderer.render(self.data, renderer_context=self.context) 58 | self.assertEqual(rendered_data, 59 | b'dict.a,dict.b,dict.c,dict.d,list.0,list.1,list.2,list.3,string,string2\r\n' 60 | b'1,2,3,,a,b,c,,"ab,c",\r\n' 61 | b',5,6,4,d,e,f,g,def,ghi\r\n') 62 | 63 | def test_render_fields_flatten_lists(self): 64 | self.renderer.concatenate_lists_sep = None 65 | self.set_request(dict(fields='string2,invalid,list.2,dict.a,list.1,string')) 66 | rendered_data = self.renderer.render(self.data, renderer_context=self.context) 67 | self.assertEqual(rendered_data, 68 | b'string2,list.2,dict.a,list.1,string\r\n' 69 | b',c,1,b,"ab,c"\r\n' 70 | b'ghi,f,,e,def\r\n') 71 | 72 | def test_render_fields_limit_headers(self): 73 | self.renderer.header = ('string2', 'invalid', 'dict.a') 74 | self.set_request(dict(fields='string2,invalid,dict.b,list,dict.a,string')) 75 | rendered_data = self.renderer.render(self.data, renderer_context=self.context) 76 | self.assertEqual(rendered_data, 77 | b'string2,invalid,dict.a\r\n' 78 | b',,1\r\n' 79 | b'ghi,,\r\n') 80 | -------------------------------------------------------------------------------- /analytics_data_api/tests/test_throttles.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | from django.core.cache import caches 4 | from django.test import override_settings 5 | 6 | from analytics_data_api.throttles import ServiceUserThrottle 7 | from analyticsdataserver.tests.utils import TestCaseWithAuthentication 8 | 9 | 10 | class RateLimitingTests(TestCaseWithAuthentication): 11 | """ 12 | Test cases for the rate limiting of analytics API calls 13 | """ 14 | 15 | worker_name = 'test_worker' 16 | 17 | def setUp(self): 18 | super().setUp() 19 | self.path = '/docs' 20 | 21 | def tearDown(self): 22 | super().tearDown() 23 | caches['default'].clear() 24 | 25 | def _make_requests(self, num_requests, throttle_rate): 26 | """ 27 | Make num_requests to an endpoint 28 | Return the response from the last request 29 | """ 30 | with patch('rest_framework.views.APIView.throttle_classes', (ServiceUserThrottle,)): 31 | with patch.object(ServiceUserThrottle, 'THROTTLE_RATES', throttle_rate): 32 | for __ in range(num_requests - 1): 33 | response = self.authenticated_get(self.path) 34 | assert response.status_code == 200 35 | response = self.authenticated_get(self.path) 36 | return response 37 | 38 | def test_rate_limiting(self): 39 | response = self._make_requests(6, {'user': '5/hour'}) 40 | assert response.status_code == 429 41 | 42 | @override_settings( 43 | ANALYTICS_API_SERVICE_USERNAMES=[worker_name], 44 | ) 45 | def test_allowed_service_user(self): 46 | self.test_user.username = self.worker_name 47 | self.test_user.save() 48 | 49 | response = self._make_requests(5, {'service_user': '10/hour', 'user': '1/hour'}) 50 | assert response.status_code == 200 51 | 52 | @override_settings( 53 | ANALYTICS_API_SERVICE_USERNAMES=[worker_name], 54 | ) 55 | def test_denied_service_user(self): 56 | self.test_user.username = self.worker_name 57 | self.test_user.save() 58 | 59 | response = self._make_requests(6, {'service_user': '5/hour', 'user': '1/hour'}) 60 | assert response.status_code == 429 61 | -------------------------------------------------------------------------------- /analytics_data_api/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from django.contrib.auth.models import User 4 | from django.core.management import CommandError, call_command 5 | from django.test import TestCase 6 | from django_dynamic_fixture import G 7 | from rest_framework.authtoken.models import Token 8 | 9 | from analytics_data_api.constants.country import UNKNOWN_COUNTRY, get_country 10 | from analytics_data_api.utils import date_range, delete_user_auth_token, set_user_auth_token 11 | 12 | 13 | class UtilsTests(TestCase): 14 | def test_delete_user_auth_token(self): 15 | # Create user and token 16 | user = G(User) 17 | G(Token, user=user) 18 | 19 | # Verify token exists 20 | self.assertTrue(Token.objects.filter(user=user).exists()) 21 | 22 | # Call delete method 23 | delete_user_auth_token(user.username) 24 | 25 | # Verify token no longer exists 26 | self.assertFalse(Token.objects.filter(user=user).exists()) 27 | 28 | def test_delete_user_auth_token_non_existing(self): 29 | user = G(User) 30 | self.assertFalse(Token.objects.filter(user=user).exists()) 31 | delete_user_auth_token(user.username) 32 | self.assertFalse(Token.objects.filter(user=user).exists()) 33 | 34 | def test_set_user_auth_token(self): 35 | user = G(User) 36 | key = "Avengers Assemble!" 37 | self.assertFalse(Token.objects.filter(user=user).exists()) 38 | set_user_auth_token(user, key) 39 | self.assertEqual(Token.objects.get(user=user).key, key) 40 | 41 | key = "Hulk Smash!" 42 | set_user_auth_token(user, key) 43 | self.assertEqual(Token.objects.get(user=user).key, key) 44 | 45 | # Verify we don't create token conflicts 46 | user2 = G(User) 47 | self.assertRaises(AttributeError, set_user_auth_token, user2, key) 48 | 49 | 50 | class SetApiKeyTests(TestCase): 51 | def test_delete_key(self): 52 | user = G(User) 53 | G(Token, user=user) 54 | self.assertTrue(Token.objects.filter(user=user).exists()) 55 | call_command('set_api_key', user.username, delete_key=True) 56 | self.assertFalse(Token.objects.filter(user=user).exists()) 57 | 58 | def test_invalid_arguments(self): 59 | self.assertRaises(CommandError, call_command, 'set_api_key') 60 | self.assertRaises(CommandError, call_command, 'set_api_key', 'username') 61 | 62 | def test_set_key(self): 63 | user = G(User) 64 | key = "Super Secret!" 65 | self.assertFalse(Token.objects.filter(user=user).exists()) 66 | call_command('set_api_key', user.username, key) 67 | self.assertEqual(Token.objects.get(user=user).key, key) 68 | 69 | key = "No one will guess this!" 70 | call_command('set_api_key', user.username, key) 71 | self.assertEqual(Token.objects.get(user=user).key, key) 72 | 73 | def test_set_key_conflict(self): 74 | key = "Super Secret!" 75 | user = G(User) 76 | user2 = G(User) 77 | G(Token, user=user, key=key) 78 | 79 | self.assertFalse(Token.objects.filter(user=user2).exists()) 80 | call_command('set_api_key', user2.username, key) 81 | self.assertFalse(Token.objects.filter(user=user2).exists()) 82 | 83 | 84 | class CountryTests(TestCase): 85 | def test_get_country(self): 86 | # Countries should be accessible 2 or 3 digit country code 87 | self.assertEqual(get_country('US'), get_country('USA')) 88 | 89 | # Use common name for Taiwan 90 | self.assertEqual(get_country('TW').name, 'Taiwan') 91 | 92 | # Return unknown country if code is invalid 93 | self.assertEqual(get_country('A1'), UNKNOWN_COUNTRY) 94 | self.assertEqual(get_country(None), UNKNOWN_COUNTRY) 95 | 96 | 97 | class DateRangeTests(TestCase): 98 | def test_empty_range(self): 99 | same_date = datetime.datetime(2016, 1, 1) 100 | self.assertEqual(list(date_range(same_date, same_date)), []) 101 | 102 | def test_range_exclusive(self): 103 | start_date = datetime.datetime(2016, 1, 1) 104 | end_date = datetime.datetime(2016, 1, 2) 105 | self.assertEqual(list(date_range(start_date, end_date)), [start_date]) 106 | 107 | def test_delta_goes_past_end_date(self): 108 | start_date = datetime.datetime(2016, 1, 1) 109 | end_date = datetime.datetime(2016, 1, 3) 110 | time_delta = datetime.timedelta(days=5) 111 | self.assertEqual(list(date_range(start_date, end_date, time_delta)), [start_date]) 112 | 113 | def test_general_range(self): 114 | start_date = datetime.datetime(2016, 1, 1) 115 | end_date = datetime.datetime(2016, 1, 5) 116 | self.assertEqual(list(date_range(start_date, end_date)), [ 117 | datetime.datetime(2016, 1, 1), 118 | datetime.datetime(2016, 1, 2), 119 | datetime.datetime(2016, 1, 3), 120 | datetime.datetime(2016, 1, 4), 121 | ]) 122 | 123 | 124 | def set_databases(cls): 125 | """ 126 | This is to be used as a class decorator to set the databases 127 | test class attribute to ensure that all databases are flushed. 128 | Please see https://docs.djangoproject.com/en/3.2/topics/testing/tools/#multi-database-support. 129 | """ 130 | def wrapper(cls): 131 | setattr(cls, 'databases', '__all__') 132 | return cls 133 | return wrapper(cls) 134 | -------------------------------------------------------------------------------- /analytics_data_api/throttles.py: -------------------------------------------------------------------------------- 1 | """ 2 | Throttle classes for edx-analytics-data-api. 3 | """ 4 | from django.conf import settings 5 | from rest_framework.throttling import UserRateThrottle 6 | 7 | SERVICE_USER_SCOPE = 'service_user' 8 | 9 | 10 | class ServiceUserThrottle(UserRateThrottle): 11 | """ 12 | A throttle allowing the service user to override rate limiting. 13 | """ 14 | 15 | def allow_request(self, request, view): 16 | """ 17 | Modify throttling for service users. 18 | Updates throttling rate if the request is coming from the service user, and 19 | defaults to UserRateThrottle's configured setting otherwise. 20 | Updated throttling rate comes from `DEFAULT_THROTTLE_RATES` key in `REST_FRAMEWORK` 21 | setting. service user throttling is specified in `DEFAULT_THROTTLE_RATES` by `service_user` key 22 | Example Setting:: 23 | REST_FRAMEWORK = { 24 | ... 25 | 'DEFAULT_THROTTLE_RATES': { 26 | ... 27 | 'service_user': '50/day' 28 | } 29 | } 30 | """ 31 | service_users = getattr(settings, 'ANALYTICS_API_SERVICE_USERNAMES', None) 32 | 33 | # User service user throttling rates for service user. 34 | if service_users and request.user.username in service_users: 35 | self.update_throttle_scope() 36 | 37 | return super().allow_request(request, view) 38 | 39 | def update_throttle_scope(self): 40 | """ 41 | Update throttle scope so that service user throttle rates are applied. 42 | """ 43 | self.scope = SERVICE_USER_SCOPE 44 | self.rate = self.get_rate() 45 | self.num_requests, self.duration = self.parse_rate(self.rate) 46 | -------------------------------------------------------------------------------- /analytics_data_api/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import include, re_path 2 | from rest_framework.urlpatterns import format_suffix_patterns 3 | 4 | app_name = 'analytics_data_api' 5 | 6 | urlpatterns = [ 7 | re_path(r'^v0/', include('analytics_data_api.v0.urls')), 8 | re_path(r'^v1/', include('analytics_data_api.v1.urls')), 9 | ] 10 | 11 | urlpatterns = format_suffix_patterns(urlpatterns) 12 | -------------------------------------------------------------------------------- /analytics_data_api/v0/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/analytics_data_api/v0/__init__.py -------------------------------------------------------------------------------- /analytics_data_api/v0/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class ApiAppConfig(AppConfig): 5 | 6 | name = 'analytics_data_api.v0' 7 | -------------------------------------------------------------------------------- /analytics_data_api/v0/exceptions.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | 4 | class BaseError(Exception, metaclass=abc.ABCMeta): 5 | """ 6 | Base error. 7 | """ 8 | 9 | message = None 10 | 11 | def __str__(self): 12 | return self.message 13 | 14 | 15 | class CourseNotSpecifiedError(BaseError): 16 | """ 17 | Raise if course not specified. 18 | """ 19 | def __init__(self, *args, **kwargs): 20 | super().__init__(*args, **kwargs) 21 | self.message = 'Course id/key not specified.' 22 | 23 | 24 | class CourseKeyMalformedError(BaseError): 25 | """ 26 | Raise if course id/key malformed. 27 | """ 28 | def __init__(self, *args, **kwargs): 29 | course_id = kwargs.pop('course_id') 30 | super().__init__(*args, **kwargs) 31 | self.message = self.message_template.format(course_id=course_id) 32 | 33 | @property 34 | def message_template(self): 35 | return 'Course id/key {course_id} malformed.' 36 | 37 | 38 | class ReportFileNotFoundError(BaseError): 39 | """ 40 | Raise if we couldn't find the file we need to produce the report 41 | """ 42 | def __init__(self, *args, **kwargs): 43 | course_id = kwargs.pop('course_id') 44 | report_name = kwargs.pop('report_name') 45 | super().__init__(*args, **kwargs) 46 | self.message = self.message_template.format(course_id=course_id, report_name=report_name) 47 | 48 | @property 49 | def message_template(self): 50 | return 'Could not find report \'{report_name}\' for course {course_id}.' 51 | 52 | 53 | class CannotCreateReportDownloadLinkError(BaseError): 54 | """ 55 | Raise if we cannot create a link for the file to be downloaded 56 | """ 57 | 58 | message = 'Could not create a downloadable link to the report.' 59 | -------------------------------------------------------------------------------- /analytics_data_api/v0/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/analytics_data_api/v0/tests/__init__.py -------------------------------------------------------------------------------- /analytics_data_api/v0/tests/test_models.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from django_dynamic_fixture import G 3 | 4 | from analytics_data_api.constants.country import UNKNOWN_COUNTRY, get_country 5 | from analytics_data_api.tests.test_utils import set_databases 6 | from analytics_data_api.v0 import models 7 | 8 | 9 | @set_databases 10 | class CourseEnrollmentByCountryTests(TestCase): 11 | def test_country(self): 12 | country = get_country('US') 13 | self.assertEqual(country.alpha2, 'US') 14 | instance = G(models.CourseEnrollmentByCountry, country_code=country.alpha2) 15 | self.assertEqual(instance.country, country) 16 | 17 | def test_invalid_country(self): 18 | instance = G(models.CourseEnrollmentByCountry, country_code='') 19 | self.assertEqual(instance.country, UNKNOWN_COUNTRY) 20 | 21 | instance = G(models.CourseEnrollmentByCountry, country_code='A1') 22 | self.assertEqual(instance.country, UNKNOWN_COUNTRY) 23 | 24 | instance = G(models.CourseEnrollmentByCountry, country_code='GobbledyGoop!') 25 | self.assertEqual(instance.country, UNKNOWN_COUNTRY) 26 | 27 | instance = G(models.CourseEnrollmentByCountry, country_code='UNKNOWN') 28 | self.assertEqual(instance.country, UNKNOWN_COUNTRY) 29 | -------------------------------------------------------------------------------- /analytics_data_api/v0/tests/test_serializers.py: -------------------------------------------------------------------------------- 1 | from datetime import date 2 | 3 | import ddt 4 | from django.test import TestCase 5 | from django_dynamic_fixture import G 6 | 7 | from analytics_data_api.tests.test_utils import set_databases 8 | from analytics_data_api.v0 import models as api_models 9 | from analytics_data_api.v0 import serializers as api_serializers 10 | 11 | 12 | class TestSerializer(api_serializers.CourseEnrollmentDailySerializer, api_serializers.DynamicFieldsModelSerializer): 13 | pass 14 | 15 | 16 | @set_databases 17 | class DynamicFieldsModelSerializerTests(TestCase): 18 | def test_fields(self): 19 | now = date.today() 20 | instance = G(api_models.CourseEnrollmentDaily, course_id='1', count=1, date=now) 21 | serialized = TestSerializer(instance) 22 | self.assertListEqual(list(serialized.data.keys()), ['course_id', 'date', 'count', 'created']) 23 | 24 | instance = G(api_models.CourseEnrollmentDaily, course_id='2', count=1, date=now) 25 | serialized = TestSerializer(instance, fields=('course_id',)) 26 | self.assertListEqual(list(serialized.data.keys()), ['course_id']) 27 | 28 | def test_exclude(self): 29 | now = date.today() 30 | instance = G(api_models.CourseEnrollmentDaily, course_id='3', count=1, date=now) 31 | serialized = TestSerializer(instance, exclude=('course_id',)) 32 | self.assertListEqual(list(serialized.data.keys()), ['date', 'count', 'created']) 33 | 34 | 35 | @set_databases 36 | @ddt.ddt 37 | class ProblemResponseAnswerDistributionSerializerTests(TestCase): 38 | 39 | @ddt.data( 40 | ('', ''), 41 | ('justastring', 'justastring'), 42 | ('25.94', '25.94'), 43 | ('[0, 100, 7, "x", "Duōshǎo"]', '[0|100|7|x|Duōshǎo]'), 44 | ('"a","b" Correct. o', '"a","b" Correct. o'), 45 | ('[{"3":"t1"},{"2":"t2"}]', '[{"3": "t1"}|{"2": "t2"}]'), 46 | ('[[1,0,0],[0,1,0],[0,0,1],[0,0,0]]', '[[1, 0, 0]|[0, 1, 0]|[0, 0, 1]|[0, 0, 0]]'), 47 | ('["(S \\to D \\to G)"]', '[(S \\to D \\to G)]'), 48 | ('\\((2,1,3)^\\n\\)', '\\((2,1,3)^\\n\\)'), 49 | ( 50 | 'MLE [mathjax]\\widehat{\\theta }_ n^{\\text {MLE}}[/mathjax]', 51 | 'MLE [mathjax]\\widehat{\\theta }_ n^{\\text {MLE}}[/mathjax]' 52 | ), 53 | ('', ''), 54 | ('
(01/30)
', '(0 1/3 0)'), 55 | ) 56 | @ddt.unpack 57 | def test_answer_formatting(self, answer_value, expected_display): 58 | instance = G( 59 | api_models.ProblemFirstLastResponseAnswerDistribution, 60 | answer_value=answer_value, 61 | value_id=answer_value 62 | ) 63 | serialized = api_serializers.ProblemFirstLastResponseAnswerDistributionSerializer(instance) 64 | self.assertEqual(serialized.data['answer_value'], expected_display) 65 | self.assertEqual(serialized.data['value_id'], expected_display) 66 | 67 | @ddt.data( 68 | ('"hello**ONEBACKSLASHQUOTE**', '"hello\\"'), 69 | ('**BACKSLASHSPACE**hello', '\\ hello'), 70 | ('**FOURBACKSLASHQUOTE****TWOBACKSLASHQUOTE**', '\\\\\\\\"\\\\"'), 71 | ) 72 | @ddt.unpack 73 | def test_answer_value_unslugged(self, answer_value, expected_display): 74 | instance = G(api_models.ProblemFirstLastResponseAnswerDistribution, answer_value=answer_value) 75 | serialized = api_serializers.ProblemFirstLastResponseAnswerDistributionSerializer(instance) 76 | self.assertEqual(serialized.data['answer_value'], expected_display) 77 | -------------------------------------------------------------------------------- /analytics_data_api/v0/tests/test_urls.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from django.urls import reverse 3 | 4 | from analytics_data_api.tests.test_utils import set_databases 5 | 6 | 7 | @set_databases 8 | class UrlRedirectTests(TestCase): 9 | api_root_path = '/api/v0/' 10 | 11 | def assertRedirectsToRootPath(self, path, **kwargs): 12 | assert_kwargs = {'status_code': 302} 13 | assert_kwargs.update(kwargs) 14 | 15 | p = f'{self.api_root_path}{path}/' 16 | response = self.client.get(p) 17 | self.assertRedirects(response, reverse(path), **assert_kwargs) 18 | 19 | def test_authenticated(self): 20 | self.assertRedirectsToRootPath('authenticated', target_status_code=401) 21 | 22 | def test_health(self): 23 | self.assertRedirectsToRootPath('health') 24 | 25 | def test_status(self): 26 | self.assertRedirectsToRootPath('status') 27 | -------------------------------------------------------------------------------- /analytics_data_api/v0/tests/utils.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import datetime 3 | 4 | import enterprise_data 5 | import pytz 6 | from django_dynamic_fixture import G 7 | 8 | from analytics_data_api.v0 import models 9 | 10 | 11 | def flatten(dictionary, parent_key='', sep='.'): 12 | """ 13 | Flatten dictionary 14 | 15 | http://stackoverflow.com/a/6027615 16 | """ 17 | items = [] 18 | for key, value in dictionary.items(): 19 | new_key = parent_key + sep + key if parent_key else key 20 | if isinstance(value, collections.MutableMapping): 21 | items.extend(list(flatten(value, new_key).items())) 22 | else: 23 | items.append((new_key, value)) 24 | return dict(items) 25 | 26 | 27 | def create_engagement(course_id, username, entity_type, event_type, entity_id, count, date=None): 28 | """Create a ModuleEngagement model""" 29 | if date is None: 30 | date = datetime.datetime(2015, 1, 1, tzinfo=pytz.utc) 31 | G( 32 | models.ModuleEngagement, 33 | course_id=course_id, 34 | username=username, 35 | date=date, 36 | entity_type=entity_type, 37 | entity_id=entity_id, 38 | event=event_type, 39 | count=count, 40 | created=date, 41 | ) 42 | 43 | 44 | def create_enterprise_user(enterprise_customer_uuid, username, enterprise_user_id=1000): 45 | """Create EnterpriseUser model data""" 46 | G( 47 | enterprise_data.models.EnterpriseUser, 48 | enterprise_id=enterprise_customer_uuid, 49 | lms_user_id=1, 50 | enterprise_user_id=enterprise_user_id, 51 | user_username=username, 52 | ) 53 | -------------------------------------------------------------------------------- /analytics_data_api/v0/tests/views/__init__.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import json 3 | from collections import OrderedDict 4 | 5 | import six 6 | from rest_framework import status 7 | from six.moves.urllib.parse import urlencode # pylint: disable=import-error 8 | 9 | from analytics_data_api.v0.tests.utils import flatten 10 | 11 | 12 | class CourseSamples: 13 | 14 | course_ids = [ 15 | 'edX/DemoX/Demo_Course', 16 | 'course-v1:edX+DemoX+Demo_2014', 17 | 'ccx-v1:edx+1.005x-CCX+rerun+ccx@15' 18 | ] 19 | 20 | program_ids = [ 21 | '482dee71-e4b9-4b42-a47b-3e16bb69e8f2', 22 | '71c14f59-35d5-41f2-a017-e108d2d9f127', 23 | 'cfc6b5ee-6aa1-4c82-8421-20418c492618' 24 | ] 25 | 26 | 27 | class VerifyCourseIdMixin: 28 | 29 | def verify_bad_course_id(self, response, course_id='malformed-course-id'): 30 | """ Assert that a course ID must be valid. """ 31 | self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) 32 | expected = { 33 | "error_code": "course_key_malformed", 34 | "developer_message": f"Course id/key {course_id} malformed." 35 | } 36 | self.assertDictEqual(json.loads(response.content.decode('utf-8')), expected) 37 | 38 | 39 | class VerifyCsvResponseMixin: 40 | 41 | def assertCsvResponseIsValid(self, response, expected_filename, expected_data): 42 | # Validate the basic response status, content type, and filename 43 | self.assertEqual(response.status_code, 200) 44 | self.assertEqual(response['Content-Type'].split(';')[0], 'text/csv') 45 | self.assertEqual(response['Content-Disposition'], f'attachment; filename={expected_filename}') 46 | 47 | data = list(map(flatten, expected_data)) 48 | 49 | # The CSV renderer sorts the headers alphabetically 50 | fieldnames = sorted(data[0].keys()) 51 | 52 | # Generate the expected CSV output 53 | expected = six.StringIO() 54 | writer = csv.DictWriter(expected, fieldnames) 55 | writer.writeheader() 56 | writer.writerows(data) 57 | self.assertEqual(response.content.decode('utf-8'), expected.getvalue()) 58 | 59 | 60 | class APIListViewTestMixin: 61 | model = None 62 | model_id = 'id' 63 | ids_param = 'ids' 64 | serializer = None 65 | expected_results = [] 66 | list_name = 'list' 67 | default_ids = [] 68 | always_exclude = ['created'] 69 | test_post_method = False 70 | 71 | def path(self, query_data=None): 72 | query_data = query_data or {} 73 | concat_query_data = { 74 | param: arg if isinstance(arg, str) else ','.join(arg) 75 | for param, arg in query_data.items() if arg 76 | } 77 | query_string = '?{}'.format(urlencode(concat_query_data)) if concat_query_data else '' 78 | return f'/api/v0/{self.list_name}/{query_string}' 79 | 80 | def validated_request(self, ids=None, fields=None, exclude=None, **extra_args): 81 | params = [self.ids_param, 'fields', 'exclude'] 82 | args = [ids, fields, exclude] 83 | data = {param: arg for param, arg in zip(params, args) if arg} 84 | data.update(extra_args) 85 | 86 | get_response = self.authenticated_get(self.path(data)) 87 | if self.test_post_method: 88 | post_response = self.authenticated_post(self.path(), data=data) 89 | self.assertEqual(get_response.status_code, post_response.status_code) 90 | if 200 <= get_response.status_code < 300: 91 | self.assertEqual(get_response.data, post_response.data) 92 | 93 | return get_response 94 | 95 | def create_model(self, model_id, **kwargs): 96 | pass # implement in subclass 97 | 98 | def generate_data(self, ids=None, **kwargs): 99 | """Generate list data""" 100 | if ids is None: 101 | ids = self.default_ids 102 | 103 | for item_id in ids: 104 | self.create_model(item_id, **kwargs) 105 | 106 | def expected_result(self, item_id, **kwargs): # pylint: disable=unused-argument 107 | result = OrderedDict([ 108 | (self.model_id, item_id), 109 | ]) 110 | return result 111 | 112 | def all_expected_results(self, ids=None, **kwargs): 113 | if ids is None: 114 | ids = self.default_ids 115 | 116 | return [self.expected_result(item_id, **kwargs) for item_id in ids] 117 | 118 | def _test_all_items(self, ids): 119 | self.generate_data() 120 | response = self.validated_request(ids=ids, exclude=self.always_exclude) 121 | self.assertEqual(response.status_code, 200) 122 | self.assertCountEqual(response.data, self.all_expected_results(ids=ids)) 123 | 124 | def _test_one_item(self, item_id): 125 | self.generate_data() 126 | response = self.validated_request(ids=[item_id], exclude=self.always_exclude) 127 | self.assertEqual(response.status_code, 200) 128 | self.assertCountEqual(response.data, [self.expected_result(item_id)]) 129 | 130 | def _test_fields(self, fields): 131 | self.generate_data() 132 | response = self.validated_request(fields=fields) 133 | self.assertEqual(response.status_code, 200) 134 | 135 | # remove fields not requested from expected results 136 | expected_results = self.all_expected_results() 137 | for expected_result in expected_results: 138 | for field_to_remove in set(expected_result.keys()) - set(fields): 139 | expected_result.pop(field_to_remove) 140 | 141 | self.assertCountEqual(response.data, expected_results) 142 | 143 | def test_no_items(self): 144 | response = self.validated_request() 145 | self.assertEqual(response.status_code, 404) 146 | 147 | def test_no_matching_items(self): 148 | self.generate_data() 149 | response = self.validated_request(ids=['no/items/found']) 150 | self.assertEqual(response.status_code, 404) 151 | -------------------------------------------------------------------------------- /analytics_data_api/v0/tests/views/test_enterprise_learner_engagements.py: -------------------------------------------------------------------------------- 1 | import ddt 2 | from django.conf import settings 3 | from rest_framework import status 4 | 5 | from analytics_data_api.constants.engagement_events import ( 6 | ATTEMPTED, 7 | COMPLETED, 8 | CONTRIBUTED, 9 | DISCUSSION, 10 | PROBLEM, 11 | VIDEO, 12 | VIEWED, 13 | ) 14 | from analytics_data_api.tests.test_utils import set_databases 15 | from analytics_data_api.v0.tests.utils import create_engagement, create_enterprise_user 16 | from analytics_data_api.v0.tests.views import CourseSamples 17 | from analyticsdataserver.tests.utils import TestCaseWithAuthentication 18 | 19 | PAGINATED_SAMPLE_DATA = [ 20 | { 21 | 'entity_id': 'entity-id', 22 | 'created': '2015-01-01T00:00:00Z', 23 | 'course_id': 'edX/DemoX/Demo_Course', 24 | 'username': 'death_stroke', 25 | 'entity_type': 'problem', 26 | 'count': 5, 27 | 'id': 1, 28 | 'event': 'attempted', 29 | 'date': '2015-01-01' 30 | }, 31 | { 32 | 'entity_id': 'entity-id', 33 | 'created': '2015-01-01T00:00:00Z', 34 | 'course_id': 'edX/DemoX/Demo_Course', 35 | 'username': 'death_stroke', 36 | 'entity_type': 'problem', 37 | 'count': 3, 38 | 'id': 2, 39 | 'event': 'completed', 40 | 'date': '2015-01-01' 41 | }, 42 | { 43 | 'entity_id': 'entity-id', 44 | 'created': '2015-01-01T00:00:00Z', 45 | 'course_id': 'edX/DemoX/Demo_Course', 46 | 'username': 'death_stroke', 47 | 'entity_type': 'video', 48 | 'count': 7, 49 | 'id': 4, 50 | 'event': 'viewed', 51 | 'date': '2015-01-01' 52 | } 53 | ] 54 | 55 | 56 | @ddt.ddt 57 | @set_databases 58 | class EnterpriseLearnerEngagementViewTests(TestCaseWithAuthentication): 59 | @classmethod 60 | def setUpClass(cls): 61 | super().setUpClass() 62 | 63 | cls.default_username = 'death_stroke' 64 | cls.enterprise_customer_uuid = '5c0dd495-e726-46fa-a6a8-2d8d26c716c9' 65 | cls.path = f'/api/v0/enterprise/{cls.enterprise_customer_uuid}/engagements/' 66 | 67 | create_enterprise_user(cls.enterprise_customer_uuid, cls.default_username) 68 | create_engagement(CourseSamples.course_ids[0], cls.default_username, PROBLEM, ATTEMPTED, 'entity-id', 5) 69 | create_engagement(CourseSamples.course_ids[0], cls.default_username, PROBLEM, COMPLETED, 'entity-id', 3) 70 | create_engagement(CourseSamples.course_ids[0], cls.default_username, DISCUSSION, CONTRIBUTED, 'entity-id', 2) 71 | create_engagement(CourseSamples.course_ids[0], cls.default_username, VIDEO, VIEWED, 'entity-id', 7) 72 | 73 | def test_enterprise_learner_engagements(self): 74 | """ 75 | Test learner engagment view. 76 | """ 77 | response = self.authenticated_get(self.path) 78 | assert response.status_code == status.HTTP_200_OK 79 | response = response.json() 80 | 81 | # verify that not allowed engagement entity_type is not present in api response 82 | for learner_engagment in response['results']: 83 | assert learner_engagment['entity_type'] not in settings.EXCLUDED_ENGAGEMENT_ENTITY_TYPES 84 | 85 | assert response == { 86 | 'results': PAGINATED_SAMPLE_DATA, 87 | 'num_pages': 1, 88 | 'next': None, 89 | 'previous': None, 90 | 'count': 3 91 | } 92 | 93 | @ddt.data( 94 | (1, 1), 95 | (2, 1), 96 | (3, 1), 97 | ) 98 | @ddt.unpack 99 | def test_enterprise_learner_engagements_paginatation(self, page, page_size): 100 | """ 101 | Test learner engagment view for paginated response. 102 | """ 103 | path = f'{self.path}?page={page}&page_size={page_size}' 104 | response = self.authenticated_get(path) 105 | assert response.status_code == status.HTTP_200_OK 106 | response = response.json() 107 | assert len(response['results']) == 1 108 | assert response['results'][0] == PAGINATED_SAMPLE_DATA[page - 1] 109 | -------------------------------------------------------------------------------- /analytics_data_api/v0/tests/views/test_programs.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import ddt 4 | from django_dynamic_fixture import G 5 | 6 | from analytics_data_api.tests.test_utils import set_databases 7 | from analytics_data_api.v0 import models, serializers 8 | from analytics_data_api.v0.tests.views import APIListViewTestMixin, CourseSamples 9 | from analyticsdataserver.tests.utils import TestCaseWithAuthentication 10 | 11 | 12 | @ddt.ddt 13 | @set_databases 14 | class ProgramsViewTests(TestCaseWithAuthentication, APIListViewTestMixin): 15 | model = models.CourseProgramMetadata 16 | model_id = 'program_id' 17 | ids_param = 'program_ids' 18 | serializer = serializers.CourseProgramMetadataSerializer 19 | expected_programs = [] 20 | list_name = 'programs' 21 | default_ids = CourseSamples.program_ids 22 | 23 | def setUp(self): 24 | super().setUp() 25 | self.now = datetime.datetime.utcnow() 26 | self.maxDiff = None 27 | self.course_id = CourseSamples.course_ids[0] 28 | 29 | def tearDown(self): 30 | self.model.objects.all().delete() 31 | 32 | def generate_data(self, ids=None, **kwargs): 33 | """Generate program list data""" 34 | if ids is None: 35 | ids = self.default_ids 36 | 37 | course_ids = kwargs.pop('course_ids', None) 38 | if course_ids is None: 39 | course_ids = [[self.course_id]] * len(ids) 40 | 41 | for item_id, course_id in zip(ids, course_ids): 42 | self.create_model(item_id, course_ids=course_id, **kwargs) 43 | 44 | def create_model(self, model_id, **kwargs): 45 | course_ids = kwargs.get('course_ids', None) 46 | if course_ids is None: 47 | course_ids = [self.course_id] 48 | 49 | for course_id in course_ids: 50 | G(self.model, course_id=course_id, program_id=model_id, program_type='Demo', program_title='Test') 51 | 52 | def all_expected_results(self, ids=None, **kwargs): 53 | if ids is None: 54 | ids = self.default_ids 55 | 56 | course_ids = kwargs.pop('course_ids', None) 57 | if course_ids is None: 58 | course_ids = [[self.course_id]] * len(ids) 59 | 60 | return [self.expected_result(item_id, course_ids=course_id, **kwargs) 61 | for item_id, course_id in zip(ids, course_ids)] 62 | 63 | def expected_result(self, item_id, **kwargs): 64 | """Expected program metadata to populate with data.""" 65 | course_ids = kwargs.get('course_ids', None) 66 | if course_ids is None: 67 | course_ids = [self.course_id] 68 | 69 | program = super().expected_result(item_id) 70 | program.update([ 71 | ('program_type', 'Demo'), 72 | ('program_title', 'Test'), 73 | ('course_ids', course_ids) 74 | ]) 75 | return program 76 | 77 | @ddt.data( 78 | None, 79 | CourseSamples.program_ids, 80 | ['not-real-program'].extend(CourseSamples.program_ids), 81 | ) 82 | def test_all_programs(self, program_ids): 83 | self._test_all_items(program_ids) 84 | 85 | @ddt.data(*CourseSamples.program_ids) 86 | def test_one_course(self, program_id): 87 | self._test_one_item(program_id) 88 | 89 | @ddt.data( 90 | ['program_id'], 91 | ['program_type', 'program_title'], 92 | ) 93 | def test_fields(self, fields): 94 | self._test_fields(fields) 95 | 96 | @ddt.data( 97 | (None, None), 98 | (CourseSamples.program_ids, [[cid] for cid in CourseSamples.course_ids]), 99 | (CourseSamples.program_ids, [CourseSamples.course_ids[1:3], 100 | CourseSamples.course_ids[0:2], 101 | CourseSamples.course_ids[0:3]]), 102 | ) 103 | @ddt.unpack 104 | def test_all_programs_multi_courses(self, program_ids, course_ids): 105 | self.generate_data(ids=program_ids, course_ids=course_ids) 106 | response = self.validated_request(ids=program_ids, exclude=self.always_exclude) 107 | self.assertEqual(response.status_code, 200) 108 | self.assertCountEqual(response.data, self.all_expected_results(ids=program_ids, course_ids=course_ids)) 109 | -------------------------------------------------------------------------------- /analytics_data_api/v0/tests/views/test_utils.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import Mock 2 | 3 | import ddt 4 | from django.http import Http404 5 | from django.test import TestCase 6 | 7 | import analytics_data_api.v0.views.utils as utils 8 | from analytics_data_api.v0.exceptions import CourseKeyMalformedError # pylint: disable=ungrouped-imports 9 | from analytics_data_api.v0.tests.views import CourseSamples 10 | 11 | 12 | @ddt.ddt 13 | class UtilsTest(TestCase): 14 | 15 | @ddt.data( 16 | None, 17 | 'not-a-key', 18 | ) 19 | def test_invalid_course_id(self, course_id): 20 | with self.assertRaises(CourseKeyMalformedError): 21 | utils.validate_course_id(course_id) 22 | 23 | @ddt.data(*CourseSamples.course_ids) 24 | def test_valid_course_id(self, course_id): 25 | try: 26 | utils.validate_course_id(course_id) 27 | except CourseKeyMalformedError: 28 | self.fail('Unexpected CourseKeyMalformedError!') 29 | 30 | def test_split_query_argument_none(self): 31 | self.assertIsNone(utils.split_query_argument(None)) 32 | 33 | @ddt.data( 34 | ('one', ['one']), 35 | ('one,two', ['one', 'two']), 36 | ) 37 | @ddt.unpack 38 | def test_split_query_argument(self, query_args, expected): 39 | self.assertListEqual(utils.split_query_argument(query_args), expected) 40 | 41 | def test_raise_404_if_none_raises_error(self): 42 | decorated_func = utils.raise_404_if_none(Mock(return_value=None)) 43 | with self.assertRaises(Http404): 44 | decorated_func(self) 45 | 46 | def test_raise_404_if_none_passes_through(self): 47 | decorated_func = utils.raise_404_if_none(Mock(return_value='Not a 404')) 48 | self.assertEqual(decorated_func(self), 'Not a 404') 49 | -------------------------------------------------------------------------------- /analytics_data_api/v0/tests/views/test_videos.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from django.conf import settings 4 | from django.utils import timezone 5 | from django_dynamic_fixture import G 6 | 7 | from analytics_data_api.tests.test_utils import set_databases 8 | from analytics_data_api.v0 import models 9 | from analyticsdataserver.tests.utils import TestCaseWithAuthentication 10 | 11 | 12 | @set_databases 13 | class VideoTimelineTests(TestCaseWithAuthentication): 14 | def _get_data(self, video_id=None): 15 | return self.authenticated_get(f'/api/v0/videos/{video_id}/timeline') 16 | 17 | def test_get(self): 18 | # add a blank row, which shouldn't be included in results 19 | G(models.VideoTimeline) 20 | 21 | video_id = 'v1d30' 22 | created = timezone.now() 23 | G(models.VideoTimeline, pipeline_video_id=video_id, segment=0, num_users=10, 24 | num_views=50, created=created) 25 | G(models.VideoTimeline, pipeline_video_id=video_id, segment=1, num_users=1, 26 | num_views=1234, created=created) 27 | 28 | alt_video_id = 'altv1d30' 29 | alt_created = created + datetime.timedelta(seconds=17) 30 | G(models.VideoTimeline, pipeline_video_id=alt_video_id, segment=0, num_users=10231, 31 | num_views=834828, created=alt_created) 32 | 33 | expected = [ 34 | { 35 | 'segment': 0, 36 | 'num_users': 10, 37 | 'num_views': 50, 38 | 'created': created.strftime(settings.DATETIME_FORMAT) 39 | }, 40 | { 41 | 'segment': 1, 42 | 'num_users': 1, 43 | 'num_views': 1234, 44 | 'created': created.strftime(settings.DATETIME_FORMAT) 45 | } 46 | ] 47 | response = self._get_data(video_id) 48 | self.assertEqual(response.status_code, 200) 49 | self.assertListEqual(response.data, expected) 50 | 51 | expected = [ 52 | { 53 | 'segment': 0, 54 | 'num_users': 10231, 55 | 'num_views': 834828, 56 | 'created': alt_created.strftime(settings.DATETIME_FORMAT) 57 | } 58 | ] 59 | response = self._get_data(alt_video_id) 60 | self.assertEqual(response.status_code, 200) 61 | self.assertListEqual(response.data, expected) 62 | 63 | def test_get_404(self): 64 | response = self._get_data('no_id') 65 | self.assertEqual(response.status_code, 404) 66 | -------------------------------------------------------------------------------- /analytics_data_api/v0/urls/__init__.py: -------------------------------------------------------------------------------- 1 | from django.urls import include, path, reverse_lazy 2 | from django.views.generic import RedirectView 3 | 4 | app_name = 'analytics_data_api.v0' 5 | 6 | COURSE_ID_PATTERN = r'(?P[^/+]+[/+][^/+]+[/+][^/]+)' 7 | 8 | urlpatterns = [ 9 | path('courses/', include('analytics_data_api.v0.urls.courses')), 10 | path('problems/', include('analytics_data_api.v0.urls.problems')), 11 | path('videos/', include('analytics_data_api.v0.urls.videos')), 12 | path('', include('analytics_data_api.v0.urls.learners')), 13 | path('', include('analytics_data_api.v0.urls.course_summaries')), 14 | path('', include('analytics_data_api.v0.urls.programs')), 15 | 16 | # pylint: disable=no-value-for-parameter 17 | path('authenticated/', RedirectView.as_view(url=reverse_lazy('authenticated')), name='authenticated'), 18 | path('health/', RedirectView.as_view(url=reverse_lazy('health')), name='health'), 19 | path('status/', RedirectView.as_view(url=reverse_lazy('status')), name='status'), 20 | ] 21 | -------------------------------------------------------------------------------- /analytics_data_api/v0/urls/course_summaries.py: -------------------------------------------------------------------------------- 1 | from django.urls import re_path 2 | 3 | from analytics_data_api.v0.views import course_summaries as views 4 | 5 | app_name = 'course_summaries' 6 | 7 | urlpatterns = [ 8 | re_path(r'^course_summaries/$', views.CourseSummariesView.as_view(), name='course_summaries'), 9 | ] 10 | -------------------------------------------------------------------------------- /analytics_data_api/v0/urls/courses.py: -------------------------------------------------------------------------------- 1 | from django.urls import re_path 2 | 3 | from analytics_data_api.v0.urls import COURSE_ID_PATTERN 4 | from analytics_data_api.v0.views import courses as views 5 | 6 | app_name = 'courses' 7 | 8 | COURSE_URLS = [ 9 | ('activity', views.CourseActivityWeeklyView, 'activity'), 10 | ('recent_activity', views.CourseActivityMostRecentWeekView, 'recent_activity'), 11 | ('enrollment', views.CourseEnrollmentView, 'enrollment_latest'), 12 | ('enrollment/mode', views.CourseEnrollmentModeView, 'enrollment_by_mode'), 13 | ('enrollment/birth_year', views.CourseEnrollmentByBirthYearView, 'enrollment_by_birth_year'), 14 | ('enrollment/education', views.CourseEnrollmentByEducationView, 'enrollment_by_education'), 15 | ('enrollment/gender', views.CourseEnrollmentByGenderView, 'enrollment_by_gender'), 16 | ('enrollment/location', views.CourseEnrollmentByLocationView, 'enrollment_by_location'), 17 | ('problems', views.ProblemsListView, 'problems'), 18 | ('problems_and_tags', views.ProblemsAndTagsListView, 'problems_and_tags'), 19 | ('videos', views.VideosListView, 'videos'), 20 | ('reports/(?P[a-zA-Z0-9_]+)', views.ReportDownloadView, 'reports'), 21 | ('user_engagement', views.UserEngagementView, 'user_engagement'), 22 | ] 23 | 24 | urlpatterns = [] 25 | 26 | for path, view, name in COURSE_URLS: 27 | regex = fr'^{COURSE_ID_PATTERN}/{path}/$' 28 | urlpatterns.append(re_path(regex, view.as_view(), name=name)) 29 | -------------------------------------------------------------------------------- /analytics_data_api/v0/urls/learners.py: -------------------------------------------------------------------------------- 1 | from django.urls import re_path 2 | 3 | from analytics_data_api.constants.learner import UUID_REGEX_PATTERN 4 | from analytics_data_api.v0.views import learners as views 5 | 6 | app_name = 'learners' 7 | 8 | USERNAME_PATTERN = r'(?P[\w.+-]+)' 9 | 10 | urlpatterns = [ 11 | re_path(fr'^enterprise/(?P{UUID_REGEX_PATTERN})/engagements/$', 12 | views.EnterpriseLearnerEngagementView.as_view(), name='engagements'), 13 | ] 14 | -------------------------------------------------------------------------------- /analytics_data_api/v0/urls/problems.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from django.urls import re_path 4 | 5 | from analytics_data_api.v0.views import problems as views 6 | 7 | app_name = 'problems' 8 | 9 | PROBLEM_URLS = [ 10 | ('answer_distribution', views.ProblemResponseAnswerDistributionView, 'answer_distribution'), 11 | ('grade_distribution', views.GradeDistributionView, 'grade_distribution'), 12 | ] 13 | 14 | urlpatterns = [ 15 | re_path(r'^(?P.+)/sequential_open_distribution/$', 16 | views.SequentialOpenDistributionView.as_view(), 17 | name='sequential_open_distribution'), 18 | ] 19 | 20 | for path, view, name in PROBLEM_URLS: 21 | urlpatterns.append(re_path(r'^(?P.+)/' + re.escape(path) + r'/$', view.as_view(), name=name)) 22 | -------------------------------------------------------------------------------- /analytics_data_api/v0/urls/programs.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | 3 | from analytics_data_api.v0.views import programs as views 4 | 5 | app_name = 'programs' 6 | 7 | urlpatterns = [ 8 | path('programs/', views.ProgramsView.as_view(), name='programs'), 9 | ] 10 | -------------------------------------------------------------------------------- /analytics_data_api/v0/urls/videos.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from django.urls import re_path 4 | 5 | from analytics_data_api.v0.views import videos as views 6 | 7 | app_name = 'videos' 8 | 9 | VIDEO_URLS = [ 10 | ('timeline', views.VideoTimelineView, 'timeline'), 11 | ] 12 | 13 | urlpatterns = [] 14 | 15 | for path, view, name in VIDEO_URLS: 16 | urlpatterns.append(re_path(r'^(?P.+)/' + re.escape(path) + r'/$', view.as_view(), name=name)) 17 | -------------------------------------------------------------------------------- /analytics_data_api/v0/views/learners.py: -------------------------------------------------------------------------------- 1 | """ 2 | API methods for module level data. 3 | """ 4 | 5 | 6 | import logging 7 | 8 | from django.conf import settings 9 | from edx_django_utils.cache import TieredCache, get_cache_key 10 | from enterprise_data.models import EnterpriseUser 11 | from rest_framework import generics 12 | 13 | from analytics_data_api.v0.models import ModuleEngagement 14 | from analytics_data_api.v0.serializers import EdxPaginationSerializer, EnterpriseLearnerEngagementSerializer 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | class EnterpriseLearnerEngagementView(generics.ListAPIView): 20 | """ 21 | Return engagement data for enterprise learners. 22 | """ 23 | serializer_class = EnterpriseLearnerEngagementSerializer 24 | pagination_class = EdxPaginationSerializer 25 | 26 | @property 27 | def cached_enterprise_learns(self): 28 | """ 29 | Caches Enterprise Learns if cache found, else get fresh copy and returns. 30 | """ 31 | enterprise_id = self.kwargs.get('enterprise_customer') 32 | cache_key = get_cache_key( 33 | resource='enterprise_users', 34 | resource_id=enterprise_id, 35 | ) 36 | enterprise_users_cache = TieredCache.get_cached_response(cache_key) 37 | if enterprise_users_cache.is_found: 38 | return enterprise_users_cache.value 39 | 40 | enterprise_users = list(EnterpriseUser.objects.filter( 41 | enterprise_id=self.kwargs.get('enterprise_customer') 42 | ).values_list( 43 | 'user_username', flat=True 44 | )) 45 | TieredCache.set_all_tiers(cache_key, enterprise_users, settings.ENGAGEMENT_CACHE_TIMEOUT) 46 | return enterprise_users 47 | 48 | def get_cached_module_engagement_count(self): 49 | """ 50 | Caches Module Engagement records count for specific enterprise. 51 | """ 52 | enterprise_id = self.kwargs.get('enterprise_customer') 53 | cache_key = get_cache_key( 54 | resource='module_engagement_count', 55 | resource_id=enterprise_id, 56 | ) 57 | module_engagement_count_cache = TieredCache.get_cached_response(cache_key) 58 | if module_engagement_count_cache.is_found: 59 | return module_engagement_count_cache.value 60 | 61 | queryset = self._get_queryset() 62 | count = queryset.count() 63 | TieredCache.set_all_tiers(cache_key, count, settings.ENGAGEMENT_CACHE_TIMEOUT) 64 | return count 65 | 66 | def _get_queryset(self): 67 | """ Return ModuleEngagement queryset""" 68 | return ModuleEngagement.objects.filter( 69 | username__in=self.cached_enterprise_learns 70 | ).exclude( 71 | entity_type__in=settings.EXCLUDED_ENGAGEMENT_ENTITY_TYPES 72 | ).order_by('id') 73 | 74 | def get_queryset(self): 75 | """ Wrapper on the _get_queryset also overrides count method to return cached count.""" 76 | query_set = self._get_queryset() 77 | setattr(query_set, 'count', self.get_cached_module_engagement_count) 78 | return query_set 79 | -------------------------------------------------------------------------------- /analytics_data_api/v0/views/programs.py: -------------------------------------------------------------------------------- 1 | from functools import reduce as functools_reduce 2 | 3 | from django.db.models import Q 4 | 5 | from analytics_data_api.v0 import models, serializers 6 | from analytics_data_api.v0.views import APIListView 7 | 8 | 9 | class ProgramsView(APIListView): 10 | """ 11 | Returns metadata information for programs. 12 | 13 | **Example Request** 14 | 15 | GET /api/v0/course_programs/?program_ids={program_id},{program_id} 16 | 17 | **Response Values** 18 | 19 | Returns metadata for every program: 20 | 21 | * program_id: The ID of the program for which data is returned. 22 | * program_type: The type of the program 23 | * program_title: The title of the program 24 | * created: The date the metadata was computed. 25 | 26 | **Parameters** 27 | 28 | Results can be filtered to the program IDs specified or limited to the fields. 29 | 30 | program_ids -- The comma-separated program identifiers for which metadata is requested. 31 | Default is to return all programs. 32 | fields -- The comma-separated fields to return in the response. 33 | For example, 'program_id,created'. Default is to return all fields. 34 | exclude -- The comma-separated fields to exclude in the response. 35 | For example, 'program_id,created'. Default is to not exclude any fields. 36 | """ 37 | serializer_class = serializers.CourseProgramMetadataSerializer 38 | model = models.CourseProgramMetadata 39 | model_id_field = 'program_id' 40 | ids_param = 'program_ids' 41 | program_meta_fields = ['program_type', 'program_title'] 42 | 43 | def base_field_dict(self, item_id): 44 | """Default program with id, empty metadata, and empty courses array.""" 45 | program = super().base_field_dict(item_id) 46 | program.update({ 47 | 'program_type': '', 48 | 'program_title': '', 49 | 'created': None, 50 | 'course_ids': [], 51 | }) 52 | return program 53 | 54 | def update_field_dict_from_model(self, model, base_field_dict=None, field_list=None): 55 | field_dict = super().update_field_dict_from_model(model, base_field_dict=base_field_dict, 56 | field_list=self.program_meta_fields) 57 | field_dict['course_ids'].append(model.course_id) 58 | 59 | # treat the most recent as the authoritative created date -- should be all the same 60 | field_dict['created'] = max(model.created, field_dict['created']) if field_dict['created'] else model.created 61 | 62 | return field_dict 63 | 64 | def get_query(self): 65 | return functools_reduce(lambda q, item_id: q | Q(program_id=item_id), self.ids, Q()) 66 | -------------------------------------------------------------------------------- /analytics_data_api/v0/views/utils.py: -------------------------------------------------------------------------------- 1 | """Utilities for view-level API logic.""" 2 | 3 | 4 | from django.http import Http404 5 | from opaque_keys import InvalidKeyError 6 | from opaque_keys.edx.keys import CourseKey 7 | 8 | from analytics_data_api.v0.exceptions import CourseKeyMalformedError 9 | 10 | 11 | def split_query_argument(argument): 12 | """ 13 | Splits a comma-separated querystring argument into a list. 14 | Returns None if the argument is empty. 15 | """ 16 | if argument: 17 | return argument.split(',') 18 | return None 19 | 20 | 21 | def raise_404_if_none(func): 22 | """ 23 | Decorator for raising Http404 if function evaluation is falsey (e.g. empty queryset). 24 | """ 25 | def func_wrapper(self): 26 | queryset = func(self) 27 | if queryset: 28 | return queryset 29 | raise Http404 30 | return func_wrapper 31 | 32 | 33 | def validate_course_id(course_id): 34 | """Raises CourseKeyMalformedError if course ID is invalid.""" 35 | try: 36 | CourseKey.from_string(course_id) 37 | except InvalidKeyError: 38 | raise CourseKeyMalformedError(course_id=course_id) 39 | -------------------------------------------------------------------------------- /analytics_data_api/v0/views/videos.py: -------------------------------------------------------------------------------- 1 | """ 2 | API methods for module level data. 3 | """ 4 | 5 | from rest_framework import generics 6 | 7 | from analytics_data_api.v0.models import VideoTimeline 8 | from analytics_data_api.v0.serializers import VideoTimelineSerializer 9 | from analytics_data_api.v0.views.utils import raise_404_if_none 10 | 11 | 12 | class VideoTimelineView(generics.ListAPIView): 13 | """ 14 | Get the counts of users and views for a video. 15 | 16 | **Example Request** 17 | 18 | GET /api/v0/videos/{video_id}/timeline/ 19 | 20 | **Response Values** 21 | 22 | Returns viewing data for each segment of a video. For each segment, 23 | the collection contains the following data. 24 | 25 | * segment: The order of the segment in the video timeline. 26 | * num_users: The number of unique users who viewed this segment. 27 | * num_views: The number of views for this segment. 28 | * created: The date the segment data was computed. 29 | """ 30 | 31 | serializer_class = VideoTimelineSerializer 32 | allow_empty = False 33 | 34 | @raise_404_if_none 35 | def get_queryset(self): 36 | """Select the view count for a specific module""" 37 | video_id = self.kwargs.get('video_id') 38 | return VideoTimeline.objects.filter(pipeline_video_id=video_id) 39 | -------------------------------------------------------------------------------- /analytics_data_api/v1/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/analytics_data_api/v1/__init__.py -------------------------------------------------------------------------------- /analytics_data_api/v1/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import include, re_path, reverse_lazy 2 | from django.views.generic import RedirectView 3 | 4 | from analytics_data_api.v0.urls import COURSE_ID_PATTERN 5 | from analytics_data_api.v0.views import courses 6 | 7 | app_name = 'v1' 8 | 9 | COURSE_URLS = [ 10 | ('activity', courses.CourseActivityWeeklyView, 'activity'), 11 | ('recent_activity', courses.CourseActivityMostRecentWeekView, 'recent_activity'), 12 | ('enrollment', courses.CourseEnrollmentView, 'enrollment_latest'), 13 | ('enrollment/mode', courses.CourseEnrollmentModeView, 'enrollment_by_mode'), 14 | ('enrollment/education', courses.CourseEnrollmentByEducationView, 'enrollment_by_education'), 15 | ('enrollment/gender', courses.CourseEnrollmentByGenderView, 'enrollment_by_gender'), 16 | ('enrollment/location', courses.CourseEnrollmentByLocationView, 'enrollment_by_location'), 17 | ('problems', courses.ProblemsListView, 'problems'), 18 | ('problems_and_tags', courses.ProblemsAndTagsListView, 'problems_and_tags'), 19 | ('videos', courses.VideosListView, 'videos'), 20 | ('reports/(?P[a-zA-Z0-9_]+)', courses.ReportDownloadView, 'reports'), 21 | ('user_engagement', courses.UserEngagementView, 'user_engagement'), 22 | ] 23 | 24 | course_urlpatterns = [] 25 | 26 | for path, view, name in COURSE_URLS: 27 | regex = fr'^courses/{COURSE_ID_PATTERN}/{path}/$' 28 | course_urlpatterns.append(re_path(regex, view.as_view(), name=name)) 29 | 30 | urlpatterns = course_urlpatterns + [ 31 | re_path(r'^problems/', include('analytics_data_api.v0.urls.problems')), 32 | re_path(r'^videos/', include('analytics_data_api.v0.urls.videos')), 33 | re_path('^', include('analytics_data_api.v0.urls.course_summaries')), 34 | re_path('^', include('analytics_data_api.v0.urls.programs')), 35 | 36 | # pylint: disable=no-value-for-parameter 37 | re_path(r'^authenticated/$', RedirectView.as_view(url=reverse_lazy('authenticated')), name='authenticated'), 38 | re_path(r'^health/$', RedirectView.as_view(url=reverse_lazy('health')), name='health'), 39 | re_path(r'^status/$', RedirectView.as_view(url=reverse_lazy('status')), name='status'), 40 | ] 41 | -------------------------------------------------------------------------------- /analyticsdataserver/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/analyticsdataserver/__init__.py -------------------------------------------------------------------------------- /analyticsdataserver/clients.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from urllib.parse import urljoin 3 | 4 | from django.conf import settings 5 | from edx_rest_api_client.client import OAuthAPIClient 6 | from opaque_keys import InvalidKeyError 7 | from opaque_keys.edx.keys import UsageKey 8 | from requests.exceptions import HTTPError, RequestException 9 | 10 | from analyticsdataserver.utils import temp_log_level 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | class CourseBlocksApiClient(OAuthAPIClient): 16 | """ 17 | This class is a sub-class of the edX Rest API Client 18 | (https://github.com/openedx/edx-rest-api-client). 19 | 20 | Details about the API itself can be found at 21 | https://openedx.atlassian.net/wiki/display/AN/Course+Structure+API. 22 | 23 | Currently, this client is only used for a local-only developer script (generate_fake_course_data). 24 | """ 25 | 26 | def all_videos(self, course_id): 27 | try: 28 | logger.debug('Retrieving course video blocks for course_id: %s', course_id) 29 | 30 | try: 31 | api_base_url = urljoin(settings.LMS_BASE_URL + '/', 'api/courses/v1/') 32 | except AttributeError: 33 | logger.warning("LMS_BASE_URL is not configured! Cannot get video ids.") 34 | return None 35 | logger.info("Assuming the Course Blocks API is hosted at: %s", api_base_url) 36 | 37 | blocks_kwargs = { 38 | 'course_id': course_id, 39 | 'all_blocks': True, 40 | 'depth': 'all', 41 | 'block_types_filter': 'video' 42 | } 43 | response = self.get(urljoin(api_base_url, 'blocks/'), params=blocks_kwargs) 44 | response.raise_for_status() 45 | data = response.json() 46 | logger.info("Successfully authenticated with the Course Blocks API.") 47 | except HTTPError as e: 48 | if e.response.status_code == 401: 49 | logger.warning("Course Blocks API failed to return video ids (%s). " 50 | "See README for instructions on how to authenticate the API with your local LMS.", 51 | e.response.status_code) 52 | elif e.response.status_code == 404: 53 | logger.warning("Course Blocks API failed to return video ids (%s). " 54 | "Does the course exist in the LMS?", 55 | e.response.status_code) 56 | else: 57 | logger.warning("Course Blocks API failed to return video ids (%s).", e.response.status_code) 58 | return None 59 | except RequestException as e: 60 | logger.warning("Course Blocks API request failed. Is the LMS running?: %s", str(e)) 61 | return None 62 | 63 | # Setup a terrible hack to silence mysterious flood of ImportErrors from stevedore inside edx-opaque-keys. 64 | # (The UsageKey utility still works despite the import errors, so I think the errors are not important). 65 | with temp_log_level('stevedore', log_level=logging.CRITICAL): 66 | videos = [] 67 | for video in data['blocks'].values(): 68 | try: 69 | encoded_id = UsageKey.from_string(video['id']).html_id() 70 | except InvalidKeyError: 71 | encoded_id = video['id'] # just pass through any wonky ids we don't understand 72 | videos.append({'video_id': course_id + '|' + encoded_id, 73 | 'video_module_id': encoded_id}) 74 | 75 | return videos 76 | -------------------------------------------------------------------------------- /analyticsdataserver/router.py: -------------------------------------------------------------------------------- 1 | # This file contains database routers used for routing database operations to the appropriate databases. 2 | # Below is a digest of the routers in this file. 3 | # AnalyticsAPIRouter: This router routes database operations based on the version of the analytics data API being used. 4 | # It allows us to route database traffic to the v1 database when using the v1 API, for example. 5 | # AnalyticsModelsRouter: This router routes database operations based on the model that is being requested. Its 6 | # primary purpose is to handle database migrations based on the multiple database settings 7 | # in the environment. 8 | # 9 | # Note that if a database router returns None for a router method or does not implement the method, the master router 10 | # delegates to the next router in the list until a value is returned. The master router falls back to a default behavior 11 | # if no custom router returns a value. 12 | 13 | from django.conf import settings 14 | 15 | from analytics_data_api.middleware import thread_data 16 | 17 | ANALYTICS_APP_LABELS = ['v0', 'v1'] 18 | ENTERPRISE_APP_LABELS = ['enterprise_data'] 19 | DJANGO_AUTH_MODELS = ['enterprisedatafeaturerole', 'enterprisedataroleassignment'] 20 | 21 | 22 | class AnalyticsAPIRouter: 23 | """ 24 | This router's role is to route database operations to the appropriate database when there is an 'analytics_database' 25 | "hint" in the local thread data. This "hint" is set by the RequestVersionMiddleware and is meant to route 26 | database operations to particular databases depending on the version of the API requested via the view. 27 | """ 28 | def _get_database(self, app_label): 29 | if app_label in ANALYTICS_APP_LABELS and hasattr(thread_data, 'analyticsapi_database'): 30 | return getattr(thread_data, 'analyticsapi_database') 31 | # If there is no analyticsapi_database database hint, then fall back to next router. 32 | return None 33 | 34 | # pylint: disable=unused-argument 35 | # pylint: disable=protected-access 36 | def db_for_read(self, model, **hints): 37 | return self._get_database(model._meta.app_label) 38 | 39 | # pylint: disable=unused-argument 40 | # pylint: disable=protected-access 41 | def db_for_write(self, model, **hints): 42 | return self._get_database(model._meta.app_label) 43 | 44 | 45 | class AnalyticsModelsRouter: 46 | """ 47 | This router's role is to route database operations. It is meant, in part, to mirror 48 | the way databases are structured in the production environment. For example, the ANALYTICS_DATABASE 49 | and ANALYTICS_DATABASE_V1 databases in production do not contain models from "non-analytics" apps, 50 | like the auth app. This is because said databases are populated by other means (e.g. EMR jobs, prefect flows). 51 | 52 | This router also ensures that analytics apps are not migrated into the default database unless the default 53 | database is the only configured database. 54 | 55 | This router also handles an edge case with the enterprise_data app where migrations exist for models that have 56 | been moved to a different app that is now in the default database. 57 | 58 | Details: 59 | The enterprise_data application has the migration 0018_enterprisedatafeaturerole_enterprisedataroleassignment, 60 | which creates a model with a ForeignKey field pointing to the auth_user model. This does not work in a multiple 61 | DB environment since MySQL does not allow cross-database relations. This model has since been moved to a 62 | different application that will migrate into the default database where auth_user exists. 63 | 64 | We do not define an allow_relation method. We fall back to Django's default behavior, which is to allow relations 65 | between model instances that were loaded from the same database. 66 | """ 67 | def _get_database(self, app_label, model_name): 68 | # select first available database if there are multiple options 69 | return self._get_databases(app_label, model_name)[0] 70 | 71 | def _get_databases(self, app_label, model_name): 72 | databases = [] 73 | if app_label in ANALYTICS_APP_LABELS: 74 | databases = [ 75 | getattr(settings, 'ANALYTICS_DATABASE', None), 76 | getattr(settings, 'ANALYTICS_DATABASE_V1', None) 77 | ] 78 | 79 | # checking against None is an unfortunate bit of code here. There are migrations in 80 | # the enterprise app that run python against auth related models which will fail on 81 | # anything other than the default database. There's no way to identify these migrations 82 | # specifically because there is no value for model. 83 | # This is brittle, however tables in the analytic databases are created by other means today. 84 | # (e.g. EMR jobs, prefect flows) We shouldn't expect migrations of this type in the 85 | # future that do need to run against the analytics database. 86 | elif ( 87 | app_label in ENTERPRISE_APP_LABELS and 88 | model_name is not None and 89 | model_name not in DJANGO_AUTH_MODELS 90 | ): 91 | databases = [getattr(settings, 'ANALYTICS_DATABASE', None)] 92 | 93 | databases = list(filter(None, databases)) 94 | if len(databases) > 0: 95 | return databases 96 | 97 | return ['default'] 98 | 99 | # pylint: disable=unused-argument 100 | # pylint: disable=protected-access 101 | def db_for_read(self, model, **hints): 102 | return self._get_database(model._meta.app_label, model._meta.model_name) 103 | 104 | # pylint: disable=unused-argument 105 | # pylint: disable=protected-access 106 | def db_for_write(self, model, **hints): 107 | return self._get_database(model._meta.app_label, model._meta.model_name) 108 | 109 | # pylint: disable=unused-argument 110 | def allow_migrate(self, database, app_label, model_name=None, **hints): 111 | databases = self._get_databases(app_label, model_name) 112 | return database in databases 113 | -------------------------------------------------------------------------------- /analyticsdataserver/settings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/analyticsdataserver/settings/__init__.py -------------------------------------------------------------------------------- /analyticsdataserver/settings/devstack.py: -------------------------------------------------------------------------------- 1 | """Devstack settings.""" 2 | 3 | import os 4 | 5 | from analyticsdataserver.settings.local import * 6 | 7 | ########## DATABASE CONFIGURATION 8 | DATABASES = { 9 | 'default': { 10 | 'ENGINE': 'django.db.backends.mysql', 11 | 'NAME': 'analytics-api', 12 | 'USER': 'api001', 13 | 'PASSWORD': 'password', 14 | 'HOST': 'edx.devstack.mysql', 15 | 'PORT': '3306', 16 | }, 17 | 'analytics_v1': { 18 | 'ENGINE': 'django.db.backends.mysql', 19 | 'NAME': 'reports_v1', 20 | 'USER': 'api001', 21 | 'PASSWORD': 'password', 22 | 'HOST': 'edx.devstack.mysql', 23 | 'PORT': '3306', 24 | }, 25 | 'analytics': { 26 | 'ENGINE': 'django.db.backends.mysql', 27 | 'NAME': 'reports', 28 | 'USER': 'reports001', 29 | 'PASSWORD': 'password', 30 | 'HOST': 'edx.devstack.mysql', 31 | 'PORT': '3306', 32 | } 33 | } 34 | 35 | ANALYTICS_DATABASE_V1 = 'analytics_v1' 36 | 37 | DB_OVERRIDES = dict( 38 | USER=os.environ.get('DB_USER', DATABASES['default']['USER']), 39 | PASSWORD=os.environ.get('DB_PASSWORD', DATABASES['default']['PASSWORD']), 40 | HOST=os.environ.get('DB_HOST', DATABASES['default']['HOST']), 41 | PORT=os.environ.get('DB_PORT', DATABASES['default']['PORT']), 42 | ) 43 | 44 | for override, value in DB_OVERRIDES.items(): 45 | DATABASES['default'][override] = value 46 | DATABASES['analytics'][override] = value 47 | DATABASES['analytics_v1'][override] = value 48 | 49 | DATABASE_ROUTERS = ['analyticsdataserver.router.AnalyticsAPIRouter', 'analyticsdataserver.router.AnalyticsModelsRouter'] 50 | 51 | ########## END DATABASE CONFIGURATION 52 | 53 | ALLOWED_HOSTS += ['edx.devstack.analyticsapi'] 54 | 55 | LMS_BASE_URL = "http://edx.devstack.lms:18000/" 56 | -------------------------------------------------------------------------------- /analyticsdataserver/settings/local.py: -------------------------------------------------------------------------------- 1 | """Development settings and globals.""" 2 | 3 | 4 | 5 | 6 | from os.path import join, normpath 7 | 8 | from corsheaders.defaults import default_headers as corsheaders_default_headers 9 | 10 | from analyticsdataserver.settings.base import * 11 | 12 | ########## DEBUG CONFIGURATION 13 | # See: https://docs.djangoproject.com/en/dev/ref/settings/#debug 14 | DEBUG = True 15 | ########## END DEBUG CONFIGURATION 16 | 17 | 18 | ########## EMAIL CONFIGURATION 19 | # See: https://docs.djangoproject.com/en/dev/ref/settings/#email-backend 20 | EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' 21 | ########## END EMAIL CONFIGURATION 22 | 23 | 24 | ########## DATABASE CONFIGURATION 25 | # See: https://docs.djangoproject.com/en/dev/ref/settings/#databases 26 | DATABASES = { 27 | 'default': { 28 | 'ENGINE': 'django.db.backends.sqlite3', 29 | 'NAME': normpath(join(DJANGO_ROOT, 'default.db')), 30 | 'USER': '', 31 | 'PASSWORD': '', 32 | 'HOST': '', 33 | 'PORT': '', 34 | }, 35 | 'analytics': { 36 | 'ENGINE': 'django.db.backends.sqlite3', 37 | 'NAME': normpath(join(DJANGO_ROOT, 'analytics.db')), 38 | 'USER': '', 39 | 'PASSWORD': '', 40 | 'HOST': '', 41 | 'PORT': '', 42 | }, 43 | 'analytics_v1': { 44 | 'ENGINE': 'django.db.backends.sqlite3', 45 | 'NAME': normpath(join(DJANGO_ROOT, 'analytics.db')), 46 | 'USER': '', 47 | 'PASSWORD': '', 48 | 'HOST': '', 49 | 'PORT': '', 50 | }, 51 | 'enterprise': { 52 | 'ENGINE': 'django.db.backends.sqlite3', 53 | 'NAME': normpath(join(DJANGO_ROOT, 'enterprise_reporting.db')), 54 | 'USER': '', 55 | 'PASSWORD': '', 56 | 'HOST': '', 57 | 'PORT': '', 58 | } 59 | } 60 | ########## END DATABASE CONFIGURATION 61 | 62 | 63 | ########## CACHE CONFIGURATION 64 | # See: https://docs.djangoproject.com/en/dev/ref/settings/#caches 65 | CACHES = { 66 | 'default': { 67 | 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', 68 | } 69 | } 70 | ########## END CACHE CONFIGURATION 71 | 72 | 73 | ########## ANALYTICS DATA API CONFIGURATION 74 | 75 | ANALYTICS_DATABASE = 'analytics' 76 | ENTERPRISE_REPORTING_DB_ALIAS = 'analytics' 77 | ANALYTICS_DATABASE_V1 = 'analytics' 78 | ENROLLMENTS_PAGE_SIZE = 10000 79 | 80 | TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' 81 | 82 | SWAGGER_SETTINGS = { 83 | 'api_key': 'edx' 84 | } 85 | 86 | # These two settings are used in generate_fake_course_data.py. 87 | # Replace with correct values to generate local fake video data. 88 | LMS_BASE_URL = 'http://localhost:18000/' # the base URL for your running local LMS instance 89 | 90 | # In Insights, we run this API as a separate service called "analyticsapi" to run acceptance/integration tests. Docker 91 | # saves the service name as a host in the Insights container so it can reach the API by requesting http://analyticsapi/. 92 | # However, in Django 1.10.3, the HTTP_HOST header of requests started to be checked against the ALLOWED_HOSTS setting 93 | # even in DEBUG=True mode. Here, we add the Docker service name "analyticsapi" to the default set of local allowed 94 | # hosts. 95 | ALLOWED_HOSTS = ['localhost', '127.0.0.1', '::1', 'analyticsapi', 'host.docker.internal'] 96 | 97 | JWT_AUTH.update({ 98 | 'JWT_SECRET_KEY': 'lms-secret', 99 | 'JWT_ISSUER': 'http://localhost:18000/oauth2', 100 | 'JWT_AUDIENCE': None, 101 | 'JWT_VERIFY_AUDIENCE': False, 102 | 'JWT_PUBLIC_SIGNING_JWK_SET': ( 103 | '{"keys": [{"kid": "devstack_key", "e": "AQAB", "kty": "RSA", "n": "smKFSYowG6nNUAdeqH1jQQnH1PmIHphzBmwJ5vRf1vu' 104 | '48BUI5VcVtUWIPqzRK_LDSlZYh9D0YFL0ZTxIrlb6Tn3Xz7pYvpIAeYuQv3_H5p8tbz7Fb8r63c1828wXPITVTv8f7oxx5W3lFFgpFAyYMmROC' 105 | '4Ee9qG5T38LFe8_oAuFCEntimWxN9F3P-FJQy43TL7wG54WodgiM0EgzkeLr5K6cDnyckWjTuZbWI-4ffcTgTZsL_Kq1owa_J2ngEfxMCObnzG' 106 | 'y5ZLcTUomo4rZLjghVpq6KZxfS6I1Vz79ZsMVUWEdXOYePCKKsrQG20ogQEkmTf9FT_SouC6jPcHLXw"}]}' 107 | ), 108 | }) 109 | 110 | CORS_ORIGIN_WHITELIST = ( 111 | 'http://localhost:1991', 112 | ) 113 | CORS_ALLOW_HEADERS = corsheaders_default_headers + ( 114 | 'use-jwt-cookie', 115 | ) 116 | CORS_ALLOW_CREDENTIALS = True 117 | 118 | ########## END ANALYTICS DATA API CONFIGURATION 119 | -------------------------------------------------------------------------------- /analyticsdataserver/settings/local_mysql.py: -------------------------------------------------------------------------------- 1 | """ 2 | A variation on the local environment that uses mysql for the analytics database. 3 | 4 | Useful for developers running both mysql ingress locally and the api locally 5 | """ 6 | 7 | 8 | from analyticsdataserver.settings.local import * 9 | 10 | ########## DATABASE CONFIGURATION 11 | # See: https://docs.djangoproject.com/en/dev/ref/settings/#databases 12 | DATABASES = { 13 | 'default': { 14 | 'ENGINE': 'django.db.backends.sqlite3', 15 | 'NAME': normpath(join(DJANGO_ROOT, 'default.db')), 16 | 'USER': '', 17 | 'PASSWORD': '', 18 | 'HOST': '', 19 | 'PORT': '', 20 | }, 21 | 'analytics': { 22 | 'ENGINE': 'django.db.backends.mysql', 23 | 'NAME': 'reports_2_0', 24 | 'USER': 'readonly001', 25 | 'PASSWORD': 'meringues unfreehold sisterize morsing', 26 | 'HOST': 'stage-edx-analytics-report-rds.edx.org', 27 | 'PORT': '3306', 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /analyticsdataserver/settings/logger.py: -------------------------------------------------------------------------------- 1 | """Logging configuration""" 2 | 3 | 4 | 5 | import os 6 | import platform 7 | import sys 8 | from logging.handlers import SysLogHandler 9 | 10 | 11 | def get_logger_config(log_dir='/var/tmp', 12 | logging_env="no_env", 13 | edx_filename="edx.log", 14 | dev_env=False, 15 | debug=False, 16 | local_loglevel='INFO', 17 | service_variant='analytics-api'): 18 | 19 | """ 20 | 21 | Return the appropriate logging config dictionary. You should assign the 22 | result of this to the LOGGING var in your settings. 23 | 24 | If dev_env is set to true logging will not be done via local rsyslogd, 25 | instead, application logs will be dropped in log_dir. 26 | 27 | "edx_filename" is ignored unless dev_env is set to true since otherwise logging is handled by rsyslogd. 28 | 29 | """ 30 | 31 | # Revert to INFO if an invalid string is passed in 32 | if local_loglevel not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']: 33 | local_loglevel = 'INFO' 34 | 35 | hostname = platform.node().split(".")[0] 36 | syslog_format = ("[service_variant={service_variant}]" 37 | "[%(name)s][env:{logging_env}] %(levelname)s " 38 | "[{hostname} %(process)d] [%(filename)s:%(lineno)d] " 39 | "- %(message)s").format( 40 | service_variant=service_variant, 41 | logging_env=logging_env, hostname=hostname) 42 | 43 | if debug: 44 | handlers = ['console'] 45 | else: 46 | handlers = ['local'] 47 | 48 | 49 | logger_config = { 50 | 'version': 1, 51 | 'disable_existing_loggers': False, 52 | 'formatters': { 53 | 'standard': { 54 | 'format': '%(asctime)s %(levelname)s %(process)d ' 55 | '[%(name)s] %(filename)s:%(lineno)d - %(message)s', 56 | }, 57 | 'syslog_format': {'format': syslog_format}, 58 | 'raw': {'format': '%(message)s'}, 59 | }, 60 | 'handlers': { 61 | 'console': { 62 | 'level': 'DEBUG' if debug else 'INFO', 63 | 'class': 'logging.StreamHandler', 64 | 'formatter': 'standard', 65 | 'stream': sys.stdout, 66 | }, 67 | }, 68 | 'loggers': { 69 | 'django': { 70 | 'handlers': handlers, 71 | 'propagate': True, 72 | 'level': 'INFO' 73 | }, 74 | '': { 75 | 'handlers': handlers, 76 | 'level': 'DEBUG', 77 | 'propagate': False 78 | }, 79 | } 80 | } 81 | 82 | if dev_env: 83 | edx_file_loc = os.path.join(log_dir, edx_filename) 84 | logger_config['handlers'].update({ 85 | 'local': { 86 | 'class': 'logging.handlers.RotatingFileHandler', 87 | 'level': local_loglevel, 88 | 'formatter': 'standard', 89 | 'filename': edx_file_loc, 90 | 'maxBytes': 1024 * 1024 * 2, 91 | 'backupCount': 5, 92 | }, 93 | }) 94 | else: 95 | logger_config['handlers'].update({ 96 | 'local': { 97 | 'level': local_loglevel, 98 | 'class': 'logging.handlers.SysLogHandler', 99 | 'address': '/dev/log', 100 | 'formatter': 'syslog_format', 101 | 'facility': SysLogHandler.LOG_LOCAL0, 102 | }, 103 | }) 104 | 105 | return logger_config 106 | -------------------------------------------------------------------------------- /analyticsdataserver/settings/production.py: -------------------------------------------------------------------------------- 1 | """Production settings and globals.""" 2 | 3 | 4 | 5 | from os import environ 6 | 7 | import django 8 | import six 9 | import yaml 10 | # Normally you should not import ANYTHING from Django directly 11 | # into your settings, but ImproperlyConfigured is an exception. 12 | from django.core.exceptions import ImproperlyConfigured 13 | 14 | from analyticsdataserver.settings.base import * 15 | from analyticsdataserver.settings.logger import get_logger_config 16 | 17 | LOGGING = get_logger_config() 18 | 19 | def get_env_setting(setting): 20 | """Get the environment setting or return exception.""" 21 | try: 22 | return environ[setting] 23 | except KeyError: 24 | error_msg = "Set the %s env variable" % setting 25 | raise ImproperlyConfigured(error_msg) 26 | 27 | ########## HOST CONFIGURATION 28 | # See: https://docs.djangoproject.com/en/1.5/releases/1.5/#allowed-hosts-required-in-production 29 | ALLOWED_HOSTS = ['*'] 30 | ########## END HOST CONFIGURATION 31 | 32 | CONFIG_FILE=get_env_setting('ANALYTICS_API_CFG') 33 | 34 | with open(CONFIG_FILE) as f: 35 | config_from_yaml = yaml.load(f, Loader=yaml.FullLoader) 36 | 37 | REPORT_DOWNLOAD_BACKEND = config_from_yaml.pop('REPORT_DOWNLOAD_BACKEND', {}) 38 | 39 | JWT_AUTH_CONFIG = config_from_yaml.pop('JWT_AUTH', {}) 40 | JWT_AUTH.update(JWT_AUTH_CONFIG) 41 | 42 | vars().update(config_from_yaml) 43 | vars().update(REPORT_DOWNLOAD_BACKEND) 44 | 45 | DB_OVERRIDES = dict( 46 | PASSWORD=environ.get('DB_MIGRATION_PASS', DATABASES['default']['PASSWORD']), 47 | ENGINE=environ.get('DB_MIGRATION_ENGINE', DATABASES['default']['ENGINE']), 48 | USER=environ.get('DB_MIGRATION_USER', DATABASES['default']['USER']), 49 | NAME=environ.get('DB_MIGRATION_NAME', DATABASES['default']['NAME']), 50 | HOST=environ.get('DB_MIGRATION_HOST', DATABASES['default']['HOST']), 51 | PORT=environ.get('DB_MIGRATION_PORT', DATABASES['default']['PORT']), 52 | ) 53 | 54 | for override, value in DB_OVERRIDES.items(): 55 | DATABASES['default'][override] = value 56 | 57 | if django.VERSION[0] >= 4: # for greater than django 3.2 use schemes. 58 | CSRF_TRUSTED_ORIGINS = CSRF_TRUSTED_ORIGINS_WITH_SCHEME 59 | -------------------------------------------------------------------------------- /analyticsdataserver/settings/test.py: -------------------------------------------------------------------------------- 1 | """Test settings and globals.""" 2 | import os 3 | 4 | from analyticsdataserver.settings.base import * 5 | 6 | ########## IN-MEMORY TEST DATABASE 7 | DATABASES = { 8 | "default": { 9 | "ENGINE": os.environ.get("DB_ENGINE", "django.db.backends.sqlite3"), 10 | "NAME": os.environ.get("DB_NAME", ":memory:"), 11 | "USER": os.environ.get("DB_USER", ""), 12 | "PASSWORD": os.environ.get("DB_PASS", ""), 13 | "HOST": os.environ.get("DB_HOST", ""), 14 | "PORT": os.environ.get("DB_PORT", ""), 15 | }, 16 | 'analytics': { 17 | 'ENGINE': os.environ.get("DB_ENGINE", "django.db.backends.sqlite3"), 18 | 'NAME': os.environ.get("DB_NAME", normpath(join(DJANGO_ROOT, 'analytics.db'))), 19 | 'USER': os.environ.get("DB_USER", ""), 20 | 'PASSWORD': os.environ.get("DB_PASS", ""), 21 | 'HOST': os.environ.get("DB_HOST", ""), 22 | 'PORT': os.environ.get("DB_PORT", ""), 23 | }, 24 | 'analytics_v1': { 25 | 'ENGINE': os.environ.get("DB_ENGINE", "django.db.backends.sqlite3"), 26 | 'NAME': os.environ.get("DB_NAME", normpath(join(DJANGO_ROOT, 'analytics_v1.db'))), 27 | 'USER': os.environ.get("DB_USER", ""), 28 | 'PASSWORD': os.environ.get("DB_PASS", ""), 29 | 'HOST': os.environ.get("DB_HOST", ""), 30 | 'PORT': os.environ.get("DB_PORT", ""), 31 | }, 32 | } 33 | 34 | ANALYTICS_DATABASE = 'analytics' 35 | ANALYTICS_DATABASE_V1 = 'analytics_v1' 36 | ENTERPRISE_REPORTING_DB_ALIAS = 'default' 37 | ENROLLMENTS_PAGE_SIZE = 10000 38 | 39 | LMS_BASE_URL = 'http://lms-host' 40 | 41 | LMS_USER_ACCOUNT_BASE_URL = 'http://lms-host' 42 | 43 | # Default the django-storage settings so we can test easily 44 | DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' 45 | AWS_ACCESS_KEY_ID = 'xxxxx' 46 | AWS_SECRET_ACCESS_KEY = 'xxxxx' 47 | AWS_STORAGE_BUCKET_NAME = 'fake-bucket' 48 | AWS_DEFAULT_ACL = None 49 | FTP_STORAGE_LOCATION = 'ftp://localhost:80/path' 50 | 51 | # Default settings for report download endpoint 52 | COURSE_REPORT_FILE_LOCATION_TEMPLATE = '/{course_id}_{report_name}.csv' 53 | COURSE_REPORT_DOWNLOAD_EXPIRY_TIME = 120 54 | 55 | # Disable throttling during most testing, as it just adds queries 56 | REST_FRAMEWORK['DEFAULT_THROTTLE_CLASSES'] = () 57 | -------------------------------------------------------------------------------- /analyticsdataserver/tests/test_clients.py: -------------------------------------------------------------------------------- 1 | import unittest.mock as mock 2 | from urllib.parse import urljoin 3 | 4 | import responses 5 | from django.conf import settings 6 | from django.test import TestCase 7 | from requests.exceptions import ConnectionError as RequestsConnectionError 8 | 9 | from analyticsdataserver.clients import CourseBlocksApiClient 10 | 11 | 12 | class ClientTests(TestCase): 13 | 14 | def setUp(self): 15 | self.client = CourseBlocksApiClient(settings.LMS_BASE_URL, 'client_id', 'client_secret') 16 | responses.add( 17 | responses.POST, 18 | urljoin(settings.LMS_BASE_URL, 'oauth2/access_token'), 19 | status=200, 20 | json={ 21 | 'access_token': 'test_access_token', 22 | 'expires_in': 10, 23 | }, 24 | content_type='application/json' 25 | ) 26 | 27 | @responses.activate 28 | def test_all_videos(self): 29 | responses.add(responses.GET, urljoin(settings.LMS_BASE_URL, 'api/courses/v1/blocks/'), json={'blocks': { 30 | 'block-v1:edX+DemoX+Demo_Course+type@video+block@5c90cffecd9b48b188cbfea176bf7fe9': { 31 | 'id': 'block-v1:edX+DemoX+Demo_Course+type@video+block@5c90cffecd9b48b188cbfea176bf7fe9' 32 | }, 33 | 'block-v1:edX+DemoX+Demo_Course+type@video+block@7e9b434e6de3435ab99bd3fb25bde807': { 34 | 'id': 'block-v1:edX+DemoX+Demo_Course+type@video+block@7e9b434e6de3435ab99bd3fb25bde807' 35 | } 36 | }}, status=200, content_type='application/json') 37 | videos = self.client.all_videos('course_id') 38 | self.assertListEqual(videos, [ 39 | { 40 | 'video_id': 'course_id|5c90cffecd9b48b188cbfea176bf7fe9', 41 | 'video_module_id': '5c90cffecd9b48b188cbfea176bf7fe9' 42 | }, 43 | { 44 | 'video_id': 'course_id|7e9b434e6de3435ab99bd3fb25bde807', 45 | 'video_module_id': '7e9b434e6de3435ab99bd3fb25bde807' 46 | } 47 | ]) 48 | 49 | @responses.activate 50 | @mock.patch('analyticsdataserver.clients.logger') 51 | def test_all_videos_401(self, logger): 52 | responses.add(responses.GET, urljoin(settings.LMS_BASE_URL, 'api/courses/v1/blocks/'), status=401, content_type='application/json') 53 | videos = self.client.all_videos('course_id') 54 | logger.warning.assert_called_with( 55 | 'Course Blocks API failed to return video ids (%s). ' 56 | 'See README for instructions on how to authenticate the API with your local LMS.', 401) 57 | self.assertEqual(videos, None) 58 | 59 | @responses.activate 60 | @mock.patch('analyticsdataserver.clients.logger') 61 | def test_all_videos_404(self, logger): 62 | responses.add(responses.GET, urljoin(settings.LMS_BASE_URL, 'api/courses/v1/blocks/'), status=404, content_type='application/json') 63 | videos = self.client.all_videos('course_id') 64 | logger.warning.assert_called_with('Course Blocks API failed to return video ids (%s). ' 65 | 'Does the course exist in the LMS?', 404) 66 | self.assertEqual(videos, None) 67 | 68 | @responses.activate 69 | @mock.patch('analyticsdataserver.clients.logger') 70 | def test_all_videos_500(self, logger): 71 | responses.add(responses.GET, urljoin(settings.LMS_BASE_URL, 'api/courses/v1/blocks/'), status=418, content_type='application/json') 72 | videos = self.client.all_videos('course_id') 73 | logger.warning.assert_called_with('Course Blocks API failed to return video ids (%s).', 418) 74 | self.assertEqual(videos, None) 75 | 76 | @responses.activate 77 | @mock.patch('analyticsdataserver.clients.logger') 78 | def test_all_videos_connection_error(self, logger): 79 | exception = RequestsConnectionError('LMS is dead') 80 | responses.add(responses.GET, urljoin(settings.LMS_BASE_URL, 'api/courses/v1/blocks/'), body=exception) 81 | videos = self.client.all_videos('course_id') 82 | logger.warning.assert_called_with('Course Blocks API request failed. Is the LMS running?: %s', str(exception)) 83 | self.assertEqual(videos, None) 84 | 85 | @responses.activate 86 | def test_all_videos_pass_through_bad_id(self): 87 | responses.add(responses.GET, urljoin(settings.LMS_BASE_URL, 'api/courses/v1/blocks/'), json={'blocks': { 88 | 'block-v1:edX+DemoX+Demo_Course+type@video+block@5c90cffecd9b48b188cbfea176bf7fe9': { 89 | 'id': 'bad_key' 90 | }, 91 | 'block-v1:edX+DemoX+Demo_Course+type@video+block@7e9b434e6de3435ab99bd3fb25bde807': { 92 | 'id': 'bad_key' 93 | } 94 | }}, status=200, content_type='application/json') 95 | responses.add(responses.GET, urljoin(settings.LMS_BASE_URL, 'api/courses/v1/blocks/'), status=200, content_type='application/json') 96 | videos = self.client.all_videos('course_id') 97 | self.assertListEqual(videos, [ 98 | { 99 | 'video_id': 'course_id|bad_key', 100 | 'video_module_id': 'bad_key' 101 | }, 102 | { 103 | 'video_id': 'course_id|bad_key', 104 | 'video_module_id': 'bad_key' 105 | } 106 | ]) 107 | -------------------------------------------------------------------------------- /analyticsdataserver/tests/test_router.py: -------------------------------------------------------------------------------- 1 | import unittest.mock as mock 2 | 3 | import ddt 4 | from django.conf import settings 5 | from django.contrib.auth import get_user_model 6 | from django.test import TestCase, override_settings 7 | 8 | from analytics_data_api.v0.models import CourseEnrollmentDaily 9 | from analyticsdataserver.router import AnalyticsAPIRouter, AnalyticsModelsRouter 10 | 11 | 12 | class AnalyticsAPIRouterTests(TestCase): 13 | def setUp(self): 14 | self.router = AnalyticsAPIRouter() 15 | 16 | @mock.patch.dict('analytics_data_api.middleware.thread_data.__dict__', {'analyticsapi_database': 'test'}) 17 | def test_db_for_read_analytics_app_thread_data_with_data(self): 18 | self.assertEqual(self.router.db_for_read(CourseEnrollmentDaily), 'test') 19 | 20 | def test_db_for_read_analytics_app_thread_data_without_data(self): 21 | self.assertEqual(self.router.db_for_read(CourseEnrollmentDaily), getattr(settings, 'ANALYTICS_DATABASE', 'analytics')) 22 | 23 | @mock.patch.dict('analytics_data_api.middleware.thread_data.__dict__', {'analyticsapi_database': 'test'}) 24 | def test_db_for_read_analytics_app_thread_data_with_data(self): 25 | self.assertEqual(self.router.db_for_read(get_user_model()), None) 26 | 27 | def test_db_for_read_not_analytics_app_thread_data_without_data(self): 28 | self.assertEqual(self.router.db_for_read(get_user_model()), None) 29 | 30 | 31 | @ddt.ddt 32 | class AnalyticsModelsRouterTests(TestCase): 33 | def setUp(self): 34 | self.router = AnalyticsModelsRouter() 35 | self.analytics_database_slug = getattr(settings, 'ANALYTICS_DATABASE', 'analytics') 36 | 37 | def test_db_for_read_analytics_app(self): 38 | self.assertEqual(self.router.db_for_read(CourseEnrollmentDaily), self.analytics_database_slug) 39 | 40 | @override_settings( 41 | ANALYTICS_DATABASE=None, 42 | ANALYTICS_DATABASE_V1=None 43 | ) 44 | def test_db_for_read_analytics_app_no_setting(self): 45 | self.assertEqual(self.router.db_for_read(CourseEnrollmentDaily), 'default') 46 | 47 | def test_db_for_read_not_analytics_app(self): 48 | self.assertEqual(self.router.db_for_read(get_user_model()), 'default') 49 | 50 | def test_db_for_write_analytics_app(self): 51 | self.assertEqual(self.router.db_for_write(CourseEnrollmentDaily), self.analytics_database_slug) 52 | 53 | @override_settings( 54 | ANALYTICS_DATABASE=None, 55 | ANALYTICS_DATABASE_V1=None 56 | ) 57 | def test_db_for_write_analytics_app_no_setting(self): 58 | self.assertEqual(self.router.db_for_write(CourseEnrollmentDaily), 'default') 59 | 60 | def test_db_for_write_not_analytics_app(self): 61 | self.assertEqual(self.router.db_for_write(get_user_model()), 'default') 62 | 63 | @ddt.data( 64 | ('default', True), 65 | (getattr(settings, 'ANALYTICS_DATABASE', 'analytics'), False), 66 | (getattr(settings, 'ANALYTICS_DATABASE_V1', 'analytics_v1'), False), 67 | ) 68 | @ddt.unpack 69 | def test_allow_migrate_not_analytics_app(self, database, expected_allow_migrate): 70 | self.assertEqual(self.router.allow_migrate(database, 'auth'), expected_allow_migrate) 71 | 72 | @ddt.data( 73 | (getattr(settings, 'ANALYTICS_DATABASE_V1', 'analytics_v1'), 'enterprise_data', False), 74 | (getattr(settings, 'default', 'default'), 'v0', False), 75 | (getattr(settings, 'ANALYTICS_DATABASE', 'analytics'), 'v0', True), 76 | (getattr(settings, 'ANALYTICS_DATABASE_V1', 'analytics_v1'), 'v0', True), 77 | ) 78 | @ddt.unpack 79 | def test_allow_migrate_analytics_app_multiple_dbs(self, database, app_label, expected_allow_migrate): 80 | self.assertEqual(self.router.allow_migrate(database, app_label), expected_allow_migrate) 81 | 82 | @ddt.data( 83 | (getattr(settings, 'ANALYTICS_DATABASE', 'analytics'), True), 84 | (getattr(settings, 'ANALYTICS_DATABASE_V1', 'analytics_v1'), False), 85 | ) 86 | @ddt.unpack 87 | @override_settings(ANALYTICS_DATABASE_V1=None) 88 | def test_does_not_migrate_database_with_no_env_setting(self, database, expected_allow_migrate): 89 | self.assertEqual(self.router.allow_migrate(database, 'v0'), expected_allow_migrate) 90 | self.assertEqual(self.router.allow_migrate(database, 'v1'), expected_allow_migrate) 91 | 92 | @ddt.data( 93 | (getattr(settings, 'default', 'default'), 'auth', True), 94 | (getattr(settings, 'default', 'default'), 'v0', True), 95 | (getattr(settings, 'default', 'default'), 'v1', True), 96 | (getattr(settings, 'ANALYTICS_DATABASE', 'analytics'), 'v0', False), 97 | (getattr(settings, 'ANALYTICS_DATABASE_V1', 'analytics_v1'), 'v0', False), 98 | (getattr(settings, 'ANALYTICS_DATABASE_V1', 'analytics_v1'), 'v1', False), 99 | ) 100 | @ddt.unpack 101 | @override_settings( 102 | ANALYTICS_DATABASE=None, 103 | ANALYTICS_DATABASE_V1=None 104 | ) 105 | def test_migrate_single_database_environment(self, database, app_label, expected_allow_migrate): 106 | self.assertEqual(self.router.allow_migrate(database, app_label), expected_allow_migrate) 107 | -------------------------------------------------------------------------------- /analyticsdataserver/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.test import TestCase 4 | 5 | from analyticsdataserver.utils import temp_log_level 6 | 7 | 8 | class UtilsTests(TestCase): 9 | def setUp(self): 10 | self.logger = logging.getLogger('test_logger') 11 | 12 | def test_temp_log_level(self): 13 | """Ensures log level is adjusted within context manager and returns to original level when exited.""" 14 | original_level = self.logger.getEffectiveLevel() 15 | with temp_log_level('test_logger'): # NOTE: defaults to logging.CRITICAL 16 | self.assertEqual(self.logger.getEffectiveLevel(), logging.CRITICAL) 17 | self.assertEqual(self.logger.getEffectiveLevel(), original_level) 18 | 19 | # test with log_level option used 20 | with temp_log_level('test_logger', log_level=logging.DEBUG): 21 | self.assertEqual(self.logger.getEffectiveLevel(), logging.DEBUG) 22 | self.assertEqual(self.logger.getEffectiveLevel(), original_level) 23 | -------------------------------------------------------------------------------- /analyticsdataserver/tests/test_views.py: -------------------------------------------------------------------------------- 1 | import unittest.mock as mock 2 | from contextlib import contextmanager 3 | 4 | from django.conf import settings 5 | from django.db.utils import ConnectionHandler 6 | from django.test.utils import override_settings 7 | 8 | from analytics_data_api.tests.test_utils import set_databases 9 | from analyticsdataserver.tests.utils import TestCaseWithAuthentication 10 | 11 | 12 | @set_databases 13 | class OperationalEndpointsTest(TestCaseWithAuthentication): 14 | def test_status(self): 15 | response = self.client.get('/status', follow=True) 16 | self.assertEqual(response.status_code, 200) 17 | 18 | def test_authentication_check_failure(self): 19 | response = self.client.get('/authenticated', follow=True) 20 | self.assertEqual(response.status_code, 401) 21 | 22 | def test_authentication_check_success(self): 23 | response = self.authenticated_get('/authenticated', follow=True) 24 | self.assertEqual(response.status_code, 200) 25 | 26 | def test_health(self): 27 | self.assert_database_health('OK') 28 | 29 | def assert_database_health( 30 | self, 31 | overall_status, 32 | default_db_status='OK', 33 | analytics_db_status='OK', 34 | status_code=200 35 | ): 36 | response = self.client.get('/health', follow=True) 37 | self.assertEqual( 38 | response.data, 39 | { 40 | 'overall_status': overall_status, 41 | 'detailed_status': { 42 | 'default_db_status': default_db_status, 43 | 'analytics_db_status': analytics_db_status, 44 | } 45 | } 46 | ) 47 | self.assertEqual(response.status_code, status_code) 48 | 49 | @staticmethod 50 | @contextmanager 51 | def override_database_connections(databases): 52 | with mock.patch('analyticsdataserver.views.connections', ConnectionHandler(databases)): 53 | yield 54 | 55 | def test_default_bad_health(self): 56 | databases = dict(settings.DATABASES) 57 | databases['default'] = {} 58 | with self.override_database_connections(databases): 59 | self.assert_database_health('UNAVAILABLE', default_db_status='UNAVAILABLE', status_code=503) 60 | 61 | @override_settings(ANALYTICS_DATABASE='reporting') 62 | def test_db_bad_health(self): 63 | databases = dict(settings.DATABASES) 64 | databases['reporting'] = {} 65 | with self.override_database_connections(databases): 66 | self.assert_database_health('UNAVAILABLE', analytics_db_status='UNAVAILABLE', status_code=503) 67 | 68 | @override_settings(ANALYTICS_DATABASE_V1='reporting_v1') 69 | def test_v1_db_bad_health(self): 70 | databases = dict(settings.DATABASES) 71 | databases['reporting_v1'] = {} 72 | with self.override_database_connections(databases): 73 | self.assert_database_health('UNAVAILABLE', analytics_db_status='UNAVAILABLE', status_code=503) 74 | -------------------------------------------------------------------------------- /analyticsdataserver/tests/utils.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import User 2 | from django.test import TestCase 3 | from rest_framework.authtoken.models import Token 4 | 5 | 6 | class TestCaseWithAuthentication(TestCase): 7 | def setUp(self): 8 | super().setUp() 9 | self.test_user = User.objects.create_user('tester', 'test@example.com', 'testpassword') 10 | self.token = Token.objects.create(user=self.test_user) 11 | 12 | def authenticated_get(self, path, data=None, follow=True, **extra): 13 | data = data or {} 14 | return self.client.get( 15 | path=path, data=data, follow=follow, HTTP_AUTHORIZATION='Token ' + self.token.key, **extra 16 | ) 17 | 18 | def authenticated_post(self, path, data=None, follow=True, **extra): 19 | data = data or {} 20 | return self.client.post( 21 | path=path, data=data, follow=follow, HTTP_AUTHORIZATION='Token ' + self.token.key, **extra 22 | ) 23 | -------------------------------------------------------------------------------- /analyticsdataserver/urls.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.urls import include, re_path 3 | from django.views.generic import RedirectView 4 | from edx_api_doc_tools import make_api_info, make_docs_ui_view 5 | from rest_framework.authtoken.views import obtain_auth_token 6 | 7 | from analyticsdataserver import views 8 | 9 | admin.site.site_header = 'Analytics Data API Service Administration' 10 | admin.site.site_title = admin.site.site_header 11 | 12 | urlpatterns = [ 13 | re_path(r'^api-auth/', include('rest_framework.urls', 'rest_framework')), 14 | re_path(r'^api-token-auth/', obtain_auth_token), 15 | 16 | re_path(r'^api/', include('analytics_data_api.urls')), 17 | re_path(r'^status/$', views.StatusView.as_view(), name='status'), 18 | re_path(r'^authenticated/$', views.AuthenticationTestView.as_view(), name='authenticated'), 19 | re_path(r'^health/$', views.HealthView.as_view(), name='health'), 20 | ] 21 | 22 | urlpatterns.append(re_path(r'', include('enterprise_data.urls'))) 23 | 24 | api_ui_view = make_docs_ui_view( 25 | api_info=make_api_info( 26 | title="edX Analytics Data API", 27 | version="v0", 28 | email="program-cosmonauts@edx.org" 29 | ), 30 | api_url_patterns=urlpatterns 31 | ) 32 | 33 | urlpatterns += [ 34 | re_path(r'^docs/$', api_ui_view, name='api-docs'), 35 | re_path(r'^$', RedirectView.as_view(url='/docs')), # pylint: disable=no-value-for-parameter 36 | ] 37 | 38 | handler500 = 'analyticsdataserver.views.handle_internal_server_error' # pylint: disable=invalid-name 39 | handler404 = 'analyticsdataserver.views.handle_missing_resource_error' # pylint: disable=invalid-name 40 | -------------------------------------------------------------------------------- /analyticsdataserver/utils.py: -------------------------------------------------------------------------------- 1 | # Put utilities that are used in managing the server or local environment here. 2 | # Utilities critical to application functionality should go under analytics_data_api. 3 | 4 | 5 | import logging 6 | from contextlib import contextmanager 7 | 8 | 9 | @contextmanager 10 | def temp_log_level(logger_name, log_level=logging.CRITICAL): 11 | """ 12 | A context manager that temporarily adjusts a logger's log level. 13 | 14 | By default, log_level is logging.CRITICAL, which will effectively silence the logger while the context 15 | manager is active. 16 | """ 17 | logger = logging.getLogger(logger_name) 18 | original_log_level = logger.getEffectiveLevel() 19 | logger.setLevel(log_level) # silences all logs up to but not including this level 20 | yield 21 | # Return log level back to what it was. 22 | logger.setLevel(original_log_level) 23 | -------------------------------------------------------------------------------- /analyticsdataserver/views.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.conf import settings 4 | from django.db import connections 5 | from django.http import HttpResponse 6 | from rest_framework import permissions 7 | from rest_framework.renderers import JSONRenderer 8 | from rest_framework.response import Response 9 | from rest_framework.views import APIView 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | def handle_internal_server_error(_request): 15 | """Notify the client that an error occurred processing the request without providing any detail.""" 16 | return _handle_error(500) 17 | 18 | 19 | def handle_missing_resource_error(_request, exception=None): # pylint: disable=unused-argument 20 | """Notify the client that the requested resource could not be found.""" 21 | return _handle_error(404) 22 | 23 | 24 | def _handle_error(status_code): 25 | info = { 26 | 'status': status_code 27 | } 28 | 29 | renderer = JSONRenderer() 30 | content_type = f'{renderer.media_type}; charset={renderer.charset}' 31 | return HttpResponse(renderer.render(info), content_type=content_type, status=status_code) 32 | 33 | 34 | class StatusView(APIView): 35 | """ 36 | Simple check to determine if the server is alive 37 | 38 | Return no data, a simple 200 OK status code is sufficient to indicate that the server is alive. This endpoint is 39 | public and does not require an authentication token to access it. 40 | 41 | """ 42 | permission_classes = (permissions.AllowAny,) 43 | 44 | def get(self, request, *args, **kwargs): # pylint: disable=unused-argument 45 | return Response({}) 46 | 47 | 48 | class AuthenticationTestView(APIView): 49 | """ 50 | Verifies that the client is authenticated 51 | 52 | Returns HTTP 200 if client is authenticated, HTTP 401 if not authenticated 53 | 54 | """ 55 | 56 | def get(self, request, *args, **kwargs): # pylint: disable=unused-argument 57 | return Response({}) 58 | 59 | 60 | class HealthView(APIView): 61 | """ 62 | A more comprehensive check to see if the system is fully operational. 63 | 64 | This endpoint is public and does not require an authentication token to access it. 65 | 66 | The returned structure contains the following fields: 67 | 68 | - overall_status: Can be either "OK" or "UNAVAILABLE". 69 | - detailed_status: More detailed information about the status of the system. 70 | - database_connection: Status of the database connection. Can be either "OK" or "UNAVAILABLE". 71 | 72 | """ 73 | permission_classes = (permissions.AllowAny,) 74 | STATUS_OK = 'OK' 75 | STATUS_UNAVAILABLE = 'UNAVAILABLE' 76 | 77 | def _get_connection_status(self, connection_name): 78 | """ 79 | With the passed in database name, try to make connection to the database. 80 | If database is able to connect successfully, return 'OK' status. 81 | Otherwise, return 'UNAVAILABLE' status 82 | """ 83 | db_status = self.STATUS_UNAVAILABLE 84 | try: 85 | cursor = connections[connection_name].cursor() 86 | try: 87 | cursor.execute("SELECT 1") 88 | cursor.fetchone() 89 | db_status = self.STATUS_OK 90 | finally: 91 | cursor.close() 92 | except Exception: # pylint: disable=broad-except 93 | pass 94 | return db_status 95 | 96 | def get(self, request, *args, **kwargs): # pylint: disable=unused-argument 97 | overall_status = self.STATUS_UNAVAILABLE 98 | analytics_db_status = self.STATUS_UNAVAILABLE 99 | 100 | # First try the default database. 101 | # The default database hosts user and auth info. Without it, no API would function 102 | default_db_status = self._get_connection_status('default') 103 | 104 | analytics_db_v1_name = getattr(settings, 'ANALYTICS_DATABASE_V1') 105 | analytics_db_name = getattr(settings, 'ANALYTICS_DATABASE') 106 | if analytics_db_v1_name: 107 | # If the v1 db is defined, we check the status of both v0 and v1 analytics db status 108 | analytics_v0_db_status = self._get_connection_status(analytics_db_name) 109 | analyitcs_v1_db_status = self._get_connection_status(analytics_db_v1_name) 110 | if analytics_v0_db_status == analyitcs_v1_db_status == self.STATUS_OK: 111 | analytics_db_status = self.STATUS_OK 112 | else: 113 | # Otherwise, only check the defined 'ANALYTICS_DATABASE' status 114 | analytics_db_status = self._get_connection_status(analytics_db_name) 115 | 116 | if default_db_status == analytics_db_status == self.STATUS_OK: 117 | overall_status = self.STATUS_OK 118 | 119 | response = { 120 | "overall_status": overall_status, 121 | "detailed_status": { 122 | 'default_db_status': default_db_status, 123 | 'analytics_db_status': analytics_db_status, 124 | } 125 | } 126 | 127 | if overall_status == self.STATUS_UNAVAILABLE: 128 | logger.error("Health check failed: %s", response) 129 | 130 | return Response(response, status=200 if overall_status == self.STATUS_OK else 503) 131 | -------------------------------------------------------------------------------- /analyticsdataserver/wsgi.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from django.core.wsgi import get_wsgi_application 4 | 5 | # We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks 6 | # if running multiple sites in the same mod_wsgi process. To fix this, use 7 | # mod_wsgi daemon mode with each site in its own daemon process, or use 8 | # os.environ["DJANGO_SETTINGS_MODULE"] = "jajaja.settings" 9 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "analyticsdataserver.settings.production") 10 | 11 | # This application object is used by any WSGI server configured to use this 12 | # file. This includes Django's development server, if the WSGI_APPLICATION 13 | # setting points here. 14 | application = get_wsgi_application() # pylint: disable=invalid-name 15 | 16 | # Apply WSGI middleware here. 17 | # from helloworld.wsgi import HelloWorldApplication 18 | # application = HelloWorldApplication(application) 19 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.2' 2 | services: 3 | 4 | elasticsearch: 5 | image: docker.elastic.co/elasticsearch/elasticsearch:7.10.1 6 | container_name: elasticsearch 7 | environment: 8 | - node.name=elasticsearch 9 | - cluster.name=docker-cluster 10 | - cluster.initial_master_nodes=elasticsearch 11 | - bootstrap.memory_lock=true 12 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 13 | - http.port=9223 14 | ulimits: 15 | memlock: 16 | soft: -1 17 | hard: -1 18 | volumes: 19 | - data01:/usr/share/elasticsearch/data 20 | ports: 21 | - "9223:9223" 22 | networks: 23 | - elastic 24 | 25 | volumes: 26 | data01: 27 | driver: local 28 | 29 | networks: 30 | elastic: 31 | driver: bridge 32 | -------------------------------------------------------------------------------- /docs/api/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS ?= 6 | SPHINXBUILD ?= sphinx-build 7 | PAPER ?= 8 | BUILDDIR ?= build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | Q_FLAG = 16 | 17 | ifeq ($(quiet), true) 18 | Q_FLAG = -Q 19 | endif 20 | 21 | # Internal variables. 22 | PAPEROPT_a4 = -D latex_paper_size=a4 23 | PAPEROPT_letter = -D latex_paper_size=letter 24 | ALLSPHINXOPTS = $(Q_FLAG) -d $(BUILDDIR)/doctrees -c source $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 25 | # the i18n builder cannot share the environment and doctrees with the others 26 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 27 | 28 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 29 | 30 | help: 31 | @echo "Please use \`make ' where is one of" 32 | @echo " html to make standalone HTML files" 33 | @echo " dirhtml to make HTML files named index.html in directories" 34 | @echo " singlehtml to make a single large HTML file" 35 | @echo " pickle to make pickle files" 36 | @echo " json to make JSON files" 37 | @echo " htmlhelp to make HTML files and a HTML help project" 38 | @echo " qthelp to make HTML files and a qthelp project" 39 | @echo " devhelp to make HTML files and a Devhelp project" 40 | @echo " epub to make an epub" 41 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 42 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 43 | @echo " text to make text files" 44 | @echo " man to make manual pages" 45 | @echo " texinfo to make Texinfo files" 46 | @echo " info to make Texinfo files and run them through makeinfo" 47 | @echo " gettext to make PO message catalogs" 48 | @echo " changes to make an overview of all changed/added/deprecated items" 49 | @echo " linkcheck to check all external links for integrity" 50 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 51 | 52 | clean: 53 | -rm -rf $(BUILDDIR)/* 54 | 55 | html: 56 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 57 | @echo 58 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 59 | 60 | dirhtml: 61 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 62 | @echo 63 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 64 | 65 | singlehtml: 66 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 67 | @echo 68 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 69 | 70 | pickle: 71 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 72 | @echo 73 | @echo "Build finished; now you can process the pickle files." 74 | 75 | json: 76 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 77 | @echo 78 | @echo "Build finished; now you can process the JSON files." 79 | 80 | htmlhelp: 81 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 82 | @echo 83 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 84 | ".hhp project file in $(BUILDDIR)/htmlhelp." 85 | 86 | qthelp: 87 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 88 | @echo 89 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 90 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 91 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/edX.qhcp" 92 | @echo "To view the help file:" 93 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/edX.qhc" 94 | 95 | devhelp: 96 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 97 | @echo 98 | @echo "Build finished." 99 | @echo "To view the help file:" 100 | @echo "# mkdir -p $$HOME/.local/share/devhelp/edX" 101 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/edX" 102 | @echo "# devhelp" 103 | 104 | epub: 105 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 106 | @echo 107 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 108 | 109 | latex: 110 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 111 | @echo 112 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 113 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 114 | "(use \`make latexpdf' here to do that automatically)." 115 | 116 | latexpdf: 117 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 118 | @echo "Running LaTeX files through pdflatex..." 119 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 120 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 121 | 122 | text: 123 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 124 | @echo 125 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 126 | 127 | man: 128 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 129 | @echo 130 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 131 | 132 | texinfo: 133 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 134 | @echo 135 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 136 | @echo "Run \`make' in that directory to run these through makeinfo" \ 137 | "(use \`make info' here to do that automatically)." 138 | 139 | info: 140 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 141 | @echo "Running Texinfo files through makeinfo..." 142 | make -C $(BUILDDIR)/texinfo info 143 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 144 | 145 | gettext: 146 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 147 | @echo 148 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 149 | 150 | changes: 151 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 152 | @echo 153 | @echo "The overview file is in $(BUILDDIR)/changes." 154 | 155 | linkcheck: 156 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 157 | @echo 158 | @echo "Link check complete; look for any errors in the above output " \ 159 | "or in $(BUILDDIR)/linkcheck/output.txt." 160 | 161 | doctest: 162 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 163 | @echo "Testing of doctests in the sources finished, look at the " \ 164 | "results in $(BUILDDIR)/doctest/output.txt." 165 | -------------------------------------------------------------------------------- /docs/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/docs/api/__init__.py -------------------------------------------------------------------------------- /docs/api/requirements.txt: -------------------------------------------------------------------------------- 1 | -r ../../requirements/doc.txt 2 | -------------------------------------------------------------------------------- /docs/api/source/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/docs/api/source/__init__.py -------------------------------------------------------------------------------- /docs/api/source/authentication.rst: -------------------------------------------------------------------------------- 1 | .. _edX Data Analytics API Authentication: 2 | 3 | ###################################### 4 | edX Data Analytics API Authentication 5 | ###################################### 6 | 7 | The edX Data Analytics API uses the Django REST framework 8 | `TokenAuthentication`_. 9 | 10 | ************************** 11 | Create a User and Token 12 | ************************** 13 | 14 | You create users and tokens in the Data Analytics API server. In the server 15 | terminal, enter: 16 | 17 | ``$ ./manage.py set_api_key `` 18 | 19 | 20 | .. include:: links.rst -------------------------------------------------------------------------------- /docs/api/source/change_log.rst: -------------------------------------------------------------------------------- 1 | ############ 2 | Change Log 3 | ############ 4 | 5 | .. list-table:: 6 | :widths: 10 70 7 | :header-rows: 1 8 | 9 | * - Date 10 | - Change 11 | * - 5 Aug 2015 12 | - Updated :ref:`Get the Course Enrollment by Mode` to add the 13 | ``cumulative_count`` response value. 14 | * - 21 May 2015 15 | - Updated :ref:`Get the Course Video Data` to add the ``users_at_start`` 16 | and ``users_at_end`` response values. 17 | * - 18 May 2015 18 | - Added :ref:`Get the Course Video Data` and :ref:`Video Data API`. 19 | -------------------------------------------------------------------------------- /docs/api/source/conf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import django 5 | from path import Path 6 | 7 | # Add any paths that contain templates here, relative to this directory. 8 | # templates_path.append('source/_templates') 9 | 10 | # Add any paths that contain custom static files (such as style sheets) here, 11 | # relative to this directory. They are copied after the builtin static files, 12 | # so a file named "default.css" will overwrite the builtin "default.css". 13 | # html_static_path.append('source/_static') 14 | 15 | master_doc = 'index' 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | root = Path('../../..').abspath() 20 | sys.path.insert(0, root) 21 | sys.path.append(root / "analytics_data_api/v0/views") 22 | sys.path.append('.') 23 | 24 | # -- General configuration ----------------------------------------------------- 25 | 26 | # django configuration - careful here 27 | os.environ['DJANGO_SETTINGS_MODULE'] = 'analyticsdataserver.settings.local' 28 | django.setup() 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be extensions 31 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 32 | extensions = [ 33 | 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 34 | 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.imgmath', 35 | 'sphinx.ext.mathjax', 'sphinx.ext.viewcode'] 36 | 37 | # List of patterns, relative to source directory, that match files and 38 | # directories to ignore when looking for source files. 39 | exclude_patterns = ['build', 'links.rst'] 40 | 41 | project = 'Open edX Data Analytics API' 42 | copyright = '2021, Axim Collaborative, Inc' 43 | 44 | # -- Options for HTML output ---------------------------------------------- 45 | 46 | # The theme to use for HTML and HTML Help pages. See the documentation for 47 | # a list of builtin themes. 48 | # 49 | html_theme = 'sphinx_book_theme' 50 | 51 | # Theme options are theme-specific and customize the look and feel of a theme 52 | # further. For a list of options available for each theme, see the 53 | # documentation. 54 | # 55 | html_theme_options = { 56 | "repository_url": "https://github.com/openedx/edx-analytics-data-api", 57 | "repository_branch": "master", 58 | "path_to_docs": "docs/api/source", 59 | "home_page_in_toc": True, 60 | "use_repository_button": True, 61 | "use_issues_button": True, 62 | "use_edit_page_button": True, 63 | # False was the default value for navigation_with_keys. However, in version 0.14.2 of pydata-sphinx-theme, this default 64 | # was removed and a warning was added that would be emitted whenever navigation_with_keys was not set. Because of the 65 | # "SPHINXOPTS = -W" configuration in tox.ini, all warnings are promoted to an error. Therefore, it's necesary to set 66 | # this value. I have set it to the default value explicitly. Please see the following GitHub comments for context. 67 | # https://github.com/pydata/pydata-sphinx-theme/issues/1539 68 | # https://github.com/pydata/pydata-sphinx-theme/issues/987#issuecomment-1277214209 69 | "navigation_with_keys": False, 70 | # Please don't change unless you know what you're doing. 71 | "extra_footer": """ 72 | 73 | Creative Commons License 77 | 78 |
79 | These works by 80 | Axim Collaborative, Inc 86 | are licensed under a 87 | Creative Commons Attribution-ShareAlike 4.0 International License. 91 | """ 92 | } 93 | 94 | # Add any paths that contain custom themes here, relative to this directory. 95 | # html_theme_path = [] 96 | -------------------------------------------------------------------------------- /docs/api/source/endpoints.rst: -------------------------------------------------------------------------------- 1 | .. _Data Analytics API: 2 | 3 | ################################# 4 | edX Data Analytics API Endpoints 5 | ################################# 6 | 7 | The edX Platform API allows you to view information about users and their course enrollments, course information, and videos and transcripts. 8 | 9 | The following tasks and endpoints are currently supported. 10 | 11 | 12 | .. list-table:: 13 | :widths: 10 70 14 | :header-rows: 1 15 | 16 | * - To: 17 | - Use this endpoint: 18 | * - :ref:`Get Weekly Course Activity` 19 | - /api/v0/courses/{course_id}/activity/ 20 | * - :ref:`Get Recent Course Activity` 21 | - /api/v0/courses/{course_id}/recent_activity/ 22 | * - :ref:`Get the Course Enrollment` 23 | - /api/v0/courses/{course_id}/enrollment/ 24 | * - :ref:`Get the Course Enrollment by Mode` 25 | - /api/v0/courses/{course_id}/enrollment/mode/ 26 | * - :ref:`Get the Course Enrollment by Birth Year` 27 | - /api/v0/courses/{course_id}/enrollment/birth_year/ 28 | * - :ref:`Get the Course Enrollment by Education Level` 29 | - /api/v0/courses/{course_id}/enrollment/education/ 30 | * - :ref:`Get the Course Enrollment by Gender` 31 | - /api/v0/courses/{course_id}/enrollment/gender/ 32 | * - :ref:`Get the Course Enrollment by Location` 33 | - /api/v0/courses/{course_id}/enrollment/location/ 34 | * - :ref:`Get the Course Video Data` 35 | - /api/v0/courses/{course_id}/videos/ 36 | * - :ref:`Get the Grade Distribution for a Course` 37 | - /api/v0/problems/{problem_id}/grade_distribution 38 | * - :ref:`Get the Answer Distribution for a Problem` 39 | - /api/v0/problems/{problem_id}/answer_distribution 40 | * - :ref:`Get the View Count for a Subsection` 41 | - /api/v0/problems/{module_id}/sequential_open_distribution 42 | * - :ref:`Get the Timeline for a Video` 43 | - /api/v0/videos/{video_id}/timeline/ 44 | -------------------------------------------------------------------------------- /docs/api/source/images/api_test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/docs/api/source/images/api_test.png -------------------------------------------------------------------------------- /docs/api/source/images/api_test_expand.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/docs/api/source/images/api_test_expand.png -------------------------------------------------------------------------------- /docs/api/source/images/api_test_response.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/docs/api/source/images/api_test_response.png -------------------------------------------------------------------------------- /docs/api/source/index.rst: -------------------------------------------------------------------------------- 1 | .. _Open EdX Data Analytics API: 2 | 3 | ################################################ 4 | Open edX Data Analytics API Version 0 5 | ################################################ 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | read_me 11 | change_log 12 | overview 13 | setup 14 | authentication 15 | endpoints 16 | courses 17 | problems 18 | videos 19 | -------------------------------------------------------------------------------- /docs/api/source/links.rst: -------------------------------------------------------------------------------- 1 | .. _TokenAuthentication: http://www.django-rest-framework.org/api-guide/authentication/#tokenauthentication 2 | 3 | .. _Data Analytics API repository: https://github.com/openedx/edx-analytics-data-api 4 | 5 | .. _ISO 3166 country codes: http://www.iso.org/iso/country_codes/country_codes -------------------------------------------------------------------------------- /docs/api/source/overview.rst: -------------------------------------------------------------------------------- 1 | .. _edX Data Analytics API Overview: 2 | 3 | ################################ 4 | edX Data Analytics API Overview 5 | ################################ 6 | 7 | The edX Data Analytics API provides the tools for building applications to view 8 | and analyze student activity in your course. 9 | 10 | The edX Platform APIs use REST design principles and support the JSON 11 | data-interchange format. 12 | 13 | **************************************** 14 | edX Data Analytics API Version 0, Alpha 15 | **************************************** 16 | 17 | The edX Data Analytics API is currently at version 0 and is an Alpha release. 18 | We plan on making significant enhancements and changes to the API. 19 | 20 | The Data Analytics API uses key-based authentication and currently has no 21 | built-in authorization mechanism. Therefore third parties cannot currently use 22 | the Data Analytics API with edx.org data. 23 | 24 | Open edX users can use the Data Analytics API with their own instances. 25 | 26 | EdX plans to make the Data Analytics API available to partners in the future, 27 | and invites feedback. 28 | 29 | ************************************** 30 | edX Data Analytics API Version 1, Beta 31 | ************************************** 32 | 33 | There is now a version 1 of the Data Analytics API. This version is largely the same 34 | as v0, but the data for v1 endpoints is sourced from a new database. The new database is populated by 35 | data outside of the current analytics pipeline. Open edX users can use version 1, but will need to populate the 36 | database being used by the v1 API. 37 | 38 | You can use version 1 of the API by updating your client URL version, for example: 39 | 40 | ``/api/v1/courses/{course_id}/activity/`` 41 | 42 | 43 | *********************************** 44 | edX Data Analytics API Capabilities 45 | *********************************** 46 | 47 | With the edX Data Analytics API, you can: 48 | 49 | * :ref:`Get Weekly Course Activity` 50 | * :ref:`Get Recent Course Activity` 51 | * :ref:`Get the Course Enrollment` 52 | * :ref:`Get the Course Enrollment by Mode` 53 | * :ref:`Get the Course Enrollment by Birth Year` 54 | * :ref:`Get the Course Enrollment by Education Level` 55 | * :ref:`Get the Course Enrollment by Gender` 56 | * :ref:`Get the Course Enrollment by Location` 57 | * :ref:`Get the Course Video Data` 58 | * :ref:`Get the Grade Distribution for a Course` 59 | * :ref:`Get the Answer Distribution for a Problem` 60 | * :ref:`Get the View Count for a Subsection` 61 | * :ref:`Get the Timeline for a Video` 62 | -------------------------------------------------------------------------------- /docs/api/source/problems.rst: -------------------------------------------------------------------------------- 1 | ######################## 2 | Problem Information API 3 | ######################## 4 | 5 | .. contents:: Section Contents 6 | :local: 7 | :depth: 1 8 | 9 | .. _Get the Grade Distribution for a Course: 10 | 11 | *************************************** 12 | Get the Grade Distribution for a Course 13 | *************************************** 14 | 15 | .. autoclass:: analytics_data_api.v0.views.problems.GradeDistributionView 16 | 17 | **Example Response** 18 | 19 | .. code-block:: 20 | 21 | HTTP 200 OK 22 | Vary: Accept 23 | Content-Type: text/html; charset=utf-8 24 | Allow: GET, HEAD, OPTIONS 25 | 26 | [ 27 | { 28 | "module_id": "i4x://edX/DemoX/Demo_Course/problem/97fd93e33a18495488578e9e74fa4cae", 29 | "course_id": "edX/DemoX/Demo_Course", 30 | "grade": 1, 31 | "max_grade": 2, 32 | "count": 5, 33 | "created": "2014-09-12T114957" 34 | }, 35 | { 36 | "module_id": "i4x://edX/DemoX/Demo_Course/problem/97fd93e33a18495488578e9e74fa4cae", 37 | "course_id": "edX/DemoX/Demo_Course", 38 | "grade": 2, 39 | "max_grade": 2, 40 | "count": 256, 41 | "created": "2014-09-12T114957" 42 | } 43 | ] 44 | 45 | .. _Get the Answer Distribution for a Problem: 46 | 47 | ******************************************* 48 | Get the Answer Distribution for a Problem 49 | ******************************************* 50 | 51 | .. autoclass:: analytics_data_api.v0.views.problems.ProblemResponseAnswerDistributionView 52 | 53 | **Example Response** 54 | 55 | .. code-block:: 56 | 57 | HTTP 200 OK 58 | Vary: Accept 59 | Content-Type: text/html; charset=utf-8 60 | Allow: GET, HEAD, OPTIONS 61 | 62 | [ 63 | { 64 | "course_id": "edX/DemoX/Demo_Course", 65 | "module_id": "i4x://edX/DemoX/Demo_Course/problem/ 66 | 268b43628e6d45f79c52453a590f9829", 67 | "part_id": "i4x-edX-DemoX-Demo_Course-problem- 68 | 268b43628e6d45f79c52453a590f9829_2_1", 69 | "correct": false, 70 | "count": 9, 71 | "value_id": "choice_0", 72 | "answer_value_text": "Russia", 73 | "answer_value_numeric": null, 74 | "problem_display_name": "Multiple Choice Problem", 75 | "question_text": "Which of the following countries has the largest 76 | population?", 77 | "variant": null, 78 | "created": "2014-12-05T225026" 79 | }, 80 | { 81 | "course_id": "edX/DemoX/Demo_Course", 82 | "module_id": "i4x://edX/DemoX/Demo_Course/problem/ 83 | 268b43628e6d45f79c52453a590f9829", 84 | "part_id": "i4x-edX-DemoX-Demo_Course-problem- 85 | 268b43628e6d45f79c52453a590f9829_2_1", 86 | "correct": true, 87 | "count": 15, 88 | "value_id": "choice_1", 89 | "answer_value_text": "Indonesia", 90 | "answer_value_numeric": null, 91 | "problem_display_name": "Multiple Choice Problem", 92 | "question_text": "Which of the following countries has the largest 93 | population?", 94 | "variant": null, 95 | "created": "2014-12-05T225026" 96 | } 97 | ] 98 | 99 | .. _Get the View Count for a Subsection: 100 | 101 | ************************************* 102 | Get the View Count for a Subsection 103 | ************************************* 104 | 105 | .. autoclass:: analytics_data_api.v0.views.problems.SequentialOpenDistributionView 106 | 107 | **Example Response** 108 | 109 | .. code-block:: 110 | 111 | HTTP 200 OK 112 | Vary: Accept 113 | Content-Type: text/html; charset=utf-8 114 | Allow: GET, HEAD, OPTIONS 115 | 116 | [ 117 | { 118 | "module_id": "i4x://edX/DemoX/Demo_Course/sequential/5c6c207e16dd47208c29bd8d3e68861e", 119 | "course_id": "edX/DemoX/Demo_Course", 120 | "count": 23, 121 | "created": "2014-09-12T114838" 122 | } 123 | ] 124 | -------------------------------------------------------------------------------- /docs/api/source/read_me.rst: -------------------------------------------------------------------------------- 1 | ######## 2 | Read Me 3 | ######## 4 | 5 | The edX Data Analytics API documentation is created using RST_ 6 | files and Sphinx_. You, the user community, can help update and revise this 7 | documentation project on GitHub: 8 | 9 | https://github.com/openedx/edx-analytics-data-api/tree/master/docs/api/source 10 | 11 | To suggest a revision, fork the project, make changes in your fork, and submit 12 | a pull request back to the original project: this is known as the `GitHub Flow`_. 13 | 14 | .. _Sphinx: http://sphinx-doc.org/ 15 | .. _LaTeX: http://www.latex-project.org/ 16 | .. _`GitHub Flow`: https://github.com/blog/1557-github-flow-in-the-browser 17 | .. _RST: http://docutils.sourceforge.net/rst.html -------------------------------------------------------------------------------- /docs/api/source/setup.rst: -------------------------------------------------------------------------------- 1 | .. _Set up the Data Analytics API Server: 2 | 3 | ###################################### 4 | Set up the Data Analytics API Server 5 | ###################################### 6 | 7 | This chapter describes how to set up and test the edX Data Analytics API 8 | server: 9 | 10 | #. `Get the Repository`_ 11 | #. `Install Server Requirements`_ 12 | #. `Run the Server`_ 13 | #. `Load Sample Data`_ 14 | #. `Test the Data Analytics API`_ 15 | 16 | Also see :ref:`edX Data Analytics API Authentication`. 17 | 18 | ************************** 19 | Get the Repository 20 | ************************** 21 | 22 | You must get the `Data Analytics API repository`_ from GitHub. 23 | 24 | From the terminal, enter: 25 | 26 | ``git clone https://github.com/openedx/edx-analytics-data-api`` 27 | 28 | You may choose to get the repository in a virtual environment. 29 | 30 | 31 | **************************** 32 | Install Server Requirements 33 | **************************** 34 | 35 | From the terminal at the top level of the server repository, enter: 36 | 37 | ``$ make develop`` 38 | 39 | Server requirements are then installed. 40 | 41 | **************************** 42 | Run the Server 43 | **************************** 44 | 45 | From the terminal at the top level of the server repository, enter: 46 | 47 | ``$ ./manage.py runserver`` 48 | 49 | The server starts. 50 | 51 | **************************** 52 | Load Sample Data 53 | **************************** 54 | 55 | From the terminal at the top level of the server repository, enter: 56 | 57 | ``$ make loaddata`` 58 | 59 | **************************** 60 | Test the Data Analytics API 61 | **************************** 62 | 63 | After you load sample data and run the server, you can test the API. 64 | 65 | #. In a browser, go to: ``http://:/docs/#!/api/`` 66 | 67 | #. Enter a valid key and click **Explore**. 68 | 69 | See :ref:`edX Data Analytics API Authentication` for information on keys. 70 | 71 | You see an interactive list of API endpoints, which you can use to get 72 | responses with the sample data. Expand the **api** section to see the 73 | available endpoints. 74 | 75 | .. image:: images/api_test.png 76 | :alt: The API test web page 77 | 78 | 3. Expand the section for an endpoint: 79 | 80 | .. image:: images/api_test_expand.png 81 | :alt: The API test web page with an expanded endpoint 82 | 83 | 4. Enter parameters as needed and click **Try it out**. The response opens: 84 | 85 | .. image:: images/api_test_response.png 86 | :alt: The API test web page with a response 87 | 88 | To get the sample enrollment data, use ``edX/DemoX/Demo_Course`` as the 89 | ``course_id``. 90 | 91 | .. include:: links.rst -------------------------------------------------------------------------------- /docs/api/source/videos.rst: -------------------------------------------------------------------------------- 1 | .. _Video Data API: 2 | 3 | ############### 4 | Video Data API 5 | ############### 6 | 7 | .. contents:: Section Contents 8 | :local: 9 | :depth: 1 10 | 11 | .. _Get the Timeline for a Video: 12 | 13 | *************************************** 14 | Get the Timeline for a Video 15 | *************************************** 16 | 17 | .. autoclass:: analytics_data_api.v0.views.videos.VideoTimelineView 18 | 19 | **Example Response** 20 | 21 | .. code-block:: 22 | 23 | HTTP 200 OK 24 | Vary: Accept 25 | Content-Type: text/html; charset=utf-8 26 | Allow: GET, HEAD, OPTIONS 27 | 28 | [ 29 | { 30 | "segment": 0, 31 | "num_users": 472, 32 | "num_views": 539, 33 | "created": "2015-05-13T050419" 34 | }, 35 | { 36 | "segment": 1, 37 | "num_users": 450, 38 | "num_views": 510, 39 | "created": "2015-05-13T050419" 40 | }, 41 | { 42 | "segment": 2, 43 | "num_users": 438, 44 | "num_views": 493, 45 | "created": "2015-05-13T050419" 46 | } 47 | ] 48 | -------------------------------------------------------------------------------- /docs/decisions/01-pipeline-choice.rst: -------------------------------------------------------------------------------- 1 | 1. Pipeline Choice 2 | ------------------ 3 | 4 | Status 5 | ------ 6 | 7 | Accepted 8 | 9 | Context 10 | ------- 11 | 12 | While migrating the analytics pipeline to a new technology, we'd like a toggle which: 13 | 14 | - Lets particular users in edX and maybe selected partner organizations see new data 15 | for validation. 16 | - Is fast for us to flip back and forth to look at new vs. old, ideally without leaving 17 | the client dashboard. 18 | 19 | We also need to make a decision on how to store the new data. In any case, this should 20 | involve a duplicate store to compare the old and new data. 21 | 22 | Store Options 23 | ------------- 24 | 25 | **New tables in original store** 26 | 27 | - Least amount of configuration needed. 28 | - Duplicate tables in the same store may get messy. (IE CourseActivity vs V1CourseActivity?) 29 | 30 | **Separate store in the same instance** 31 | 32 | - Allows for complete duplication with the same table names. 33 | - Cleaner separation of data. 34 | 35 | API Options 36 | ----------- 37 | 38 | **New V1 API** 39 | 40 | - Cleaner separation, but there will be duplication of code. 41 | - Easier for us to maintain the original API for Edge and Open edX. (and deprecate later, 42 | if needed) 43 | - Simple to switch between; we can implement a check on the client dashboard to determine 44 | which base URL to use. 45 | 46 | **Alter the existing API** 47 | 48 | - Less code duplication. 49 | - The check for which data to use will have to be passed down from the client to the API. 50 | 51 | Decision 52 | -------- 53 | 54 | To house the new data, we will create a separate database in the same MySQL instance. We 55 | will access this database using a new V1 API, and implement a simple toggle on the client 56 | dashboard to switch between the two APIs. 57 | 58 | This toggle can be controlled by a query string (IE `?new_data=true`), and converted to 59 | a Django setting once the new API is fully rolled out. 60 | 61 | Consequences 62 | ------------ 63 | 64 | - We will be maintaining two databases and two APIs until the old API is retired (if at all). 65 | - We will rely on the client (in our case, 66 | `edx-analytics-dashboard `_) in order to 67 | determine which API to use. 68 | 69 | References 70 | ---------- 71 | 72 | - `Insights Replumbing `_ 73 | -------------------------------------------------------------------------------- /docs/decisions/02-no-pii.rst: -------------------------------------------------------------------------------- 1 | 2. Only Aggregated Learner Data In Analytics 2 | -------------------------------------------- 3 | 4 | Status 5 | ------ 6 | 7 | Accepted 8 | 9 | Context 10 | ------- 11 | 12 | Most endpoints in the analytics data API provide aggregate user data. The exceptions are the learner and engagement timeline calls which provide individual learner data. That data goes on to populate the Insights learner view. That learner view includes username, email address, and individual performance information. 13 | 14 | Peronally identifiable information (PII) belonging to learners requires more careful treatment than aggregated learner data. Individual performance is also more sensitive than averages or aggregates. These pieces of data are more sensitive not only in the UI but during every stage of the analytics pipeline. 15 | 16 | Since the learner view is the most expensive to support and one of the least used we are deprecating it. That creates an opportunity to better define what analytics data is allowed. 17 | 18 | Decision 19 | -------- 20 | 21 | The analytics API will not contain any personally identifiable learner information such as names or emails. Further, the analytics API will only provide aggregate or average information across a class of learners. 22 | 23 | Consequences 24 | ------------ 25 | 26 | Learner view and API deprecation is already underway. In addition we will deprecate the user engagement view which uses the same analytics database tables. 27 | 28 | This decision constrains future analytics work to only aggregate data. If we want to expose individual learner data to instructors, we will use a different channel with access control appropriate to more sensitive information. 29 | 30 | 31 | References 32 | ---------- 33 | 34 | - `Learner view removal discussion `_ 35 | - `Learner view DEPR ticket `_ 36 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "analyticsdataserver.settings.local") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /openedx.yaml: -------------------------------------------------------------------------------- 1 | # This file describes this Open edX repo, as described in OEP-2: 2 | # http://open-edx-proposals.readthedocs.io/en/latest/oeps/oep-0002.html#specification 3 | 4 | nick: aap 5 | oeps: 6 | oep-2: true 7 | oep-7: true 8 | oep-18: true 9 | openedx-release: {ref: master} 10 | tags: [analytics] 11 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = --ds=analyticsdataserver.settings.test --cov analytics_data_api --cov-report term-missing --cov-config=.coveragerc --no-cov-on-fail -p no:randomly 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # This file is here because many Platforms as a Service look for 2 | # requirements.txt in the root directory of a project. 3 | -r requirements/production.txt 4 | -------------------------------------------------------------------------------- /requirements/base.in: -------------------------------------------------------------------------------- 1 | # Core dependencies go here 2 | 3 | -c constraints.txt 4 | 5 | edx-api-doc-tools 6 | boto # MIT 7 | boto3 8 | coreapi 9 | Django # BSD License 10 | django-countries # MIT 11 | python-memcached # Python Software Foundation License v2 12 | pymemcache 13 | djangorestframework # BSD 14 | djangorestframework-csv # BSD 15 | django-storages # BSD 16 | html5lib # MIT 17 | ordered-set # MIT 18 | tqdm # MIT 19 | urllib3 # MIT 20 | Markdown # BSD:markdown is used by swagger for rendering the api docs 21 | edx-ccx-keys 22 | edx-django-release-util 23 | edx-opaque-keys 24 | edx-django-utils 25 | edx-rest-api-client # Apache 2.0 26 | edx-drf-extensions 27 | edx-enterprise-data 28 | django-cors-headers # Used to allow to configure CORS headers for cross-domain requests 29 | -------------------------------------------------------------------------------- /requirements/base.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.8 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | asgiref==3.8.1 8 | # via 9 | # django 10 | # django-cors-headers 11 | # django-countries 12 | backports-zoneinfo==0.2.1 13 | # via 14 | # django 15 | # djangorestframework 16 | boto==2.49.0 17 | # via -r requirements/base.in 18 | boto3==1.34.89 19 | # via -r requirements/base.in 20 | botocore==1.34.89 21 | # via 22 | # boto3 23 | # s3transfer 24 | certifi==2024.2.2 25 | # via requests 26 | cffi==1.16.0 27 | # via 28 | # cryptography 29 | # pynacl 30 | charset-normalizer==3.3.2 31 | # via requests 32 | click==8.1.7 33 | # via edx-django-utils 34 | coreapi==2.3.3 35 | # via -r requirements/base.in 36 | coreschema==0.0.4 37 | # via coreapi 38 | cryptography==42.0.5 39 | # via 40 | # django-fernet-fields-v2 41 | # pyjwt 42 | django==4.2.11 43 | # via 44 | # -c requirements/constraints.txt 45 | # -r requirements/base.in 46 | # django-cors-headers 47 | # django-crum 48 | # django-fernet-fields-v2 49 | # django-filter 50 | # django-model-utils 51 | # django-storages 52 | # django-waffle 53 | # djangorestframework 54 | # drf-jwt 55 | # drf-yasg 56 | # edx-api-doc-tools 57 | # edx-django-release-util 58 | # edx-django-utils 59 | # edx-drf-extensions 60 | # edx-enterprise-data 61 | # edx-rbac 62 | django-cors-headers==4.3.1 63 | # via -r requirements/base.in 64 | django-countries==7.6.1 65 | # via -r requirements/base.in 66 | django-crum==0.7.9 67 | # via 68 | # edx-django-utils 69 | # edx-rbac 70 | django-fernet-fields-v2==0.9 71 | # via edx-enterprise-data 72 | django-filter==24.2 73 | # via edx-enterprise-data 74 | django-model-utils==4.5.0 75 | # via 76 | # edx-enterprise-data 77 | # edx-rbac 78 | django-storages==1.10.1 79 | # via 80 | # -c requirements/constraints.txt 81 | # -r requirements/base.in 82 | django-waffle==4.1.0 83 | # via 84 | # edx-django-utils 85 | # edx-drf-extensions 86 | djangorestframework==3.15.1 87 | # via 88 | # -r requirements/base.in 89 | # djangorestframework-csv 90 | # drf-jwt 91 | # drf-yasg 92 | # edx-api-doc-tools 93 | # edx-drf-extensions 94 | djangorestframework-csv==3.0.2 95 | # via 96 | # -r requirements/base.in 97 | # edx-enterprise-data 98 | drf-jwt==1.19.2 99 | # via edx-drf-extensions 100 | drf-yasg==1.21.7 101 | # via edx-api-doc-tools 102 | edx-api-doc-tools==1.8.0 103 | # via -r requirements/base.in 104 | edx-ccx-keys==1.3.0 105 | # via -r requirements/base.in 106 | edx-django-release-util==1.4.0 107 | # via -r requirements/base.in 108 | edx-django-utils==5.12.0 109 | # via 110 | # -r requirements/base.in 111 | # edx-drf-extensions 112 | # edx-enterprise-data 113 | # edx-rest-api-client 114 | edx-drf-extensions==10.3.0 115 | # via 116 | # -r requirements/base.in 117 | # edx-enterprise-data 118 | # edx-rbac 119 | edx-enterprise-data==6.2.0 120 | # via -r requirements/base.in 121 | edx-opaque-keys==2.8.0 122 | # via 123 | # -r requirements/base.in 124 | # edx-ccx-keys 125 | # edx-drf-extensions 126 | # edx-enterprise-data 127 | edx-rbac==1.9.0 128 | # via edx-enterprise-data 129 | edx-rest-api-client==5.7.0 130 | # via 131 | # -r requirements/base.in 132 | # edx-enterprise-data 133 | factory-boy==3.3.0 134 | # via edx-enterprise-data 135 | faker==24.11.0 136 | # via factory-boy 137 | html5lib==1.1 138 | # via -r requirements/base.in 139 | idna==3.7 140 | # via requests 141 | importlib-metadata==7.1.0 142 | # via markdown 143 | inflection==0.5.1 144 | # via drf-yasg 145 | itypes==1.2.0 146 | # via coreapi 147 | jinja2==3.1.3 148 | # via coreschema 149 | jmespath==1.0.1 150 | # via 151 | # boto3 152 | # botocore 153 | markdown==3.6 154 | # via -r requirements/base.in 155 | markupsafe==2.1.5 156 | # via jinja2 157 | newrelic==9.9.0 158 | # via edx-django-utils 159 | ordered-set==4.1.0 160 | # via -r requirements/base.in 161 | packaging==24.0 162 | # via drf-yasg 163 | pbr==6.0.0 164 | # via stevedore 165 | psutil==5.9.8 166 | # via edx-django-utils 167 | pycparser==2.22 168 | # via cffi 169 | pyjwt[crypto]==2.8.0 170 | # via 171 | # drf-jwt 172 | # edx-drf-extensions 173 | # edx-rest-api-client 174 | pymemcache==4.0.0 175 | # via -r requirements/base.in 176 | pymongo==3.13.0 177 | # via edx-opaque-keys 178 | pynacl==1.5.0 179 | # via edx-django-utils 180 | python-dateutil==2.9.0.post0 181 | # via 182 | # botocore 183 | # faker 184 | python-memcached==1.62 185 | # via -r requirements/base.in 186 | pytz==2024.1 187 | # via drf-yasg 188 | pyyaml==6.0.1 189 | # via 190 | # drf-yasg 191 | # edx-django-release-util 192 | requests==2.31.0 193 | # via 194 | # coreapi 195 | # edx-drf-extensions 196 | # edx-enterprise-data 197 | # edx-rest-api-client 198 | # slumber 199 | rules==3.3 200 | # via edx-enterprise-data 201 | s3transfer==0.10.1 202 | # via boto3 203 | semantic-version==2.10.0 204 | # via edx-drf-extensions 205 | six==1.16.0 206 | # via 207 | # edx-ccx-keys 208 | # edx-django-release-util 209 | # edx-rbac 210 | # html5lib 211 | # python-dateutil 212 | slumber==0.7.1 213 | # via edx-rest-api-client 214 | sqlparse==0.5.0 215 | # via django 216 | stevedore==5.2.0 217 | # via 218 | # edx-django-utils 219 | # edx-opaque-keys 220 | tqdm==4.66.2 221 | # via -r requirements/base.in 222 | typing-extensions==4.11.0 223 | # via 224 | # asgiref 225 | # django-countries 226 | # edx-opaque-keys 227 | # faker 228 | uritemplate==4.1.1 229 | # via 230 | # coreapi 231 | # drf-yasg 232 | urllib3==1.26.18 233 | # via 234 | # -r requirements/base.in 235 | # botocore 236 | # requests 237 | webencodings==0.5.1 238 | # via html5lib 239 | zipp==3.18.1 240 | # via importlib-metadata 241 | 242 | # The following packages are considered to be unsafe in a requirements file: 243 | # setuptools 244 | -------------------------------------------------------------------------------- /requirements/ci.in: -------------------------------------------------------------------------------- 1 | # Basic requirements for Travis CI 2 | 3 | -c constraints.txt 4 | 5 | -------------------------------------------------------------------------------- /requirements/ci.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.8 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | -------------------------------------------------------------------------------- /requirements/constraints.txt: -------------------------------------------------------------------------------- 1 | # Version constraints for pip-installation. 2 | # 3 | # This file doesn't install any packages. It specifies version constraints 4 | # that will be applied if a package is needed. 5 | # 6 | # When pinning something here, please provide an explanation of why. Ideally, 7 | # link to other information that will help people in the future to remove the 8 | # pin when possible. Writing an issue against the offending project and 9 | # linking to it here is good. 10 | 11 | # TODO: Many pinned dependencies should be unpinned and/or moved to this constraints file. 12 | django<4.3 13 | 14 | # django-storages version 1.9 drops support for boto storage backend. 15 | # pinning before django upgrade, will unpin after that 16 | django-storages<1.11 # BSD 17 | 18 | # elasticsearch library cannot be greater than 7.14.0. 7.14.0 library is not compatible with the 7.10 server in stage and prod 19 | elasticsearch < 7.14.0 20 | 21 | # elasticsearch-dsl depends on elasticsearch >=7.8.0,<8.0.0 22 | elasticsearch-dsl>=7.2.1,<8.0.0 23 | 24 | # Use same version of edx-lint 25 | pylint==2.4.4 26 | pylint-django==2.0.11 27 | -------------------------------------------------------------------------------- /requirements/dev.in: -------------------------------------------------------------------------------- 1 | # Local development dependencies go here 2 | 3 | -r base.in # Core dependencies of edx-analytics-data-api 4 | mysqlclient 5 | -------------------------------------------------------------------------------- /requirements/dev.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.8 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | asgiref==3.8.1 8 | # via 9 | # django 10 | # django-cors-headers 11 | # django-countries 12 | backports-zoneinfo==0.2.1 13 | # via 14 | # django 15 | # djangorestframework 16 | boto==2.49.0 17 | # via -r requirements/base.in 18 | boto3==1.34.89 19 | # via -r requirements/base.in 20 | botocore==1.34.89 21 | # via 22 | # boto3 23 | # s3transfer 24 | certifi==2024.2.2 25 | # via requests 26 | cffi==1.16.0 27 | # via 28 | # cryptography 29 | # pynacl 30 | charset-normalizer==3.3.2 31 | # via requests 32 | click==8.1.7 33 | # via edx-django-utils 34 | coreapi==2.3.3 35 | # via -r requirements/base.in 36 | coreschema==0.0.4 37 | # via coreapi 38 | cryptography==42.0.5 39 | # via 40 | # django-fernet-fields-v2 41 | # pyjwt 42 | django==4.2.11 43 | # via 44 | # -c requirements/constraints.txt 45 | # -r requirements/base.in 46 | # django-cors-headers 47 | # django-crum 48 | # django-fernet-fields-v2 49 | # django-filter 50 | # django-model-utils 51 | # django-storages 52 | # django-waffle 53 | # djangorestframework 54 | # drf-jwt 55 | # drf-yasg 56 | # edx-api-doc-tools 57 | # edx-django-release-util 58 | # edx-django-utils 59 | # edx-drf-extensions 60 | # edx-enterprise-data 61 | # edx-rbac 62 | django-cors-headers==4.3.1 63 | # via -r requirements/base.in 64 | django-countries==7.6.1 65 | # via -r requirements/base.in 66 | django-crum==0.7.9 67 | # via 68 | # edx-django-utils 69 | # edx-rbac 70 | django-fernet-fields-v2==0.9 71 | # via edx-enterprise-data 72 | django-filter==24.2 73 | # via edx-enterprise-data 74 | django-model-utils==4.5.0 75 | # via 76 | # edx-enterprise-data 77 | # edx-rbac 78 | django-storages==1.10.1 79 | # via 80 | # -c requirements/constraints.txt 81 | # -r requirements/base.in 82 | django-waffle==4.1.0 83 | # via 84 | # edx-django-utils 85 | # edx-drf-extensions 86 | djangorestframework==3.15.1 87 | # via 88 | # -r requirements/base.in 89 | # djangorestframework-csv 90 | # drf-jwt 91 | # drf-yasg 92 | # edx-api-doc-tools 93 | # edx-drf-extensions 94 | djangorestframework-csv==3.0.2 95 | # via 96 | # -r requirements/base.in 97 | # edx-enterprise-data 98 | drf-jwt==1.19.2 99 | # via edx-drf-extensions 100 | drf-yasg==1.21.7 101 | # via edx-api-doc-tools 102 | edx-api-doc-tools==1.8.0 103 | # via -r requirements/base.in 104 | edx-ccx-keys==1.3.0 105 | # via -r requirements/base.in 106 | edx-django-release-util==1.4.0 107 | # via -r requirements/base.in 108 | edx-django-utils==5.12.0 109 | # via 110 | # -r requirements/base.in 111 | # edx-drf-extensions 112 | # edx-enterprise-data 113 | # edx-rest-api-client 114 | edx-drf-extensions==10.3.0 115 | # via 116 | # -r requirements/base.in 117 | # edx-enterprise-data 118 | # edx-rbac 119 | edx-enterprise-data==6.2.0 120 | # via -r requirements/base.in 121 | edx-opaque-keys==2.8.0 122 | # via 123 | # -r requirements/base.in 124 | # edx-ccx-keys 125 | # edx-drf-extensions 126 | # edx-enterprise-data 127 | edx-rbac==1.9.0 128 | # via edx-enterprise-data 129 | edx-rest-api-client==5.7.0 130 | # via 131 | # -r requirements/base.in 132 | # edx-enterprise-data 133 | factory-boy==3.3.0 134 | # via edx-enterprise-data 135 | faker==24.11.0 136 | # via factory-boy 137 | html5lib==1.1 138 | # via -r requirements/base.in 139 | idna==3.7 140 | # via requests 141 | importlib-metadata==7.1.0 142 | # via markdown 143 | inflection==0.5.1 144 | # via drf-yasg 145 | itypes==1.2.0 146 | # via coreapi 147 | jinja2==3.1.3 148 | # via coreschema 149 | jmespath==1.0.1 150 | # via 151 | # boto3 152 | # botocore 153 | markdown==3.6 154 | # via -r requirements/base.in 155 | markupsafe==2.1.5 156 | # via jinja2 157 | mysqlclient==2.2.4 158 | # via -r requirements/dev.in 159 | newrelic==9.9.0 160 | # via edx-django-utils 161 | ordered-set==4.1.0 162 | # via -r requirements/base.in 163 | packaging==24.0 164 | # via drf-yasg 165 | pbr==6.0.0 166 | # via stevedore 167 | psutil==5.9.8 168 | # via edx-django-utils 169 | pycparser==2.22 170 | # via cffi 171 | pyjwt[crypto]==2.8.0 172 | # via 173 | # drf-jwt 174 | # edx-drf-extensions 175 | # edx-rest-api-client 176 | pymemcache==4.0.0 177 | # via -r requirements/base.in 178 | pymongo==3.13.0 179 | # via edx-opaque-keys 180 | pynacl==1.5.0 181 | # via edx-django-utils 182 | python-dateutil==2.9.0.post0 183 | # via 184 | # botocore 185 | # faker 186 | python-memcached==1.62 187 | # via -r requirements/base.in 188 | pytz==2024.1 189 | # via drf-yasg 190 | pyyaml==6.0.1 191 | # via 192 | # drf-yasg 193 | # edx-django-release-util 194 | requests==2.31.0 195 | # via 196 | # coreapi 197 | # edx-drf-extensions 198 | # edx-enterprise-data 199 | # edx-rest-api-client 200 | # slumber 201 | rules==3.3 202 | # via edx-enterprise-data 203 | s3transfer==0.10.1 204 | # via boto3 205 | semantic-version==2.10.0 206 | # via edx-drf-extensions 207 | six==1.16.0 208 | # via 209 | # edx-ccx-keys 210 | # edx-django-release-util 211 | # edx-rbac 212 | # html5lib 213 | # python-dateutil 214 | slumber==0.7.1 215 | # via edx-rest-api-client 216 | sqlparse==0.5.0 217 | # via django 218 | stevedore==5.2.0 219 | # via 220 | # edx-django-utils 221 | # edx-opaque-keys 222 | tqdm==4.66.2 223 | # via -r requirements/base.in 224 | typing-extensions==4.11.0 225 | # via 226 | # asgiref 227 | # django-countries 228 | # edx-opaque-keys 229 | # faker 230 | uritemplate==4.1.1 231 | # via 232 | # coreapi 233 | # drf-yasg 234 | urllib3==1.26.18 235 | # via 236 | # -r requirements/base.in 237 | # botocore 238 | # requests 239 | webencodings==0.5.1 240 | # via html5lib 241 | zipp==3.18.1 242 | # via importlib-metadata 243 | 244 | # The following packages are considered to be unsafe in a requirements file: 245 | # setuptools 246 | -------------------------------------------------------------------------------- /requirements/django.txt: -------------------------------------------------------------------------------- 1 | django==4.2.11 2 | -------------------------------------------------------------------------------- /requirements/doc.in: -------------------------------------------------------------------------------- 1 | # Dependencies for building documentation 2 | 3 | -r base.in # Core dependencies of edx-analytics-data-api 4 | 5 | Sphinx # Developer documentation builder 6 | path 7 | sphinx-book-theme -------------------------------------------------------------------------------- /requirements/pip.in: -------------------------------------------------------------------------------- 1 | # Core dependencies for installing other packages 2 | 3 | pip 4 | setuptools 5 | wheel 6 | -------------------------------------------------------------------------------- /requirements/pip.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.8 3 | # by the following command: 4 | # 5 | # pip-compile --allow-unsafe --output-file=requirements/pip.txt requirements/pip.in 6 | # 7 | wheel==0.43.0 8 | # via -r requirements/pip.in 9 | 10 | # The following packages are considered to be unsafe in a requirements file: 11 | pip==24.0 12 | # via -r requirements/pip.in 13 | setuptools==69.5.1 14 | # via -r requirements/pip.in 15 | -------------------------------------------------------------------------------- /requirements/pip_tools.in: -------------------------------------------------------------------------------- 1 | # Dependencies to run compile tools 2 | pip-tools # Contains pip-compile, used to generate pip requirements files 3 | six 4 | -------------------------------------------------------------------------------- /requirements/pip_tools.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.8 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | build==1.2.1 8 | # via pip-tools 9 | click==8.1.7 10 | # via pip-tools 11 | importlib-metadata==7.1.0 12 | # via build 13 | packaging==24.0 14 | # via build 15 | pip-tools==7.4.1 16 | # via -r requirements/pip_tools.in 17 | pyproject-hooks==1.0.0 18 | # via 19 | # build 20 | # pip-tools 21 | six==1.16.0 22 | # via -r requirements/pip_tools.in 23 | tomli==2.0.1 24 | # via 25 | # build 26 | # pip-tools 27 | # pyproject-hooks 28 | wheel==0.43.0 29 | # via pip-tools 30 | zipp==3.18.1 31 | # via importlib-metadata 32 | 33 | # The following packages are considered to be unsafe in a requirements file: 34 | # pip 35 | # setuptools 36 | -------------------------------------------------------------------------------- /requirements/production.in: -------------------------------------------------------------------------------- 1 | # Pro-tip: Try not to put anything here. There should be no dependency in 2 | # production that isn't in development. 3 | 4 | -r base.in # Core dependencies of edx-analytics-data-api 5 | mysqlclient # GPL License 6 | PyYAML # MIT 7 | gevent 8 | gunicorn # MIT 9 | path.py==8.2.1 10 | newrelic # New Relic agent for performance monitoring 11 | -------------------------------------------------------------------------------- /requirements/test.in: -------------------------------------------------------------------------------- 1 | # Test dependencies go here. 2 | 3 | -r base.in # Core dependencies of edx-analytics-data-api 4 | 5 | coverage 6 | ddt 7 | diff-cover 8 | django-dynamic-fixture 9 | freezegun 10 | pycodestyle 11 | pydocstyle 12 | pylint 13 | pytest-cov 14 | pytest-django 15 | pytz 16 | responses 17 | -------------------------------------------------------------------------------- /requirements/tox.in: -------------------------------------------------------------------------------- 1 | # Used for tests 2 | -c constraints.txt 3 | 4 | tox 5 | -------------------------------------------------------------------------------- /requirements/tox.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.8 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | cachetools==5.3.3 8 | # via tox 9 | chardet==5.2.0 10 | # via tox 11 | colorama==0.4.6 12 | # via tox 13 | distlib==0.3.8 14 | # via virtualenv 15 | filelock==3.13.4 16 | # via 17 | # tox 18 | # virtualenv 19 | packaging==24.0 20 | # via 21 | # pyproject-api 22 | # tox 23 | platformdirs==4.2.0 24 | # via 25 | # tox 26 | # virtualenv 27 | pluggy==1.5.0 28 | # via tox 29 | pyproject-api==1.6.1 30 | # via tox 31 | tomli==2.0.1 32 | # via 33 | # pyproject-api 34 | # tox 35 | tox==4.14.2 36 | # via -r requirements/tox.in 37 | virtualenv==20.25.3 38 | # via tox 39 | -------------------------------------------------------------------------------- /scripts/post-pip-compile.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # Replace the instructions for regenerating requirements/*.txt files 5 | 6 | function show_help { 7 | echo "Usage: post-pip-compile.sh file ..." 8 | echo "Replace the instructions for regenerating the given requirements file(s)." 9 | } 10 | 11 | function clean_file { 12 | FILE_PATH=$1 13 | TEMP_FILE=${FILE_PATH}.tmp 14 | sed "s/pip-compile --output-file.*/make upgrade/" ${FILE_PATH} > ${TEMP_FILE} 15 | mv ${TEMP_FILE} ${FILE_PATH} 16 | } 17 | 18 | for i in "$@"; do 19 | case ${i} in 20 | -h|--help) 21 | # help or unknown option 22 | show_help 23 | exit 0 24 | ;; 25 | *) 26 | clean_file ${i} 27 | ;; 28 | esac 29 | done 30 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [pycodestyle] 2 | ignore=W504,E501,E741 3 | max_line_length=119 4 | exclude=settings,analyticsdataserver/wsgi.py 5 | 6 | [isort] 7 | include_trailing_comma = True 8 | indent = ' ' 9 | line_length = 120 10 | multi_line_output = 3 11 | -------------------------------------------------------------------------------- /static/css/edx-swagger.css: -------------------------------------------------------------------------------- 1 | #header form#api_selector .input a#explore { 2 | color: white; 3 | background-color: #126F9E; 4 | box-shadow: 0 2px 1px 0 #0a4a67; 5 | } 6 | 7 | #header form#api_selector .input a#explore:hover { 8 | background-color: #1790c7; 9 | } 10 | 11 | #header { 12 | border-bottom: 1px solid #8a8c8f; 13 | box-shadow: 0 1px 5px 0 rgba(0, 0, 0, 0.1); 14 | background: #fff; 15 | } 16 | 17 | #header a#logo { 18 | background: transparent url(../images/header-logo.png) no-repeat left center; 19 | padding-left: 100px; 20 | } 21 | 22 | body { 23 | background: #f5f5f5; 24 | } 25 | 26 | #swagger-ui-container, #message-bar { 27 | background: white; 28 | } 29 | 30 | #swagger-ui-container { 31 | padding-bottom: 20px; 32 | } 33 | -------------------------------------------------------------------------------- /static/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/static/images/favicon.ico -------------------------------------------------------------------------------- /static/images/header-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx-unsupported/edx-analytics-data-api/7e2aa56c18911010acd609017d22853f698a9c04/static/images/header-logo.png -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skipsdist = True 3 | envlist = py38-django{42} 4 | 5 | [testenv] 6 | passenv = 7 | ELASTICSEARCH_LEARNERS_HOST 8 | COVERAGE_DIR 9 | setenv = 10 | tests: DJANGO_SETTINGS_MODULE = analyticsdataserver.settings.test 11 | NODE_BIN = ./node_modules/.bin 12 | PATH = $PATH:$NODE_BIN 13 | deps = 14 | django42: Django>=4.2,<4.3 15 | -r requirements/test.txt 16 | commands = 17 | {posargs:pytest} 18 | 19 | [testenv:docs] 20 | deps = 21 | -r{toxinidir}/requirements/doc.txt 22 | allowlist_externals = 23 | make 24 | env 25 | setenv = 26 | SPHINXOPTS = -W 27 | commands = 28 | make -e -C docs/api clean 29 | make -e -C docs/api html 30 | --------------------------------------------------------------------------------