├── .dockerignore ├── .github └── dependabot.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .taskcluster.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── backend ├── Dockerfile ├── README.md ├── VERSION ├── build.sh ├── ci │ ├── pg_hba.conf │ └── setup_postgres.sh ├── code_review_backend │ ├── __init__.py │ ├── app │ │ ├── __init__.py │ │ ├── settings.py │ │ ├── urls.py │ │ └── wsgi.py │ ├── issues │ │ ├── __init__.py │ │ ├── admin.py │ │ ├── api.py │ │ ├── apps.py │ │ ├── compare.py │ │ ├── management │ │ │ ├── __init__.py │ │ │ └── commands │ │ │ │ ├── __init__.py │ │ │ │ ├── cleanup_issues.py │ │ │ │ ├── load_in_patch.py │ │ │ │ └── load_issues.py │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ ├── 0002_compare_issues.py │ │ │ ├── 0003_diff_repository.py │ │ │ ├── 0004_issue_in_patch.py │ │ │ ├── 0005_rename_check_issue_analyzer_check.py │ │ │ ├── 0006_issuelink_initial.py │ │ │ ├── 0007_issuelink_swap.py │ │ │ ├── 0008_revision_base_head_references.py │ │ │ ├── 0009_revision_optional_phab_references.py │ │ │ ├── 0010_alter_revision_id.py │ │ │ ├── 0011_indexes_revision_issue_path.py │ │ │ ├── 0012_move_issues_attributes_part_1.py │ │ │ ├── 0013_move_issues_attributes_part_2.py │ │ │ ├── 0014_unique_hash.py │ │ │ ├── 0015_remove_repository_phid_alter_repository_id.py │ │ │ └── __init__.py │ │ ├── models.py │ │ ├── serializers.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ ├── commands │ │ │ ├── __init__.py │ │ │ └── test_cleanup_issues.py │ │ │ ├── test_api.py │ │ │ ├── test_compare.py │ │ │ ├── test_diff.py │ │ │ ├── test_dockerflow.py │ │ │ ├── test_issue.py │ │ │ ├── test_repository.py │ │ │ ├── test_revision.py │ │ │ └── test_stats.py │ └── version.json ├── fixtures │ └── repositories.json ├── manage.py ├── requirements-dev.txt ├── requirements.txt └── setup.py ├── bot ├── .dockerignore ├── README.md ├── VERSION ├── code_review_bot │ ├── __init__.py │ ├── analysis.py │ ├── backend.py │ ├── cli.py │ ├── config.py │ ├── mercurial.py │ ├── report │ │ ├── __init__.py │ │ ├── base.py │ │ ├── debug.py │ │ ├── lando.py │ │ ├── mail.py │ │ ├── mail_builderrors.py │ │ └── phabricator.py │ ├── retrigger.py │ ├── revisions.py │ ├── stats.py │ ├── tasks │ │ ├── __init__.py │ │ ├── base.py │ │ ├── clang_format.py │ │ ├── clang_tidy.py │ │ ├── clang_tidy_external.py │ │ ├── coverage.py │ │ ├── default.py │ │ ├── docupload.py │ │ ├── lint.py │ │ └── tgdiff.py │ ├── tools │ │ ├── __init__.py │ │ ├── libmozdata.py │ │ ├── log.py │ │ └── treeherder.py │ └── workflow.py ├── docker │ └── Dockerfile ├── requirements-dev.txt ├── requirements.txt ├── setup.py ├── taskcluster-hook.json ├── tests │ ├── conftest.py │ ├── fixtures │ │ ├── clang_format.diff │ │ ├── infer_artifact_0.16.0.json │ │ ├── infer_artifact_0.17.0.json │ │ ├── mozlint_license_no_check.json │ │ ├── sentry_event_after.json │ │ └── sentry_event_before.json │ ├── mocks │ │ ├── config.yaml │ │ ├── hgmo_hello1 │ │ ├── hgmo_integration-autoland_deadbeef123.json │ │ ├── phabricator_auth.json │ │ ├── phabricator_build_search.json │ │ ├── phabricator_buildable_search.json │ │ ├── phabricator_createinline.json │ │ ├── phabricator_diff_query.json │ │ ├── phabricator_diff_search_PHID-DIFF-test.json │ │ ├── phabricator_diff_search_PHID-DIFF-testABcd12.json │ │ ├── phabricator_diff_search_PHID-DREV-azcDeadbeef.json │ │ ├── phabricator_diff_search_PHID-DREV-zzzzz-updated.json │ │ ├── phabricator_diff_search_PHID-DREV-zzzzz.json │ │ ├── phabricator_edge_search.json │ │ ├── phabricator_patch.diff │ │ ├── phabricator_project_search.json │ │ ├── phabricator_repository_search.json │ │ ├── phabricator_revision_search.json │ │ ├── phabricator_send_message.json │ │ ├── phabricator_target_search.json │ │ ├── phabricator_transaction_search.json │ │ └── zero_coverage_report.json │ ├── test_analysis.py │ ├── test_artifacts.py │ ├── test_autoland.py │ ├── test_backend.py │ ├── test_clang.py │ ├── test_coverage.py │ ├── test_default.py │ ├── test_docupload_task.py │ ├── test_hash.py │ ├── test_issues.py │ ├── test_lint.py │ ├── test_mercurial.py │ ├── test_patch.py │ ├── test_remote.py │ ├── test_reporter_debug.py │ ├── test_reporter_lando.py │ ├── test_reporter_mail.py │ ├── test_reporter_phabricator.py │ ├── test_revisions.py │ ├── test_stats.py │ ├── test_tgdiff_task.py │ ├── test_tools.py │ └── test_workflow.py └── tools │ ├── __init__.py │ ├── copy_diff.py │ ├── fix_missing.py │ ├── test_validator.py │ └── validator.py ├── docker-compose.yml ├── docs ├── README.md ├── analysis_format.md ├── architecture.drawio ├── architecture.md ├── architecture.png ├── ci-cd │ ├── README.md │ ├── community.mermaid │ ├── community.png │ ├── firefox-ci.mermaid │ └── firefox-ci.png ├── configuration.md ├── debugging.md ├── docker.md ├── new_repository.md ├── phabricator.md ├── phabricator.mermaid ├── phabricator.png ├── projects │ ├── backend.md │ ├── backend.png │ ├── bot.md │ ├── bot.mermaid │ ├── bot.png │ └── frontend.md ├── publication.md └── trigger.md ├── frontend ├── .eslintrc.js ├── .gitignore ├── README.md ├── babel.config.json ├── package-lock.json ├── package.json ├── src │ ├── App.vue │ ├── Bool.vue │ ├── Check.vue │ ├── Choice.vue │ ├── Diffs.vue │ ├── Issues.vue │ ├── Pagination.vue │ ├── Progress.vue │ ├── Revision.vue │ ├── RevisionDiffs.vue │ ├── Stats.vue │ ├── Tasks.vue │ ├── index.html │ ├── index.js │ ├── mixins.js │ ├── routes.js │ └── store.js ├── test │ └── simple_test.js └── webpack.config.js ├── integration ├── VERSION ├── docker │ └── Dockerfile ├── patches │ └── nss.diff ├── requirements-dev.txt ├── requirements.txt ├── run.py ├── taskcluster-hook.json └── tests │ ├── conftest.py │ ├── test_hooks.py │ └── test_workflow.py ├── pyproject.toml └── tools ├── code_review_tools ├── __init__.py ├── heroku.py ├── libmozdata.py ├── log.py └── treeherder.py ├── docker ├── bootstrap-mercurial.sh └── hgrc ├── requirements.txt └── setup.py /.dockerignore: -------------------------------------------------------------------------------- 1 | */node_modules 2 | *.sqlite* 3 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "/tools" 5 | schedule: 6 | interval: weekly 7 | open-pull-requests-limit: 99 8 | labels: 9 | - tools 10 | - package-ecosystem: pip 11 | directory: "/integration" 12 | schedule: 13 | interval: weekly 14 | open-pull-requests-limit: 99 15 | labels: 16 | - integration 17 | - package-ecosystem: docker 18 | directory: "/bot/docker" 19 | schedule: 20 | interval: weekly 21 | open-pull-requests-limit: 99 22 | labels: 23 | - bot 24 | - package-ecosystem: docker 25 | directory: "/integration/docker" 26 | schedule: 27 | interval: weekly 28 | open-pull-requests-limit: 99 29 | - package-ecosystem: docker 30 | directory: "/backend" 31 | schedule: 32 | interval: weekly 33 | open-pull-requests-limit: 99 34 | - package-ecosystem: npm 35 | directory: "/frontend" 36 | schedule: 37 | interval: weekly 38 | open-pull-requests-limit: 99 39 | labels: 40 | - frontend 41 | - package-ecosystem: pip 42 | directory: "/bot" 43 | schedule: 44 | interval: weekly 45 | open-pull-requests-limit: 99 46 | labels: 47 | - bot 48 | - package-ecosystem: pip 49 | directory: "/backend" 50 | schedule: 51 | interval: weekly 52 | open-pull-requests-limit: 99 53 | labels: 54 | - backend 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | *.pyc 3 | *.egg-info 4 | *.sqlite* 5 | backend/hgmo 6 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/astral-sh/ruff-pre-commit 3 | rev: v0.5.4 4 | hooks: 5 | # Run the linter. 6 | - id: ruff 7 | args: [--fix] 8 | # Run the formatter. 9 | - id: ruff-format 10 | - repo: https://github.com/pre-commit/mirrors-prettier 11 | rev: v2.7.1 12 | hooks: 13 | - id: prettier 14 | exclude: fixtures|mocks 15 | - repo: https://github.com/pre-commit/pre-commit-hooks 16 | rev: v4.4.0 17 | hooks: 18 | - id: check-ast 19 | - id: check-docstring-first 20 | - id: check-executables-have-shebangs 21 | - id: check-merge-conflict 22 | - id: check-symlinks 23 | - id: debug-statements 24 | - id: trailing-whitespace 25 | - id: check-yaml 26 | - id: mixed-line-ending 27 | - id: name-tests-test 28 | args: ["--django"] 29 | - id: check-json 30 | - id: requirements-txt-fixer 31 | - id: check-vcs-permalinks 32 | - repo: https://github.com/codespell-project/codespell 33 | rev: v2.2.4 34 | hooks: 35 | - id: codespell 36 | exclude_types: [json] 37 | - repo: https://github.com/marco-c/taskcluster_yml_validator 38 | rev: v0.0.11 39 | hooks: 40 | - id: taskcluster_yml 41 | - repo: https://github.com/asottile/yesqa 42 | rev: v1.4.0 43 | hooks: 44 | - id: yesqa 45 | - repo: meta 46 | hooks: 47 | - id: check-useless-excludes 48 | default_language_version: 49 | python: python3 50 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Community Participation Guidelines 2 | 3 | This repository is governed by Mozilla's code of conduct and etiquette guidelines. 4 | For more details, please read the 5 | [Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). 6 | 7 | ## How to Report 8 | 9 | For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. 10 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing 2 | 3 | Discussions happen in #static-analyzers on irc.mozilla.org. 4 | 5 | 1. [Issues marked as `good-first-bug`](https://github.com/mozilla/code-review/labels/good-first-bug) are self-contained enough that a contributor should be able to work on them. 6 | 2. Issues are considered not assigned, until there is a PR linked to them. 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mozilla Code Review 2 | 3 | The **Code Review Bot** aims to give early feedback to Mozilla developers about their patches. We automate code analyzers and publish detected issues on Phabricator as soon as possible, and for all revisions. 4 | 5 | This project has 4 parts: 6 | 7 | - `bot` is a Python script running as a Taskcluster task, triggering Try pushes from Phabricator notifications and then reporting issues found in analyzer tasks, 8 | - `backend` is a Django web API used to store issues detected by the bot, 9 | - `frontend` is an administration frontend (in Vue.js) displaying detailed information about analyses and issues, 10 | 11 | :blue_book: Documentation is available in this repository [in the docs folder](docs/README.md). A good starting point is the [architecture description](docs/architecture.md). 12 | 13 | :loudspeaker: You can contact the code review bot's developers [on Matrix](https://chat.mozilla.org/#/room/#code-review-bot:mozilla.org) or on Slack in #code-review-bot. 14 | -------------------------------------------------------------------------------- /backend/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.13.3-slim 2 | 3 | ADD tools /src/tools 4 | ADD backend /src/backend 5 | 6 | # Setup tools 7 | RUN cd /src/tools && pip install --disable-pip-version-check --no-cache-dir --quiet . 8 | 9 | WORKDIR /src/backend 10 | 11 | # Activate Django settings for in docker image 12 | ENV DEBUG=false 13 | 14 | RUN pip install --disable-pip-version-check --no-cache-dir --quiet . 15 | 16 | # Collect all static files 17 | RUN ./manage.py collectstatic --no-input 18 | 19 | CMD gunicorn code_review_backend.app.wsgi 20 | -------------------------------------------------------------------------------- /backend/README.md: -------------------------------------------------------------------------------- 1 | # Code Review Backend 2 | 3 | ## Developer setup 4 | 5 | ### Run the application 6 | 7 | You may want to install dependencies in a virtual environment an run the development test server with a base fixure for development purpose: 8 | 9 | ``` 10 | mkvirtualenv code-review-backend 11 | pip install -e ./tools -e ./backend -r ./backend/requirements-dev.txt 12 | cd backend 13 | ./manage.py migrate 14 | ./manage.py createsuperuser 15 | ./manage.py loaddata fixtures/repositories.json 16 | ./manage.py runserver 17 | ``` 18 | 19 | At this point, you can log into http://127.0.0.1:8000/admin/ with the credentials you mentioned during the `createsuperuser` step. 20 | 21 | ### Debugging tools 22 | 23 | Run `pip install -r requirements-dev.txt` to install all the available optional dev tools for the backend. 24 | 25 | [Django Debug Toolbar](https://django-debug-toolbar.readthedocs.io/en/latest/) provides you with a neat debug sidebar that will help diagnosing slow API endpoints. 26 | 27 | [Django Extensions](https://django-extensions.readthedocs.io/en/latest/) adds a _lot_ of `manage.py` commands ; the most important one is `./manage.py shell_plus` which runs the usual shell but with all the available models pre-imported. 28 | 29 | You may also want to use IPython (`pip install ipython`) to get a nicer shell with syntax highlighting, auto reloading and much more via `./manage.py shell`. 30 | 31 | ## Load existing issues 32 | 33 | To load remote issues from production (default configuration): 34 | 35 | ``` 36 | ./manage.py load_issues 37 | ``` 38 | 39 | To load already retrieved issues 40 | 41 | ``` 42 | ./manage.py load_issues --offline 43 | ``` 44 | 45 | To load from testing 46 | 47 | ``` 48 | ./manage.py load_issues --environment=testing 49 | ``` 50 | 51 | ## Use a DB dump from testing or production 52 | 53 | You can retrieve a Database dump from an Heroku instance on your computer using (process [documented on Heroku](https://devcenter.heroku.com/articles/heroku-postgres-import-export)): 54 | 55 | ``` 56 | heroku pg:backups:capture -a code-review-backend-testing 57 | heroku pg:backups:download -a code-review-backend-testing 58 | ``` 59 | 60 | This will produce a local Postgres binary dump named `latest.dump`. 61 | 62 | To use this dump, you'll need a local PostgreSQL instance running. The following Docker configuration works well for local development: 63 | 64 | - a `code_review` database is created, 65 | - with user/password credentials `postgres` / `crdev1234`, 66 | - data is stored in a Docker volume named `code_review_postgres`, 67 | - default Postgres port **5432** on the host is mapped to the container. 68 | 69 | ``` 70 | docker run --rm -p 5432:5432 \ 71 | -e POSTGRES_DB=code_review \ 72 | -e POSTGRES_PASSWORD=crdev1234 \ 73 | -v code_review_postgres:/var/lib/postgresql/data \ 74 | postgres 75 | ``` 76 | 77 | To restore the dump, use the following command (using the password used to start the database): 78 | 79 | ``` 80 | pg_restore --verbose --clean --no-acl --no-owner -h localhost -U postgres -d code_review latest.dump 81 | ``` 82 | 83 | It's also possible to use a direct one-step command from Heroku, but you need to have a compatible version of pg_dump on your system (moght be tricky in some scenarios): 84 | 85 | ``` 86 | heroku pg:pull postgresql-concave-XXX --app code-review-backend-production postgresql://postgres@localhost:5432/prod 87 | ``` 88 | 89 | The postgresql database name can be found through the CLI `pg:info` tool, or on the Heroku dashboard. More information on the [official documentation](https://devcenter.heroku.com/articles/heroku-postgresql#pg-push-and-pg-pull) 90 | 91 | Finally you can use that database with the backend as: 92 | 93 | ``` 94 | DATABASE_URL=psql://devuser:devdata@localhost/code_review_dev ./manage.py runserver 95 | ``` 96 | -------------------------------------------------------------------------------- /backend/VERSION: -------------------------------------------------------------------------------- 1 | 1.10.2 2 | -------------------------------------------------------------------------------- /backend/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | COMMIT_SHA=$1 3 | VERSION=$2 4 | SOURCE=$3 5 | CHANNEL=$4 6 | 7 | # Create a version.json per https://github.com/mozilla-services/Dockerflow/blob/master/docs/version_object.md 8 | printf '{"commit": "%s", "version": "%s", "source": "%s", "build": "%s"}\n' \ 9 | "$COMMIT_SHA" \ 10 | "$VERSION" \ 11 | "$SOURCE" \ 12 | "${TASKCLUSTER_ROOT_URL}/tasks/${TASK_ID}" > backend/code_review_backend/version.json 13 | 14 | # Run 'taskboot build' with our local copy of the Git repository where we updated the version.json with correct values. 15 | # To do so, we use '--target /path/to/existing/clone' instead of passing environment variables (GIT_REPOSITORY, GIT_REVISION) 16 | # to taskboot that would activate an automated clone. 17 | taskboot --target /code-review build --image mozilla/code-review --tag "$CHANNEL" --tag "$COMMIT_SHA" --write /backend.tar backend/Dockerfile 18 | -------------------------------------------------------------------------------- /backend/ci/pg_hba.conf: -------------------------------------------------------------------------------- 1 | # Database administrative login by Unix domain socket 2 | local all postgres peer 3 | 4 | # Only tester user can connect locally, without password 5 | host all tester 127.0.0.1/32 trust 6 | -------------------------------------------------------------------------------- /backend/ci/setup_postgres.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | export PG_VERSION=15 3 | 4 | # Silent apt 5 | export DEBIAN_FRONTEND=noninteractive 6 | 7 | # Install postgresql 8 | apt update 9 | apt install -qq -y postgresql-$PG_VERSION 10 | 11 | # Setup access rights 12 | cp $(dirname $0)/pg_hba.conf /etc/postgresql/$PG_VERSION/main/pg_hba.conf 13 | 14 | # Start postgresql 15 | pg_ctlcluster $PG_VERSION main start 16 | 17 | # Create user & database 18 | su postgres -c 'createuser --createdb tester' 19 | su postgres -c 'createdb --owner=tester code-review' 20 | -------------------------------------------------------------------------------- /backend/code_review_backend/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/backend/code_review_backend/__init__.py -------------------------------------------------------------------------------- /backend/code_review_backend/app/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | from taskcluster.helper import TaskclusterConfig 5 | 6 | taskcluster = TaskclusterConfig("https://firefox-ci-tc.services.mozilla.com") 7 | -------------------------------------------------------------------------------- /backend/code_review_backend/app/urls.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | from django.conf import settings 6 | from django.contrib import admin 7 | from django.shortcuts import redirect 8 | from django.urls import include, path 9 | from drf_yasg import openapi 10 | from drf_yasg.views import get_schema_view 11 | from rest_framework import permissions 12 | 13 | from code_review_backend.issues import api 14 | 15 | # Build Swagger schema view 16 | schema_view = get_schema_view( 17 | openapi.Info( 18 | title="Mozilla Code Review API", 19 | default_version="v1", 20 | description="Mozilla Code Review Backend API", 21 | contact=openapi.Contact(email="release-mgmt-analysis@mozilla.com"), 22 | license=openapi.License(name="MPL2"), 23 | ), 24 | public=True, 25 | permission_classes=(permissions.AllowAny,), 26 | ) 27 | 28 | urlpatterns = [ 29 | path("", lambda request: redirect("docs/", permanent=False)), 30 | path("v1/", include(api.urls)), 31 | path("admin/", admin.site.urls), 32 | path( 33 | r"docs\.json|\.yaml)", 34 | schema_view.without_ui(cache_timeout=0), 35 | name="schema-json", 36 | ), 37 | path( 38 | r"docs/", 39 | schema_view.with_ui("swagger", cache_timeout=0), 40 | name="schema-swagger-ui", 41 | ), 42 | ] 43 | 44 | if settings.DEBUG: 45 | import debug_toolbar 46 | 47 | urlpatterns += [path("__debug__/", include(debug_toolbar.urls))] 48 | -------------------------------------------------------------------------------- /backend/code_review_backend/app/wsgi.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | """ 6 | WSGI config for backend project. 7 | 8 | It exposes the WSGI callable as a module-level variable named ``application``. 9 | 10 | For more information on this file, see 11 | https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/ 12 | """ 13 | 14 | import os 15 | 16 | from django.core.wsgi import get_wsgi_application 17 | 18 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "code_review_backend.app.settings") 19 | 20 | application = get_wsgi_application() 21 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/backend/code_review_backend/issues/__init__.py -------------------------------------------------------------------------------- /backend/code_review_backend/issues/admin.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | from django.contrib import admin 6 | 7 | from code_review_backend.issues.models import Diff, Issue, Repository, Revision 8 | 9 | 10 | class RepositoryAdmin(admin.ModelAdmin): 11 | list_display = ("slug", "url") 12 | 13 | 14 | class DiffInline(admin.TabularInline): 15 | # Read only inline 16 | model = Diff 17 | readonly_fields = ("id", "repository", "mercurial_hash", "phid", "review_task_id") 18 | 19 | 20 | class RevisionAdmin(admin.ModelAdmin): 21 | list_display = ( 22 | "id", 23 | "phabricator_id", 24 | "title", 25 | "bugzilla_id", 26 | "base_repository", 27 | "head_repository", 28 | ) 29 | list_filter = ("base_repository", "head_repository") 30 | inlines = (DiffInline,) 31 | 32 | 33 | class IssueAdmin(admin.ModelAdmin): 34 | list_filter = ("analyzer",) 35 | list_display = ( 36 | "id", 37 | "path", 38 | "level", 39 | "analyzer", 40 | "analyzer_check", 41 | "created", 42 | ) 43 | search_fields = ("line", "analyzer", "path") 44 | ordering = ("-created",) 45 | 46 | 47 | admin.site.register(Repository, RepositoryAdmin) 48 | admin.site.register(Revision, RevisionAdmin) 49 | admin.site.register(Issue, IssueAdmin) 50 | 51 | # Naming 52 | admin.site.site_header = "Mozilla Code Review Backend" 53 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/apps.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | from django.apps import AppConfig 6 | 7 | 8 | class IssuesConfig(AppConfig): 9 | name = "code_review_backend.issues" 10 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/compare.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | from code_review_backend.issues.models import Diff, IssueLink 5 | 6 | 7 | def detect_new_for_revision(diff: Diff, path: str, hash: str) -> bool: 8 | """ 9 | Detect if an issue identified by its path and hash are new for a revision, from its diff 10 | This function ignores pre-existing issues outside of that revision ! 11 | """ 12 | assert diff is not None, "Missing diff" 13 | return not IssueLink.objects.filter( 14 | revision_id=diff.revision_id, 15 | diff_id__lt=diff.id, 16 | issue__path=path, 17 | issue__hash=hash, 18 | ).exists() 19 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/backend/code_review_backend/issues/management/__init__.py -------------------------------------------------------------------------------- /backend/code_review_backend/issues/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/backend/code_review_backend/issues/management/commands/__init__.py -------------------------------------------------------------------------------- /backend/code_review_backend/issues/management/commands/load_in_patch.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import logging 6 | from multiprocessing import Pool 7 | 8 | import requests 9 | from django import db 10 | from django.core.management.base import BaseCommand 11 | from parsepatch.patch import Patch 12 | 13 | from code_review_backend.app.settings import BACKEND_USER_AGENT 14 | from code_review_backend.issues.models import Diff, IssueLink 15 | 16 | logging.basicConfig(level=logging.INFO) 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | 21 | def load_hgmo_patch(diff): 22 | # Load the parent info as we have the try-task-config commit 23 | url = f"{diff.repository.url}/json-rev/{diff.mercurial_hash}" 24 | logging.info(f"Downloading {url}") 25 | resp = requests.get(url, headers={"user-agent": BACKEND_USER_AGENT}) 26 | resp.raise_for_status() 27 | meta = resp.json() 28 | if meta["desc"].startswith("try_task_config"): 29 | patch_rev = resp.json()["parents"][0] 30 | else: 31 | patch_rev = diff.mercurial_hash 32 | 33 | # Load the parent patch 34 | url = f"{diff.repository.url}/raw-rev/{patch_rev}" 35 | logging.info(f"Downloading {url}") 36 | resp = requests.get(url, headers={"user-agent": BACKEND_USER_AGENT}) 37 | resp.raise_for_status() 38 | 39 | patch = Patch.parse_patch(resp.text, skip_comments=False) 40 | assert patch != {}, "Empty patch" 41 | lines = { 42 | # Use all changes in new files 43 | filename: diff.get("touched", []) + diff.get("added", []) 44 | for filename, diff in patch.items() 45 | } 46 | 47 | return lines 48 | 49 | 50 | def detect_in_patch(issue_link, lines): 51 | """From the code-review bot revisions.py contains() method""" 52 | modified_lines = lines.get(issue_link.issue.path) 53 | 54 | if modified_lines is None: 55 | # File not in patch 56 | issue_link.in_patch = False 57 | 58 | elif issue_link.issue.line is None: 59 | # Empty line means full file 60 | issue_link.in_patch = True 61 | 62 | else: 63 | # Detect if this issue is in the patch 64 | chunk_lines = set( 65 | range( 66 | issue_link.issue.line, issue_link.issue.line + issue_link.issue.nb_lines 67 | ) 68 | ) 69 | issue_link.in_patch = not chunk_lines.isdisjoint(modified_lines) 70 | return issue_link 71 | 72 | 73 | def process_diff(diff: Diff): 74 | """This function needs to be on the top level in order to be usable by the pool""" 75 | try: 76 | lines = load_hgmo_patch(diff) 77 | 78 | issue_links = [ 79 | detect_in_patch(issue_link, lines) for issue_link in diff.issue_links.all() 80 | ] 81 | logging.info( 82 | f"Found {len([i for i in issue_links if i.in_patch])} issue link in patch for {diff.id}" 83 | ) 84 | IssueLink.objects.bulk_update(issue_links, ["in_patch"]) 85 | except Exception as e: 86 | logging.info(f"Failure on diff {diff.id}: {e}") 87 | 88 | 89 | class Command(BaseCommand): 90 | help = "Load issues from remote taskcluster reports" 91 | 92 | def add_arguments(self, parser): 93 | parser.add_argument( 94 | "--nb-processes", 95 | type=int, 96 | help="Number of processes used to process the diffs", 97 | default=1, 98 | ) 99 | 100 | def handle(self, *args, **options): 101 | # Only apply on diffs with issues that are not already processed 102 | diffs = ( 103 | Diff.objects.filter(issue_links__in_patch__isnull=True) 104 | .order_by("id") 105 | .distinct() 106 | ) 107 | logger.debug(f"Will process {diffs.count()} diffs") 108 | 109 | # Close all DB connection so each process get its own 110 | db.connections.close_all() 111 | 112 | # Process all the diffs in parallel 113 | with Pool(processes=options["nb_processes"]) as pool: 114 | pool.map(process_diff, diffs, chunksize=20) 115 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/migrations/0002_compare_issues.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | # Generated by Django 2.2.6 on 2019-10-31 15:57 6 | 7 | from django.db import migrations, models 8 | 9 | from code_review_backend.issues.compare import detect_new_for_revision 10 | 11 | 12 | def set_existing_as_new(apps, schema_editor): 13 | """ 14 | Update default value for existing issues 15 | """ 16 | Issue = apps.get_model("issues", "Issue") 17 | issues = Issue.objects.filter(diff__isnull=False).prefetch_related("diff") 18 | nb = issues.count() 19 | if not nb: 20 | return 21 | 22 | last_percent = 0 23 | for i, issue in enumerate(issues): 24 | issue.new_for_revision = detect_new_for_revision( 25 | issue.diff, issue.path, issue.hash 26 | ) 27 | issue.save() 28 | 29 | # Display progress 30 | percent = int(100.0 * i / nb) 31 | if percent > last_percent: 32 | nb_new = Issue.objects.filter(new_for_revision=True).count() 33 | print(f"{percent}% : {i}/{nb} issues with {nb_new} new issues") 34 | last_percent = percent 35 | 36 | 37 | class Migration(migrations.Migration): 38 | dependencies = [("issues", "0001_initial")] 39 | 40 | operations = [ 41 | migrations.AddField( 42 | model_name="issue", 43 | name="new_for_revision", 44 | field=models.BooleanField(null=True), 45 | ), 46 | migrations.RunPython(set_existing_as_new, reverse_code=lambda x, y: None), 47 | ] 48 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/migrations/0003_diff_repository.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | # Generated by Django 2.2.6 on 2019-11-27 10:23 6 | 7 | import django.db.models.deletion 8 | from django.db import migrations, models 9 | 10 | 11 | def _create_diff_repos(apps, schema_editor): 12 | """ 13 | Initialize the repositories for existing Diff instances 14 | We currently only have try repositories, so it's easy: 15 | - a revision on MC has diffs on Try 16 | - a revision on NSS has diffs on NSS-Try 17 | """ 18 | Repository = apps.get_model("issues", "Repository") 19 | Diff = apps.get_model("issues", "Diff") 20 | 21 | # Create try repositories 22 | repo_try = Repository.objects.create( 23 | id=100, slug="try", url="https://hg.mozilla.org/try" 24 | ) 25 | repo_nss_try = Repository.objects.create( 26 | id=101, slug="nss-try", url="https://hg.mozilla.org/projects/nss-try" 27 | ) 28 | 29 | # MC revisions have diffs on Try 30 | Diff.objects.filter(revision__repository__slug="mozilla-central").update( 31 | repository=repo_try 32 | ) 33 | 34 | # NSS revisions have diffs on NSS Try 35 | Diff.objects.filter(revision__repository__slug="nss").update( 36 | repository=repo_nss_try 37 | ) 38 | 39 | 40 | class Migration(migrations.Migration): 41 | dependencies = [("issues", "0002_compare_issues")] 42 | 43 | operations = [ 44 | migrations.AlterField( 45 | model_name="repository", 46 | name="phid", 47 | field=models.CharField(blank=True, max_length=40, null=True), 48 | ), 49 | migrations.AddField( 50 | model_name="diff", 51 | name="repository", 52 | field=models.ForeignKey( 53 | null=True, 54 | on_delete=django.db.models.deletion.CASCADE, 55 | related_name="diffs", 56 | to="issues.Repository", 57 | ), 58 | preserve_default=False, 59 | ), 60 | migrations.RunPython(_create_diff_repos), 61 | migrations.AlterField( 62 | model_name="diff", 63 | name="repository", 64 | field=models.ForeignKey( 65 | null=False, 66 | on_delete=django.db.models.deletion.CASCADE, 67 | related_name="diffs", 68 | to="issues.Repository", 69 | ), 70 | preserve_default=False, 71 | ), 72 | ] 73 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/migrations/0004_issue_in_patch.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | # Generated by Django 2.2.8 on 2020-01-06 10:19 6 | 7 | from django.db import migrations, models 8 | 9 | 10 | class Migration(migrations.Migration): 11 | dependencies = [("issues", "0003_diff_repository")] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name="issue", name="in_patch", field=models.BooleanField(null=True) 16 | ) 17 | ] 18 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/migrations/0005_rename_check_issue_analyzer_check.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | # Generated by Django 4.0.5 on 2022-06-30 12:13 6 | 7 | from django.db import migrations 8 | 9 | 10 | class Migration(migrations.Migration): 11 | dependencies = [ 12 | ("issues", "0004_issue_in_patch"), 13 | ] 14 | 15 | operations = [ 16 | migrations.RenameField( 17 | model_name="issue", 18 | old_name="check", 19 | new_name="analyzer_check", 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/migrations/0007_issuelink_swap.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | # Generated by Django 4.1.2 on 2022-12-07 16:59 6 | 7 | from math import ceil 8 | 9 | from django.db import migrations 10 | 11 | ISSUES_INSERT_SIZE = 1000 12 | 13 | 14 | def generate_issue_links(apps, schema_editor): 15 | """Generate the IssueLink M2M table from issues' FK to the diff of a revision. 16 | Issues are migrated from the most recent to the oldest. 17 | The generation is done in small transactions, so the tables ar not locked for a 18 | long period of time (allowing to ingest new issues) and can be resumed later on. 19 | """ 20 | Issue = apps.get_model("issues", "Issue") 21 | IssueLink = apps.get_model("issues", "IssueLink") 22 | 23 | # Search for the IssueLink referencing the older issue 24 | older_issue_link = IssueLink.objects.order_by("issue__created").first() 25 | issue_filters = {} 26 | if older_issue_link is not None: 27 | issue_filters = {"created__lt": older_issue_link.issue.created} 28 | 29 | qs = ( 30 | # This ensures we do not handle newly created issues 31 | # during the migration so the order is preserved 32 | Issue.objects.filter(diff__isnull=False) 33 | .filter(**issue_filters) 34 | .order_by("-created", "id") 35 | .values("id", "diff_id", "diff__revision_id") 36 | ) 37 | 38 | issues_count = qs.count() 39 | slices = ceil(issues_count / ISSUES_INSERT_SIZE) 40 | 41 | for index in range(0, slices): 42 | current = min((index + 1) * ISSUES_INSERT_SIZE, issues_count) 43 | print( 44 | f"[{current}/{issues_count}] Initializing Issues references on the M2M table." 45 | ) 46 | issues = qs[index * ISSUES_INSERT_SIZE : (index + 1) * ISSUES_INSERT_SIZE] 47 | IssueLink.objects.bulk_create( 48 | IssueLink( 49 | issue_id=issue["id"], 50 | diff_id=issue["diff_id"], 51 | revision_id=issue["diff__revision_id"], 52 | ) 53 | for issue in issues 54 | ) 55 | 56 | 57 | class Migration(migrations.Migration): 58 | atomic = False 59 | 60 | dependencies = [ 61 | ("issues", "0006_issuelink_initial"), 62 | ] 63 | 64 | operations = [ 65 | # Fill the M2M table 66 | migrations.RunPython( 67 | generate_issue_links, 68 | reverse_code=None, 69 | elidable=True, 70 | atomic=False, 71 | ), 72 | # Drop old FK 73 | migrations.RemoveField( 74 | model_name="issue", 75 | name="diff", 76 | ), 77 | ] 78 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/migrations/0009_revision_optional_phab_references.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.5 on 2023-04-14 15:37 2 | 3 | from django.db import migrations, models 4 | from django.db.models import F 5 | 6 | 7 | def populate_phabricator_id(apps, schema_editor): 8 | """ 9 | Before that migration, revisions used the Phabricator 10 | numerical ID as their primary key. 11 | """ 12 | Revision = apps.get_model("issues", "Revision") 13 | Revision.objects.all().update(phabricator_id=F("id")) 14 | 15 | 16 | class Migration(migrations.Migration): 17 | dependencies = [ 18 | ("issues", "0008_revision_base_head_references"), 19 | ] 20 | 21 | operations = [ 22 | migrations.AlterModelOptions( 23 | name="revision", 24 | options={"ordering": ("phabricator_id", "id")}, 25 | ), 26 | migrations.AddField( 27 | model_name="revision", 28 | name="phabricator_id", 29 | field=models.PositiveIntegerField(blank=True, null=True, unique=True), 30 | ), 31 | migrations.AlterField( 32 | model_name="revision", 33 | name="phid", 34 | field=models.CharField(blank=True, max_length=40, null=True, unique=True), 35 | ), 36 | migrations.RenameField( 37 | model_name="revision", 38 | old_name="phid", 39 | new_name="phabricator_phid", 40 | ), 41 | migrations.RunPython( 42 | populate_phabricator_id, 43 | reverse_code=migrations.RunPython.noop, 44 | elidable=True, 45 | ), 46 | migrations.AlterField( 47 | model_name="revision", 48 | name="id", 49 | field=models.AutoField(primary_key=True, serialize=False), 50 | ), 51 | migrations.AddConstraint( 52 | model_name="revision", 53 | constraint=models.UniqueConstraint( 54 | condition=models.Q(("phabricator_id__isnull", False)), 55 | fields=("phabricator_id",), 56 | name="revision_unique_phab_id", 57 | ), 58 | ), 59 | migrations.AddConstraint( 60 | model_name="revision", 61 | constraint=models.UniqueConstraint( 62 | condition=models.Q(("phabricator_phid__isnull", False)), 63 | fields=("phabricator_phid",), 64 | name="revision_unique_phab_phabid", 65 | ), 66 | ), 67 | ] 68 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/migrations/0010_alter_revision_id.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | # Generated by Django 2.2.6 on 2019-10-31 15:57 6 | 7 | from django.db import migrations, models 8 | 9 | 10 | class Migration(migrations.Migration): 11 | dependencies = [ 12 | ("issues", "0009_revision_optional_phab_references"), 13 | ] 14 | 15 | operations = [ 16 | migrations.AlterField( 17 | model_name="revision", 18 | name="id", 19 | field=models.BigAutoField(primary_key=True, serialize=False), 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/migrations/0011_indexes_revision_issue_path.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.1 on 2024-09-10 11:18 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("issues", "0010_alter_revision_id"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AddIndex( 13 | model_name="issue", 14 | index=models.Index(fields=["path"], name="issues_issu_path_ee2627_idx"), 15 | ), 16 | migrations.AddIndex( 17 | model_name="revision", 18 | index=models.Index( 19 | fields=["head_repository", "head_changeset"], 20 | name="issues_revi_head_re_7be180_idx", 21 | ), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/migrations/0012_move_issues_attributes_part_1.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.1.1 on 2024-10-14 13:05 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("issues", "0011_indexes_revision_issue_path"), 9 | ] 10 | 11 | operations = [ 12 | # Create new fields on IssueLink 13 | migrations.AddField( 14 | model_name="issuelink", 15 | name="in_patch", 16 | field=models.BooleanField(null=True), 17 | ), 18 | migrations.AddField( 19 | model_name="issuelink", 20 | name="new_for_revision", 21 | field=models.BooleanField(null=True), 22 | ), 23 | migrations.AddField( 24 | model_name="issuelink", 25 | name="char", 26 | field=models.PositiveIntegerField(null=True), 27 | ), 28 | migrations.AddField( 29 | model_name="issuelink", 30 | name="line", 31 | field=models.PositiveIntegerField(null=True), 32 | ), 33 | migrations.AddField( 34 | model_name="issuelink", 35 | name="nb_lines", 36 | field=models.PositiveIntegerField(null=True), 37 | ), 38 | ] 39 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/migrations/0013_move_issues_attributes_part_2.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.1.1 on 2024-10-14 13:05 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("issues", "0012_move_issues_attributes_part_1"), 9 | ] 10 | 11 | operations = [ 12 | # Move all their values 13 | migrations.RunSQL( 14 | """ 15 | UPDATE issues_issuelink as link 16 | SET 17 | in_patch=i.in_patch, 18 | new_for_revision=i.new_for_revision, 19 | char=i.char, 20 | line=i.line, 21 | nb_lines=i.nb_lines 22 | FROM issues_issue as i 23 | WHERE i.id = link.issue_id 24 | """, 25 | reverse_sql=""" 26 | UPDATE issues_issue as i 27 | SET 28 | in_patch=link.in_patch, 29 | new_for_revision=link.new_for_revision, 30 | char=link.char, 31 | line=link.line, 32 | nb_lines=link.nb_lines 33 | FROM issues_issuelink as link 34 | WHERE i.id = link.issue_id 35 | """, 36 | ), 37 | # Remove old fields from Issue 38 | migrations.RemoveField( 39 | model_name="issue", 40 | name="in_patch", 41 | ), 42 | migrations.RemoveField( 43 | model_name="issue", 44 | name="new_for_revision", 45 | ), 46 | migrations.RemoveField( 47 | model_name="issue", 48 | name="char", 49 | ), 50 | migrations.RemoveField( 51 | model_name="issue", 52 | name="line", 53 | ), 54 | migrations.RemoveField( 55 | model_name="issue", 56 | name="nb_lines", 57 | ), 58 | # Update constraints 59 | migrations.RemoveConstraint( 60 | model_name="issuelink", 61 | name="issue_link_unique_revision", 62 | ), 63 | migrations.RemoveConstraint( 64 | model_name="issuelink", 65 | name="issue_link_unique_diff", 66 | ), 67 | migrations.AddConstraint( 68 | model_name="issuelink", 69 | constraint=models.UniqueConstraint( 70 | condition=models.Q(("diff__isnull", True)), 71 | fields=("issue", "revision", "line", "nb_lines", "char"), 72 | name="issue_link_unique_revision", 73 | ), 74 | ), 75 | migrations.AddConstraint( 76 | model_name="issuelink", 77 | constraint=models.UniqueConstraint( 78 | condition=models.Q(("diff__isnull", False)), 79 | fields=("issue", "revision", "diff", "line", "nb_lines", "char"), 80 | name="issue_link_unique_diff", 81 | ), 82 | ), 83 | ] 84 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/migrations/0014_unique_hash.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.1.1 on 2024-10-14 13:36 2 | 3 | from django.db import migrations, models 4 | 5 | # This query aims to update all IssueLink in-place 6 | # moving them from duplicated issues (by hash) towards the single one remaining 7 | # It uses a first simple query `dupes` to partition issues by hash, identifying 8 | # the remaining issue as (idx=1) 9 | # The second query `translations` is a mapping of duplicates (idx > 1) 10 | # towards the remaining issue for every hash (idx=1) 11 | # That mapping is finally used to update all issue links in a single query 12 | QUERY_UPDATE_LINKS = """ 13 | with dupes as ( 14 | select id, hash, row_number() over (partition by hash order by id) as idx 15 | from issues_issue 16 | ), 17 | translations as ( 18 | select src.id as src_id, dest.id as dest_id 19 | from dupes as src 20 | inner join dupes as dest on (src.hash=dest.hash and dest.idx=1) 21 | where src.idx > 1 22 | ) 23 | update issues_issuelink as l 24 | set issue_id = t.dest_id 25 | from translations as t 26 | where t.src_id = l.issue_id; 27 | """ 28 | 29 | # This query runs after the update described above and reuse the exact same 30 | # `dupes` query to identify then delete duplicated issues (idx > 1) 31 | QUERY_DELETE_ISSUES = """ 32 | with dupes as ( 33 | select id, hash, row_number() over (partition by hash order by id) as idx 34 | from issues_issue 35 | ) 36 | delete from issues_issue where id in (select id from dupes where idx > 1); 37 | """ 38 | 39 | 40 | class Migration(migrations.Migration): 41 | atomic = False 42 | 43 | dependencies = [ 44 | ("issues", "0013_move_issues_attributes_part_2"), 45 | ] 46 | 47 | operations = [ 48 | migrations.AddIndex( 49 | model_name="issue", 50 | index=models.Index(fields=["hash"], name="issue_hash_idx"), 51 | ), 52 | migrations.RunSQL(QUERY_UPDATE_LINKS), 53 | migrations.RunSQL(QUERY_DELETE_ISSUES), 54 | migrations.AlterField( 55 | model_name="issue", 56 | name="hash", 57 | field=models.CharField(max_length=32, unique=True), 58 | ), 59 | ] 60 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/migrations/0015_remove_repository_phid_alter_repository_id.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.1.2 on 2024-11-18 15:53 2 | 3 | from django.conf import settings 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | dependencies = [ 9 | ("issues", "0014_unique_hash"), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name="repository", 15 | name="phid", 16 | ), 17 | migrations.AlterField( 18 | model_name="repository", 19 | name="id", 20 | field=models.AutoField(primary_key=True, serialize=False), 21 | ), 22 | migrations.AlterModelOptions( 23 | name="repository", 24 | options={"ordering": ("id",), "verbose_name_plural": "repositories"}, 25 | ), 26 | ] 27 | if "postgresql" in settings.DATABASES["default"]["ENGINE"]: 28 | print( 29 | "Adding sequence initialization for Repository PK to issues.0015 with PostgreSQL backend" 30 | ) 31 | operations.append( 32 | migrations.RunSQL( 33 | """ 34 | SELECT setval( 35 | pg_get_serial_sequence('issues_repository', 'id'), 36 | coalesce(max(id)+1, 1), 37 | false 38 | ) FROM issues_repository; 39 | """, 40 | reverse_sql=migrations.RunSQL.noop, 41 | ) 42 | ) 43 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/backend/code_review_backend/issues/migrations/__init__.py -------------------------------------------------------------------------------- /backend/code_review_backend/issues/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/backend/code_review_backend/issues/tests/__init__.py -------------------------------------------------------------------------------- /backend/code_review_backend/issues/tests/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/backend/code_review_backend/issues/tests/commands/__init__.py -------------------------------------------------------------------------------- /backend/code_review_backend/issues/tests/test_compare.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import hashlib 6 | import random 7 | 8 | from django.contrib.auth.models import User 9 | from rest_framework.test import APITestCase 10 | 11 | from code_review_backend.issues.compare import detect_new_for_revision 12 | from code_review_backend.issues.models import Diff, Issue, Repository 13 | 14 | 15 | class CompareAPITestCase(APITestCase): 16 | def setUp(self): 17 | # Create a user 18 | self.user = User.objects.create(username="crash_user") 19 | 20 | # Create a repo & its try counterpart 21 | self.repo = Repository.objects.create( 22 | id=1, slug="myrepo", url="http://repo.test/myrepo" 23 | ) 24 | self.repo_try = Repository.objects.create( 25 | id=2, slug="myrepo-try", url="http://repo.test/try" 26 | ) 27 | 28 | # Create a simple stack with 2 diffs 29 | self.revision = self.repo_try.head_revisions.create( 30 | phabricator_id=1, 31 | phabricator_phid="PHID-DREV-1", 32 | title="Revision XYZ", 33 | bugzilla_id=1234567, 34 | base_repository=self.repo, 35 | ) 36 | for i in range(2): 37 | self.revision.diffs.create( 38 | id=i + 1, 39 | phid=f"PHID-DIFF-{i+1}", 40 | review_task_id=f"task-{i}", 41 | mercurial_hash=hashlib.sha1(f"hg {i}".encode()).hexdigest(), 42 | repository=self.repo_try, 43 | ) 44 | 45 | # Add 3 issues on first diff 46 | for i in range(3): 47 | self.build_issue(1, i) 48 | 49 | def build_issue(self, diff_id, hash_id): 50 | issue, _ = Issue.objects.get_or_create( 51 | hash=self.build_hash(hash_id), 52 | defaults={ 53 | "path": "path/to/file", 54 | "level": "warning", 55 | "message": None, 56 | "analyzer": "analyzer-x", 57 | "analyzer_check": "check-y", 58 | }, 59 | ) 60 | # Link the issue to the specific diff 61 | issue.issue_links.create( 62 | diff_id=diff_id, 63 | revision=self.revision, 64 | line=random.randint(1, 100), 65 | nb_lines=random.randint(1, 100), 66 | char=None, 67 | ) 68 | return issue 69 | 70 | def build_hash(self, content): 71 | """Produce a dummy hash from some content""" 72 | return hashlib.md5(bytes(content)).hexdigest() 73 | 74 | def test_detect_new_for_revision(self): 75 | """ 76 | Check the detection of a new issue in a revision 77 | """ 78 | # No issues on second diff at first 79 | self.assertFalse(Issue.objects.filter(diffs=2).exists()) 80 | 81 | # All issues on top diff are new 82 | top_diff = Diff.objects.get(pk=1) 83 | for issue in top_diff.issues.all(): 84 | self.assertTrue(detect_new_for_revision(top_diff, issue.path, issue.hash)) 85 | 86 | # Adding an issue with same hash on second diff will be set as existing 87 | second_diff = Diff.objects.get(pk=2) 88 | issue = self.build_issue(2, 1) 89 | self.assertFalse(detect_new_for_revision(second_diff, issue.path, issue.hash)) 90 | 91 | # But adding an issue with a different hash on second diff will be set as new 92 | issue = self.build_issue(2, 12345) 93 | self.assertTrue(detect_new_for_revision(second_diff, issue.path, issue.hash)) 94 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/tests/test_dockerflow.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import json 6 | 7 | from django.conf import settings 8 | from django.test import TestCase 9 | 10 | 11 | class DockerflowEndpointsTestCase(TestCase): 12 | def setUp(self): 13 | self.old_setting = settings.DEBUG 14 | 15 | def tearDown(self): 16 | settings.DEBUG = self.old_setting 17 | 18 | def test_get_version(self): 19 | response = self.client.get("/__version__") 20 | self.assertEqual(response.status_code, 200) 21 | 22 | with open(f"{settings.BASE_DIR}/version.json") as version_file: 23 | self.assertEqual(response.json(), json.loads(version_file.read())) 24 | 25 | def test_get_heartbeat_debug(self): 26 | settings.DEBUG = True 27 | 28 | response = self.client.get("/__heartbeat__") 29 | self.assertEqual(response.status_code, 200) 30 | 31 | # In DEBUG mode, we can retrieve checks details 32 | heartbeat = response.json() 33 | self.assertEqual(heartbeat["status"], "ok") 34 | self.assertTrue("checks" in heartbeat) 35 | self.assertTrue("details" in heartbeat) 36 | 37 | def test_get_heartbeat(self): 38 | settings.DEBUG = False 39 | 40 | response = self.client.get("/__heartbeat__") 41 | self.assertEqual(response.status_code, 200) 42 | 43 | # When DEBUG is False, we can't retrieve checks details and the status is certainly 44 | # equal to "warning" because of the deployment checks that are added: 45 | # https://github.com/mozilla-services/python-dockerflow/blob/e316f0c5f0aa6d176a6d08d1f568f83658b51339/src/dockerflow/django/views.py#L45 46 | self.assertEqual(response.json(), {"status": "warning"}) 47 | 48 | def test_get_lbheartbeat(self): 49 | response = self.client.get("/__lbheartbeat__") 50 | self.assertEqual(response.status_code, 200) 51 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/tests/test_repository.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | from rest_framework import status 6 | from rest_framework.test import APITestCase 7 | 8 | from code_review_backend.issues.models import Repository 9 | 10 | 11 | class RepositoryAPITestCase(APITestCase): 12 | fixtures = ["fixtures/repositories.json"] 13 | 14 | def test_list_repositories(self): 15 | """ 16 | Check we can list all repositories in database 17 | """ 18 | response = self.client.get("/v1/repository/") 19 | self.assertEqual(Repository.objects.count(), 5) 20 | self.assertEqual(response.status_code, status.HTTP_200_OK) 21 | self.assertDictEqual( 22 | response.json(), 23 | { 24 | "count": 5, 25 | "next": None, 26 | "previous": None, 27 | "results": [ 28 | { 29 | "id": 102, 30 | "slug": "autoland", 31 | "url": "https://hg.mozilla.org/integration/autoland", 32 | }, 33 | { 34 | "id": 1, 35 | "slug": "mozilla-central", 36 | "url": "https://hg.mozilla.org/mozilla-central", 37 | }, 38 | { 39 | "id": 8, 40 | "slug": "nss", 41 | "url": "https://hg.mozilla.org/projects/nss", 42 | }, 43 | { 44 | "id": 101, 45 | "slug": "nss-try", 46 | "url": "https://hg.mozilla.org/projects/nss-try", 47 | }, 48 | { 49 | "id": 100, 50 | "slug": "try", 51 | "url": "https://hg.mozilla.org/try", 52 | }, 53 | ], 54 | }, 55 | ) 56 | -------------------------------------------------------------------------------- /backend/code_review_backend/issues/tests/test_revision.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | from django.conf import settings 6 | from rest_framework.test import APITestCase 7 | 8 | from code_review_backend.issues.models import Repository, Revision 9 | 10 | 11 | class RevisionAPITestCase(APITestCase): 12 | def setUp(self): 13 | self.repo = Repository.objects.create( 14 | id=1, 15 | slug="myrepo", 16 | url="http://repo.test/myrepo", 17 | ) 18 | 19 | def test_phabricator_url(self): 20 | rev = Revision.objects.create( 21 | phabricator_id=12, 22 | phabricator_phid="PHID-REV-12345", 23 | base_repository=self.repo, 24 | head_repository=self.repo, 25 | ) 26 | 27 | # Default settings 28 | self.assertEqual( 29 | rev.phabricator_url, "https://phabricator.services.mozilla.com/D12" 30 | ) 31 | 32 | # Override host with /api 33 | settings.PHABRICATOR_HOST = "http://phab.test/api" 34 | self.assertEqual(rev.phabricator_url, "http://phab.test/D12") 35 | 36 | # Override host with complex url 37 | settings.PHABRICATOR_HOST = "http://anotherphab.test/api123/?custom" 38 | self.assertEqual(rev.phabricator_url, "http://anotherphab.test/D12") 39 | -------------------------------------------------------------------------------- /backend/code_review_backend/version.json: -------------------------------------------------------------------------------- 1 | { 2 | "commit": "stub", 3 | "version": "stub", 4 | "source": "https://github.com/mozilla/code-review", 5 | "build": "https://tools.taskcluster.net/task-inspector/#XXXXXXXXXXXXXXXXXX" 6 | } 7 | -------------------------------------------------------------------------------- /backend/fixtures/repositories.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "model": "issues.repository", 4 | "pk": 1, 5 | "fields": { 6 | "created": "2019-10-17T07:17:06.396Z", 7 | "updated": "2019-10-17T07:17:06.396Z", 8 | "slug": "mozilla-central", 9 | "url": "https://hg.mozilla.org/mozilla-central" 10 | } 11 | }, 12 | { 13 | "model": "issues.repository", 14 | "pk": 8, 15 | "fields": { 16 | "created": "2019-10-17T07:17:32.970Z", 17 | "updated": "2019-10-17T07:17:32.970Z", 18 | "slug": "nss", 19 | "url": "https://hg.mozilla.org/projects/nss" 20 | } 21 | }, 22 | { 23 | "model": "issues.repository", 24 | "pk": 100, 25 | "fields": { 26 | "created": "2019-11-27T10:42:35.034Z", 27 | "updated": "2019-11-27T10:42:35.034Z", 28 | "slug": "try", 29 | "url": "https://hg.mozilla.org/try" 30 | } 31 | }, 32 | { 33 | "model": "issues.repository", 34 | "pk": 101, 35 | "fields": { 36 | "created": "2019-11-27T10:42:35.035Z", 37 | "updated": "2019-11-27T10:42:35.035Z", 38 | "slug": "nss-try", 39 | "url": "https://hg.mozilla.org/projects/nss-try" 40 | } 41 | }, 42 | { 43 | "model": "issues.repository", 44 | "pk": 102, 45 | "fields": { 46 | "created": "2019-12-02T16:01:23.231Z", 47 | "updated": "2019-12-02T16:01:23.231Z", 48 | "slug": "autoland", 49 | "url": "https://hg.mozilla.org/integration/autoland" 50 | } 51 | } 52 | ] 53 | -------------------------------------------------------------------------------- /backend/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this 4 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | """Django's command-line utility for administrative tasks.""" 7 | 8 | import os 9 | import sys 10 | 11 | 12 | def main(): 13 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "code_review_backend.app.settings") 14 | try: 15 | from django.core.management import execute_from_command_line 16 | except ImportError as exc: 17 | raise ImportError( 18 | "Couldn't import Django. Are you sure it's installed and " 19 | "available on your PYTHONPATH environment variable? Did you " 20 | "forget to activate a virtual environment?" 21 | ) from exc 22 | execute_from_command_line(sys.argv) 23 | 24 | 25 | if __name__ == "__main__": 26 | main() 27 | -------------------------------------------------------------------------------- /backend/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | django-debug-toolbar==5.2.0 2 | django-extensions==4.1 3 | pre-commit==4.2.0 4 | -------------------------------------------------------------------------------- /backend/requirements.txt: -------------------------------------------------------------------------------- 1 | Django==5.1.7 2 | django-cors-headers==4.7.0 3 | django-environ==0.12.0 4 | djangorestframework==3.16.0 5 | dockerflow==2024.4.2 6 | drf-yasg==1.21.10 7 | gunicorn==23.0.0 8 | parsepatch==0.1.3 9 | psycopg2-binary==2.9.10 10 | sentry-sdk==2.29.1 11 | setuptools==80.8.0 12 | sqlparse==0.5.3 13 | taskcluster==84.0.2 14 | tqdm==4.67.1 15 | whitenoise==6.9.0 16 | -------------------------------------------------------------------------------- /backend/setup.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import setuptools 6 | 7 | 8 | def read_requirements(file_): 9 | lines = [] 10 | with open(file_) as f: 11 | for line in f.readlines(): 12 | line = line.strip() 13 | if ( 14 | line.startswith("-e ") 15 | or line.startswith("http://") 16 | or line.startswith("https://") 17 | ): 18 | extras = "" 19 | if "[" in line: 20 | extras = "[" + line.split("[")[1].split("]")[0] + "]" 21 | line = line.split("#")[1].split("egg=")[1] + extras 22 | elif line == "" or line.startswith("#") or line.startswith("-"): 23 | continue 24 | line = line.split("#")[0].strip() 25 | lines.append(line) 26 | return sorted(list(set(lines))) 27 | 28 | 29 | with open("VERSION") as f: 30 | VERSION = f.read().strip() 31 | 32 | 33 | setuptools.setup( 34 | name="code_review_backend", 35 | version=VERSION, 36 | description="Store and compare issues found in Mozilla code review tasks", 37 | author="Mozilla Release Management", 38 | author_email="release-mgmt-analysis@mozilla.com", 39 | url="https://github.com/mozilla/code-review", 40 | tests_require=read_requirements("requirements-dev.txt"), 41 | install_requires=read_requirements("requirements.txt"), 42 | packages=setuptools.find_packages(), 43 | include_package_data=True, 44 | zip_safe=False, 45 | license="MPL2", 46 | ) 47 | -------------------------------------------------------------------------------- /bot/.dockerignore: -------------------------------------------------------------------------------- 1 | tests 2 | -------------------------------------------------------------------------------- /bot/README.md: -------------------------------------------------------------------------------- 1 | # Static Analysis Bot 2 | 3 | ## Developer setup 4 | 5 | The code review bot is a Python 3 application, so it should be easy to bootstrap on your computer: 6 | 7 | ``` 8 | mkvirtualenv -p /usr/bin/python3 code-review 9 | pip install -r requirements.txt -r requirements-dev.txt 10 | pip install -e . 11 | ``` 12 | 13 | You should now be able to run linting and unit tests: 14 | 15 | ``` 16 | flake8 17 | pytest 18 | ``` 19 | 20 | If those tests are OK, you can run the bot locally, by using a local configuration file with your Phabricator API token (see details at the end of this README), and a task reference to analyze. 21 | 22 | ``` 23 | export TRY_TASK_ID=XXX 24 | export TRY_TASK_GROUP_ID=XXX 25 | code-review-bot --configuration=path/to/config.yaml 26 | ``` 27 | 28 | ## Configuration 29 | 30 | The code review bot is configured through the [Taskcluster secrets service](https://firefox-ci-tc.services.mozilla.com/secrets) or a local YAML configuration file (the latter is preferred for new contributors as it's easier to setup) 31 | 32 | The following configuration variables are currently supported: 33 | 34 | - `APP_CHANNEL` **[required]** is provided by the common configuration (staging or production) 35 | - `REPORTERS` **[required]** lists all the reporting tools to use when a code review is completed (details below) 36 | - `PHABRICATOR` **[required]** holds the credentials to make API calls on Phabricator. 37 | - `ZERO_COVERAGE_ENABLED` is a boolean value enabling or disabling the zero coverage warning report. 38 | - `PAPERTRAIL_HOST` is the optional Papertrail host configuration, used for logging. 39 | - `PAPERTRAIL_PORT` is the optional Papertrail port configuration, used for logging. 40 | - `SENTRY_DSN` is the optional Sentry full url to report runtime errors. 41 | 42 | The `REPORTERS` configuration is a list of dictionaries describing which reporting tool to use at the end of the patches code review. 43 | Supported reporting tools are emails (for admins) and Phabricator. 44 | 45 | Each reporter configuration must contain a `reporter` key with a unique name per tool. Each tool has its own configuration requirement. 46 | 47 | You can view a [full configuration sample here](/docs/configuration.md). 48 | 49 | ## Phabricator credentials 50 | 51 | They are required, and must be set like this: 52 | 53 | ``` 54 | PHABRICATOR: 55 | url: 'https://phabricator.services.mozilla.com/api/' 56 | api_key: api-XXXX 57 | ``` 58 | 59 | ## Reporter: Mail 60 | 61 | Key `reporter` is `mail` 62 | 63 | The emails are sent through Taskcluster notify service, the hook must have `notify:email:*` in its scopes (enabled on our staging & production instances) 64 | 65 | Only one configuration is required: `emails` is a list of emails addresses receiving the admin output for each analysis. 66 | 67 | This reporter will send detailed information about every issue. 68 | 69 | ## Reporter: Phabricator 70 | 71 | Key `reporter` is `phabricator` 72 | 73 | Configuration: 74 | 75 | - `analyzers_skipped` : The analyzers that will **not** be published on Phabricator. 76 | 77 | This reporter will send detailed information about every **publishable** issue. 78 | 79 | ## Example configuration 80 | 81 | ```yaml 82 | --- 83 | common: 84 | APP_CHANNEL: development 85 | PHABRICATOR: 86 | url: https://dev.phabricator.mozilla.com 87 | api_key: deadbeef123456 88 | 89 | bot: 90 | REPORTERS: 91 | - reporter: phabricator 92 | ``` 93 | -------------------------------------------------------------------------------- /bot/VERSION: -------------------------------------------------------------------------------- 1 | 1.10.2 2 | -------------------------------------------------------------------------------- /bot/code_review_bot/mercurial.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import fcntl 6 | import os 7 | import time 8 | 9 | import hglib 10 | import structlog 11 | 12 | logger = structlog.get_logger(__name__) 13 | 14 | 15 | def hg_run(cmd): 16 | """ 17 | Run a mercurial command without an hglib instance 18 | Useful for initial custom clones 19 | Redirects stdout & stderr to python's logger 20 | 21 | This code has been copied from the libmozevent library 22 | https://github.com/mozilla/libmozevent/blob/fd0b3689c50c3d14ac82302b31115d0046c6e7c8/libmozevent/utils.py#L77 23 | """ 24 | 25 | def _log_process(output, name): 26 | # Read and display every line 27 | out = output.read() 28 | if out is None: 29 | return 30 | text = filter(None, out.decode("utf-8").splitlines()) 31 | for line in text: 32 | logger.info(f"{name}: {line}") 33 | 34 | # Start process 35 | main_cmd = cmd[0] 36 | proc = hglib.util.popen([hglib.HGPATH] + cmd) 37 | 38 | # Set process outputs as non blocking 39 | for output in (proc.stdout, proc.stderr): 40 | fcntl.fcntl( 41 | output.fileno(), 42 | fcntl.F_SETFL, 43 | fcntl.fcntl(output, fcntl.F_GETFL) | os.O_NONBLOCK, 44 | ) 45 | 46 | while proc.poll() is None: 47 | _log_process(proc.stdout, main_cmd) 48 | _log_process(proc.stderr, f"{main_cmd} (err)") 49 | time.sleep(2) 50 | 51 | out, err = proc.communicate() 52 | if proc.returncode != 0: 53 | logger.error(f"Mercurial {main_cmd} failure", out=out, err=err, exc_info=True) 54 | raise hglib.error.CommandError(cmd, proc.returncode, out, err) 55 | 56 | return out 57 | 58 | 59 | def robust_checkout( 60 | repo_url, checkout_dir, sharebase_dir, revision, repo_upstream_url=None 61 | ): 62 | cmd = hglib.util.cmdbuilder( 63 | "robustcheckout", 64 | repo_url, 65 | checkout_dir, 66 | purge=True, 67 | sharebase=sharebase_dir, 68 | revision=revision, 69 | upstream=repo_upstream_url, 70 | ) 71 | hg_run(cmd) 72 | -------------------------------------------------------------------------------- /bot/code_review_bot/report/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import structlog 6 | 7 | from code_review_bot.report.lando import LandoReporter 8 | from code_review_bot.report.mail import MailReporter 9 | from code_review_bot.report.mail_builderrors import BuildErrorsReporter 10 | from code_review_bot.report.phabricator import PhabricatorReporter 11 | 12 | logger = structlog.get_logger(__name__) 13 | 14 | 15 | def get_reporters(configuration): 16 | """ 17 | Load reporters using Taskcluster configuration 18 | """ 19 | assert isinstance(configuration, list) 20 | reporters = { 21 | "lando": LandoReporter, 22 | "mail": MailReporter, 23 | "build_error": BuildErrorsReporter, 24 | "phabricator": PhabricatorReporter, 25 | } 26 | 27 | out = {} 28 | for conf in configuration: 29 | try: 30 | if "reporter" not in conf: 31 | raise Exception("Missing reporter declaration") 32 | name = conf["reporter"] 33 | cls = reporters.get(name) 34 | if cls is None: 35 | raise Exception("Missing reporter class {}".format(conf["reporter"])) 36 | out[name] = cls(conf) 37 | except Exception as e: 38 | logger.warning(f"Failed to create reporter: {e}") 39 | 40 | return out 41 | -------------------------------------------------------------------------------- /bot/code_review_bot/report/base.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import itertools 6 | 7 | from code_review_bot import Level 8 | 9 | 10 | class Reporter: 11 | """ 12 | Common interface to post reports on a website 13 | Will configure & build reports 14 | """ 15 | 16 | def __init__(self, configuration): 17 | """ 18 | Configure reporter using Taskcluster credentials and configuration 19 | """ 20 | raise NotImplementedError 21 | 22 | def publish(self, issues, revision): 23 | """ 24 | Publish a new report 25 | """ 26 | raise NotImplementedError 27 | 28 | def requires(self, configuration, *keys): 29 | """ 30 | Check all configuration necessary keys are present 31 | """ 32 | assert isinstance(configuration, dict) 33 | 34 | out = [] 35 | for key in keys: 36 | assert key in configuration, f"Missing {self.__class__.__name__} {key}" 37 | out.append(configuration[key]) 38 | 39 | return out 40 | 41 | def calc_stats(self, issues): 42 | """ 43 | Calc stats about issues: 44 | * group issues by analyzer 45 | * count their total number 46 | * count their publishable number 47 | """ 48 | 49 | groups = itertools.groupby( 50 | sorted(issues, key=lambda i: i.analyzer.name), lambda i: i.analyzer 51 | ) 52 | 53 | def stats(analyzer, items): 54 | _items = list(items) 55 | paths = list({i.path for i in _items if i.is_publishable()}) 56 | 57 | publishable = sum(i.is_publishable() for i in _items) 58 | build_errors = sum(i.is_build_error() for i in _items) 59 | 60 | return { 61 | "analyzer": analyzer.display_name, 62 | "help": analyzer.build_help_message(paths), 63 | "total": len(_items), 64 | "publishable": publishable, 65 | "publishable_paths": paths, 66 | # Split results for normal publishable issues and build errors 67 | "nb_defects": publishable - build_errors, 68 | "nb_build_errors": build_errors, 69 | "nb_warnings": sum(i.level == Level.Warning for i in _items), 70 | "nb_errors": sum(i.level == Level.Error for i in _items), 71 | } 72 | 73 | return [stats(analyzer, items) for analyzer, items in groups] 74 | -------------------------------------------------------------------------------- /bot/code_review_bot/report/debug.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import json 6 | import os.path 7 | import time 8 | 9 | import structlog 10 | 11 | from code_review_bot.report.base import Reporter 12 | 13 | logger = structlog.get_logger(__name__) 14 | 15 | 16 | class DebugReporter(Reporter): 17 | """ 18 | Debug the issues found and report through the logs 19 | Build a json file with all issues details, stored as a TC artifact 20 | """ 21 | 22 | def __init__(self, output_dir): 23 | assert os.path.isdir(output_dir), "Invalid output dir" 24 | self.report_path = os.path.join(output_dir, "report.json") 25 | 26 | def publish(self, issues, revision, task_failures, links, reviewers): 27 | """ 28 | Display issues choices 29 | """ 30 | # Simply output issues details through logging 31 | logger.info("Debug revision", rev=str(revision)) 32 | for issue in issues: 33 | logger.info( 34 | "Issue {}".format( 35 | "publishable" if issue.is_publishable() else "silent" 36 | ), 37 | issue=str(issue), 38 | ) 39 | for task in task_failures: 40 | logger.info("Task failure detected", name=task.name, task=task.id) 41 | for patch in revision.improvement_patches: 42 | logger.info(f"Patch {patch}") 43 | 44 | # Output json report in public directory 45 | report = { 46 | "time": time.time(), 47 | "revision": revision.as_dict(), 48 | "issues": [issue.as_dict() for issue in issues], 49 | "patches": { 50 | patch.analyzer.name: patch.url or patch.path 51 | for patch in revision.improvement_patches 52 | }, 53 | "task_failures": [ 54 | {"name": task.name, "id": task.id} for task in task_failures 55 | ], 56 | } 57 | with open(self.report_path, "w") as f: 58 | json.dump(report, f) 59 | -------------------------------------------------------------------------------- /bot/code_review_bot/report/lando.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import structlog 6 | 7 | from code_review_bot import Level 8 | from code_review_bot.report.base import Reporter 9 | 10 | logger = structlog.get_logger(__name__) 11 | 12 | LANDO_MESSAGE = "The code review bot found {errors} {errors_noun} which should be fixed to avoid backout and {warnings} {warnings_noun}." 13 | 14 | 15 | class LandoReporter(Reporter): 16 | """ 17 | Update lando with a warning message 18 | """ 19 | 20 | def __init__(self, configuration): 21 | self.lando_api = None 22 | 23 | def setup_api(self, lando_api): 24 | logger.info("Publishing warnings to lando is enabled by the bot!") 25 | self.lando_api = lando_api 26 | 27 | def publish(self, issues, revision, task_failures, links, reviewers): 28 | """ 29 | Send an email to administrators 30 | """ 31 | assert ( 32 | revision.phabricator_id and revision.phabricator_phid and revision.diff 33 | ), "Revision must have a Phabricator ID, a PHID and a diff" 34 | 35 | if self.lando_api is None: 36 | logger.info("Lando integration is not set!") 37 | return 38 | 39 | nb_publishable = len([i for i in issues if i.is_publishable()]) 40 | nb_publishable_errors = sum( 41 | 1 for i in issues if i.is_publishable() and i.level == Level.Error 42 | ) 43 | 44 | nb_publishable_warnings = nb_publishable - nb_publishable_errors 45 | 46 | logger.info( 47 | f"Publishing warnings to lando for {nb_publishable_errors} errors and {nb_publishable_warnings} warnings", 48 | revision=revision.phabricator_id, 49 | diff=revision.diff["id"], 50 | ) 51 | 52 | try: 53 | # code-review.events sends an initial warning message to lando to specify that the analysis is in progress, 54 | # we should remove it 55 | self.lando_api.del_all_warnings( 56 | revision.phabricator_id, revision.diff["id"] 57 | ) 58 | 59 | if nb_publishable > 0: 60 | self.lando_api.add_warning( 61 | LANDO_MESSAGE.format( 62 | errors=nb_publishable_errors, 63 | errors_noun="error" if nb_publishable_errors == 1 else "errors", 64 | warnings=nb_publishable_warnings, 65 | warnings_noun="warning" 66 | if nb_publishable_warnings == 1 67 | else "warnings", 68 | ), 69 | revision.phabricator_id, 70 | revision.diff["id"], 71 | ) 72 | except Exception as ex: 73 | logger.error(str(ex), exc_info=True) 74 | -------------------------------------------------------------------------------- /bot/code_review_bot/report/mail.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import structlog 6 | 7 | from code_review_bot import taskcluster 8 | from code_review_bot.config import settings 9 | from code_review_bot.report.base import Reporter 10 | 11 | logger = structlog.get_logger(__name__) 12 | 13 | 14 | EMAIL_STATS_LINE = "* **{analyzer}**: {publishable} publishable ({total} total)" 15 | 16 | EMAIL_HEADER = """ 17 | # Found {publishable} publishable issues ({total} total) 18 | 19 | {stats} 20 | 21 | Review Url: {review_url} 22 | 23 | """ 24 | EMAIL_HEADER_PATCH = "* Improvement patch from {}" 25 | 26 | 27 | class MailReporter(Reporter): 28 | """ 29 | Send an email to admins through Taskcluster service 30 | """ 31 | 32 | def __init__(self, configuration): 33 | (self.emails,) = self.requires(configuration, "emails") 34 | assert len(self.emails) > 0, "Missing emails data" 35 | 36 | # Load TC services & secrets 37 | self.notify = taskcluster.get_service("notify") 38 | 39 | logger.info("Mail report enabled", emails=self.emails) 40 | 41 | def publish(self, issues, revision, task_failures, links, reviewers): 42 | """ 43 | Send an email to administrators 44 | """ 45 | 46 | # For no issues do not publish anything 47 | if len(issues) == 0: 48 | return 49 | 50 | # Build stats display for all issues 51 | # One line per issues class 52 | stats = "\n".join( 53 | [ 54 | EMAIL_STATS_LINE.format( 55 | analyzer=stat["analyzer"], 56 | total=stat["total"], 57 | publishable=stat["publishable"], 58 | ) 59 | for stat in self.calc_stats(issues) 60 | ] 61 | ) 62 | 63 | content = EMAIL_HEADER.format( 64 | total=len(issues), 65 | publishable=sum([i.is_publishable() for i in issues]), 66 | stats=stats, 67 | review_url=revision.url, 68 | ) 69 | if revision.improvement_patches: 70 | content += "## Improvement patches:\n\n{}\n\n".format( 71 | "\n".join( 72 | EMAIL_HEADER_PATCH.format(patch) 73 | for patch in revision.improvement_patches 74 | ) 75 | ) 76 | content += "\n\n".join([i.as_markdown() for i in issues]) 77 | if len(content) > 102400: 78 | # Content is 102400 chars max 79 | content = content[:102000] + "\n\n... Content max limit reached!" 80 | subject = f"[{settings.app_channel}] New Static Analysis {revision}" 81 | for email in self.emails: 82 | self.notify.email( 83 | { 84 | "address": email, 85 | "subject": subject, 86 | "content": content, 87 | "template": "fullscreen", 88 | } 89 | ) 90 | -------------------------------------------------------------------------------- /bot/code_review_bot/report/mail_builderrors.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import structlog 6 | 7 | from code_review_bot import taskcluster 8 | from code_review_bot.report.base import Reporter 9 | 10 | logger = structlog.get_logger(__name__) 11 | 12 | EMAIL_SUBJECT = ( 13 | """Code Review bot found {build_errors} build errors on D{phabricator_id}""" 14 | ) 15 | 16 | EMAIL_HEADER = """ 17 | # [Code Review bot](https://github.com/mozilla/code-review) found {build_errors} build errors on [D{phabricator_id}]({review_url}) 18 | 19 | {content}""" 20 | 21 | 22 | class BuildErrorsReporter(Reporter): 23 | """ 24 | Send an email to the author of the revision in case there are build errors 25 | """ 26 | 27 | def __init__(self, configuration): 28 | # Load TC services 29 | self.notify = taskcluster.get_service("notify") 30 | 31 | logger.info("BuildErrorsReporter report enabled.") 32 | 33 | def publish(self, issues, revision, task_failures, links, reviewers): 34 | """ 35 | Send an email to the author of the revision 36 | """ 37 | assert ( 38 | revision.phabricator_id and revision.phabricator_phid 39 | ), "Revision must have a Phabricator ID and PHID" 40 | assert ( 41 | "attachments" in revision.diff 42 | ), f"Unable to find the commits for revision with phid {revision.phabricator_phid}." 43 | 44 | attachments = revision.diff["attachments"] 45 | 46 | if "commits" not in attachments and "commits" not in attachments["commits"]: 47 | logger.info( 48 | f"Unable to find the commits for revision with phid {revision.phabricator_phid}." 49 | ) 50 | return 51 | 52 | build_errors = [issue for issue in issues if issue.is_build_error()] 53 | 54 | if not build_errors: 55 | logger.info("No build errors encountered.") 56 | return 57 | 58 | content = EMAIL_HEADER.format( 59 | build_errors=len(build_errors), 60 | phabricator_id=revision.phabricator_id, 61 | review_url=revision.url, 62 | content="\n".join([i.as_error() for i in build_errors]), 63 | ) 64 | 65 | if len(content) > 102400: 66 | # Content is 102400 chars max 67 | content = content[:102000] + "\n\n... Content max limit reached!" 68 | 69 | # Get the last commit 70 | commit = attachments["commits"]["commits"][-1] 71 | 72 | if "author" not in commit: 73 | logger.info("Unable to find the author for commit.") 74 | return 75 | 76 | logger.info("Send build error email", to=commit["author"]["email"]) 77 | 78 | # Since we nw know that there is an "author" field we assume that we have "email" 79 | self.notify.email( 80 | { 81 | "address": commit["author"]["email"], 82 | "subject": EMAIL_SUBJECT.format( 83 | build_errors=len(build_errors), 84 | phabricator_id=revision.phabricator_id, 85 | ), 86 | "content": content, 87 | } 88 | ) 89 | -------------------------------------------------------------------------------- /bot/code_review_bot/retrigger.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import requests 6 | 7 | from code_review_bot import taskcluster 8 | from code_review_bot.config import GetAppUserAgent 9 | 10 | TC_INDEX_URL = "https://firefox-ci-tc.services.mozilla.com/api/index/v1/tasks/project.relman.{}.code-review.phabricator" 11 | 12 | 13 | def list_tasks(env): 14 | url = TC_INDEX_URL.format(env) 15 | resp = requests.get(url, headers=GetAppUserAgent()) 16 | resp.raise_for_status() 17 | return list(map(lambda t: t["data"], resp.json()["tasks"])) 18 | 19 | 20 | def is_mach_failure(issue): 21 | return issue["state"] == "error" and issue.get("error_code") == "mach" 22 | 23 | 24 | def is_not_error(issue): 25 | return issue["state"] != "error" 26 | 27 | 28 | def main(env): 29 | taskcluster.auth() 30 | hooks = taskcluster.get_service("hooks") 31 | 32 | # List all tasks on the env 33 | all_tasks = list_tasks(env) 34 | 35 | # List non erroneous tasks 36 | skip_phids = [t["diff_phid"] for t in filter(is_not_error, all_tasks)] 37 | 38 | # Get tasks with a mach failure 39 | tasks = list(filter(is_mach_failure, all_tasks)) 40 | 41 | # Trigger all mach error tasks 42 | total = 0 43 | for task in tasks: 44 | phid = task["diff_phid"] 45 | print("Triggering {} > {}".format(phid, task["title"])) 46 | 47 | if phid in skip_phids: 48 | print(f">> Skipping, phid {phid} has already a non-erroneous task") 49 | continue 50 | 51 | extra_env = {"ANALYSIS_SOURCE": "phabricator", "ANALYSIS_ID": phid} 52 | task = hooks.triggerHook("project-relman", f"code-review-{env}", extra_env) 53 | print(">> New task {}".format(task["status"]["taskId"])) 54 | total += 1 55 | 56 | print(f"Triggered {total} tasks") 57 | 58 | 59 | if __name__ == "__main__": 60 | main("production") 61 | -------------------------------------------------------------------------------- /bot/code_review_bot/stats.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import atexit 6 | import time 7 | from contextlib import contextmanager 8 | from datetime import datetime 9 | 10 | import structlog 11 | from influxdb import InfluxDBClient 12 | 13 | from code_review_bot.config import settings 14 | from code_review_bot.tasks.base import AnalysisTask 15 | 16 | logger = structlog.get_logger(__name__) 17 | 18 | 19 | class InfluxDb: 20 | """ 21 | Log metrics using InfluxDb REST api 22 | """ 23 | 24 | def __init__(self): 25 | self.client = None 26 | self.metrics = [] 27 | 28 | # Always flush at the end of the execution 29 | atexit.register(self.flush) 30 | 31 | def auth(self, conf): 32 | assert settings.app_channel is not None, "Missing app channel" 33 | self.client = InfluxDBClient( 34 | conf["host"], 35 | conf["port"], 36 | conf["username"], 37 | conf["password"], 38 | conf["database"], 39 | ssl=conf.get("ssl", False), 40 | verify_ssl=conf.get("ssl", False), 41 | ) 42 | assert self.client.ping() 43 | logger.info( 44 | "InfluxDb reporting enabled", database=conf["database"], host=conf["host"] 45 | ) 46 | 47 | def add_metric(self, name, value=1, tags={}): 48 | """ 49 | Store a metric in memory, using InfluxDb point format 50 | """ 51 | tags.update({"app": "code-review-bot", "channel": settings.app_channel}) 52 | self.metrics.append( 53 | { 54 | "measurement": f"code-review.{name}", 55 | "tags": tags, 56 | "time": datetime.utcnow().isoformat(), 57 | "fields": {"value": value}, 58 | } 59 | ) 60 | 61 | def flush(self): 62 | """ 63 | Publish all metrics in memory to influxdb 64 | """ 65 | if self.client is None: 66 | logger.warning( 67 | "InfluxDb client not connected: metrics will not be reported" 68 | ) 69 | return 70 | if not self.metrics: 71 | return 72 | 73 | logger.info("Flushing stats metrics", nb=len(self.metrics)) 74 | # TODO: add a retry ? 75 | self.client.write_points(self.metrics) 76 | self.metrics = [] 77 | 78 | def report_task(self, task, issues): 79 | """ 80 | Aggregate statistics about issues from a remote analysis task 81 | """ 82 | assert isinstance(task, AnalysisTask) 83 | tags = {"task": task.name} 84 | 85 | # Report all issues found 86 | self.add_metric("issues", len(issues), tags) 87 | 88 | # Report publishable issues 89 | self.add_metric( 90 | "issues.publishable", sum(i.is_publishable() for i in issues), tags 91 | ) 92 | 93 | # Report total paths 94 | self.add_metric("issues.paths", len({i.path for i in issues}), tags) 95 | 96 | @contextmanager 97 | def timer(self, name): 98 | """ 99 | A context manager tracking the contained code's runtime 100 | """ 101 | start = time.perf_counter() 102 | try: 103 | yield 104 | finally: 105 | end = time.perf_counter() 106 | self.add_metric(name, end - start) 107 | -------------------------------------------------------------------------------- /bot/code_review_bot/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/bot/code_review_bot/tasks/__init__.py -------------------------------------------------------------------------------- /bot/code_review_bot/tasks/coverage.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import structlog 4 | 5 | from code_review_bot import Issue, Level 6 | from code_review_bot.config import settings 7 | from code_review_bot.tasks.base import AnalysisTask 8 | 9 | logger = structlog.get_logger(__name__) 10 | 11 | ISSUE_MARKDOWN = """ 12 | ## coverage problem 13 | 14 | - **Path**: {path} 15 | - **Publishable**: {publishable} 16 | 17 | ``` 18 | {message} 19 | ``` 20 | """ 21 | 22 | 23 | class CoverageIssue(Issue): 24 | def __init__(self, analyzer, path, lineno, message, revision): 25 | super().__init__( 26 | analyzer, 27 | revision, 28 | path, 29 | line=lineno and int(lineno) or None, 30 | nb_lines=1, 31 | check="no-coverage", 32 | level=Level.Warning, 33 | message=message, 34 | ) 35 | 36 | def is_publishable(self): 37 | """ 38 | Coverage issues are always publishable, unless they are in header files 39 | """ 40 | return self.validates() 41 | 42 | def validates(self): 43 | """ 44 | Coverage issues are always publishable, unless they are in header files 45 | """ 46 | _, ext = os.path.splitext(self.path) 47 | return ext.lower() in settings.cpp_extensions.union(settings.js_extensions) 48 | 49 | def as_text(self): 50 | """ 51 | Build the text content for reporters 52 | """ 53 | return self.message 54 | 55 | def as_markdown(self): 56 | """ 57 | Build the Markdown content for the debug email 58 | """ 59 | return ISSUE_MARKDOWN.format( 60 | path=self.path, 61 | message=self.message, 62 | publishable=self.is_publishable() and "yes" or "no", 63 | ) 64 | 65 | 66 | class ZeroCoverageTask(AnalysisTask): 67 | """ 68 | List all issues found by coverage analysis on specified files 69 | Uses the most recent data from the code coverage bot 70 | """ 71 | 72 | route = "project.relman.code-coverage.production.cron.latest" 73 | artifacts = ["public/zero_coverage_report.json"] 74 | 75 | @property 76 | def display_name(self): 77 | return "code coverage analysis" 78 | 79 | def parse_issues(self, artifacts, revision): 80 | zero_coverage_files = { 81 | file_info["name"] 82 | for artifact in artifacts.values() 83 | for file_info in artifact["files"] 84 | if file_info["uncovered"] 85 | } 86 | 87 | return [ 88 | CoverageIssue(self, path, 0, "This file is uncovered", revision) 89 | for path in revision.files 90 | if path in zero_coverage_files 91 | ] 92 | -------------------------------------------------------------------------------- /bot/code_review_bot/tasks/default.py: -------------------------------------------------------------------------------- 1 | import structlog 2 | 3 | from code_review_bot import Issue, Level, taskcluster 4 | from code_review_bot.tasks.base import AnalysisTask 5 | 6 | logger = structlog.get_logger(__name__) 7 | 8 | ISSUE_MARKDOWN = """ 9 | ## issue {analyzer} 10 | 11 | - **Path**: {path} 12 | - **Level**: {level} 13 | - **Check**: {check} 14 | - **Line**: {line} 15 | - **Publishable**: {publishable} 16 | 17 | ``` 18 | {message} 19 | ``` 20 | """ 21 | 22 | 23 | class DefaultIssue(Issue): 24 | def validates(self): 25 | """ 26 | Default issues are valid as long as they match the format 27 | """ 28 | return True 29 | 30 | def as_text(self): 31 | """ 32 | Build the text content for reporters 33 | """ 34 | return f"{self.level.name}: {self.message} [{self.check}]" 35 | 36 | def as_markdown(self): 37 | """ 38 | Build the Markdown content for debug email 39 | """ 40 | return ISSUE_MARKDOWN.format( 41 | analyzer=self.analyzer.name, 42 | path=self.path, 43 | check=self.check, 44 | level=self.level.value, 45 | line=self.line, 46 | message=self.message, 47 | publishable=self.is_publishable() and "yes" or "no", 48 | ) 49 | 50 | 51 | class DefaultTask(AnalysisTask): 52 | """ 53 | Support issues using the code review format 54 | https://github.com/mozilla/code-review/blob/master/docs/analysis_format.md 55 | """ 56 | 57 | artifacts = ["public/code-review/issues.json"] 58 | 59 | def parse_issues(self, artifacts, revision): 60 | """ 61 | Parse issues from a log file content 62 | """ 63 | assert isinstance(artifacts, dict) 64 | 65 | def default_check(issue): 66 | # Use analyzer name when check is not provided 67 | # This happens for analyzers who only have one rule 68 | # This logic could become the standard once most analyzers 69 | # use that format 70 | check = issue.get("check") 71 | if check: 72 | return check 73 | return issue.get("analyzer", self.name) 74 | 75 | return [ 76 | DefaultIssue( 77 | analyzer=self, 78 | revision=revision, 79 | path=issue["path"], 80 | line=issue["line"], 81 | column=issue["column"], 82 | nb_lines=issue.get("nb_lines", 1), 83 | level=Level(issue["level"]), 84 | check=default_check(issue), 85 | message=issue["message"], 86 | ) 87 | for artifact in artifacts.values() 88 | for _, path_issues in artifact.items() 89 | for issue in path_issues 90 | ] 91 | 92 | @staticmethod 93 | def matches(task_id): 94 | """ 95 | Check if the default task can work on a task 96 | * Lookup the available latest artifacts 97 | * Check if any artifact matches the official default path 98 | """ 99 | queue = taskcluster.get_service("queue") 100 | result = queue.listLatestArtifacts(task_id) 101 | if "artifacts" not in result: 102 | return False 103 | 104 | names = set(artifact["name"] for artifact in result["artifacts"]) 105 | return len(names.intersection(DefaultTask.artifacts)) > 0 106 | -------------------------------------------------------------------------------- /bot/code_review_bot/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/bot/code_review_bot/tools/__init__.py -------------------------------------------------------------------------------- /bot/code_review_bot/tools/libmozdata.py: -------------------------------------------------------------------------------- 1 | from importlib.metadata import version 2 | 3 | import structlog 4 | from libmozdata.config import Config, set_config 5 | 6 | logger = structlog.get_logger(__name__) 7 | 8 | 9 | class LocalConfig(Config): 10 | """ 11 | Provide required configuration for libmozdata 12 | using in-memory class instead of an INI file 13 | """ 14 | 15 | def __init__(self, name, version): 16 | self.user_agent = f"{name}/{version}" 17 | logger.debug(f"User agent is {self.user_agent}") 18 | 19 | def get(self, section, option, default=None, **kwargs): 20 | if section == "User-Agent" and option == "name": 21 | return self.user_agent 22 | 23 | return default 24 | 25 | 26 | def setup(package_name): 27 | # Get version for main package 28 | package_version = version(package_name) 29 | 30 | # Provide to custom libzmodata configuration 31 | set_config(LocalConfig(package_name, package_version)) 32 | -------------------------------------------------------------------------------- /bot/code_review_bot/tools/treeherder.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import urlencode 2 | 3 | JOBS_URL = "https://treeherder.mozilla.org/#/jobs" 4 | 5 | 6 | def get_job_url(repository, revision, task_id=None, run_id=None, **params): 7 | """Build a Treeherder job url for a given Taskcluster task""" 8 | assert isinstance(repository, str) and repository, "Missing repository" 9 | assert isinstance(revision, str) and revision, "Missing revision" 10 | assert "repo" not in params, "repo cannot be set in params" 11 | assert "revision" not in params, "revision cannot be set in params" 12 | 13 | params.update({"repo": repository, "revision": revision}) 14 | 15 | if task_id is not None and run_id is not None: 16 | params["selectedTaskRun"] = f"{task_id}-{run_id}" 17 | 18 | return f"{JOBS_URL}?{urlencode(params)}" 19 | -------------------------------------------------------------------------------- /bot/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.13.3-slim 2 | 3 | # TODO: remove by switching to modern pyproject.toml 4 | RUN pip install setuptools==74.1.2 --disable-pip-version-check --no-cache-dir --quiet 5 | 6 | ADD bot /src/bot 7 | RUN cd /src/bot && python setup.py install 8 | 9 | # Add mercurial & robustcheckout 10 | ADD tools/docker /src/tools/docker 11 | RUN /src/tools/docker/bootstrap-mercurial.sh 12 | 13 | CMD ["code-review-bot"] 14 | -------------------------------------------------------------------------------- /bot/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pre-commit==4.2.0 2 | pytest==8.3.5 3 | pytest-responses==0.5.1 4 | pytest-structlog==1.1 5 | responses==0.25.7 6 | -------------------------------------------------------------------------------- /bot/requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp<4 2 | influxdb==5.3.2 3 | libmozdata==0.2.10 4 | libmozevent==1.1.33 5 | python-hglib==2.6.2 6 | pyyaml==6.0.2 7 | sentry-sdk==2.29.1 8 | setuptools==80.8.0 9 | structlog==25.3.0 10 | -------------------------------------------------------------------------------- /bot/setup.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import setuptools 6 | 7 | 8 | def read_requirements(file_): 9 | lines = [] 10 | with open(file_) as f: 11 | for line in f.readlines(): 12 | line = line.strip() 13 | if ( 14 | line.startswith("-e ") 15 | or line.startswith("http://") 16 | or line.startswith("https://") 17 | ): 18 | extras = "" 19 | if "[" in line: 20 | extras = "[" + line.split("[")[1].split("]")[0] + "]" 21 | line = line.split("#")[1].split("egg=")[1] + extras 22 | elif line == "" or line.startswith("#") or line.startswith("-"): 23 | continue 24 | line = line.split("#")[0].strip() 25 | lines.append(line) 26 | return sorted(list(set(lines))) 27 | 28 | 29 | with open("VERSION") as f: 30 | VERSION = f.read().strip() 31 | 32 | 33 | setuptools.setup( 34 | name="code_review_bot", 35 | version=VERSION, 36 | description="Reports issues found in Mozilla code review tasks", 37 | author="Mozilla Release Management", 38 | author_email="release-mgmt-analysis@mozilla.com", 39 | url="https://github.com/mozilla/code-review", 40 | tests_require=read_requirements("requirements-dev.txt"), 41 | install_requires=read_requirements("requirements.txt"), 42 | packages=setuptools.find_packages(), 43 | include_package_data=True, 44 | zip_safe=False, 45 | license="MPL2", 46 | entry_points={"console_scripts": ["code-review-bot = code_review_bot.cli:main"]}, 47 | ) 48 | -------------------------------------------------------------------------------- /bot/taskcluster-hook.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { 4 | "exchange": "exchange/taskcluster-queue/v1/task-completed", 5 | "routingKeyPattern": "route.project.relman.codereview.v1.try_ending" 6 | } 7 | ], 8 | "metadata": { 9 | "description": "Automatically create code review publication tasjs", 10 | "emailOnError": true, 11 | "name": "Code review hook (CHANNEL)", 12 | "owner": "babadie@mozilla.com" 13 | }, 14 | "schedule": [], 15 | "task": { 16 | "created": { 17 | "$fromNow": "0 seconds" 18 | }, 19 | "deadline": { 20 | "$fromNow": "2 hours" 21 | }, 22 | "expires": { 23 | "$fromNow": "1 month" 24 | }, 25 | "extra": {}, 26 | "metadata": { 27 | "description": "Publish issues detected in remote tasks", 28 | "name": "Code review publication (CHANNEL)", 29 | "owner": "babadie@mozilla.com", 30 | "source": "https://github.com/mozilla/code-review" 31 | }, 32 | "payload": { 33 | "artifacts": { 34 | "public/results": { 35 | "path": "/tmp/results", 36 | "type": "directory" 37 | } 38 | }, 39 | "cache": {}, 40 | "capabilities": {}, 41 | "command": [ 42 | "code-review-bot", 43 | "--taskcluster-secret", 44 | "project/relman/code-review/runtime-CHANNEL" 45 | ], 46 | "env": { 47 | "$merge": [ 48 | { 49 | "$if": "firedBy == 'triggerHook'", 50 | "else": {}, 51 | "then": { 52 | "$eval": "payload" 53 | } 54 | }, 55 | { 56 | "$if": "firedBy == 'pulseMessage'", 57 | "else": {}, 58 | "then": { 59 | "TRY_RUN_ID": { 60 | "$eval": "payload.runId" 61 | }, 62 | "TRY_TASK_GROUP_ID": { 63 | "$eval": "payload.status.taskGroupId" 64 | }, 65 | "TRY_TASK_ID": { 66 | "$eval": "payload.status.taskId" 67 | } 68 | } 69 | } 70 | ] 71 | }, 72 | "features": { 73 | "taskclusterProxy": true 74 | }, 75 | "image": "mozilla/code-review:REVISION", 76 | "maxRunTime": 7200 77 | }, 78 | "priority": "normal", 79 | "provisionerId": "aws-provisioner-v1", 80 | "retries": 3, 81 | "routes": ["index.project.relman.CHANNEL.code-review.latest"], 82 | "schedulerId": "-", 83 | "scopes": [ 84 | "secrets:get:project/relman/code-review/runtime-CHANNEL", 85 | "index:insert-task:project.relman.CHANNEL.code-review.*", 86 | "notify:email:*" 87 | ], 88 | "tags": {}, 89 | "workerType": "relman-svc" 90 | }, 91 | "triggerSchema": { 92 | "additionalProperties": true, 93 | "type": "object" 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /bot/tests/fixtures/clang_format.diff: -------------------------------------------------------------------------------- 1 | --- gfx/2d/Factory.cpp 2020-03-31 13:19:38.060336000 +0000 2 | +++ gfx/2d/Factory.cpp 2020-03-31 13:22:36.680336000 +0000 3 | @@ -613,8 +613,12 @@ 4 | #ifdef XP_DARWIN 5 | // Trigger clang-format 6 | already_AddRefed Factory::CreateScaledFontForMacFont( 7 | - CGFontRef aCGFont, const RefPtr& aUnscaledFont, Float aSize, const DeviceColor& aFontSmoothingBackgroundColor, bool aUseFontSmoothing, bool aApplySyntheticBold) { 8 | - return MakeAndAddRef(aCGFont, aUnscaledFont, aSize, false, aFontSmoothingBackgroundColor, aUseFontSmoothing, aApplySyntheticBold); 9 | + CGFontRef aCGFont, const RefPtr& aUnscaledFont, Float aSize, 10 | + const DeviceColor& aFontSmoothingBackgroundColor, bool aUseFontSmoothing, 11 | + bool aApplySyntheticBold) { 12 | + return MakeAndAddRef(aCGFont, aUnscaledFont, aSize, false, 13 | + aFontSmoothingBackgroundColor, 14 | + aUseFontSmoothing, aApplySyntheticBold); 15 | } 16 | #endif 17 | 18 | 19 | --- dom/canvas/ClientWebGLContext.cpp 2020-03-31 13:19:30.392336000 +0000 20 | +++ dom/canvas/ClientWebGLContext.cpp 2020-03-31 13:22:36.280336000 +0000 21 | @@ -115,10 +115,10 @@ 22 | aHandle.Value()); 23 | return mNotLost->outOfProcess->mWebGLChild->SendUpdateCompositableHandle( 24 | aLayerTransaction, aHandle); 25 | - }else { 26 | - // Comment to trigger readability-else-after-return 27 | - const auto x = "aa"; 28 | - } 29 | + } else { 30 | + // Comment to trigger readability-else-after-return 31 | + const auto x = "aa"; 32 | + } 33 | return true; 34 | } 35 | 36 | 37 | --- accessible/xul/XULAlertAccessible.cpp 2020-03-31 13:19:24.204336000 +0000 38 | +++ accessible/xul/XULAlertAccessible.cpp 2020-03-31 13:22:35.132336000 +0000 39 | @@ -33,7 +33,7 @@ 40 | // Screen readers need to read contents of alert, not the accessible name. 41 | // If we have both some screen readers will read the alert twice. 42 | aName.Truncate(); 43 | - if (false) return true; 44 | + if (false) return true; 45 | return eNameOK; 46 | } 47 | 48 | 49 | -------------------------------------------------------------------------------- /bot/tests/fixtures/mozlint_license_no_check.json: -------------------------------------------------------------------------------- 1 | { 2 | "intl/locale/rust/unic-langid-ffi/src/lib.rs": [ 3 | { 4 | "rule": null, 5 | "linter": "license", 6 | "column": null, 7 | "lineoffset": null, 8 | "lineno": 0, 9 | "path": "intl/locale/rust/unic-langid-ffi/src/lib.rs", 10 | "relpath": "intl/locale/rust/unic-langid-ffi/src/lib.rs", 11 | "level": "error", 12 | "diff": null, 13 | "source": null, 14 | "hint": null, 15 | "message": "No matching license strings found in tools/lint/license/valid-licenses.txt" 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /bot/tests/fixtures/sentry_event_after.json: -------------------------------------------------------------------------------- 1 | { 2 | "breadcrumbs": { 3 | "values": [ 4 | { 5 | "category": "test.log", 6 | "data": { 7 | "asctime": "2024-08-01 16:31:37" 8 | }, 9 | "level": "info", 10 | "message": "[info ] This is Info ", 11 | "timestamp": "2024-08-01T14:31:37.784756Z", 12 | "type": "log" 13 | }, 14 | { 15 | "category": "test.log", 16 | "data": { 17 | "asctime": "2024-08-01 16:31:37" 18 | }, 19 | "level": "warning", 20 | "message": "[warning ] This is warning ", 21 | "timestamp": "2024-08-01T14:31:37.785234Z", 22 | "type": "log" 23 | } 24 | ] 25 | }, 26 | "contexts": { 27 | "runtime": { 28 | "build": "3.10.12 (main, Mar 22 2024, 16:50:05) [GCC 11.4.0]", 29 | "name": "CPython", 30 | "version": "3.10.12" 31 | }, 32 | "trace": { 33 | "dynamic_sampling_context": { 34 | "environment": "dev", 35 | "public_key": "3e4688ab0490667ddf863da3c8b074b3", 36 | "release": "1.6.2", 37 | "trace_id": "d6c4832c2380477c9f28efb0c5e36cdb" 38 | }, 39 | "parent_span_id": null, 40 | "span_id": "83670bdf42178356", 41 | "trace_id": "d6c4832c2380477c9f28efb0c5e36cdb" 42 | } 43 | }, 44 | "environment": "dev", 45 | "event_id": "6d93a346184743359d003cefb0371ce5", 46 | "extra": { 47 | "asctime": "2024-08-01 16:31:37", 48 | "sys.argv": [ 49 | "log.py" 50 | ] 51 | }, 52 | "level": "error", 53 | "logentry": { 54 | "message": "[error ] This is error my=arg test=ok", 55 | "params": [] 56 | }, 57 | "logger": "test.log", 58 | "modules": { 59 | "aiohttp": "3.9.5", 60 | "aiosignal": "1.3.1", 61 | "attrs": "22.2.0", 62 | "certifi": "2022.12.7", 63 | "charset-normalizer": "2.1.1", 64 | "code-review-bot": "1.6.2", 65 | "code-review-tools": "0.2.0", 66 | "frozenlist": "1.3.3", 67 | "icalendar": "5.0.4", 68 | "idna": "3.4", 69 | "influxdb": "5.3.1", 70 | "libmozdata": "0.1.83", 71 | "mohawk": "1.1.0", 72 | "msgpack": "1.0.4", 73 | "multidict": "6.0.5", 74 | "pip": "22.0.2", 75 | "python-dateutil": "2.8.2", 76 | "python-hglib": "2.6.2", 77 | "pytz": "2022.7.1", 78 | "pyyaml": "6.0", 79 | "requests": "2.28.2", 80 | "requests-futures": "1.0.0", 81 | "rs_parsepatch": "0.4.0", 82 | "sentry-sdk": "2.11.0", 83 | "setuptools": "59.6.0", 84 | "six": "1.16.0", 85 | "slugid": "2.0.0", 86 | "structlog": "24.4.0", 87 | "taskcluster": "67.1.0", 88 | "taskcluster-urls": "13.0.1", 89 | "treeherder-client": "5.0.0", 90 | "urllib3": "1.26.14", 91 | "whatthepatch": "1.0.4", 92 | "wheel": "0.37.1", 93 | "yarl": "1.9.4" 94 | }, 95 | "platform": "python", 96 | "release": "1.6.2", 97 | "sdk": { 98 | "integrations": [ 99 | "aiohttp", 100 | "argv", 101 | "atexit", 102 | "dedupe", 103 | "excepthook", 104 | "logging", 105 | "modules", 106 | "stdlib", 107 | "threading" 108 | ], 109 | "name": "sentry.python.aiohttp", 110 | "packages": [ 111 | { 112 | "name": "pypi:sentry-sdk", 113 | "version": "2.11.0" 114 | } 115 | ], 116 | "version": "2.11.0" 117 | }, 118 | "server_name": "bot", 119 | "tags": { 120 | "site": "unknown" 121 | }, 122 | "timestamp": "2024-08-01T14:31:37.819483Z", 123 | "transaction_info": {} 124 | } -------------------------------------------------------------------------------- /bot/tests/mocks/config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | target: obj-x86_64-pc-linux-gnu 3 | clang_checkers: 4 | - name: -* 5 | publish: !!bool no 6 | - name: clang-analyzer-deadcode.DeadStores 7 | publish: !!bool yes 8 | - name: clang-analyzer-security.* 9 | publish: !!bool no 10 | - name: modernize-use-nullptr 11 | publish: !!bool yes 12 | reason: "Modernize our code base to C++11" 13 | 14 | third_party: 3rdparty.txt 15 | -------------------------------------------------------------------------------- /bot/tests/mocks/hgmo_hello1: -------------------------------------------------------------------------------- 1 | def hello(): 2 | print("Hello !") 3 | 4 | # Same line as above, without indent 5 | print("Hello !") 6 | -------------------------------------------------------------------------------- /bot/tests/mocks/hgmo_integration-autoland_deadbeef123.json: -------------------------------------------------------------------------------- 1 | { 2 | "node": "deadbeef123", 3 | "date": [ 4 | 1575279254.0, 5 | 0 6 | ], 7 | "desc": "Bug XXYYZZ - Demo bug\nDifferential Revision: https://phabricator.services.mozilla.com/D123", 8 | "backedoutby": "", 9 | "branch": "default", 10 | "bookmarks": [], 11 | "tags": [], 12 | "user": "John Doe ", 13 | "parents": [ 14 | "7e66846136d8871b7c0620cd7b5314e7c8d76a18" 15 | ], 16 | "phase": "public", 17 | "pushid": 112233, 18 | "pushdate": [ 19 | 1575284673, 20 | 0 21 | ], 22 | "pushuser": "john@doe.com", 23 | "landingsystem": "lando" 24 | } 25 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "phid": "PHID-USER-test1234", 4 | "userName": "Tester", 5 | "realName": "Mr. Tester", 6 | "image": "https://cdn.test/avatar.png", 7 | "uri": "https://phabricator.test/p/Tester/", 8 | "roles": [ 9 | "verified", 10 | "approved", 11 | "activated" 12 | ], 13 | "primaryEmail": "test@mozilla.com" 14 | }, 15 | "error_code": null, 16 | "error_info": null 17 | } 18 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_build_search.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "data": [ 4 | { 5 | "fields": { 6 | "buildablePHID": "PHID-HMXX-test" 7 | }, 8 | "phid": "PHID-HMBD-test" 9 | } 10 | ], 11 | "maps": {}, 12 | "query": { 13 | "queryKey": null 14 | }, 15 | "cursor": { 16 | "limit": 100, 17 | "after": null, 18 | "before": null, 19 | "order": null 20 | } 21 | }, 22 | "error_code": null, 23 | "error_info": null 24 | } 25 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_buildable_search.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "data": [ 4 | { 5 | "fields": { 6 | "objectPHID": "PHID-DIFF-test" 7 | }, 8 | "phid": "PHID-HMBB-test" 9 | } 10 | ], 11 | "maps": {}, 12 | "query": { 13 | "queryKey": null 14 | }, 15 | "cursor": { 16 | "limit": 100, 17 | "after": null, 18 | "before": null, 19 | "order": null 20 | } 21 | }, 22 | "error_code": null, 23 | "error_info": null 24 | } 25 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_createinline.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "id": 17073, 4 | "authorPHID": "PHID-USER-qvozgl7osyqxo226fzye", 5 | "filePath": ".arcconfig", 6 | "isNewFile": false, 7 | "lineNumber": 2, 8 | "lineLength": 0, 9 | "diffID": "1158", 10 | "content": "Such comment, wow" 11 | }, 12 | "error_code": null, 13 | "error_info": null 14 | } -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_diff_query.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": [ 3 | { 4 | "id": "51", 5 | "phid": "PHID-DREV-wsdktdyp2prfxmlzu4vm", 6 | "title": "Bug 1387052 - Make Firefox code worse intentionally. r=clangbot", 7 | "uri": "https://phabricator.services.mozilla.com/D389", 8 | "dateCreated": "1516026749", 9 | "dateModified": "1516035024", 10 | "authorPHID": "PHID-USER-j3olszo6rgfclrn7abj4", 11 | "status": "0", 12 | "statusName": "Needs Review", 13 | "properties": [], 14 | "branch": "default", 15 | "summary": "MozReview-Commit-ID: 65EixGF10HH", 16 | "testPlan": "", 17 | "lineCount": "255", 18 | "activeDiffPHID": "PHID-DIFF-xiiqglwqw2tjrqdb3g3w", 19 | "diffs": [ 20 | "42" 21 | ], 22 | "commits": [], 23 | "reviewers": [], 24 | "ccs": [ 25 | "PHID-USER-2pljsekqcs4ysqgc7pxm" 26 | ], 27 | "hashes": [ 28 | [ 29 | "hgcm", 30 | "coffeedeadbeef123456789" 31 | ] 32 | ], 33 | "auxiliary": { 34 | "bugzilla.bug-id": "", 35 | "phabricator:projects": [], 36 | "phabricator:depends-on": [] 37 | }, 38 | "repositoryPHID": "PHID-REPO-saax4qdxlbbhahhp2kg5" 39 | } 40 | ], 41 | "error_code": null, 42 | "error_info": null 43 | } 44 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_diff_search_PHID-DIFF-test.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "data": [ 4 | { 5 | "id": 42, 6 | "type": "DIFF", 7 | "phid": "PHID-DIFF-testABcd12", 8 | "fields": { 9 | "revisionPHID": "PHID-DREV-zzzzz", 10 | "authorPHID": "PHID-USER-xxxxx", 11 | "repositoryPHID": "PHID-REPO-aaaaaa", 12 | "refs": [ 13 | { 14 | "type": "branch", 15 | "name": "default" 16 | }, 17 | { 18 | "type": "base", 19 | "identifier": "coffeedeadbeef123456789" 20 | } 21 | ], 22 | "dateCreated": 1510251135, 23 | "dateModified": 1510251138, 24 | "policy": { 25 | "view": "public" 26 | } 27 | }, 28 | "attachments": { 29 | "commits": { 30 | "commits": [ 31 | { 32 | "identifier": "dcfebedebe9f18bce45f9bfc0339fe8255e5468b", 33 | "tree": null, 34 | "parents": [ 35 | "083ea767da9b11d52d71c227415d2a1b4fab6173" 36 | ], 37 | "author": { 38 | "name": "test", 39 | "email": "test@mozilla.com", 40 | "raw": "\"test\" ", 41 | "epoch": 0 42 | }, 43 | "message": "create update for phabsend test\n\nDifferential Revision: https://phabricator-dev.allizom.org/D1354" 44 | } 45 | ] 46 | } 47 | } 48 | } 49 | ], 50 | "maps": {}, 51 | "query": { 52 | "queryKey": null 53 | }, 54 | "cursor": { 55 | "limit": 100, 56 | "after": null, 57 | "before": null, 58 | "order": null 59 | } 60 | }, 61 | "error_code": null, 62 | "error_info": null 63 | } 64 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_diff_search_PHID-DIFF-testABcd12.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "data": [ 4 | { 5 | "id": 456, 6 | "type": "DIFF", 7 | "phid": "PHID-DIFF-testABcd12", 8 | "fields": { 9 | "revisionPHID": "PHID-DREV-zzzzz", 10 | "authorPHID": "PHID-USER-xxxxx", 11 | "repositoryPHID": "PHID-REPO-autoland", 12 | "refs": [ 13 | { 14 | "type": "branch", 15 | "name": "default" 16 | }, 17 | { 18 | "type": "base", 19 | "identifier": "deadbeef123" 20 | } 21 | ], 22 | "dateCreated": 1510251135, 23 | "dateModified": 1510251138, 24 | "policy": { 25 | "view": "public" 26 | } 27 | }, 28 | "attachments": { 29 | "commits": { 30 | "commits": [ 31 | { 32 | "identifier": "deadbeef123", 33 | "tree": null, 34 | "parents": [ 35 | "someNode" 36 | ], 37 | "author": { 38 | "name": "test", 39 | "email": "test@mozilla.com", 40 | "raw": "\"test\" ", 41 | "epoch": 0 42 | }, 43 | "message": "Random commit message" 44 | } 45 | ] 46 | } 47 | } 48 | } 49 | ], 50 | "maps": {}, 51 | "query": { 52 | "queryKey": null 53 | }, 54 | "cursor": { 55 | "limit": 100, 56 | "after": null, 57 | "before": null, 58 | "order": null 59 | } 60 | }, 61 | "error_code": null, 62 | "error_info": null 63 | } 64 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_diff_search_PHID-DREV-azcDeadbeef.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "data": [ 4 | { 5 | "id": 1, 6 | "type": "DIFF", 7 | "phid": "PHID-DIFF-autoland", 8 | "fields": { 9 | "revisionPHID": "PHID-DREV-azcDeadbeef", 10 | "authorPHID": "PHID-USER-xxxxx", 11 | "repositoryPHID": "PHID-REPO-autoland", 12 | "refs": [ 13 | { 14 | "type": "branch", 15 | "name": "default" 16 | }, 17 | { 18 | "type": "base", 19 | "identifier": "deadbeef123" 20 | } 21 | ], 22 | "dateCreated": 1510251135, 23 | "dateModified": 1510251138, 24 | "policy": { 25 | "view": "public" 26 | } 27 | }, 28 | "attachments": { 29 | "commits": { 30 | "commits": [ 31 | { 32 | "identifier": "deadbeef123", 33 | "tree": null, 34 | "parents": [ 35 | "someNode" 36 | ], 37 | "author": { 38 | "name": "test", 39 | "email": "test@mozilla.com", 40 | "raw": "\"test\" ", 41 | "epoch": 0 42 | }, 43 | "message": "Random commit message" 44 | } 45 | ] 46 | } 47 | } 48 | } 49 | ], 50 | "maps": {}, 51 | "query": { 52 | "queryKey": null 53 | }, 54 | "cursor": { 55 | "limit": 100, 56 | "after": null, 57 | "before": null, 58 | "order": null 59 | } 60 | }, 61 | "error_code": null, 62 | "error_info": null 63 | } 64 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_diff_search_PHID-DREV-zzzzz-updated.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "data": [ 4 | { 5 | "id": 43, 6 | "type": "DIFF", 7 | "phid": "PHID-DIFF-autoland", 8 | "fields": { 9 | "revisionPHID": "PHID-DREV-zzzzz", 10 | "authorPHID": "PHID-USER-xxxxx", 11 | "repositoryPHID": "PHID-REPO-autoland", 12 | "refs": [ 13 | { 14 | "type": "branch", 15 | "name": "default" 16 | }, 17 | { 18 | "type": "base", 19 | "identifier": "deadbeef123" 20 | } 21 | ], 22 | "dateCreated": 1510251135, 23 | "dateModified": 1510251138, 24 | "policy": { 25 | "view": "public" 26 | } 27 | }, 28 | "attachments": { 29 | "commits": { 30 | "commits": [ 31 | { 32 | "identifier": "deadbeef123", 33 | "tree": null, 34 | "parents": [ 35 | "someNode" 36 | ], 37 | "author": { 38 | "name": "test", 39 | "email": "test@mozilla.com", 40 | "raw": "\"test\" ", 41 | "epoch": 0 42 | }, 43 | "message": "Random commit message" 44 | } 45 | ] 46 | } 47 | } 48 | } 49 | ], 50 | "maps": {}, 51 | "query": { 52 | "queryKey": null 53 | }, 54 | "cursor": { 55 | "limit": 100, 56 | "after": null, 57 | "before": null, 58 | "order": null 59 | } 60 | }, 61 | "error_code": null, 62 | "error_info": null 63 | } 64 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_diff_search_PHID-DREV-zzzzz.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "data": [ 4 | { 5 | "id": 1, 6 | "type": "DIFF", 7 | "phid": "PHID-DIFF-autoland", 8 | "fields": { 9 | "revisionPHID": "PHID-DREV-zzzzz", 10 | "authorPHID": "PHID-USER-xxxxx", 11 | "repositoryPHID": "PHID-REPO-autoland", 12 | "refs": [ 13 | { 14 | "type": "branch", 15 | "name": "default" 16 | }, 17 | { 18 | "type": "base", 19 | "identifier": "deadbeef123" 20 | } 21 | ], 22 | "dateCreated": 1510251135, 23 | "dateModified": 1510251138, 24 | "policy": { 25 | "view": "public" 26 | } 27 | }, 28 | "attachments": { 29 | "commits": { 30 | "commits": [ 31 | { 32 | "identifier": "deadbeef123", 33 | "tree": null, 34 | "parents": [ 35 | "someNode" 36 | ], 37 | "author": { 38 | "name": "test", 39 | "email": "test@mozilla.com", 40 | "raw": "\"test\" ", 41 | "epoch": 0 42 | }, 43 | "message": "Random commit message" 44 | } 45 | ] 46 | } 47 | } 48 | } 49 | ], 50 | "maps": {}, 51 | "query": { 52 | "queryKey": null 53 | }, 54 | "cursor": { 55 | "limit": 100, 56 | "after": null, 57 | "before": null, 58 | "order": null 59 | } 60 | }, 61 | "error_code": null, 62 | "error_info": null 63 | } 64 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_edge_search.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "data": [ 4 | ], 5 | "maps": {}, 6 | "query": { 7 | }, 8 | "cursor": { 9 | "limit": 100, 10 | "after": null, 11 | "before": null, 12 | "order": null 13 | } 14 | }, 15 | "error_code": null, 16 | "error_info": null 17 | } 18 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_patch.diff: -------------------------------------------------------------------------------- 1 | diff --git a/test.txt b/test.txt 2 | index 557db03..5eb0bec 100644 3 | --- a/test.txt 4 | +++ b/test.txt 5 | @@ -1 +1,2 @@ 6 | Hello World 7 | +Second line 8 | diff --git a/test.cpp b/test.cpp 9 | new file mode 100644 10 | index 000000..5eb0bec 100644 11 | --- a/test.cpp 12 | +++ b/test.cpp 13 | @@ -1 +1,2 @@ 14 | +Hello World 15 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_project_search.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "data": [ 4 | { 5 | "id": 1, 6 | "type": "PROJ", 7 | "phid": "PHID-PROJ-secure", 8 | "fields": { 9 | "name": "secure-revision", 10 | "slug": "secure-revision", 11 | "subtype": "default", 12 | "milestone": null, 13 | "depth": 0, 14 | "parent": null, 15 | "icon": { 16 | "key": "policy", 17 | "name": "Policy", 18 | "icon": "fa-lock" 19 | }, 20 | "color": { 21 | "key": "blue", 22 | "name": "Blue" 23 | }, 24 | "spacePHID": null, 25 | "dateCreated": 1519855816, 26 | "dateModified": 1683298448, 27 | "policy": { 28 | "view": "public", 29 | "edit": "admin", 30 | "join": "admin" 31 | }, 32 | "description": "The revision contains sensitive information and email should not be sent in the clear." 33 | }, 34 | "attachments": {} 35 | } 36 | ], 37 | "maps": {}, 38 | "query": { 39 | "queryKey": null 40 | }, 41 | "cursor": { 42 | "limit": 100, 43 | "after": null, 44 | "before": null, 45 | "order": null 46 | } 47 | }, 48 | "error_code": null, 49 | "error_info": null 50 | } 51 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_repository_search.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "data": [ 4 | { 5 | "id": 1, 6 | "type": "REPO", 7 | "phid": "PHID-REPO-mc", 8 | "fields": { 9 | "name": "mozilla-central", 10 | "vcs": "hg", 11 | "callsign": "MOZILLACENTRAL", 12 | "shortName": "mozilla-central", 13 | "status": "active", 14 | "isImporting": false, 15 | "almanacServicePHID": null, 16 | "refRules": { 17 | "fetchRules": [], 18 | "trackRules": [], 19 | "permanentRefRules": [] 20 | }, 21 | "defaultBranch": "default", 22 | "description": { 23 | "raw": "" 24 | }, 25 | "spacePHID": null, 26 | "dateCreated": 1502986064, 27 | "dateModified": 1558793245, 28 | "policy": { 29 | "view": "public", 30 | "edit": "admin", 31 | "diffusion.push": "no-one" 32 | } 33 | }, 34 | "attachments": {} 35 | } 36 | ], 37 | "maps": {}, 38 | "query": { 39 | "queryKey": null 40 | }, 41 | "cursor": { 42 | "limit": 100, 43 | "after": null, 44 | "before": null, 45 | "order": null 46 | } 47 | }, 48 | "error_code": null, 49 | "error_info": null 50 | } 51 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_revision_search.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "data": [ 4 | { 5 | "id": 51, 6 | "type": "DREV", 7 | "phid": "PHID-DREV-azcDeadbeef", 8 | "fields": { 9 | "title": "Static Analysis tests", 10 | "authorPHID": "PHID-USER-crasher", 11 | "status": { 12 | "value": "needs-review", 13 | "name": "Needs Review", 14 | "closed": false, 15 | "color.ansi": "magenta" 16 | }, 17 | "repositoryPHID": "PHID-REPO-coffeee", 18 | "diffPHID": "PHID-DIFF-test", 19 | "summary": "Nice summary", 20 | "dateCreated": 1512389294, 21 | "dateModified": 1512401613, 22 | "policy": { 23 | "view": "public", 24 | "edit": "users" 25 | }, 26 | "bugzilla.bug-id": "1234567" 27 | }, 28 | "attachments": { 29 | "projects": { 30 | "projectPHIDs": [ 31 | "PHID-PROJ-A" 32 | ] 33 | } 34 | } 35 | } 36 | ], 37 | "maps": {}, 38 | "query": { 39 | "queryKey": null 40 | }, 41 | "cursor": { 42 | "limit": 100, 43 | "after": null, 44 | "before": null, 45 | "order": null 46 | } 47 | }, 48 | "error_code": null, 49 | "error_info": null 50 | } 51 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_send_message.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "ok": true 4 | }, 5 | "error_code": null, 6 | "error_info": null 7 | } 8 | -------------------------------------------------------------------------------- /bot/tests/mocks/phabricator_target_search.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "data": [ 4 | { 5 | "fields": { 6 | "buildPHID": "PHID-HMBB-xxx" 7 | }, 8 | "phid": "PHID-HMBT-test" 9 | } 10 | ], 11 | "maps": {}, 12 | "query": { 13 | "queryKey": null 14 | }, 15 | "cursor": { 16 | "limit": 100, 17 | "after": null, 18 | "before": null, 19 | "order": null 20 | } 21 | }, 22 | "error_code": null, 23 | "error_info": null 24 | } 25 | -------------------------------------------------------------------------------- /bot/tests/mocks/zero_coverage_report.json: -------------------------------------------------------------------------------- 1 | { 2 | "github_revision": "00c0d068ece99717bea7475f7dc07e61f7f35984", 3 | "hg_revision": "2bf86657a4482f75eef4469686d2eb246ee55dd2", 4 | "files": [ 5 | { 6 | "size": 5518, 7 | "first_push_date": "2017-06-01", 8 | "last_push_date": "2018-08-29", 9 | "commits": 2, 10 | "name": "my/path/file1.cpp", 11 | "funcs": 9, 12 | "uncovered": true 13 | }, 14 | { 15 | "size": 2998, 16 | "first_push_date": "2014-08-31", 17 | "last_push_date": "2019-02-06", 18 | "commits": 20, 19 | "name": "my/path/file2.js", 20 | "funcs": 7, 21 | "uncovered": false 22 | }, 23 | { 24 | "size": 30786, 25 | "first_push_date": "2008-03-20", 26 | "last_push_date": "2018-12-14", 27 | "commits": 113, 28 | "name": "test/dummy/thirdparty.c", 29 | "funcs": 32, 30 | "uncovered": true 31 | }, 32 | { 33 | "size": 30786, 34 | "first_push_date": "2008-03-20", 35 | "last_push_date": "2018-12-14", 36 | "commits": 113, 37 | "name": "path/not/in/patch.java", 38 | "funcs": 32, 39 | "uncovered": true 40 | }, 41 | { 42 | "size": 5518, 43 | "first_push_date": "2017-06-01", 44 | "last_push_date": "2018-08-29", 45 | "commits": 2, 46 | "name": "my/path/header.h", 47 | "funcs": 9, 48 | "uncovered": true 49 | }] 50 | } 51 | -------------------------------------------------------------------------------- /bot/tests/test_artifacts.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | from conftest import MockQueue 6 | 7 | from code_review_bot.tasks.base import AnalysisTask 8 | 9 | 10 | class TestTask(AnalysisTask): 11 | def parse_issues(*args, **kwargs): 12 | return [] 13 | 14 | 15 | def test_loading_artifacts(log): 16 | """ 17 | Test Taskcluster artifacts loading workflow 18 | """ 19 | assert log.events == [] 20 | 21 | task = TestTask( 22 | "testTask", 23 | { 24 | "task": {"metadata": {"name": "test-task"}}, 25 | "status": {"state": "xxx", "runs": [{"runId": 0}]}, 26 | }, 27 | ) 28 | 29 | # Add a dummy artifact to load 30 | queue = MockQueue() 31 | queue.configure({"testTask": {"artifacts": {"test.txt": "Hello World"}}}) 32 | task.artifacts = ["test.txt"] 33 | 34 | # Unsupported task state 35 | assert task.load_artifacts(queue) is None 36 | assert task.state == "xxx" 37 | assert log.has("Invalid task state", state="xxx", level="warning") 38 | 39 | # Invalid task state 40 | log.events = [] 41 | task.status["state"] = "running" 42 | assert task.load_artifacts(queue) is None 43 | assert task.state == "running" 44 | assert log.has("Invalid task state", state="running", level="warning") 45 | 46 | # Valid task state 47 | log.events = [] 48 | task.status["state"] = "completed" 49 | assert task.load_artifacts(queue) == {"test.txt": b"Hello World"} 50 | assert task.state == "completed" 51 | assert log.has( 52 | "Load artifact", task_id="testTask", artifact="test.txt", level="info" 53 | ) 54 | 55 | # Skip completed tasks 56 | task.valid_states = ("failed",) 57 | task.skipped_states = ("completed",) 58 | 59 | log.events = [] 60 | task.status["state"] = "completed" 61 | assert task.load_artifacts(queue) is None 62 | assert task.state == "completed" 63 | assert log.has("Skipping task", id="testTask", name="test-task", level="info") 64 | -------------------------------------------------------------------------------- /bot/tests/test_autoland.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | from code_review_bot.revisions import Revision 6 | 7 | 8 | def test_revision(mock_autoland_task, mock_phabricator, mock_hgmo): 9 | """ 10 | Validate the creation of an autoland Revision 11 | """ 12 | 13 | with mock_phabricator as api: 14 | revision = Revision.from_decision_task(mock_autoland_task, api) 15 | 16 | assert revision.as_dict() == { 17 | "bugzilla_id": None, 18 | "diff_id": None, 19 | "diff_phid": None, 20 | "has_clang_files": False, 21 | "id": None, 22 | "mercurial_revision": "deadbeef123", 23 | "phid": None, 24 | "repository": "https://hg.mozilla.org/integration/autoland", 25 | "target_repository": "https://hg.mozilla.org/mozilla-unified", 26 | "title": "Changeset deadbeef123 (https://hg.mozilla.org/integration/autoland)", 27 | "url": None, 28 | "head_repository": "https://hg.mozilla.org/integration/autoland", 29 | "base_repository": "https://hg.mozilla.org/mozilla-unified", 30 | "head_changeset": "deadbeef123", 31 | "base_changeset": "123deadbeef", 32 | } 33 | -------------------------------------------------------------------------------- /bot/tests/test_default.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import pytest 6 | 7 | from code_review_bot import Level 8 | from code_review_bot.tasks.default import DefaultIssue, DefaultTask 9 | 10 | 11 | @pytest.mark.parametrize( 12 | "path, matches", 13 | [ 14 | ("public/code-review/issues.json", True), 15 | ("private/code-review/issues.json", False), 16 | ("public/code-review/mozlint.json", False), 17 | ], 18 | ) 19 | def test_matches(path, matches, mock_taskcluster_config): 20 | """Test that DefaultTask matches tasks with valid artifacts""" 21 | 22 | queue = mock_taskcluster_config.get_service("queue") 23 | queue.configure( 24 | { 25 | "testDefaultTask": { 26 | "name": "some-analyzer", 27 | "state": "failed", 28 | "artifacts": {path: {}}, 29 | } 30 | } 31 | ) 32 | assert DefaultTask.matches("testDefaultTask") is matches 33 | 34 | 35 | def test_parser(mock_workflow, mock_revision, mock_hgmo, mock_backend): 36 | """Test the default format parser""" 37 | mock_workflow.setup_mock_tasks( 38 | { 39 | "remoteTryTask": {"dependencies": ["analyzer-A", "analyzer-B"]}, 40 | "analyzer-A": {}, 41 | "analyzer-B": { 42 | "name": "any-analyzer-name", 43 | "state": "failed", 44 | "artifacts": { 45 | "nope.log": "No issues here !", 46 | "still-nope.txt": "xxxxx", 47 | "public/code-review/issues.json": { 48 | "test.cpp": [ 49 | { 50 | "path": "test.cpp", 51 | "line": 42, 52 | "column": 51, 53 | "level": "error", 54 | "check": "XYZ", 55 | "message": "A random issue happened here", 56 | } 57 | ] 58 | }, 59 | }, 60 | }, 61 | "extra-task": {}, 62 | } 63 | ) 64 | # Bypass revision publication check 65 | mock_workflow.backend_api.publish_revision = lambda rev: {} 66 | mock_revision.id = 1337 67 | 68 | issues = mock_workflow.run(mock_revision) 69 | assert len(issues) == 1 70 | issue = issues.pop() 71 | 72 | assert isinstance(issue, DefaultIssue) 73 | assert str(issue) == "any-analyzer-name issue XYZ@error test.cpp line 42" 74 | assert issue.path == "test.cpp" 75 | assert issue.line == 42 76 | assert issue.nb_lines == 1 77 | assert issue.column == 51 78 | assert issue.level == Level.Error 79 | assert issue.check == "XYZ" 80 | assert issue.message == "A random issue happened here" 81 | assert issue.as_text() == "Error: A random issue happened here [XYZ]" 82 | assert ( 83 | issue.as_markdown() 84 | == """ 85 | ## issue any-analyzer-name 86 | 87 | - **Path**: test.cpp 88 | - **Level**: error 89 | - **Check**: XYZ 90 | - **Line**: 42 91 | - **Publishable**: yes 92 | 93 | ``` 94 | A random issue happened here 95 | ``` 96 | """ 97 | ) 98 | 99 | assert issue.hash == "533d1aefc79ef542b3e7d677c1c5724e" 100 | assert issue.as_dict() == { 101 | "analyzer": "any-analyzer-name", 102 | "check": "XYZ", 103 | "column": 51, 104 | "hash": "533d1aefc79ef542b3e7d677c1c5724e", 105 | "in_patch": False, 106 | "level": "error", 107 | "line": 42, 108 | "message": "A random issue happened here", 109 | "nb_lines": 1, 110 | "path": "test.cpp", 111 | "publishable": True, 112 | "validates": True, 113 | "fix": None, 114 | } 115 | -------------------------------------------------------------------------------- /bot/tests/test_issues.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | from code_review_bot.tasks.clang_format import ClangFormatIssue, ClangFormatTask 6 | 7 | 8 | def test_allowed_paths(mock_config, mock_revision, mock_task): 9 | """ 10 | Test allowed paths for ClangFormatIssue 11 | The test config has these 2 rules: dom/* and tests/*.py 12 | """ 13 | 14 | def _allowed(path): 15 | # Build an issue and check its validation 16 | # that will trigger the path validation 17 | lines = [ 18 | (1, None, b"deletion"), 19 | (None, 1, b"change here"), 20 | ] 21 | issue = ClangFormatIssue( 22 | mock_task(ClangFormatTask, "mock-clang-format"), path, lines, mock_revision 23 | ) 24 | return issue.validates() 25 | 26 | checks = { 27 | "nope.cpp": False, 28 | "dom/whatever.cpp": True, 29 | "dom/sub/folders/whatever.cpp": True, 30 | "dom/noext": True, 31 | "dom_fail.h": False, 32 | "tests/xxx.pyc": False, 33 | "tests/folder/part/1.py": True, 34 | } 35 | for path, result in checks.items(): 36 | assert _allowed(path) is result 37 | 38 | 39 | def test_backend_publication(mock_revision, mock_task): 40 | """ 41 | Test the backend publication status modifies an issue publication 42 | """ 43 | 44 | lines = [ 45 | (1, None, b"deletion"), 46 | (None, 1, b"change here"), 47 | ] 48 | issue = ClangFormatIssue( 49 | mock_task(ClangFormatTask, "mock-clang-format"), 50 | "dom/somefile.cpp", 51 | lines, 52 | mock_revision, 53 | ) 54 | assert issue.validates() 55 | 56 | # At first backend data is empty 57 | assert issue.on_backend is None 58 | 59 | # Not publishable as not in patch 60 | assert mock_revision.lines == {} 61 | assert not mock_revision.contains(issue) 62 | assert not issue.is_publishable() 63 | 64 | # The backend data takes precedence over local in patch 65 | issue.on_backend = {"publishable": True} 66 | assert issue.is_publishable() 67 | -------------------------------------------------------------------------------- /bot/tests/test_mercurial.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | from code_review_bot import mercurial 6 | 7 | 8 | class STDOutputMock: 9 | def fileno(self): 10 | return 4 11 | 12 | content = "" 13 | 14 | 15 | class PopenMock: 16 | stdout = STDOutputMock() 17 | stderr = STDOutputMock() 18 | returncode = 0 19 | 20 | def poll(self): 21 | return True 22 | 23 | def communicate(self): 24 | out = self.stdout.content = "Hello world" 25 | err = self.stderr.content = "An error occurred" 26 | return out, err 27 | 28 | def __call__(self, command): 29 | self.command = command 30 | return self 31 | 32 | 33 | def test_hg_run(monkeypatch): 34 | popen_mock = PopenMock() 35 | monkeypatch.setattr("hglib.util.popen", popen_mock) 36 | mercurial.hg_run(["checkout", "https://hg.repo/", "--test"]) 37 | assert popen_mock.command == ["hg", "checkout", "https://hg.repo/", "--test"] 38 | assert popen_mock.stdout.content == "Hello world" 39 | assert popen_mock.stderr.content == "An error occurred" 40 | 41 | 42 | def test_robustcheckout(monkeypatch): 43 | popen_mock = PopenMock() 44 | monkeypatch.setattr("hglib.util.popen", popen_mock) 45 | 46 | mercurial.robust_checkout( 47 | repo_url="https://hg.repo/try", 48 | repo_upstream_url="https://hg.repo/mc", 49 | revision="deadbeef1234", 50 | checkout_dir="/tmp/checkout", 51 | sharebase_dir="/tmp/shared", 52 | ) 53 | 54 | assert popen_mock.command == [ 55 | "hg", 56 | "robustcheckout", 57 | b"--purge", 58 | b"--sharebase=/tmp/shared", 59 | b"--revision=deadbeef1234", 60 | b"--upstream=https://hg.repo/mc", 61 | b"--", 62 | "https://hg.repo/try", 63 | "/tmp/checkout", 64 | ] 65 | -------------------------------------------------------------------------------- /bot/tests/test_patch.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import responses 6 | 7 | from code_review_bot.config import settings 8 | from code_review_bot.revisions import ImprovementPatch 9 | from code_review_bot.tasks.default import DefaultTask 10 | 11 | 12 | def test_publication( 13 | monkeypatch, mock_taskcluster_config, mock_repositories, mock_task 14 | ): 15 | """ 16 | Check a patch publication through Taskcluster services 17 | """ 18 | 19 | # Setup local config as running in a real Taskcluster task with proxy 20 | monkeypatch.setenv("TASK_ID", "fakeTaskId") 21 | monkeypatch.setenv("RUN_ID", "0") 22 | monkeypatch.setenv("TASKCLUSTER_PROXY_URL", "http://proxy") 23 | settings.setup("test", [], mock_repositories) 24 | 25 | # Mock the storage response 26 | responses.add( 27 | responses.PUT, 28 | "http://storage.test/public/patch/mock-analyzer-test-improvement.diff", 29 | json={}, 30 | headers={"ETag": "test123"}, 31 | ) 32 | 33 | patch = ImprovementPatch( 34 | mock_task(DefaultTask, "mock-analyzer"), "test-improvement", "This is good code" 35 | ) 36 | assert patch.url is None 37 | 38 | patch.publish() 39 | assert ( 40 | patch.url 41 | == "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/fakeTaskId/runs/0/artifacts/public/patch/mock-analyzer-test-improvement.diff" 42 | ) 43 | 44 | # Check the mock has been called 45 | assert [c.request.url for c in responses.calls] == [ 46 | "http://storage.test/public/patch/mock-analyzer-test-improvement.diff" 47 | ] 48 | -------------------------------------------------------------------------------- /bot/tests/test_reporter_debug.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | import json 5 | import os.path 6 | 7 | from code_review_bot.tasks.clang_tidy import ClangTidyTask 8 | 9 | 10 | def test_publication(tmpdir, mock_issues, mock_revision): 11 | """ 12 | Test debug publication and report analysis 13 | """ 14 | from code_review_bot.report.debug import DebugReporter 15 | 16 | report_dir = str(tmpdir.mkdir("public").realpath()) 17 | report_path = os.path.join(report_dir, "report.json") 18 | assert not os.path.exists(report_path) 19 | 20 | status = {"task": {"metadata": {"name": "mock-clang-tidy"}}, "status": {}} 21 | task = ClangTidyTask("someTaskId", status) 22 | 23 | r = DebugReporter(report_dir) 24 | r.publish(mock_issues, mock_revision, [task], [], []) 25 | 26 | assert os.path.exists(report_path) 27 | with open(report_path) as f: 28 | report = json.load(f) 29 | 30 | assert "issues" in report 31 | assert report["issues"] == [{"nb": 0}, {"nb": 1}, {"nb": 2}, {"nb": 3}, {"nb": 4}] 32 | 33 | assert "revision" in report 34 | assert report["revision"] == { 35 | "id": 51, 36 | "diff_id": 42, 37 | "url": "https://phabricator.test/D51", 38 | "bugzilla_id": 1234567, 39 | "diff_phid": "PHID-DIFF-test", 40 | "phid": "PHID-DREV-zzzzz", 41 | "title": "Static Analysis tests", 42 | "has_clang_files": False, 43 | "repository": "https://hg.mozilla.org/try", 44 | "target_repository": "https://hg.mozilla.org/mozilla-central", 45 | "mercurial_revision": "deadc0ffee", 46 | "head_repository": "https://hg.mozilla.org/try", 47 | "base_repository": "https://hg.mozilla.org/mozilla-central", 48 | "head_changeset": "deadc0ffee", 49 | "base_changeset": "c0ffeedead", 50 | } 51 | 52 | assert "task_failures" in report 53 | assert report["task_failures"] == [{"id": "someTaskId", "name": "mock-clang-tidy"}] 54 | 55 | assert "time" in report 56 | assert isinstance(report["time"], float) 57 | -------------------------------------------------------------------------------- /bot/tests/test_reporter_lando.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | from code_review_bot.report.lando import LANDO_MESSAGE, LandoReporter 6 | 7 | MOCK_LANDO_API_URL = "http://api.lando.test" 8 | MOCK_LANDO_TOKEN = "Some Test Token" 9 | 10 | 11 | class MockLandoWarnings: 12 | """ 13 | LandoWarnings Mock class 14 | """ 15 | 16 | def __init__(self, api_url, api_key): 17 | self.api_url = MOCK_LANDO_API_URL 18 | self.api_key = MOCK_LANDO_TOKEN 19 | 20 | def del_warnings(self, warnings): 21 | pass 22 | 23 | def add_warning(self, warning, revision_id, diff_id): 24 | self.revision_id = revision_id 25 | self.diff_id = diff_id 26 | self.warning = warning 27 | 28 | def del_all_warnings(self, revision_id, diff_id): 29 | self.revision_id = revision_id 30 | self.diff_id = diff_id 31 | 32 | 33 | def test_lando(log, mock_clang_tidy_issues, mock_revision): 34 | """ 35 | Test lando reporter 36 | """ 37 | 38 | # empty config should be OK 39 | assert LandoReporter({}).lando_api is None 40 | 41 | r = LandoReporter({}) 42 | 43 | lando_api = MockLandoWarnings(api_url=MOCK_LANDO_API_URL, api_key=MOCK_LANDO_TOKEN) 44 | 45 | r.setup_api(lando_api) 46 | 47 | assert r.lando_api == lando_api 48 | 49 | assert log.has("Publishing warnings to lando is enabled by the bot!") 50 | 51 | r.publish(mock_clang_tidy_issues, mock_revision, [], [], []) 52 | 53 | assert lando_api.revision_id == mock_revision.revision["id"] 54 | assert lando_api.diff_id == mock_revision.diff_id 55 | assert lando_api.warning == LANDO_MESSAGE.format( 56 | errors=1, errors_noun="error", warnings=0, warnings_noun="warnings" 57 | ) 58 | 59 | assert log.has("Publishing warnings to lando for 1 errors and 0 warnings") 60 | -------------------------------------------------------------------------------- /bot/tests/test_stats.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | from datetime import datetime 6 | 7 | from code_review_bot import stats 8 | 9 | 10 | class MockInflux: 11 | """ 12 | Mock the InfluxDb python client to retrieve data sent 13 | """ 14 | 15 | points = [] 16 | 17 | def write_points(self, points): 18 | self.points += points 19 | 20 | 21 | def test_base_stats(): 22 | """ 23 | Test simple stat management 24 | """ 25 | # Reset stats 26 | stats.metrics = [] 27 | 28 | stats.add_metric("test.a.b.c", 12) 29 | 30 | assert len(stats.metrics) == 1 31 | metric = stats.metrics[0] 32 | assert metric["fields"] == {"value": 12} 33 | assert metric["measurement"] == "code-review.test.a.b.c" 34 | assert metric["tags"] == {"app": "code-review-bot", "channel": "test"} 35 | assert datetime.strptime(metric["time"], "%Y-%m-%dT%H:%M:%S.%f") 36 | 37 | # Flush without client does not do anything (no crash) 38 | stats.flush() 39 | assert len(stats.metrics) == 1 40 | 41 | # Point are sent on flush 42 | stats.client = MockInflux() 43 | assert len(stats.client.points) == 0 44 | stats.flush() 45 | assert len(stats.metrics) == 0 46 | assert len(stats.client.points) == 1 47 | -------------------------------------------------------------------------------- /bot/tests/test_tgdiff_task.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from code_review_bot.tasks.tgdiff import TaskGraphDiffTask 4 | 5 | 6 | @pytest.fixture 7 | def mock_taskgraph_diff_task(): 8 | tg_task_status = { 9 | "task": {"metadata": {"name": "source-test-taskgraph-diff"}}, 10 | "status": { 11 | "taskId": "12345deadbeef", 12 | "state": "completed", 13 | "runs": [{"runId": 0}], 14 | }, 15 | } 16 | return TaskGraphDiffTask("12345deadbeef", tg_task_status) 17 | 18 | 19 | def test_load_artifacts_no_summary(mock_taskcluster_config, mock_taskgraph_diff_task): 20 | queue = mock_taskcluster_config.get_service("queue") 21 | queue.options = {"rootUrl": "http://taskcluster.test"} 22 | queue.configure( 23 | { 24 | "12345deadbeef": { 25 | "artifacts": { 26 | "public/taskgraph/diffs/diff_mc-onpush.txt": "Some diff", 27 | "public/taskgraph/not_a_diff.txt": "Not a diff", 28 | } 29 | } 30 | } 31 | ) 32 | 33 | mock_taskgraph_diff_task.load_artifacts(queue) 34 | 35 | assert mock_taskgraph_diff_task.artifact_urls == { 36 | "public/taskgraph/diffs/diff_mc-onpush.txt": "http://tc.test/12345deadbeef/0/artifacts/public/taskgraph/diffs/diff_mc-onpush.txt" 37 | } 38 | assert mock_taskgraph_diff_task.extra_reviewers_groups == [] 39 | 40 | 41 | def test_load_artifacts_ok_summary(mock_taskcluster_config, mock_taskgraph_diff_task): 42 | queue = mock_taskcluster_config.get_service("queue") 43 | queue.options = {"rootUrl": "http://taskcluster.test"} 44 | queue.configure( 45 | { 46 | "12345deadbeef": { 47 | "artifacts": { 48 | "public/taskgraph/diffs/diff_mc-onpush.txt": "Some diff", 49 | "public/taskgraph/diffs/summary.json": { 50 | "files": {}, 51 | "status": "OK", 52 | "threshold": 20, 53 | }, 54 | "public/taskgraph/not_a_diff.txt": "Not a diff", 55 | } 56 | } 57 | } 58 | ) 59 | 60 | mock_taskgraph_diff_task.load_artifacts(queue) 61 | 62 | # The summary.json is OK, so no extra reviewer group is added and we don't load it in the artifact_urls list 63 | assert mock_taskgraph_diff_task.artifact_urls == { 64 | "public/taskgraph/diffs/diff_mc-onpush.txt": "http://tc.test/12345deadbeef/0/artifacts/public/taskgraph/diffs/diff_mc-onpush.txt" 65 | } 66 | assert mock_taskgraph_diff_task.extra_reviewers_groups == [] 67 | 68 | 69 | def test_load_artifacts_warning_summary( 70 | mock_taskcluster_config, mock_taskgraph_diff_task 71 | ): 72 | queue = mock_taskcluster_config.get_service("queue") 73 | queue.options = {"rootUrl": "http://taskcluster.test"} 74 | queue.configure( 75 | { 76 | "12345deadbeef": { 77 | "artifacts": { 78 | "public/taskgraph/diffs/diff_mc-onpush.txt": "Some diff", 79 | "public/taskgraph/diffs/summary.json": { 80 | "files": {}, 81 | "status": "WARNING", 82 | "threshold": 20, 83 | }, 84 | "public/taskgraph/not_a_diff.txt": "Not a diff", 85 | } 86 | } 87 | } 88 | ) 89 | 90 | mock_taskgraph_diff_task.load_artifacts(queue) 91 | 92 | # The summary.json is WARNING, so an extra reviewer group is added and we don't load it in the artifact_urls list 93 | assert mock_taskgraph_diff_task.artifact_urls == { 94 | "public/taskgraph/diffs/diff_mc-onpush.txt": "http://tc.test/12345deadbeef/0/artifacts/public/taskgraph/diffs/diff_mc-onpush.txt" 95 | } 96 | assert mock_taskgraph_diff_task.extra_reviewers_groups == ["taskgraph-reviewers"] 97 | -------------------------------------------------------------------------------- /bot/tests/test_tools.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import pytest 6 | from taskcluster.helper import TaskclusterConfig 7 | 8 | from code_review_bot.tools.log import remove_color_codes 9 | 10 | 11 | def test_taskcluster_service(): 12 | """ 13 | Test taskcluster service loader 14 | """ 15 | taskcluster = TaskclusterConfig("http://tc.test") 16 | 17 | assert taskcluster.get_service("secrets") is not None 18 | assert taskcluster.get_service("hooks") is not None 19 | assert taskcluster.get_service("index") is not None 20 | with pytest.raises(AssertionError) as e: 21 | taskcluster.get_service("nope") 22 | assert str(e.value) == "Invalid Taskcluster service nope" 23 | 24 | 25 | def test_remove_color_codes(sentry_event_with_colors, sentry_event_without_colors): 26 | """ 27 | Test the removal of color codes from Sentry payloads 28 | """ 29 | assert ( 30 | remove_color_codes(sentry_event_with_colors, hint=None) 31 | == sentry_event_without_colors 32 | ) 33 | -------------------------------------------------------------------------------- /bot/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/bot/tools/__init__.py -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | services: 3 | backend: 4 | container_name: code-review-backend 5 | build: 6 | dockerfile: ./backend/Dockerfile 7 | context: . 8 | 9 | environment: 10 | DATABASE_URL: psql://devuser:devdata@db/code_review_dev 11 | 12 | # Setup environment like on Heroku 13 | DYNO: 1 14 | HOST: 0.0.0.0 15 | PORT: 80 16 | SECRET_KEY: randomSecretKey123 17 | 18 | # Special marker to skip SSL for Postgres 19 | DATABASE_NO_SSL: 1 20 | 21 | ports: 22 | - 127.0.0.1:8000:80 23 | 24 | depends_on: 25 | - db 26 | 27 | db: 28 | container_name: code-review-db 29 | image: postgres:16-alpine 30 | 31 | ports: 32 | - 127.0.0.1:5432:5432 33 | volumes: 34 | - pgdata:/var/lib/postgresql/data 35 | environment: 36 | POSTGRES_USER: devuser 37 | POSTGRES_PASSWORD: devdata 38 | POSTGRES_DB: code_review_dev 39 | 40 | volumes: 41 | pgdata: 42 | driver: local 43 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Code review bot for developers 2 | 3 | This documentation is targeted towards developers of the code review bot itself, or developers of analyzers who want to integrate it in our bot. 4 | 5 | Here is the overall summary of available documentation: 6 | 7 | - [System Architecture](architecture.md) (good starting point) 8 | - [System Configuration](configuration.md) 9 | - [How to debug the code review bot](debugging.md) and other tips for maintainers 10 | - [CI/CD Pipeline](ci-cd/README.md) with Taskcluster 11 | - [Phabricator](phabricator.md) integrations, lessons learned, tips & tricks 12 | - Projects details: 13 | - [backend](projects/backend.md) 14 | - [bot](projects/bot.md) 15 | - [frontend](projects/bot.md) 16 | - How to add a new analyzer: 17 | - [How to trigger your task on new diffs](trigger.md) 18 | - [How to publish issues on Phabricator](publication.md) 19 | - [Analyzer output format](analysis_format.md) 20 | - [How to hook up a new repository](new_repository.md) 21 | 22 | You can contact the code review bot's developers directly, see [README](../README.md) for contact info. 23 | -------------------------------------------------------------------------------- /docs/analysis_format.md: -------------------------------------------------------------------------------- 1 | # The default analysis format 2 | 3 | Each analysis **must** build a JSON Taskcluster artifact publicly available, containing all the issues detected on the patch. 4 | 5 | The file **must** be available as `public/code-review/issues.json`. 6 | 7 | The [default format](https://github.com/mozilla/code-review/blob/1.0.5/bot/code_review_bot/tasks/default.py#L170) (loosely based on Mozlint format) has the following fields for each issue: 8 | 9 | - `path` **relative** to the repository 10 | - `column` & `line` where the issue is happening in the file. They must be positive integers, or `null` when unknown or for an issue linked to a full file. 11 | - `nb_lines` (optional) is a positive integer when your issue spans across several lines. It will default to 1 line. 12 | - `level` (warning | error) of the issue 13 | - `check` (optional) describing the issue detected (often a unique shorthand code). It can be optional when the analyzer only produce one type of issues. In that case, the analyzer name will be used instead. 14 | - `message` with all the details to provide to the developer 15 | - `analyzer` (optional) if you have multiple analyzers using the same format. It will default to the task name. 16 | 17 | The issues should be grouped by relative paths, as a list of issues per file. 18 | 19 | :warning: You need to provide relative paths to the repository. The bot does not support any absolute path resolution as it's not using the same setup as your own task. 20 | 21 | Here is an example of an analysis using this format: 22 | 23 | ```json 24 | { 25 | "path/to/file.py": [ 26 | { 27 | "path": "path/to/file.py", 28 | "line": 51, 29 | "nb_lines": 1, 30 | "column": 42, 31 | "check": "bad_issue.XXX123", 32 | "level": "error", 33 | "message": "This is a really bad issue in your code", 34 | "analyzer": "python_analyzer" 35 | } 36 | ] 37 | } 38 | ``` 39 | 40 | We have built a validation tool, available in this repository as `bot/tools/validator.py`. It has no extra dependencies, and can run using any Python version. You can download it directly on your computer to troubleshoot your payloads: 41 | 42 | ``` 43 | wget https://raw.githubusercontent.com/mozilla/code-review/master/bot/tools/validator.py 44 | python validator.py path/to/issues.json 45 | ``` 46 | 47 | If your format is valid, no error should be displayed and the exit status should be `0`. If you encounter an error, you can get more information by adding the `--verbose` (or `-v`) flag to the command line. 48 | 49 | To have more information about Mozlint, please see the [mozlint documentation](https://firefox-source-docs.mozilla.org/tools/lint/index.html) 50 | -------------------------------------------------------------------------------- /docs/architecture.drawio: -------------------------------------------------------------------------------- 1 | 7Vtdd5s4EP01fkwPCPP1mOJ4e07bbXqyPWkeBciYDUZeWcR2f/0KEGCBiHFsIEnrhwSNhIxm7tyZEfJEc1a7vwhcL79iH0UToPi7iTabAGBagP1NBftcoKomlwQk9LmsEtyFvxAXKlyahD7aCAMpxhEN16LQw3GMPCrIICF4Kw5b4Ej81jUMUENw58GoKb0PfbrMpRYwK/knFAbL4ptVw857VrAYzFeyWUIfbw9E2s1EcwjGNL9a7RwUpcor9JLfN2/pLR+MoJh2uWHr//i+ib8/mNH0788z/3Oyv/Kv+CxPMEr4gifAiNh8HxeYTcuemu65Koz/Elx0XG0yQ12zAaqx3lWdmSUiTIQ7JkBT2Gc+PxQZQfr/dgldEnqQslv4F7MV5N+dj+DKKx8DEJzEPkoXpbLu7TKk6G4NvbR3yzDIZEu6inh3OtMcrsIoxd8PN4lpkkrDKHLKx9R8iKyFx+QbSvAjOugxPAu5i/IpnhChaNeqf7W0KnMHhFeIkj0bshMRzT2hxMW2gpVayJYHkDK4DHIkB+XMlbHZBbf3CbYHMuPXlB0wba9bF8+dDrrFcOVkpZgdtGJJtKKpek9qMY5r5TQIimBznPnccWRgs4CrGUZXsD1j0qa2x9MmOEYwLXThOIrSpIvavYtc59U47uFH6Wg6s23Z/PfIbaWhXOySuuQYX1ESwjhIW8eQQjCFNMQxa9qKhKZ0ZPnTgZGzE0mIu+lUb3ipIYGV1Rd1aQOFrTacnI7D/L6viHgJw0O6wHtMHlF71Hsp3M7hpjERpokIU60mxIYNj+oFciOrBrKLE9YnxAyVdKYitKMiAERDxzhGNVRwEYzCIOUlj9mYgVb7mCKA5W3RNe9Yhb7fynEVJpU+IVTPJSQQmkoQBPpC0LRhAOSz4oI3MaFLHOAYRjeVtKaqaswXjNfcZv8iSve8UoIJxaJF0S6kPw+uH9KpPui8NdvxmbPGvmjEbLk/q4Fp8+Gwr7ota+0FS6aLeokdmWZwQrhDPJs9UEgC9NyEmhwXBEUsoD6JT3f5FEeW4wycR+v68Txasw1Z5tcbf077TaTn1g37yIKVa+lT/UymAUcT6aHVqb+icDSfpwV9Mxz9AzePXpRs6DOJzXuLSS1IKQtcoxaUmo45aFACxhhR6cWRouCMY5FCNTqGCtUaNVaYo+QEZXzvGt1B2bxFJGRrTx3svJh/eUuao1rS+s39yB5T+6psl+4ye0NtsW2GvHCT7s5cfN+nltgsLA950u3oHhObYhoxWqnq2Bs92ih8yWuoolB6OLjOyNIWSqhp0apzZQvvVkXWpeuqzl5udvRyTZHDZSAvNwf38muGk/2v7unrZgnX6aWXuB083c2R+cUtBdB7DDK8fktoFKZ5bSb3IXn8xqYJaYaJD4ouCkEmfW3kAcQiVG+Shz3kDp426gbMueTRPwlYb4MErD8k8IZJQAVjs4D+PhN1+214r/3He9+y99pje68yOH5uEzdKjyS991qv/lJfxtSDFntgFKbusUjr/JpL68jleXk+2pbXBY5dTC/1CsJxDEP6CiJTyB0iT90jwDmvoRaLBZB7sG+4ht75zMSR0xG26Kua5ATOwKcHxymuxnfWrr4KRn0prR09vfJiz1MUVZWd5f3IEiEU+7KjS5LzTDNIoQs36OQsbc9yLD9733epTG2U48CWoQserUtOq5gSh1aV3lIt0FD673depTMLKF3LrxYYdKaB7NZrQuD+YMAahzHdHMx8mwoqdNlKLV5wdM07jtfF8ewif4IKXeVSzgDcyC9DTwPPS4A6BuDeJN6KU+lt481nh/cEz1FOSryCDEftiLSW8vLcBIc1q19F5casflum3fwP -------------------------------------------------------------------------------- /docs/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/docs/architecture.png -------------------------------------------------------------------------------- /docs/ci-cd/community.mermaid: -------------------------------------------------------------------------------- 1 | graph TD 2 | 3 | push --> check_lint 4 | push --> backend_check_tests 5 | push --> frontend_build 6 | push --> integration_check_tests 7 | check_lint --> backend_build 8 | backend_check_tests --> backend_build 9 | check_lint --> integration_build 10 | integration_check_tests --> integration_build 11 | 12 | frontend_build --> frontend_deploy 13 | backend_build --> backend_deploy 14 | integration_build --> integration_deploy 15 | 16 | subgraph deployment 17 | frontend_deploy 18 | backend_deploy 19 | integration_deploy 20 | integration_deploy --> integration_hook 21 | end 22 | 23 | 24 | backend_build --> github_release 25 | frontend_build --> github_release 26 | integration_build --> github_release 27 | 28 | subgraph release 29 | github_release 30 | end 31 | -------------------------------------------------------------------------------- /docs/ci-cd/community.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/docs/ci-cd/community.png -------------------------------------------------------------------------------- /docs/ci-cd/firefox-ci.mermaid: -------------------------------------------------------------------------------- 1 | graph TD 2 | 3 | push --> check_lint 4 | push --> bot_check_tests 5 | check_lint --> bot_build_dind 6 | bot_check_tests --> bot_build_dind 7 | -------------------------------------------------------------------------------- /docs/ci-cd/firefox-ci.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/docs/ci-cd/firefox-ci.png -------------------------------------------------------------------------------- /docs/docker.md: -------------------------------------------------------------------------------- 1 | # Docker stack 2 | 3 | A `docker-compose.yml` file is available to reproduce locally the code-review stack. 4 | 5 | Run it with `docker-compose up` 6 | 7 | ## Use the backend 8 | 9 | A backend instance will be available as http://localhost:8000 10 | 11 | You can initialize the database with: 12 | 13 | ``` 14 | docker exec code-review-backend python manage.py migrate 15 | ``` 16 | 17 | You can create an admin account: 18 | 19 | ``` 20 | docker exec -it code-review-backend python manage.py createsuperuser 21 | ``` 22 | 23 | Then you can login on http://localhost:8000/admin/ 24 | 25 | ## Restore a backend postgres dump 26 | 27 | The database must be empty (no `migrate`) to be able to restore a backup. 28 | 29 | You can download the backup from the Heroku datastore dedicated page. 30 | 31 | ```bash 32 | export PGPASSWORD=devdata 33 | pg_restore -h localhost --user devuser -d code_review_dev path/to/dump 34 | ``` 35 | -------------------------------------------------------------------------------- /docs/new_repository.md: -------------------------------------------------------------------------------- 1 | # How to hook up a new repository 2 | 3 | ## Requirements 4 | 5 | 1. Your repository must use [Mozilla Phabricator's instance](https://phabricator.services.mozilla.com/) 6 | 2. Your repository must use [Taskcluster as CI](https://community-tc.services.mozilla.com/docs) (at least one task must start on each push). 7 | 3. You are adding a Taskcluster task that runs on each push, analyzes the modified source code and lists potential issues. 8 | 9 | ## Contact us 10 | 11 | As each new repository has different needs and constraints, please reach out to our team. Look in [README](../README.md) for contact information. 12 | -------------------------------------------------------------------------------- /docs/phabricator.mermaid: -------------------------------------------------------------------------------- 1 | graph TD 2 | Revision --> Diff 3 | Revision --> Comment 4 | Diff --> build[Harbormaster Build] 5 | build --> LintResult 6 | build --> UnitResult 7 | -------------------------------------------------------------------------------- /docs/phabricator.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/docs/phabricator.png -------------------------------------------------------------------------------- /docs/projects/backend.md: -------------------------------------------------------------------------------- 1 | # Code review Backend 2 | 3 | This is a Python REST API, powered by [Django](https://docs.djangoproject.com/) and [django-rest-framework](https://www.django-rest-framework.org/). 4 | 5 | The technical information on how to get started is available in the [project README](/backend/README.md). 6 | 7 | This project is used by the bot and the frontend: the bot publishes issues at the end of each execution, and the frontend fetches data to display it nicely to users (admins or developers). 8 | 9 | ## Deployment 10 | 11 | The application is hosted on Heroku (more information in [debugging](/docs/debugging.md) to get access). 12 | 13 | It uses currently a single web dyno on each environment: 14 | 15 | - https://api.code-review.moz.tools on production 16 | - https://api.code-review.testing.moz.tools on testing 17 | 18 | It is deployed with [task-boot](https://github.com/mozilla/task-boot) on every push to `testing` or `production` branch by an administrator. 19 | The application has no state in its docker image, it can restart immediately and reuse the same database. 20 | 21 | ### Database migration 22 | 23 | If you want to make a Database migration (adding or altering a model), you'll need to make a new deployment, then run the following command from your computer using the [Heroku cli](https://devcenter.heroku.com/articles/heroku-cli). 24 | 25 | Make sure you're authenticated first (it will use your browser to get your SSO credentials): 26 | 27 | ```bash 28 | heroku login 29 | ``` 30 | 31 | Then run the migration on the desired application: 32 | 33 | ```bash 34 | heroku run -a code-review-backend-production ./manage.py migrate 35 | ``` 36 | 37 | It's also possible to do that through the web shell on the Heroku dashboard. 38 | 39 | ### Administration 40 | 41 | You can also use the heroku cli application to add a new administrator account: 42 | 43 | ```bash 44 | heroku run -a code-review-backend-production ./manage.py createsuperuser 45 | ``` 46 | 47 | The admin dashboard is available as `/admin/` on each instance, login with the credentials from the above command. 48 | 49 | ## Models 50 | 51 | ![](backend.png) 52 | 53 | Generated with [django-extensions](https://django-extensions.readthedocs.io/en/latest/graph_models.html) 54 | 55 | TODO: explain a bit Issue models 56 | 57 | ## Endpoints 58 | 59 | All endpoints are described in the generated [OpenAPI documentation](https://api.code-review.moz.tools/docs). 60 | 61 | The OpenAPI raw file is available as [JSON](https://api.code-review.moz.tools/docs.json) and [YAML](https://api.code-review.moz.tools/docs.yaml) 62 | -------------------------------------------------------------------------------- /docs/projects/backend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/docs/projects/backend.png -------------------------------------------------------------------------------- /docs/projects/bot.mermaid: -------------------------------------------------------------------------------- 1 | graph TD 2 | pulse[Pulse Trigger] -- task id --> new 3 | new[New bot task] --> tasks 4 | tasks{List all tasks} -->|artifact| clang-tidy 5 | tasks{List all tasks} -->|artifact| mozlint 6 | tasks{List all tasks} -->|artifact| analyzer-X 7 | clang-tidy --> issues[List of issues] 8 | mozlint --> issues 9 | analyzer-X --> issues 10 | issues --> reporters{Publication} 11 | reporters --> Phabricator 12 | reporters --> Mail 13 | -------------------------------------------------------------------------------- /docs/projects/bot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/docs/projects/bot.png -------------------------------------------------------------------------------- /docs/projects/frontend.md: -------------------------------------------------------------------------------- 1 | # Code Review Frontend 2 | 3 | The frontend is a Single Page Application, built with [Vue.JS](https://vuejs.org), using [vuex](https://vuex.vuejs.org/) as a store for data shared across components in the application. 4 | 5 | It's really a pretty simple application, not much complexity: 6 | 7 | - few views, and few components 8 | - uses [axios](https://github.com/axios/axios) to retrieve data from the backend 9 | - no authentication, everything is public 10 | - uses [vue-chartjs](https://vue-chartjs.org/) to build the stats graph 11 | - uses [vue-router](https://router.vuejs.org/) to handle routing 12 | 13 | The application is built with [neutrino](https://neutrinojs.org/) (a Mozilla project) using its defaults for Vue.js applications. 14 | 15 | On every Github push (pull request or branch), the frontend is built, and even usable from the Taskcluster artifacts (it uses the testing environment as its default source). 16 | 17 | The application is then deployed with [task-boot](https://github.com/mozilla/task-boot/) on an Amazon S3 bucket, exposed through a Cloudfront configuration (this is managed by the Cloudops team at Mozilla). 18 | 19 | Finally the application is currently available at: 20 | 21 | - https://code-review.moz.tools/ on production (uses production backend) 22 | - https://code-review.testing.moz.tools/ on testing (uses testing backend) 23 | -------------------------------------------------------------------------------- /docs/publication.md: -------------------------------------------------------------------------------- 1 | # How to publish issues on Phabricator 2 | 3 | ## Build analysis output 4 | 5 | The code-review bot currently supports 6 different formats to report issues (clang-tidy, clang-format, zero coverage and mozlint). 6 | We are in the process of standardizing toward [a single format described here](analysis_format.md). 7 | 8 | ### Taskcluster artifacts 9 | 10 | Your analyzer task needs to produce a public Taskcluster artifact listing all the issues found in the patch. 11 | 12 | That means your task **must** write a JSON file on the local file system at a specified path. The task definition will take care of configuring Taskcluster for the storage of your file, so it becomes a publicly available file online. 13 | 14 | Here is [an example implementation](https://hg.mozilla.org/mozilla-central/file/tip/taskcluster/ci/source-test/clang.yml#l58) from the `clang-tidy` task in mozilla-central: 15 | 16 | ```yaml 17 | worker: 18 | artifacts: 19 | - type: file 20 | name: public/code-review/clang-tidy.json 21 | path: /builds/worker/clang-tidy.json 22 | ``` 23 | 24 | Here, the analyzer produces its JSON output as `/builds/worker/clang-tidy.json`, but Taskcluster will expose it on its own public hostname as `https://taskcluster-artifacts.net///public/code-review/clang-tidy.json` 25 | 26 | ## Publish results on Phabricator 27 | 28 | Once your task is triggered with the `code-review` attribute, its analysis artifact will be retrieved automatically by the bot. All issues found will be filtered using those basic rules: 29 | 30 | - if the issue is not in a modifided line of a file in the patch, it will be discarded. 31 | - if the issue is in a third party path, it will be discarded. 32 | 33 | We have [plans](https://bugzilla.mozilla.org/show_bug.cgi?id=1555721) to remove the first filter, by using a two pass approach and comparing the issues found before vs. after applying the patch. 34 | 35 | ## Troubleshooting 36 | 37 | 1. Check that your task is triggered by the decision task on new diffs 38 | 2. Check that your task is present in the task group published by the code review bot as `CI (Treeherder) Jobs` 39 | 3. Check that your task produces the expected analysis artifact 40 | 4. Check that the `code-review-issues` is present in that task group (for mozilla-central tasks) 41 | 5. Check that your test diff is available on [our dashboard](https://code-review.moz.tools/) by searching its revision ID or title (it can take several seconds to load all the tasks available) 42 | 6. Reach out to us, see [README](../README.md) for contact info 43 | -------------------------------------------------------------------------------- /docs/trigger.md: -------------------------------------------------------------------------------- 1 | # How to trigger your task on new diffs 2 | 3 | ## Step for mozilla-central 4 | 5 | :warning: This step is specific to mozilla-central and its taskgraph. Other repositories have different decision tasks and mechanisms. 6 | 7 | Once your task is setup in the repository taskgraph, usually in [taskcluster/ci/source-test](https://github.com/mozilla/release-services/issues/2254), you'll need to add an attribute to the task definition so the code-review bot will automatically start your task on new diffs. 8 | 9 | It's simple to add: 10 | 11 | ```yaml 12 | attributes: 13 | code-review: true 14 | ``` 15 | 16 | Here is an example for [clang tasks](https://hg.mozilla.org/mozilla-central/file/177ac92fb734b80f07c04710ec70f0b89a073351/taskcluster/ci/source-test/clang.yml#l12) 17 | 18 | Ask the [linter-reviewers group](https://phabricator.services.mozilla.com/project/view/119/) for review on that step, especially if you add a task in a different namespace than `source-test` 19 | 20 | ## Step for NSS 21 | 22 | [NSS](https://phabricator.services.mozilla.com/source/nss/) is already integrated in the code-review bot workflow, using its own decision task (no taskgraph here). 23 | 24 | To add a new analyzer, you'll need to add a task in `automation/taskcluster/graph/src/extend.js`, with the tag `code-review`. You can lookup the `clang-tidy` for a sample implementation. 25 | -------------------------------------------------------------------------------- /frontend/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | extends: ["standard", "plugin:vue/base"], 4 | globals: { 5 | process: true, 6 | BACKEND_URL: true, 7 | }, 8 | 9 | parser: "vue-eslint-parser", 10 | parserOptions: { 11 | ecmaFeatures: { 12 | generators: true, 13 | impliedStrict: true, 14 | objectLiteralDuplicateProperties: false, 15 | }, 16 | ecmaVersion: 2017, 17 | parser: "@babel/eslint-parser", 18 | sourceType: "module", 19 | }, 20 | plugins: ["babel", "vue"], 21 | rules: { 22 | "babel/new-cap": [ 23 | "error", 24 | { 25 | newIsCap: true, 26 | }, 27 | ], 28 | "babel/object-curly-spacing": ["error", "always"], 29 | "new-cap": "off", 30 | "object-curly-spacing": "off", 31 | }, 32 | settings: {}, 33 | }; 34 | -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | node_modules/ 3 | -------------------------------------------------------------------------------- /frontend/README.md: -------------------------------------------------------------------------------- 1 | # Code Review Frontend 2 | 3 | This is a simple Vue.JS administration frontend, the production instance is publicly available at https://code-review.moz.tools/ 4 | 5 | You'll need Node 16+ to be able to build it. 6 | 7 | ## Developer setup 8 | 9 | ``` 10 | npm install 11 | npm run build # to build once in production mode 12 | npm run build:dev # to build once in development mode 13 | npm run start # to start a dev server on port 8010 14 | ``` 15 | 16 | ## Linting 17 | 18 | eslint is available through: 19 | 20 | - `npm run lint` to list potential errors, 21 | - `npm run lint:fix` to automatically fix these errors. 22 | -------------------------------------------------------------------------------- /frontend/babel.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "plugins": ["@babel/plugin-syntax-dynamic-import"], 3 | "presets": [ 4 | [ 5 | "@babel/preset-env", 6 | { 7 | "debug": false, 8 | "exclude": ["transform-regenerator", "transform-async-to-generator"], 9 | "modules": false, 10 | "targets": { 11 | "browsers": [ 12 | "last 2 Chrome versions", 13 | "last 2 Firefox versions", 14 | "last 2 Edge versions", 15 | "last 2 Opera versions", 16 | "last 2 Safari versions", 17 | "last 2 iOS versions" 18 | ] 19 | }, 20 | "useBuiltIns": false 21 | } 22 | ] 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "code-review-frontend", 3 | "version": "1.10.2", 4 | "repository": "https://github.com/mozilla/code-review", 5 | "author": "", 6 | "description": "Mozilla Code Review frontend", 7 | "main": "index.js", 8 | "dependencies": { 9 | "axios": "^1.7.2", 10 | "bulma": "^1.0.4", 11 | "lodash": "^4.17.21", 12 | "vue": "^2.7.15", 13 | "vue-chartjs": "^5.3.2", 14 | "vue-router": "^3.6.5", 15 | "vue-template-compiler": "^2.7.16", 16 | "vuex": "^3.6.2" 17 | }, 18 | "devDependencies": { 19 | "@babel/eslint-parser": "^7.27.1", 20 | "@babel/plugin-syntax-dynamic-import": "^7.8.3", 21 | "@babel/preset-env": "^7.27.2", 22 | "babel-loader": "^10.0.0", 23 | "clean-webpack-plugin": "^4.0.0", 24 | "eslint": "^8.57.1", 25 | "eslint-config-standard": "^17.1.0", 26 | "eslint-plugin-babel": "^5.3.1", 27 | "eslint-plugin-import": "^2.31.0", 28 | "eslint-plugin-n": "^16.6.2", 29 | "eslint-plugin-promise": "^6.6.0", 30 | "eslint-plugin-vue": "^10.1.0", 31 | "html-webpack-plugin": "^5.6.3", 32 | "mini-css-extract-plugin": "^2.9.2", 33 | "process": "^0.11.10", 34 | "style-loader": "^4.0.0", 35 | "vue-loader": "^15.11.1", 36 | "webpack": "^5.99.8", 37 | "webpack-cli": "^6.0.1", 38 | "webpack-dev-server": "^5.2.1", 39 | "webpack-merge": "^6.0.1" 40 | }, 41 | "scripts": { 42 | "build": "webpack --mode=production", 43 | "build:dev": "webpack --mode=development", 44 | "start": "webpack serve --mode=development", 45 | "lint": "eslint *.js src/**/*.js src/**/*.vue", 46 | "lint:fix": "eslint --fix *.js src/**/*.js src/**/*.vue" 47 | }, 48 | "keywords": [], 49 | "license": "MPLv2" 50 | } 51 | -------------------------------------------------------------------------------- /frontend/src/App.vue: -------------------------------------------------------------------------------- 1 | 11 | 12 | 101 | 102 | 142 | -------------------------------------------------------------------------------- /frontend/src/Bool.vue: -------------------------------------------------------------------------------- 1 | 7 | 8 | 14 | -------------------------------------------------------------------------------- /frontend/src/Choice.vue: -------------------------------------------------------------------------------- 1 | 33 | 34 | 80 | -------------------------------------------------------------------------------- /frontend/src/Pagination.vue: -------------------------------------------------------------------------------- 1 | 47 | 48 | 71 | 72 | 85 | -------------------------------------------------------------------------------- /frontend/src/Progress.vue: -------------------------------------------------------------------------------- 1 | 11 | 12 | 44 | -------------------------------------------------------------------------------- /frontend/src/Revision.vue: -------------------------------------------------------------------------------- 1 | 43 | 44 | 114 | 115 | 120 | -------------------------------------------------------------------------------- /frontend/src/RevisionDiffs.vue: -------------------------------------------------------------------------------- 1 | 45 | 46 | 58 | -------------------------------------------------------------------------------- /frontend/src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | <%= htmlWebpackPlugin.options.title %> 6 | 7 | 8 |
9 | 10 | 11 | -------------------------------------------------------------------------------- /frontend/src/index.js: -------------------------------------------------------------------------------- 1 | import Vue from "vue"; 2 | import "bulma/css/bulma.css"; 3 | import App from "./App.vue"; 4 | import store from "./store.js"; 5 | import router from "./routes.js"; 6 | 7 | export default new Vue({ 8 | store, 9 | router, 10 | el: "#root", 11 | render: (h) => h(App), 12 | }); 13 | -------------------------------------------------------------------------------- /frontend/src/mixins.js: -------------------------------------------------------------------------------- 1 | export default { 2 | query: { 3 | methods: { 4 | update_query(name, value) { 5 | console.log("update query", name, value); 6 | const query = Object.assign({}, this.$route.query); 7 | if (value !== null && value !== "") { 8 | query[name] = value; 9 | } else if (name in query) { 10 | delete query[name]; 11 | } 12 | if (this.$router) { 13 | this.$router.push({ query }); 14 | } 15 | }, 16 | }, 17 | }, 18 | stats: { 19 | computed: { 20 | stats() { 21 | return this.$store.state.stats; 22 | }, 23 | progress() { 24 | if (!this.$store.state.total_stats) { 25 | return 0; 26 | } 27 | return (100 * this.stats.length) / this.$store.state.total_stats; 28 | }, 29 | }, 30 | }, 31 | date: { 32 | filters: { 33 | // Display time since elapsed in a human format 34 | since(datetime) { 35 | const dspStep = (t, name) => { 36 | const x = Math.round(t); 37 | if (x === 0) { 38 | return ""; 39 | } 40 | return x + " " + name + (x > 1 ? "s" : ""); 41 | }; 42 | 43 | let diff = (new Date() - new Date(datetime)) / 1000; 44 | const steps = [ 45 | [60, "second"], 46 | [60, "minute"], 47 | [24, "hour"], 48 | [30, "day"], 49 | [12, "month"], 50 | ]; 51 | let prev = ""; 52 | for (const [t, name] of steps) { 53 | if (diff > t) { 54 | prev = dspStep(diff % t, name); 55 | diff = diff / t; 56 | } else { 57 | return dspStep(diff, name) + " " + prev; 58 | } 59 | } 60 | return "Too long ago"; 61 | }, 62 | }, 63 | }, 64 | }; 65 | -------------------------------------------------------------------------------- /frontend/src/routes.js: -------------------------------------------------------------------------------- 1 | import Vue from "vue"; 2 | import VueRouter from "vue-router"; 3 | import Diffs from "./Diffs.vue"; 4 | import Tasks from "./Tasks.vue"; 5 | import Revision from "./Revision.vue"; 6 | import Issues from "./Issues.vue"; 7 | import Stats from "./Stats.vue"; 8 | import Check from "./Check.vue"; 9 | 10 | Vue.use(VueRouter); 11 | 12 | export default new VueRouter({ 13 | routes: [ 14 | { 15 | path: "/", 16 | name: "diffs", 17 | component: Diffs, 18 | }, 19 | { 20 | path: "/tasks", 21 | name: "tasks", 22 | component: Tasks, 23 | }, 24 | { 25 | path: "/revision/:revisionId", 26 | name: "revision", 27 | component: Revision, 28 | }, 29 | { 30 | path: "/diff/:diffId", 31 | name: "diff", 32 | component: Issues, 33 | }, 34 | { 35 | path: "/stats", 36 | name: "stats", 37 | component: Stats, 38 | }, 39 | { 40 | path: "/check/:repository/:analyzer/:check", 41 | name: "check", 42 | component: Check, 43 | }, 44 | ], 45 | }); 46 | -------------------------------------------------------------------------------- /frontend/test/simple_test.js: -------------------------------------------------------------------------------- 1 | import assert from "assert"; 2 | 3 | describe("simple", () => { 4 | it("should be sane", () => { 5 | assert.equal(true, !false); 6 | }); 7 | }); 8 | -------------------------------------------------------------------------------- /frontend/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require("path"); 2 | const HtmlWebpackPlugin = require("html-webpack-plugin"); 3 | const { merge } = require("webpack-merge"); 4 | const webpack = require("webpack"); 5 | const { CleanWebpackPlugin } = require("clean-webpack-plugin"); 6 | const MiniCssExtractPlugin = require("mini-css-extract-plugin"); 7 | 8 | const { VueLoaderPlugin } = require("vue-loader"); 9 | 10 | const common = { 11 | context: path.resolve(__dirname), 12 | entry: ["./src/index.js"], 13 | 14 | resolve: { 15 | extensions: [".js", ".vue"], 16 | }, 17 | 18 | output: { 19 | path: path.resolve(__dirname, "build"), 20 | filename: "[name].bundle.js", 21 | }, 22 | 23 | plugins: [ 24 | new VueLoaderPlugin(), 25 | 26 | new HtmlWebpackPlugin({ 27 | title: "Mozilla Code Review Bot", 28 | filename: "index.html", 29 | template: "./src/index.html", 30 | }), 31 | 32 | new webpack.ProvidePlugin({ 33 | process: "process/browser", 34 | }), 35 | 36 | // Define backend url as constant 37 | // using an environment variable with fallback for devs 38 | new webpack.DefinePlugin({ 39 | BACKEND_URL: JSON.stringify( 40 | process.env.BACKEND_URL || "http://localhost:8000" 41 | ), 42 | }), 43 | 44 | new MiniCssExtractPlugin({ 45 | filename: "[name].[contenthash:8].css", 46 | }), 47 | ], 48 | 49 | module: { 50 | rules: [ 51 | { 52 | test: /\.vue$/, 53 | loader: "vue-loader", 54 | }, 55 | { 56 | test: /\.js$/, 57 | exclude: /node_modules/, 58 | use: ["babel-loader"], 59 | }, 60 | { 61 | test: /\.(scss|css)$/, 62 | use: [ 63 | MiniCssExtractPlugin.loader, 64 | { 65 | loader: "css-loader", 66 | options: { 67 | importLoaders: 0, 68 | }, 69 | }, 70 | ], 71 | }, 72 | 73 | // Images: Copy image files to build folder 74 | { test: /\.(?:ico|gif|png|jpg|jpeg)$/i, type: "asset/resource" }, 75 | 76 | // Fonts and SVGs: Inline files 77 | { test: /\.(woff(2)?|eot|ttf|otf|svg|)$/, type: "asset/inline" }, 78 | ], 79 | }, 80 | }; 81 | 82 | const development = { 83 | mode: "development", 84 | 85 | devtool: "eval-cheap-module-source-map", 86 | 87 | devServer: { 88 | port: 8010, 89 | hot: true, 90 | historyApiFallback: true, 91 | open: true, 92 | }, 93 | }; 94 | 95 | const production = { 96 | mode: "production", 97 | 98 | devtool: "source-map", 99 | 100 | optimization: { 101 | minimize: true, 102 | splitChunks: { 103 | chunks: "all", 104 | maxInitialRequests: 5, 105 | name: false, 106 | }, 107 | runtimeChunk: "single", 108 | }, 109 | 110 | performance: { 111 | hints: "error", 112 | maxAssetSize: 1782579.2, 113 | maxEntrypointSize: 2621440, 114 | }, 115 | 116 | plugins: [ 117 | new CleanWebpackPlugin({ 118 | verbose: false, 119 | }), 120 | ], 121 | }; 122 | 123 | module.exports = (env, args) => { 124 | switch (args.mode) { 125 | case "development": 126 | return merge(common, development); 127 | case "production": 128 | return merge(common, production); 129 | default: 130 | throw new Error("No matching configuration was found!"); 131 | } 132 | }; 133 | -------------------------------------------------------------------------------- /integration/VERSION: -------------------------------------------------------------------------------- 1 | 1.10.2 2 | -------------------------------------------------------------------------------- /integration/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.13.3-slim 2 | 3 | ADD tools /src/tools 4 | ADD integration /src/integration 5 | 6 | RUN pip install --disable-pip-version-check --no-cache-dir --quiet /src/tools 7 | RUN pip install --disable-pip-version-check --no-cache-dir --quiet -r /src/integration/requirements.txt 8 | 9 | # Setup mercurial 10 | RUN /src/tools/docker/bootstrap-mercurial.sh 11 | 12 | CMD ["/src/integration/run.py"] 13 | -------------------------------------------------------------------------------- /integration/patches/nss.diff: -------------------------------------------------------------------------------- 1 | diff --git a/lib/mozpkix/lib/pkixnames.cpp b/lib/mozpkix/lib/pkixnames.cpp 2 | --- a/lib/mozpkix/lib/pkixnames.cpp 3 | +++ b/lib/mozpkix/lib/pkixnames.cpp 4 | @@ -77,6 +77,7 @@ ReadGeneralName(Reader& reader, 5 | /*out*/ GeneralNameType& generalNameType, 6 | /*out*/ Input& value) 7 | { 8 | + int a = null; 9 | uint8_t tag; 10 | Result rv = der::ReadTagAndGetValue(reader, tag, value); 11 | if (rv != Success) { 12 | -------------------------------------------------------------------------------- /integration/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | json-e==4.8.0 2 | jsonschema==4.23.0 3 | pytest==8.3.5 4 | 5 | -------------------------------------------------------------------------------- /integration/requirements.txt: -------------------------------------------------------------------------------- 1 | MozPhab==1.9.0 2 | pyyaml==6.0.2 3 | structlog==25.3.0 4 | taskcluster==84.0.2 5 | -------------------------------------------------------------------------------- /integration/taskcluster-hook.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [], 3 | "metadata": { 4 | "description": "Run code-review integration tests once every day", 5 | "emailOnError": true, 6 | "name": "Code review hook (CHANNEL)", 7 | "owner": "babadie@mozilla.com" 8 | }, 9 | "schedule": ["0 0 7 * * *"], 10 | "task": { 11 | "created": { 12 | "$fromNow": "0 seconds" 13 | }, 14 | "deadline": { 15 | "$fromNow": "2 hours" 16 | }, 17 | "expires": { 18 | "$fromNow": "1 month" 19 | }, 20 | "extra": {}, 21 | "metadata": { 22 | "description": "Integration tests for the code-review bot workflow", 23 | "name": "Code review integration tests (CHANNEL)", 24 | "owner": "babadie@mozilla.com", 25 | "source": "https://github.com/mozilla/code-review" 26 | }, 27 | "payload": { 28 | "artifacts": {}, 29 | "cache": { 30 | "code-review-integration-CHANNEL": "/cache" 31 | }, 32 | "capabilities": {}, 33 | "env": { 34 | "CLONE_DIR": "/cache", 35 | "TASKCLUSTER_SECRET": "project/relman/code-review/integration-CHANNEL" 36 | }, 37 | "features": { 38 | "taskclusterProxy": true 39 | }, 40 | "image": "mozilla/code-review:integration-REVISION", 41 | "maxRunTime": 7200 42 | }, 43 | "priority": "normal", 44 | "provisionerId": "proj-relman", 45 | "retries": 1, 46 | "routes": [], 47 | "schedulerId": "-", 48 | "scopes": [ 49 | "secrets:get:project/relman/code-review/integration-CHANNEL", 50 | "docker-worker:cache:code-review-integration-CHANNEL", 51 | "generic-worker:cache:code-review-integration-CHANNEL", 52 | "notify:email:*" 53 | ], 54 | "tags": {}, 55 | "workerType": "ci" 56 | }, 57 | "triggerSchema": { 58 | "additionalProperties": false, 59 | "type": "object" 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /integration/tests/conftest.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import importlib 6 | import sys 7 | 8 | import pytest 9 | 10 | 11 | @pytest.fixture 12 | def workflow(): 13 | """Import manually run.py file as we do not use a Python module""" 14 | spec = importlib.util.spec_from_file_location("run", "run.py") 15 | run = importlib.util.module_from_spec(spec) 16 | sys.modules["run"] = run 17 | spec.loader.exec_module(run) 18 | return run 19 | 20 | 21 | @pytest.fixture 22 | def mock_taskcluster(workflow): 23 | """ 24 | Mock the taskcluster secret 25 | """ 26 | workflow.taskcluster.secrets = { 27 | "phabricator": {"url": "http://phab.test/api/", "token": "cli-fakeToken"} 28 | } 29 | -------------------------------------------------------------------------------- /integration/tests/test_hooks.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import json 6 | import os 7 | 8 | import jsone 9 | import jsonschema 10 | 11 | with open(os.path.join("VERSION")) as f: 12 | version = f.read().strip() 13 | 14 | 15 | def test_jsone_validates(tmp_path): 16 | payload = {} 17 | hook_file = os.path.realpath("taskcluster-hook.json") 18 | assert os.path.exists(hook_file) 19 | 20 | content = open(hook_file).read() 21 | content = content.replace("CHANNEL", "dev") 22 | content = content.replace("VERSION", version) 23 | 24 | hook_content = json.loads(content) 25 | 26 | jsonschema.validate(instance=payload, schema=hook_content["triggerSchema"]) 27 | 28 | jsone.render(hook_content, context={"payload": payload}) 29 | -------------------------------------------------------------------------------- /integration/tests/test_workflow.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | import json 5 | import subprocess 6 | 7 | 8 | def test_publish(monkeypatch, workflow, tmpdir, mock_taskcluster): 9 | # Fake repo 10 | repo_dir = tmpdir.realpath() 11 | hg = tmpdir.mkdir(".hg") 12 | 13 | # Fake moz-phab 14 | def moz_phab(cmd, **kwargs): 15 | assert cmd == ["moz-phab", "submit", "--yes", "--no-lint", "--no-bug", "1234"] 16 | assert kwargs == {"cwd": repo_dir} 17 | 18 | return b"some random output...\n-> http://phab.test/D1" 19 | 20 | monkeypatch.setattr(subprocess, "check_output", moz_phab) 21 | 22 | # Run publication 23 | revision_url = workflow.publish(repo_dir, "TEST", 1234) 24 | assert revision_url == "http://phab.test/D1" 25 | 26 | # Check the arc auth is setup 27 | arcconfig = json.loads(hg.join(".arcconfig").read()) 28 | assert arcconfig == { 29 | "phabricator.uri": "http://phab.test/", 30 | "repository.callsign": "TEST", 31 | } 32 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools==74.1.2", "setuptools-scm==8.1.0"] 3 | 4 | [tool.ruff] 5 | 6 | [tool.ruff.lint] 7 | select = ["W", "E", "F", "I", "T10", "UP"] 8 | ignore = [ 9 | # Line too long 10 | "E501", 11 | ] 12 | 13 | [tool.ruff.lint.isort] 14 | known-first-party = ["code_review_backend", "code_review_bot", "code_review_tools"] 15 | -------------------------------------------------------------------------------- /tools/code_review_tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/code-review/b4a335bed1f6e5e8ea18a871080a833d4715b490/tools/code_review_tools/__init__.py -------------------------------------------------------------------------------- /tools/code_review_tools/heroku.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | # Heroku Documentation: https://devcenter.heroku.com/articles/dynos#local-environment-variables 6 | 7 | import os 8 | 9 | 10 | def in_dyno(): 11 | """Detects if the process is running in an Heroku Dyno""" 12 | return "DYNO" in os.environ 13 | 14 | 15 | def in_web_dyno(): 16 | """Detects if the process is running in an Heroku web Dyno""" 17 | return "PORT" in os.environ and os.environ.get("DYNO", "").startswith("web.") 18 | 19 | 20 | def in_worker_dyno(): 21 | """Detects if the process is running in an Heroku web Dyno""" 22 | return os.environ.get("DYNO", "").startswith("worker.") 23 | -------------------------------------------------------------------------------- /tools/code_review_tools/libmozdata.py: -------------------------------------------------------------------------------- 1 | from importlib.metadata import version 2 | 3 | import structlog 4 | from libmozdata.config import Config, set_config 5 | 6 | logger = structlog.get_logger(__name__) 7 | 8 | 9 | class LocalConfig(Config): 10 | """ 11 | Provide required configuration for libmozdata 12 | using in-memory class instead of an INI file 13 | """ 14 | 15 | def __init__(self, name, version): 16 | self.user_agent = f"{name}/{version}" 17 | logger.debug(f"User agent is {self.user_agent}") 18 | 19 | def get(self, section, option, default=None, **kwargs): 20 | if section == "User-Agent" and option == "name": 21 | return self.user_agent 22 | 23 | return default 24 | 25 | 26 | def setup(package_name): 27 | # Get version for main package 28 | package_version = version(package_name) 29 | 30 | # Provide to custom libzmodata configuration 31 | set_config(LocalConfig(package_name, package_version)) 32 | -------------------------------------------------------------------------------- /tools/code_review_tools/treeherder.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import urlencode 2 | 3 | JOBS_URL = "https://treeherder.mozilla.org/#/jobs" 4 | 5 | 6 | def get_job_url(repository, revision, task_id=None, run_id=None, **params): 7 | """Build a Treeherder job url for a given Taskcluster task""" 8 | assert isinstance(repository, str) and repository, "Missing repository" 9 | assert isinstance(revision, str) and revision, "Missing revision" 10 | assert "repo" not in params, "repo cannot be set in params" 11 | assert "revision" not in params, "revision cannot be set in params" 12 | 13 | params.update({"repo": repository, "revision": revision}) 14 | 15 | if task_id is not None and run_id is not None: 16 | params["selectedTaskRun"] = f"{task_id}-{run_id}" 17 | 18 | return f"{JOBS_URL}?{urlencode(params)}" 19 | -------------------------------------------------------------------------------- /tools/docker/bootstrap-mercurial.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | MERCURIAL_VERSION="6.8.2" 3 | VERSION_CONTROL_TOOLS_REV="5e36b6028416b2fbefcbee2bb9960c8cacb18caa" 4 | 5 | # Check source hgrc is available 6 | HGRC=/src/tools/docker/hgrc 7 | if [[ ! -f $HGRC ]]; then 8 | echo "Missing hgrc in $HGRC" 9 | exit 1 10 | fi 11 | 12 | apt-get update 13 | apt-get install --no-install-recommends -y curl python-dev-is-python3 gcc openssh-client libjemalloc2 14 | 15 | pip install --disable-pip-version-check --quiet --no-cache-dir mercurial==$MERCURIAL_VERSION 16 | 17 | # Setup mercurial with needed extensions 18 | hg clone -r $VERSION_CONTROL_TOOLS_REV https://hg.mozilla.org/hgcustom/version-control-tools /src/version-control-tools/ 19 | mkdir -p /etc/mercurial/hgrc.d 20 | ln -s $HGRC /etc/mercurial/hgrc.d/code-review.rc 21 | 22 | # Cleanup 23 | apt-get purge -y gcc curl python-dev-is-python3 24 | apt-get autoremove -y 25 | rm -rf /var/lib/apt/lists/* 26 | rm -rf /src/version-control-tools/.hg /src/version-control-tools/ansible /src/version-control-tools/docs /src/version-control-tools/testing 27 | -------------------------------------------------------------------------------- /tools/docker/hgrc: -------------------------------------------------------------------------------- 1 | [ui] 2 | username = Code Review Bot 3 | 4 | [extensions] 5 | hgmo = /src/version-control-tools/hgext/hgmo 6 | pushlog = /src/version-control-tools/hgext/pushlog 7 | robustcheckout = /src/version-control-tools/hgext/robustcheckout 8 | strip = 9 | -------------------------------------------------------------------------------- /tools/requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.11.18 2 | 3 | # Limit idna to avid conflicts 4 | idna>=2.5,<3.11 5 | libmozdata==0.2.10 6 | multidict==6.4.4 7 | rs_parsepatch==0.4.4 8 | sentry-sdk==2.29.1 9 | structlog==25.3.0 10 | taskcluster==84.0.2 11 | treeherder-client==5.0.0 12 | yarl==1.20.0 13 | -------------------------------------------------------------------------------- /tools/setup.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import setuptools 6 | 7 | 8 | def read_requirements(file_): 9 | lines = [] 10 | with open(file_) as f: 11 | for line in f.readlines(): 12 | line = line.strip() 13 | if ( 14 | line.startswith("-e ") 15 | or line.startswith("http://") 16 | or line.startswith("https://") 17 | ): 18 | extras = "" 19 | if "[" in line: 20 | extras = "[" + line.split("[")[1].split("]")[0] + "]" 21 | line = line.split("#")[1].split("egg=")[1] + extras 22 | elif line == "" or line.startswith("#") or line.startswith("-"): 23 | continue 24 | line = line.split("#")[0].strip() 25 | lines.append(line) 26 | return sorted(list(set(lines))) 27 | 28 | 29 | setuptools.setup( 30 | name="code-review-tools", 31 | version="0.2.0", 32 | description="Support tools for Mozilla code review", 33 | author="Mozilla Release Management", 34 | author_email="release-mgmt-analysis@mozilla.com", 35 | install_requires=read_requirements("requirements.txt"), 36 | packages=setuptools.find_packages(), 37 | include_package_data=True, 38 | zip_safe=False, 39 | license="MPL2", 40 | ) 41 | --------------------------------------------------------------------------------