├── .github └── workflows │ └── main.yml ├── .gitignore ├── .gitreview ├── COPYING ├── HACKING ├── INSTALL.md ├── README.md ├── app ├── .coveragerc ├── .gitignore ├── __init__.py ├── handlers │ ├── __init__.py │ ├── app.py │ ├── base.py │ ├── batch.py │ ├── bisect.py │ ├── build.py │ ├── build_logs.py │ ├── callback.py │ ├── common │ │ ├── __init__.py │ │ ├── lab.py │ │ ├── query.py │ │ ├── request.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── test_lab.py │ │ │ ├── test_query.py │ │ │ └── test_token.py │ │ └── token.py │ ├── count.py │ ├── count_distinct.py │ ├── dbindexes.py │ ├── distinct.py │ ├── job.py │ ├── job_logs.py │ ├── lab.py │ ├── report.py │ ├── response.py │ ├── send.py │ ├── stats.py │ ├── test_base.py │ ├── test_case.py │ ├── test_group.py │ ├── test_regression.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_batch_handler.py │ │ ├── test_bisect_handler.py │ │ ├── test_build_handler.py │ │ ├── test_build_logs_handler.py │ │ ├── test_callback_handler.py │ │ ├── test_count_handler.py │ │ ├── test_handler_base.py │ │ ├── test_handler_response.py │ │ ├── test_job_handler.py │ │ ├── test_job_logs_handler.py │ │ ├── test_lab_handler.py │ │ ├── test_report_handler.py │ │ ├── test_send_handler.py │ │ ├── test_stats_handler.py │ │ ├── test_test_case_handler.py │ │ ├── test_test_group_handler.py │ │ ├── test_token_handler.py │ │ ├── test_upload_handler.py │ │ └── test_version_handler.py │ ├── token.py │ ├── upload.py │ └── version.py ├── models │ ├── __init__.py │ ├── base.py │ ├── bisect.py │ ├── build.py │ ├── error_log.py │ ├── error_summary.py │ ├── job.py │ ├── lab.py │ ├── report.py │ ├── stats.py │ ├── test_case.py │ ├── test_group.py │ ├── test_regression.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_bisect_model.py │ │ ├── test_build_model.py │ │ ├── test_error_log_model.py │ │ ├── test_error_summary_model.py │ │ ├── test_job_model.py │ │ ├── test_lab_model.py │ │ ├── test_report_model.py │ │ ├── test_stats_model.py │ │ ├── test_test_case_model.py │ │ ├── test_test_group_model.py │ │ ├── test_test_regression_model.py │ │ └── test_token_model.py │ └── token.py ├── server.py ├── taskqueue │ ├── __init__.py │ ├── celery.py │ ├── celeryconfig.py │ ├── serializer.py │ └── tasks │ │ ├── __init__.py │ │ ├── bisect.py │ │ ├── build.py │ │ ├── callback.py │ │ ├── common.py │ │ ├── kcidb.py │ │ ├── report.py │ │ ├── stats.py │ │ └── test.py ├── tests │ └── __init__.py ├── urls.py └── utils │ ├── __init__.py │ ├── batch │ ├── __init__.py │ ├── batch_op.py │ ├── common.py │ └── tests │ │ ├── __init__.py │ │ └── test_batch_common.py │ ├── bisect │ ├── __init__.py │ ├── common.py │ ├── defconfig.py │ ├── test.py │ └── tests │ │ ├── __init__.py │ │ └── test_bisect.py │ ├── build │ ├── __init__.py │ └── tests │ │ ├── __init__.py │ │ └── test_build_import.py │ ├── callback │ ├── __init__.py │ ├── lava.py │ ├── lava_filters.py │ └── tests │ │ ├── __init__.py │ │ ├── data │ │ ├── auto-login-action.json │ │ ├── lava-json-jetson-tk1.json │ │ ├── lava-json-meson-gxbb-p200.json │ │ ├── login-action.json │ │ └── unhandled_fault-login-status.json │ │ └── test_lava_callback.py │ ├── database │ ├── __init__.py │ └── redisdb.py │ ├── db.py │ ├── emails.py │ ├── errors.py │ ├── kci_test │ ├── __init__.py │ ├── regressions.py │ └── tests │ │ ├── __init__.py │ │ └── test_tests.py │ ├── kcidb.py │ ├── lava_log_parser.py │ ├── log.py │ ├── log_parser.py │ ├── logs │ ├── __init__.py │ └── build.py │ ├── report │ ├── __init__.py │ ├── bisect.py │ ├── build.py │ ├── common.py │ ├── error.py │ ├── templates │ │ ├── bisect.txt │ │ ├── build.html │ │ ├── build.txt │ │ ├── multiple_emails.txt │ │ ├── templates.yaml │ │ ├── test.txt │ │ └── v4l2-compliance.txt │ ├── test.py │ └── tests │ │ ├── __init__.py │ │ ├── test_build_report.py │ │ └── test_report_common.py │ ├── scripts │ ├── __init__.py │ ├── create-data.py │ ├── migrate-git-branch.py │ └── update-defconfig.py │ ├── stats │ ├── __init__.py │ ├── daily.py │ └── tests │ │ ├── __init__.py │ │ └── test_daily_stats.py │ ├── tests │ ├── __init__.py │ ├── assets │ │ ├── build_log_0.log │ │ └── upload_file.txt │ ├── test_base.py │ ├── test_emails.py │ ├── test_log_parser.py │ ├── test_upload.py │ └── test_validator.py │ ├── upload.py │ └── validator.py ├── doc ├── .gitignore ├── Makefile ├── collection-batch.rst ├── collection-boot.rst ├── collection-build.rst ├── collection-count.rst ├── collection-job.rst ├── collection-lab.rst ├── collection-report.rst ├── collection-send.rst ├── collection-test-case.rst ├── collection-test-group.rst ├── collection-test.rst ├── collection-token.rst ├── collection-upload.rst ├── collection-version.rst ├── collections.rst ├── conf.py ├── examples.rst ├── examples │ ├── create-new-lab.py │ ├── get-all-boot-reports-with-jobid.py │ ├── get-all-boot-reports.py │ ├── get-all-failed-boot-reports.py │ ├── handling-compressed-data.py │ ├── import-tests-all-embedded.py │ ├── post-boot-report.py │ ├── single-tests-registration.py │ ├── trigger-boot-email-report.py │ ├── upload-multiple-files-2.py │ ├── upload-multiple-files.py │ └── upload-single-file.py ├── images │ ├── test-user-case0.svg │ ├── test-user-case1.svg │ └── test-user-case2.svg ├── index.rst ├── intro.rst ├── schema-attachment.rst ├── schema-batch.rst ├── schema-boot.rst ├── schema-build-logs-summary.rst ├── schema-build-logs.rst ├── schema-build.rst ├── schema-job.rst ├── schema-lab.rst ├── schema-measurement.rst ├── schema-report.rst ├── schema-send.rst ├── schema-test-case.rst ├── schema-test-defect.rst ├── schema-test-group.rst ├── schema-test.rst ├── schema-token.rst ├── schema.rst └── schema │ ├── 1.0 │ ├── attachment.json │ ├── error_log.json │ ├── get_boot.json │ ├── get_build.json │ ├── get_build_logs.json │ ├── get_build_logs_summary.json │ ├── get_job.json │ ├── get_lab.json │ ├── get_report.json │ ├── get_test_case.json │ ├── get_test_group.json │ ├── get_token.json │ ├── measurement.json │ ├── post_batch.json │ ├── post_boot.json │ ├── post_build.json │ ├── post_job.json │ ├── post_lab.json │ ├── post_send.json │ ├── post_test_case.json │ ├── post_test_group.json │ ├── post_token.json │ └── test_defect.json │ ├── 1.1 │ ├── get_boot.json │ ├── get_build.json │ ├── get_build_logs.json │ ├── get_build_logs_summary.json │ ├── get_job.json │ ├── post_boot.json │ ├── post_build.json │ ├── post_job.json │ └── post_send.json │ └── 1.2 │ └── post_test_group.json ├── requirements-dev.txt └── requirements.txt /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: CI 4 | 5 | # Controls when the action will run. 6 | on: 7 | # Triggers the workflow on push or pull request events but only for the 8 | # main branch 9 | push: 10 | branches: [ main ] 11 | pull_request: 12 | branches: [ main ] 13 | 14 | # Allows you to run this workflow manually from the Actions tab 15 | workflow_dispatch: 16 | 17 | jobs: 18 | check: 19 | runs-on: ubuntu-18.04 20 | strategy: 21 | matrix: 22 | python-version: [2.7] 23 | 24 | steps: 25 | 26 | # Checks out the repository under $GITHUB_WORKSPACE 27 | - uses: actions/checkout@v2 28 | 29 | - name: Install Debian dependencies 30 | run: | 31 | sudo apt-get update 32 | sudo apt-get install -y python-setuptools pycodestyle python-wheel 33 | 34 | - name: Install pip dependencies 35 | run: | 36 | pip install -r requirements-dev.txt 37 | 38 | - name: Run unit tests 39 | run: | 40 | cd app 41 | for dir in \ 42 | models/tests \ 43 | utils/tests \ 44 | utils/build/tests \ 45 | utils/bisect/tests \ 46 | handlers/common/tests \ 47 | handlers/tests \ 48 | utils/report/tests 49 | do 50 | echo Running tests in $dir 51 | python -m unittest discover $dir || exit 1 52 | done 53 | 54 | - name: Run pycodestyle 55 | run: | 56 | cd app 57 | pycodestyle . 58 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.pyo 3 | .coverage 4 | *.pid 5 | -------------------------------------------------------------------------------- /.gitreview: -------------------------------------------------------------------------------- 1 | [gerrit] 2 | host=review.linaro.org 3 | port=29418 4 | project=infrastructure/kernel-ci-backend 5 | -------------------------------------------------------------------------------- /HACKING: -------------------------------------------------------------------------------- 1 | Tests 2 | ===== 3 | 4 | Tests run best from a virtualenv using python2.7. Though most tests use 5 | fakeredis, some do actually require an actual redis running locally. 6 | 7 | From the 'app/' folder, run: 8 | 9 | python -m unittest discover 10 | 11 | or 12 | 13 | python -m unittest tests.test_suite 14 | 15 | Example Test Setup 16 | ------------------ 17 | 18 | $ redis-cli ping # ensure redis is available 19 | $ cd app 20 | $ virtualenv -p python2.7 venv 21 | $ . venv/bin/activate 22 | $ pip install -r ../requirements-dev.txt 23 | $ python -m unittest discover 24 | ... 25 | Ran 787 tests in 1.852s 26 | 27 | OK 28 | 29 | 30 | Test Code Coverage 31 | ================== 32 | 33 | To have a test coverage report, install the `coverage` package via pip, and 34 | then execute: 35 | 36 | cd app/ 37 | coverage run -m unittest tests.test_suite 2>/dev/null && coverage html 38 | 39 | The output will be stored in the `kernel-ci-backend-coverage` directory. 40 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | # Install 2 | 3 | All installation docs are now on the dedicated [kernelci-backend-config](https://github.com/kernelci/kernelci-backend-config/blob/master/INSTALL.md) 4 | 5 | # Configuration/Administration 6 | 7 | ## Main configurations files 8 | /etc/kernelci/kernelci-frontend.cfg 9 | /etc/kernelci/kernelci-backend.cfg 10 | /etc/kernelci/kernelci-celery.cfg 11 | 12 | They are filled with informations from secrets.yml. 13 | 14 | ## Troubleshooting/Main log file 15 | Celery and kernelci-backend services logs via syslog. 16 | Nginx logs in /var/log/nginx 17 | Uwsgi logs via syslog on Centos (and in /var/log/uwsgi on debian) 18 | 19 | ## Network access 20 | Only the nginx service needs to be accessible from the external network. All other components needs to be available only from localhost. 21 | 22 | ## Data / backups 23 | Only mongodb stores data and need to be backuped. 24 | 25 | # Run the server 26 | 27 | From the 'app/' folder, run: 28 | 29 | python server.py 30 | 31 | ## The Celery worker 32 | 33 | To run external tasks, Celery is used. It will start automatically at boot 34 | time. In the ansible playbook, there is the upstart configuration file to be 35 | used. 36 | 37 | It is possible to pass an external configuration file (key=value) using an 38 | environment variable: CELERY_CONFIG_MODULE. 39 | 40 | To manually run it, from the 'app/' folder, run: 41 | 42 | celery worker --autoscale=10,0 --app=taskqueue --loglevel=INFO 43 | 44 | At the moment the Celery worker is based on redis.io: it needs to be installed 45 | as well to make it work. 46 | 47 | There is support for MongoDB, but is still experimental and has not been 48 | tested yet with this application. 49 | 50 | # Basic interactions 51 | 52 | All operations now need a token or they will not be valid: the server will 53 | reply with a 403 error. 54 | 55 | In order to create the first token, use the master key set in the application 56 | and create an admin token. Admin tokens can perform all actions: GET, POST, 57 | DELETE and create new token as well. 58 | 59 | For other application usage, you should create a superuser token. 60 | 61 | ## Create Token 62 | 63 | To create a token at the beginning: 64 | 65 | curl -X POST -H "Content-Type: application/json" -H "Authorization: $MASTER_KEY" -d "{"email": "you@example.net", "admin": 1}" localhost:8888/token 66 | 67 | The command will return the token created. Use it to create new tokens. 68 | The master key should be used only the first time. 69 | 70 | ## GET 71 | 72 | curl localhost:8888/job 73 | curl localhost:8888/build 74 | curl localhost:8888/boot 75 | curl localhost:8888/count 76 | 77 | curl localhost:8888/job/$JOB_ID 78 | curl localhost:8888/build/$DEFCONF_ID 79 | 80 | curl localhost:8888/job?limit=40 81 | curl localhost:8888/job?limit=40&skip=20 82 | 83 | ## POST 84 | 85 | POST requests work for: 86 | 87 | /job 88 | /boot 89 | 90 | This command will tell the application to parse the directory located at 91 | 'stable/v3.12.14' and import everything there: 92 | 93 | curl -X POST -H "Content-Type: application/json" -H "Authorization: foo" -d '{"job": "stable", "kernel": "v3.12.14"}' localhost:8888/job 94 | 95 | The content of the `Authorization` is not important at the moment. All requests 96 | without that header will be discarded though. 97 | 98 | ## DELETE 99 | 100 | curl -X DELETE -H "Authorization: foo" localhost:8888/job/job-id 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![License: AGPL v3](https://img.shields.io/badge/License-AGPL%20v3-blue.svg)](http://www.gnu.org/licenses/agpl-3.0) 2 | 3 | # Kernel CI Backend 4 | 5 | This is the backend engine, Python and Tornado powered, that makes up all the 6 | available API for the Kernel CI reporting dashboard. 7 | 8 | For more info, see: http://api.kernelci.org/ 9 | 10 | # Installation 11 | 12 | See the [INSTALL](./INSTALL.md) files. 13 | -------------------------------------------------------------------------------- /app/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = . 4 | omit = 5 | server.py 6 | setup* 7 | */tests/* 8 | utils/scripts/* 9 | handlers/dbindexes.py 10 | 11 | [report] 12 | precision = 2 13 | show_missing = True 14 | 15 | [html] 16 | title = Code Coverage of kernel-ci-backend 17 | directory = kernel-ci-backend-coverage 18 | -------------------------------------------------------------------------------- /app/.gitignore: -------------------------------------------------------------------------------- 1 | .coverage 2 | kernel-ci-backend-coverage/ 3 | -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/__init__.py -------------------------------------------------------------------------------- /app/handlers/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "2022.2.0" 2 | __versionfull__ = __version__ 3 | -------------------------------------------------------------------------------- /app/handlers/app.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2014,2015 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """ 19 | A very simple RequestHandler used as the default one for the Tornado 20 | application. 21 | """ 22 | 23 | import tornado.web 24 | 25 | 26 | class AppHandler(tornado.web.RequestHandler): 27 | """This handler is used to provide custom error messages. 28 | 29 | It is used to provide JSON response on errors, and the only implemented 30 | method is `write_error'. 31 | """ 32 | 33 | def __init__(self, application, request, **kwargs): 34 | super(AppHandler, self).__init__(application, request, **kwargs) 35 | 36 | def write_error(self, status_code, **kwargs): 37 | self.set_status(404, "Resource not found") 38 | self.write(dict(code=404, message="Resource not found")) 39 | self.finish() 40 | -------------------------------------------------------------------------------- /app/handlers/build_logs.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2015 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """The RequestHandler for /build//logs URLs.""" 19 | 20 | import bson 21 | 22 | import handlers.base as hbase 23 | import handlers.common.query 24 | import handlers.response as hresponse 25 | import models 26 | import models.error_log as merrlog 27 | import utils.db 28 | 29 | 30 | # pylint: disable=too-many-public-methods 31 | class BuildLogsHandler(hbase.BaseHandler): 32 | """Retrieve the parsed build logs of a single build ID.""" 33 | 34 | def __init__(self, application, request, **kwargs): 35 | super(BuildLogsHandler, self).__init__(application, request, **kwargs) 36 | 37 | @property 38 | def collection(self): 39 | return self.db[models.ERROR_LOGS_COLLECTION] 40 | 41 | @staticmethod 42 | def _valid_keys(method): 43 | return merrlog.ERROR_LOG_VALID_KEYS.get(method, None) 44 | 45 | def execute_delete(self, *args, **kwargs): 46 | """Not implemented.""" 47 | return hresponse.HandlerResponse(501) 48 | 49 | def execute_put(self, *args, **kwargs): 50 | """Not implemented.""" 51 | return hresponse.HandlerResponse(501) 52 | 53 | def execute_post(self, *args, **kwargs): 54 | """Not implemented.""" 55 | return hresponse.HandlerResponse(501) 56 | 57 | def _get_one(self, doc_id, **kwargs): 58 | response = hresponse.HandlerResponse() 59 | result = None 60 | 61 | try: 62 | obj_id = bson.objectid.ObjectId(doc_id) 63 | result = utils.db.find_one2( 64 | self.collection, 65 | {models.BUILD_ID_KEY: obj_id}, 66 | fields=handlers.common.query.get_query_fields( 67 | self.get_query_arguments) 68 | ) 69 | 70 | if result: 71 | # result here is returned as a dictionary from mongodb 72 | response.result = result 73 | else: 74 | response.status_code = 404 75 | response.reason = "Resource '%s' not found" % doc_id 76 | except bson.errors.InvalidId, ex: 77 | self.log.exception(ex) 78 | self.log.error("Provided doc ID '%s' is not valid", doc_id) 79 | response.status_code = 400 80 | response.reason = "Wrong ID value provided" 81 | 82 | return response 83 | -------------------------------------------------------------------------------- /app/handlers/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/handlers/common/__init__.py -------------------------------------------------------------------------------- /app/handlers/common/request.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2015 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | import utils 19 | 20 | 21 | def has_valid_content_type(headers, remote_ip, content_type=None): 22 | """Check if the Content-Type header is set correctly. 23 | 24 | :param headers: The headers of the request to check. 25 | :type headers: dict 26 | :param remote_ip: The remote IP address. 27 | :type remote_ip: str 28 | :param content_type: The content-type that should be found. Default to 29 | application/json. 30 | :type content_type: str 31 | :return True or False. 32 | """ 33 | is_valid = False 34 | 35 | if not content_type: 36 | content_type = "application/json" 37 | 38 | if "Content-Type" in headers.viewkeys(): 39 | if headers["Content-Type"].startswith(content_type): 40 | is_valid = True 41 | else: 42 | utils.LOG.error( 43 | "Received wrong content type ('%s') from IP '%s'", 44 | headers["Content-Type"], remote_ip) 45 | 46 | return is_valid 47 | 48 | 49 | def valid_post_request(headers, remote_ip, content_type=None): 50 | """Check that a POST request is valid. 51 | 52 | :param headers: The headers of the request. 53 | :type headers: dict 54 | :param remote_ip: The remote IP address. 55 | :type remote_ip: str 56 | :param content_type: The content type that should be found. 57 | :type content_type: str 58 | :return 200 in case is valid, 415 if not. 59 | """ 60 | return_code = 200 61 | 62 | if not has_valid_content_type( 63 | headers, remote_ip, content_type=content_type): 64 | return_code = 415 65 | 66 | return return_code 67 | -------------------------------------------------------------------------------- /app/handlers/common/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/handlers/common/tests/__init__.py -------------------------------------------------------------------------------- /app/handlers/job.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2014,2015,2016,2017 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """The RequestHandler for /job URLs.""" 19 | 20 | import bson 21 | 22 | import handlers.base as hbase 23 | import handlers.response as hresponse 24 | import models 25 | import taskqueue.tasks.build as taskb 26 | import utils.db 27 | 28 | JOB_NOT_FOUND = "Job '%s-%s (branch %s)' not found" 29 | INTERNAL_ERROR = \ 30 | "Internal error while searching/updating job '%s-%s' (branch %s)" 31 | JOB_UPDATED = "Job '%s-%s' (branch %s) marked as '%s'" 32 | INVALID_STATUS = "Status value '%s' is not valid, should be one of: %s" 33 | 34 | 35 | # pylint: disable=too-many-public-methods 36 | class JobHandler(hbase.BaseHandler): 37 | """Handle the /job URLs.""" 38 | 39 | def __init__(self, application, request, **kwargs): 40 | super(JobHandler, self).__init__(application, request, **kwargs) 41 | 42 | @property 43 | def collection(self): 44 | return self.db[models.JOB_COLLECTION] 45 | 46 | @staticmethod 47 | def _valid_keys(method): 48 | return models.JOB_VALID_KEYS.get(method, None) 49 | 50 | def _post(self, *args, **kwargs): 51 | response = hresponse.HandlerResponse() 52 | 53 | obj = kwargs["json_obj"] 54 | 55 | job = obj.get(models.JOB_KEY) 56 | kernel = obj.get(models.KERNEL_KEY) 57 | git_branch = obj.get(models.GIT_BRANCH_KEY) 58 | status = obj.get(models.STATUS_KEY, None) 59 | 60 | if not status: 61 | status = models.PASS_STATUS 62 | 63 | if (status in models.VALID_JOB_STATUS): 64 | ret_val = utils.db.find_and_update( 65 | self.collection, 66 | { 67 | models.GIT_BRANCH_KEY: git_branch, 68 | models.JOB_KEY: job, 69 | models.KERNEL_KEY: kernel 70 | }, 71 | {models.STATUS_KEY: status} 72 | ) 73 | 74 | if ret_val == 404: 75 | response.status_code = 404 76 | response.reason = JOB_NOT_FOUND % (job, kernel, git_branch) 77 | elif ret_val == 500: 78 | response.status_code = 500 79 | response.reason = INTERNAL_ERROR % (job, kernel, git_branch) 80 | else: 81 | response.reason = \ 82 | JOB_UPDATED % (job, kernel, git_branch, status) 83 | # Create the build logs summary file. 84 | taskb.create_build_logs_summary.apply_async( 85 | [job, kernel, git_branch]) 86 | else: 87 | response.status_code = 400 88 | response.reason = \ 89 | INVALID_STATUS % (status, str(models.VALID_JOB_STATUS)) 90 | 91 | return response 92 | -------------------------------------------------------------------------------- /app/handlers/job_logs.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited -0700 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """The RequestHandler for /job//logs URLs.""" 19 | 20 | import bson 21 | 22 | import handlers.base as hbase 23 | import handlers.common.query 24 | import handlers.response as hresponse 25 | import models 26 | import models.error_summary as errsummary 27 | import utils.db 28 | 29 | 30 | # pylint: disable=too-many-public-methods 31 | class JobLogsHandler(hbase.BaseHandler): 32 | """Retrieve the parsed build logs of a single job ID.""" 33 | 34 | def __init__(self, application, request, **kwargs): 35 | super(JobLogsHandler, self).__init__(application, request, **kwargs) 36 | 37 | @property 38 | def collection(self): 39 | return self.db[models.ERRORS_SUMMARY_COLLECTION] 40 | 41 | @staticmethod 42 | def _valid_keys(method): 43 | return errsummary.ERROR_SUMMARY_VALID_KEYS.get(method, None) 44 | 45 | def execute_delete(self, *args, **kwargs): 46 | """Not implemented.""" 47 | return hresponse.HandlerResponse(501) 48 | 49 | def execute_put(self, *args, **kwargs): 50 | """Not implemented.""" 51 | return hresponse.HandlerResponse(501) 52 | 53 | def execute_post(self, *args, **kwargs): 54 | """Not implemented.""" 55 | return hresponse.HandlerResponse(501) 56 | 57 | def _get_one(self, doc_id, **kwargs): 58 | response = hresponse.HandlerResponse() 59 | result = None 60 | 61 | try: 62 | obj_id = bson.objectid.ObjectId(doc_id) 63 | result = utils.db.find_one2( 64 | self.collection, 65 | {models.JOB_ID_KEY: obj_id}, 66 | fields=handlers.common.query.get_query_fields( 67 | self.get_query_arguments) 68 | ) 69 | 70 | if result: 71 | # result here is returned as a dictionary from mongodb 72 | response.result = result 73 | else: 74 | response.status_code = 404 75 | response.reason = "Resource '%s' not found" % doc_id 76 | except bson.errors.InvalidId, ex: 77 | self.log.exception(ex) 78 | self.log.error("Provided doc ID '%s' is not valid", doc_id) 79 | response.status_code = 400 80 | response.reason = "Wrong ID value provided" 81 | 82 | return response 83 | -------------------------------------------------------------------------------- /app/handlers/report.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2015 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """The RequestHandler for /report URLs.""" 19 | 20 | import handlers.base as hbase 21 | import handlers.common.token 22 | import handlers.response as hresponse 23 | import models 24 | 25 | 26 | class ReportHandler(hbase.BaseHandler): 27 | """Handle the /report[s] URLs.""" 28 | 29 | def __init__(self, application, request, **kwargs): 30 | super(ReportHandler, self).__init__(application, request, **kwargs) 31 | 32 | @property 33 | def collection(self): 34 | return self.db[models.REPORT_COLLECTION] 35 | 36 | @staticmethod 37 | def _valid_keys(method): 38 | return models.REPORT_VALID_KEYS.get(method, None) 39 | 40 | @staticmethod 41 | def _token_validation_func(): 42 | return handlers.common.token.valid_token_th 43 | 44 | def execute_delete(self, *args, **kwargs): 45 | """Perform DELETE pre-operations. 46 | 47 | Check that the DELETE request is OK. 48 | """ 49 | response = None 50 | valid_token, _ = self.validate_req_token("DELETE") 51 | 52 | if valid_token: 53 | response = hresponse.HandlerResponse(501) 54 | else: 55 | response = hresponse.HandlerResponse(403) 56 | 57 | return response 58 | 59 | def execute_post(self, *args, **kwargs): 60 | """Execute the POST pre-operations. 61 | 62 | Checks that everything is OK to perform a POST. 63 | """ 64 | response = None 65 | valid_token, _ = self.validate_req_token("POST") 66 | 67 | if valid_token: 68 | response = hresponse.HandlerResponse(501) 69 | else: 70 | response = hresponse.HandlerResponse(403) 71 | 72 | return response 73 | -------------------------------------------------------------------------------- /app/handlers/stats.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2015 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """The request handler for the /statistics URL.""" 19 | 20 | import handlers.base as hbase 21 | import handlers.response as hresponse 22 | import models 23 | 24 | 25 | class StatisticsHandler(hbase.BaseHandler): 26 | """Handle request to the statistics API resource.""" 27 | 28 | def __init__(self, application, request, **kwargs): 29 | super(StatisticsHandler, self).__init__(application, request, **kwargs) 30 | 31 | @property 32 | def collection(self): 33 | return self.db[models.DAILY_STATS_COLLECTION] 34 | 35 | @staticmethod 36 | def _valid_keys(method): 37 | return models.STATISTICS_VALID_KEYS.get(method, None) 38 | 39 | def execute_delete(self, *args, **kwargs): 40 | """Perform DELETE pre-operations. 41 | 42 | Check that the DELETE request is OK. 43 | """ 44 | response = None 45 | valid_token, _ = self.validate_req_token("DELETE") 46 | 47 | if valid_token: 48 | response = hresponse.HandlerResponse(501) 49 | else: 50 | response = hresponse.HandlerResponse(403) 51 | 52 | return response 53 | 54 | def execute_post(self, *args, **kwargs): 55 | """Execute the POST pre-operations. 56 | 57 | Checks that everything is OK to perform a POST. 58 | """ 59 | response = None 60 | valid_token, _ = self.validate_req_token("POST") 61 | 62 | if valid_token: 63 | response = hresponse.HandlerResponse(501) 64 | else: 65 | response = hresponse.HandlerResponse(403) 66 | 67 | return response 68 | 69 | def execute_put(self, *args, **kwargs): 70 | """Execute the PUT pre-operations. 71 | 72 | Checks that everything is OK to perform a PUT. 73 | """ 74 | response = None 75 | valid_token, _ = self.validate_req_token("PUT") 76 | 77 | if valid_token: 78 | response = hresponse.HandlerResponse(501) 79 | else: 80 | response = hresponse.HandlerResponse(403) 81 | 82 | return response 83 | -------------------------------------------------------------------------------- /app/handlers/test_case.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Collabora Limited 2017, 2018, 2019, 2020 2 | # Author: Guillaume Tucker 3 | # Author: Ana Guerrero Lopez 4 | # Author: Michal Galka 5 | # 6 | # Copyright (C) Linaro Limited 2015,2016 7 | # Author: Milo Casagrande 8 | # 9 | # This program is free software; you can redistribute it and/or modify it under 10 | # the terms of the GNU Lesser General Public License as published by the Free 11 | # Software Foundation; either version 2.1 of the License, or (at your option) 12 | # any later version. 13 | # 14 | # This program is distributed in the hope that it will be useful, but WITHOUT 15 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 17 | # details. 18 | # 19 | # You should have received a copy of the GNU Lesser General Public License 20 | # along with this library; if not, write to the Free Software Foundation, Inc., 21 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 22 | 23 | """The RequestHandler for /test/case URLs.""" 24 | 25 | import handlers.test_base as htbase 26 | import models 27 | 28 | 29 | # pylint: disable=too-many-public-methods 30 | class TestCaseHandler(htbase.TestBaseHandler): 31 | """The test set request handler.""" 32 | 33 | def __init__(self, application, request, **kwargs): 34 | super(TestCaseHandler, self).__init__(application, request, **kwargs) 35 | 36 | @property 37 | def collection(self): 38 | return self.db[models.TEST_CASE_COLLECTION] 39 | 40 | @staticmethod 41 | def _valid_keys(method): 42 | return models.TEST_CASE_VALID_KEYS.get(method, None) 43 | -------------------------------------------------------------------------------- /app/handlers/test_regression.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Collabora Limited 2020 2 | # Author: Guillaume Tucker 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """The RequestHandler for /test/regression URLs.""" 19 | 20 | import handlers.test_base 21 | import models 22 | 23 | 24 | # pylint: disable=too-many-public-methods 25 | class TestRegressionHandler(handlers.test_base.TestBaseHandler): 26 | """The test regression request handler.""" 27 | 28 | def __init__(self, application, request, **kwargs): 29 | super(TestRegressionHandler, self).__init__( 30 | application, request, **kwargs) 31 | 32 | @property 33 | def collection(self): 34 | return self.db[models.TEST_REGRESSION_COLLECTION] 35 | 36 | @staticmethod 37 | def _valid_keys(method): 38 | return models.TEST_REGRESSION_VALID_KEYS.get(method) 39 | -------------------------------------------------------------------------------- /app/handlers/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/handlers/tests/__init__.py -------------------------------------------------------------------------------- /app/handlers/tests/test_bisect_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2014,2015 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Test module for the BisectHandler handler.""" 19 | 20 | import mock 21 | import tornado 22 | 23 | import urls 24 | 25 | from handlers.tests.test_handler_base import TestHandlerBase 26 | 27 | 28 | class TestBisectHandler(TestHandlerBase): 29 | 30 | def get_app(self): 31 | return tornado.web.Application([urls._BISECT_URL], **self.settings) 32 | 33 | def test_bisect_wrong_collection(self): 34 | headers = {"Authorization": "foo"} 35 | 36 | response = self.fetch("/bisect/bisect_id", headers=headers) 37 | self.assertEqual(response.code, 400) 38 | -------------------------------------------------------------------------------- /app/handlers/tests/test_count_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2014,2015 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Test module for the CountHandler handler.""" 19 | 20 | import json 21 | import tornado 22 | 23 | import urls 24 | 25 | from handlers.tests.test_handler_base import TestHandlerBase 26 | 27 | 28 | class TestCountHandler(TestHandlerBase): 29 | 30 | def get_app(self): 31 | return tornado.web.Application([urls._COUNT_URL], **self.settings) 32 | 33 | def test_post(self): 34 | body = json.dumps(dict(job="job", kernel="kernel")) 35 | 36 | response = self.fetch("/count", method="POST", body=body) 37 | 38 | self.assertEqual(response.code, 501) 39 | self.assertEqual( 40 | response.headers["Content-Type"], self.content_type) 41 | 42 | def test_delete(self): 43 | response = self.fetch("/count", method="DELETE") 44 | 45 | self.assertEqual(response.code, 501) 46 | self.assertEqual( 47 | response.headers["Content-Type"], self.content_type) 48 | 49 | def test_get_wrong_resource(self): 50 | headers = {"Authorization": "foo"} 51 | 52 | response = self.fetch("/count/foobar", headers=headers) 53 | 54 | self.assertEqual(response.code, 404) 55 | self.assertEqual( 56 | response.headers["Content-Type"], self.content_type) 57 | 58 | def test_get_count_all(self): 59 | headers = {"Authorization": "foo"} 60 | response = self.fetch("/count", headers=headers) 61 | 62 | self.assertEqual(response.code, 200) 63 | self.assertEqual( 64 | response.headers["Content-Type"], self.content_type) 65 | 66 | def test_get_count_all_with_query(self): 67 | headers = {"Authorization": "foo"} 68 | response = self.fetch( 69 | "/count?board=foo&status=FAIL", headers=headers) 70 | 71 | self.assertEqual(response.code, 200) 72 | self.assertEqual( 73 | response.headers["Content-Type"], self.content_type) 74 | 75 | def test_get_count_collection(self): 76 | headers = {"Authorization": "foo"} 77 | response = self.fetch("/count/build", headers=headers) 78 | 79 | self.assertEqual(response.code, 200) 80 | self.assertEqual( 81 | response.headers["Content-Type"], self.content_type) 82 | 83 | def test_get_count_collection_with_query(self): 84 | headers = {"Authorization": "foo"} 85 | response = self.fetch("/count/build?arch=foo", headers=headers) 86 | 87 | self.assertEqual(response.code, 200) 88 | self.assertEqual( 89 | response.headers["Content-Type"], self.content_type) 90 | -------------------------------------------------------------------------------- /app/handlers/tests/test_handler_base.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2015,2016,2017 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Base module for testing all handlers.""" 19 | 20 | import concurrent.futures 21 | import fakeredis 22 | import mock 23 | import mongomock 24 | import random 25 | import string 26 | import tornado 27 | 28 | from tornado.testing import ( 29 | AsyncHTTPTestCase, 30 | LogTrapTestCase 31 | ) 32 | 33 | import handlers.app 34 | import models.token as mtoken 35 | 36 | 37 | class TestHandlerBase(AsyncHTTPTestCase, LogTrapTestCase): 38 | 39 | def setUp(self): 40 | # Default Content-Type header returned by Tornado. 41 | self.content_type = "application/json; charset=UTF-8" 42 | self.req_token = mtoken.Token() 43 | self.database = mongomock.MongoClient()["kernel-ci"] 44 | self.redisdb = fakeredis.FakeStrictRedis() 45 | self.dboptions = { 46 | "mongodb_password": "", 47 | "mongodb_user": "" 48 | } 49 | self.mail_options = { 50 | "smtp_port": 465, 51 | "smtp_host": "localhost", 52 | "smtp_user": "mailuser", 53 | "smtp_password": "mailpassword", 54 | "smtp_sender": "me@example.net", 55 | "smtp_sender_desc": "Me Email" 56 | } 57 | self.settings = { 58 | "dboptions": self.dboptions, 59 | "mailoptions": self.mail_options, 60 | "redis_connection": self.redisdb, 61 | "database": self.database, 62 | "executor": concurrent.futures.ThreadPoolExecutor(max_workers=1), 63 | "default_handler_class": handlers.app.AppHandler, 64 | "debug": False, 65 | "version": "foo", 66 | "master_key": "bar", 67 | "senddelay": 60 * 60 68 | } 69 | 70 | super(TestHandlerBase, self).setUp() 71 | 72 | patched_find_token = mock.patch( 73 | "handlers.common.token.find_token") 74 | self.find_token = patched_find_token.start() 75 | self.find_token.return_value = self.req_token 76 | 77 | patched_validate_token = mock.patch( 78 | "handlers.common.token.validate_token") 79 | self.validate_token = patched_validate_token.start() 80 | self.validate_token.return_value = (True, self.req_token) 81 | 82 | self.addCleanup(patched_find_token.stop) 83 | self.addCleanup(patched_validate_token.stop) 84 | 85 | self.doc_id = "".join( 86 | [random.choice(string.digits) for x in xrange(24)]) 87 | 88 | def tearDown(self): 89 | self.redisdb.flushall() 90 | 91 | def get_new_ioloop(self): 92 | return tornado.ioloop.IOLoop.instance() 93 | -------------------------------------------------------------------------------- /app/handlers/tests/test_report_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2015 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Test module for the ReportHandler.""" 19 | 20 | import mock 21 | import tornado 22 | 23 | import urls 24 | 25 | from handlers.tests.test_handler_base import TestHandlerBase 26 | 27 | 28 | class TestReportHandler(TestHandlerBase): 29 | 30 | def get_app(self): 31 | return tornado.web.Application([urls._REPORT_URL], **self.settings) 32 | 33 | def test_post(self): 34 | headers = {"Authorization": "foo"} 35 | response = self.fetch( 36 | "/reports", method="POST", body="", headers=headers) 37 | 38 | self.assertEqual(response.code, 501) 39 | 40 | def test_post_no_token(self): 41 | response = self.fetch("/reports", method="POST", body="") 42 | self.assertEqual(response.code, 403) 43 | 44 | def test_delete(self): 45 | headers = {"Authorization": "foo"} 46 | response = self.fetch( 47 | "/reports", method="DELETE", headers=headers) 48 | 49 | self.assertEqual(response.code, 501) 50 | 51 | def test_delete_no_token(self): 52 | response = self.fetch("/reports", method="DELETE") 53 | self.assertEqual(response.code, 403) 54 | 55 | def test_get_no_token(self): 56 | response = self.fetch("/reports") 57 | self.assertEqual(response.code, 403) 58 | 59 | @mock.patch("utils.db.find") 60 | @mock.patch("utils.db.count") 61 | def test_get(self, mock_count, mock_find): 62 | mock_count.return_value = 0 63 | mock_find.return_value = [] 64 | 65 | headers = {"Authorization": "foo"} 66 | response = self.fetch("/reports", headers=headers) 67 | 68 | self.assertEqual(response.code, 200) 69 | self.assertEqual( 70 | response.headers['Content-Type'], self.content_type) 71 | 72 | @mock.patch("utils.db.find") 73 | @mock.patch("utils.db.count") 74 | def test_get_with_keys(self, mock_count, mock_find): 75 | mock_count.return_value = 0 76 | mock_find.return_value = [] 77 | 78 | headers = {"Authorization": "foo"} 79 | response = self.fetch( 80 | "/reports?job=job&kernel=kernel", headers=headers) 81 | 82 | self.assertEqual(response.code, 200) 83 | self.assertEqual( 84 | response.headers['Content-Type'], self.content_type) 85 | -------------------------------------------------------------------------------- /app/handlers/tests/test_stats_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2015 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Test module for the JobHandler handler.""" 19 | 20 | try: 21 | import simplejson as json 22 | except ImportError: 23 | import json 24 | 25 | import tornado 26 | 27 | import urls 28 | 29 | from handlers.tests.test_handler_base import TestHandlerBase 30 | 31 | 32 | class TestStatsHandler(TestHandlerBase): 33 | 34 | def get_app(self): 35 | return tornado.web.Application([urls._STATS_URL], **self.settings) 36 | 37 | def test_post(self): 38 | headers = {"Authorization": "foo", "Content-Type": "application/json"} 39 | body = json.dumps({"foo": "bar"}) 40 | 41 | response = self.fetch( 42 | "/statistics", method="POST", headers=headers, body=body) 43 | 44 | self.assertEqual(response.code, 501) 45 | self.assertEqual( 46 | response.headers["Content-Type"], self.content_type) 47 | 48 | def test_post_wrong_token(self): 49 | self.validate_token.return_value = (False, None) 50 | headers = {"Authorization": "foo", "Content-Type": "application/json"} 51 | body = json.dumps({"foo": "bar"}) 52 | 53 | response = self.fetch( 54 | "/statistics", method="POST", headers=headers, body=body) 55 | 56 | self.assertEqual(response.code, 403) 57 | self.assertEqual( 58 | response.headers["Content-Type"], self.content_type) 59 | 60 | def test_put(self): 61 | headers = {"Authorization": "foo", "Content-Type": "application/json"} 62 | body = json.dumps({"foo": "bar"}) 63 | 64 | response = self.fetch( 65 | "/statistics", method="PUT", headers=headers, body=body) 66 | 67 | self.assertEqual(response.code, 501) 68 | self.assertEqual( 69 | response.headers["Content-Type"], self.content_type) 70 | 71 | def test_put_wrong_token(self): 72 | self.validate_token.return_value = (False, None) 73 | headers = {"Authorization": "foo", "Content-Type": "application/json"} 74 | body = json.dumps({"foo": "bar"}) 75 | 76 | response = self.fetch( 77 | "/statistics", method="PUT", headers=headers, body=body) 78 | 79 | self.assertEqual(response.code, 403) 80 | self.assertEqual( 81 | response.headers["Content-Type"], self.content_type) 82 | 83 | def test_delete(self): 84 | headers = {"Authorization": "foo"} 85 | 86 | response = self.fetch( 87 | "/statistics", method="DELETE", headers=headers) 88 | 89 | self.assertEqual(response.code, 501) 90 | self.assertEqual( 91 | response.headers["Content-Type"], self.content_type) 92 | 93 | def test_delete_wrong_token(self): 94 | self.validate_token.return_value = (False, None) 95 | headers = {"Authorization": "foo"} 96 | 97 | response = self.fetch("/statistics", method="DELETE", headers=headers) 98 | 99 | self.assertEqual(response.code, 403) 100 | self.assertEqual( 101 | response.headers["Content-Type"], self.content_type) 102 | -------------------------------------------------------------------------------- /app/handlers/tests/test_test_case_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Collabora Limited 2018 2 | # Author: Guillaume Tucker 3 | # 4 | # Copyright (C) Linaro Limited 2015,2016,2017 5 | # Author: Milo Casagrande 6 | # 7 | # This program is free software; you can redistribute it and/or modify it under 8 | # the terms of the GNU Lesser General Public License as published by the Free 9 | # Software Foundation; either version 2.1 of the License, or (at your option) 10 | # any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, but WITHOUT 13 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this library; if not, write to the Free Software Foundation, Inc., 19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 20 | 21 | """Test module for the TestCaseHandler handler.""" 22 | 23 | import json 24 | import mock 25 | import tornado 26 | 27 | import urls 28 | 29 | from handlers.tests.test_handler_base import TestHandlerBase 30 | 31 | 32 | class TestTestCaseHandler(TestHandlerBase): 33 | 34 | def get_app(self): 35 | return tornado.web.Application([urls._TEST_CASE_URL], **self.settings) 36 | 37 | @mock.patch("utils.db.find_and_count") 38 | def test_get(self, mock_find): 39 | mock_find.return_value = ([{"foo": "bar"}], 1) 40 | 41 | headers = {"Authorization": "foo"} 42 | response = self.fetch("/test/case/", headers=headers) 43 | 44 | self.assertEqual(response.code, 200) 45 | self.assertEqual( 46 | response.headers["Content-Type"], self.content_type) 47 | 48 | @mock.patch("bson.objectid.ObjectId") 49 | @mock.patch("handlers.test_case.TestCaseHandler.collection") 50 | def test_get_by_id_not_found(self, collection, mock_id): 51 | mock_id.return_value = "suite-id" 52 | collection.find_one = mock.MagicMock() 53 | collection.find_one.return_value = None 54 | 55 | headers = {"Authorization": "foo"} 56 | response = self.fetch("/test/case/case-id", headers=headers) 57 | 58 | self.assertEqual(response.code, 404) 59 | self.assertEqual( 60 | response.headers["Content-Type"], self.content_type) 61 | 62 | @mock.patch("bson.objectid.ObjectId") 63 | @mock.patch("handlers.test_case.TestCaseHandler.collection") 64 | def test_get_by_id_not_found_empty_list(self, collection, mock_id): 65 | mock_id.return_value = "case-id" 66 | collection.find_one = mock.MagicMock() 67 | collection.find_one.return_value = [] 68 | 69 | headers = {"Authorization": "foo"} 70 | response = self.fetch("/test/case/case-id", headers=headers) 71 | 72 | self.assertEqual(response.code, 404) 73 | self.assertEqual( 74 | response.headers["Content-Type"], self.content_type) 75 | 76 | @mock.patch("bson.objectid.ObjectId") 77 | @mock.patch("handlers.test_case.TestCaseHandler.collection") 78 | def test_get_by_id_found(self, collection, mock_id): 79 | mock_id.return_value = "case-id" 80 | collection.find_one = mock.MagicMock() 81 | collection.find_one.return_value = {"_id": "case-id"} 82 | 83 | headers = {"Authorization": "foo"} 84 | response = self.fetch("/test/case/case-id", headers=headers) 85 | 86 | self.assertEqual(response.code, 200) 87 | self.assertEqual( 88 | response.headers["Content-Type"], self.content_type) 89 | -------------------------------------------------------------------------------- /app/handlers/tests/test_upload_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2015 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Test module for the UploadHandler.""" 19 | 20 | import tornado 21 | 22 | import urls 23 | 24 | from handlers.tests.test_handler_base import TestHandlerBase 25 | 26 | 27 | class TestUploadHandler(TestHandlerBase): 28 | 29 | def get_app(self): 30 | return tornado.web.Application([urls._UPLOAD_URL], **self.settings) 31 | 32 | def test_get(self): 33 | headers = {"Authorization": "foo"} 34 | response = self.fetch( 35 | "/upload", method="GET", headers=headers) 36 | 37 | self.assertEqual(response.code, 501) 38 | self.assertEqual( 39 | response.headers["Content-Type"], self.content_type) 40 | 41 | def test_get_no_token(self): 42 | response = self.fetch("/upload", method="GET") 43 | self.assertEqual(response.code, 403) 44 | self.assertEqual( 45 | response.headers["Content-Type"], self.content_type) 46 | 47 | def test_delete(self): 48 | headers = {"Authorization": "foo"} 49 | response = self.fetch( 50 | "/upload", method="DELETE", headers=headers) 51 | 52 | self.assertEqual(response.code, 501) 53 | self.assertEqual( 54 | response.headers["Content-Type"], self.content_type) 55 | 56 | def test_delete_no_token(self): 57 | response = self.fetch("/upload", method="DELETE") 58 | self.assertEqual(response.code, 403) 59 | self.assertEqual( 60 | response.headers["Content-Type"], self.content_type) 61 | 62 | def test_post_no_token(self): 63 | response = self.fetch("/upload", method="POST", body="") 64 | self.assertEqual(response.code, 403) 65 | self.assertEqual( 66 | response.headers["Content-Type"], self.content_type) 67 | 68 | def test_post_token_wrong_content(self): 69 | headers = { 70 | "Authorization": "foo", 71 | "Content-Type": "application/json" 72 | } 73 | response = self.fetch( 74 | "/upload", method="POST", body="", headers=headers) 75 | self.assertEqual(response.code, 415) 76 | self.assertEqual( 77 | response.headers["Content-Type"], self.content_type) 78 | 79 | def test_post_token_missing_content(self): 80 | headers = { 81 | "Authorization": "foo" 82 | } 83 | response = self.fetch( 84 | "/upload", method="POST", body="", headers=headers) 85 | self.assertEqual(response.code, 415) 86 | self.assertEqual( 87 | response.headers["Content-Type"], self.content_type) 88 | -------------------------------------------------------------------------------- /app/handlers/tests/test_version_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2014,2015 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Test module for the JobHandler handler.""" 19 | 20 | import tornado 21 | 22 | import urls 23 | 24 | from handlers.tests.test_handler_base import TestHandlerBase 25 | 26 | 27 | class TestVersionHandler(TestHandlerBase): 28 | 29 | def get_app(self): 30 | return tornado.web.Application([urls._VERSION_URL], **self.settings) 31 | 32 | def test_get(self): 33 | response = self.fetch("/version", method="GET") 34 | self.assertEqual(response.code, 200,) 35 | 36 | def test_post(self): 37 | response = self.fetch("/version", method="POST", body="") 38 | self.assertEqual(response.code, 501) 39 | 40 | def test_delete(self): 41 | response = self.fetch("/version", method="DELETE") 42 | self.assertEqual(response.code, 501) 43 | -------------------------------------------------------------------------------- /app/handlers/version.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2014 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Provide a simple /version handler.""" 19 | 20 | import handlers 21 | import handlers.base as hbase 22 | import handlers.response as hresponse 23 | import models 24 | 25 | 26 | # pylint: disable=too-many-public-methods 27 | class VersionHandler(hbase.BaseHandler): 28 | """Handle request to the /version URL. 29 | 30 | Provide the backend version number in use. 31 | """ 32 | 33 | def __init__(self, application, request, **kwargs): 34 | super(VersionHandler, self).__init__(application, request, **kwargs) 35 | 36 | def execute_get(self, *args, **kwargs): 37 | response = hresponse.HandlerResponse() 38 | response.result = [ 39 | { 40 | models.VERSION_FULL_KEY: handlers.__versionfull__, 41 | models.VERSION_KEY: handlers.__version__, 42 | } 43 | ] 44 | return response 45 | 46 | def execute_post(self, *args, **kwargs): 47 | return hresponse.HandlerResponse(501) 48 | 49 | def execute_delete(self, *args, **kwargs): 50 | return hresponse.HandlerResponse(501) 51 | -------------------------------------------------------------------------------- /app/models/base.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2014,2015 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """The base document abstract model that represents a mongodb document.""" 19 | 20 | import abc 21 | 22 | 23 | # pylint: disable=abstract-class-not-used 24 | class BaseDocument(object): 25 | """The base document abstract model for all other documents. 26 | 27 | It defines the necessary methods and properties that all documents must 28 | implement. 29 | """ 30 | 31 | __metaclass__ = abc.ABCMeta 32 | 33 | id_doc = ( 34 | """ 35 | The ID of this document as returned by mongodb. 36 | 37 | This should only be set with values returned by mongodb since it is 38 | an internal used field. 39 | """ 40 | ) 41 | 42 | # pylint: disable=invalid-name 43 | id = abc.abstractproperty(None, None, doc=id_doc) 44 | 45 | @abc.abstractproperty 46 | def collection(self): 47 | """The collection this document belongs to.""" 48 | return None 49 | 50 | created_on_doc = ( 51 | """ 52 | The date this document was created. 53 | 54 | A datetime object with UTC time zone. 55 | """ 56 | ) 57 | 58 | created_on = abc.abstractproperty(None, None, doc=created_on_doc) 59 | 60 | version_doc = ( 61 | """ 62 | The schema version number of this object. 63 | """ 64 | ) 65 | 66 | version = abc.abstractproperty(None, None, doc=version_doc) 67 | 68 | @abc.abstractmethod 69 | def to_dict(self): 70 | """Return a dictionary view of the document that can be serialized.""" 71 | raise NotImplementedError( 72 | "Class '%s' doesn't implement to_dict()" % self.__class__.__name__ 73 | ) 74 | 75 | @staticmethod 76 | @abc.abstractmethod 77 | def from_json(json_obj): 78 | """Build a document from a JSON object. 79 | 80 | The passed `json_obj` must be a valid Python dictionary. No checks are 81 | performed on its type. 82 | 83 | :param json_obj: The JSON object from which to build this object. 84 | :type json_obj: dict 85 | """ 86 | raise NotImplementedError("This class doesn't implement from_json()") 87 | -------------------------------------------------------------------------------- /app/models/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/models/tests/__init__.py -------------------------------------------------------------------------------- /app/models/tests/test_error_summary_model.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2015,2017 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | import unittest 19 | 20 | import models.base as modb 21 | import models.error_summary as msumm 22 | 23 | 24 | class TestErrorSummaryModel(unittest.TestCase): 25 | 26 | def test_doc_valid_instance(self): 27 | doc = msumm.ErrorSummaryDocument("job_id", "1.1") 28 | self.assertIsInstance(doc, modb.BaseDocument) 29 | self.assertIsInstance(doc, msumm.ErrorSummaryDocument) 30 | 31 | def test_doc_collection(self): 32 | doc = msumm.ErrorSummaryDocument("job_id", "1.1") 33 | self.assertEqual("errors_summary", doc.collection) 34 | 35 | def test_doc_wrong_lists(self): 36 | doc = msumm.ErrorSummaryDocument("job_id", "1.1") 37 | 38 | self.assertRaises(TypeError, setattr, doc, "errors", {}) 39 | self.assertRaises(TypeError, setattr, doc, "errors", "") 40 | self.assertRaises(TypeError, setattr, doc, "errors", 0) 41 | self.assertRaises(TypeError, setattr, doc, "errors", ()) 42 | 43 | self.assertRaises(TypeError, setattr, doc, "warnings", {}) 44 | self.assertRaises(TypeError, setattr, doc, "warnings", "") 45 | self.assertRaises(TypeError, setattr, doc, "warnings", 0) 46 | self.assertRaises(TypeError, setattr, doc, "warnings", ()) 47 | 48 | self.assertRaises(TypeError, setattr, doc, "mismatches", {}) 49 | self.assertRaises(TypeError, setattr, doc, "mismatches", "") 50 | self.assertRaises(TypeError, setattr, doc, "mismatches", 0) 51 | self.assertRaises(TypeError, setattr, doc, "mismatches", ()) 52 | 53 | def test_doc_to_dict(self): 54 | doc = msumm.ErrorSummaryDocument("job_id", "1.1") 55 | doc.git_branch = "branch" 56 | doc.errors = [("error1", 1)] 57 | doc.job = "job" 58 | doc.kernel = "kernel" 59 | doc.mismatches = [("mismatch1", 1)] 60 | doc.warnings = [("warning1", 1)] 61 | doc.created_on = "today" 62 | doc.version = "1.1" 63 | 64 | expected = { 65 | "created_on": "today", 66 | "errors": [("error1", 1)], 67 | "git_branch": "branch", 68 | "job": "job", 69 | "job_id": "job_id", 70 | "kernel": "kernel", 71 | "mismatches": [("mismatch1", 1)], 72 | "version": "1.1", 73 | "warnings": [("warning1", 1)] 74 | } 75 | 76 | self.assertDictEqual(expected, doc.to_dict()) 77 | 78 | doc.id = "id" 79 | expected["_id"] = "id" 80 | self.assertDictEqual(expected, doc.to_dict()) 81 | 82 | def test_doc_from_json(self): 83 | json_obj = { 84 | "_id": "id", 85 | "created_on": "today", 86 | "errors": [("error1", 1)], 87 | "job": "job", 88 | "job_id": "job_id", 89 | "kernel": "kernel", 90 | "mismatches": [("mismatch1", 1)], 91 | "name": "job_id", 92 | "version": "1.1", 93 | "warnings": [("warning1", 1)] 94 | } 95 | 96 | self.assertIsNone(msumm.ErrorSummaryDocument.from_json(json_obj)) 97 | -------------------------------------------------------------------------------- /app/models/tests/test_stats_model.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2015 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | import unittest 19 | 20 | import models.base as mbase 21 | import models.stats as mstats 22 | 23 | 24 | class TestStatsModel(unittest.TestCase): 25 | 26 | def test_daily_stats_document_instance(self): 27 | daily_stats = mstats.DailyStats() 28 | self.assertIsInstance(daily_stats, mbase.BaseDocument) 29 | 30 | def test_daily_stats_collection_name(self): 31 | daily_stats = mstats.DailyStats() 32 | self.assertEqual("daily_stats", daily_stats.collection) 33 | 34 | def test_daily_stats_to_dict(self): 35 | daily_stats = mstats.DailyStats() 36 | daily_stats.created_on = "now" 37 | daily_stats.id = "stats-id" 38 | daily_stats.total_builds = 100 39 | daily_stats.total_jobs = 1000 40 | daily_stats.version = "foo" 41 | daily_stats.start_date = "yesterday" 42 | 43 | expected = { 44 | "_id": "stats-id", 45 | "biweekly_total_builds": 0, 46 | "biweekly_total_jobs": 0, 47 | "biweekly_unique_archs": 0, 48 | "biweekly_unique_boards": 0, 49 | "biweekly_unique_kernels": 0, 50 | "biweekly_unique_machs": 0, 51 | "biweekly_unique_trees": 0, 52 | "biweekly_unique_defconfigs": 0, 53 | "created_on": "now", 54 | "daily_total_builds": 0, 55 | "daily_total_jobs": 0, 56 | "daily_unique_archs": 0, 57 | "daily_unique_boards": 0, 58 | "daily_unique_kernels": 0, 59 | "daily_unique_machs": 0, 60 | "daily_unique_trees": 0, 61 | "daily_unique_defconfigs": 0, 62 | "start_date": "yesterday", 63 | "total_builds": 100, 64 | "total_jobs": 1000, 65 | "total_unique_archs": 0, 66 | "total_unique_boards": 0, 67 | "total_unique_kernels": 0, 68 | "total_unique_machs": 0, 69 | "total_unique_trees": 0, 70 | "total_unique_defconfigs": 0, 71 | "version": "foo", 72 | "weekly_total_builds": 0, 73 | "weekly_total_jobs": 0, 74 | "weekly_unique_archs": 0, 75 | "weekly_unique_boards": 0, 76 | "weekly_unique_kernels": 0, 77 | "weekly_unique_machs": 0, 78 | "weekly_unique_trees": 0, 79 | "weekly_unique_defconfigs": 0, 80 | } 81 | 82 | self.assertDictEqual(expected, daily_stats.to_dict()) 83 | 84 | def test_daily_stats_from_json(self): 85 | self.assertIsNone(mstats.DailyStats.from_json({})) 86 | self.assertIsNone(mstats.DailyStats.from_json(())) 87 | self.assertIsNone(mstats.DailyStats.from_json("")) 88 | self.assertIsNone(mstats.DailyStats.from_json([])) 89 | -------------------------------------------------------------------------------- /app/taskqueue/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/taskqueue/__init__.py -------------------------------------------------------------------------------- /app/taskqueue/celeryconfig.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2014,2015,2016,2017,2018 2 | # Author: Matt Hart 3 | # Author: Milo Casagrande 4 | # 5 | # This program is free software; you can redistribute it and/or modify it under 6 | # the terms of the GNU Lesser General Public License as published by the Free 7 | # Software Foundation; either version 2.1 of the License, or (at your option) 8 | # any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 12 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 13 | # details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this library; if not, write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 | 19 | """Celery configuration values.""" 20 | 21 | BROKER_URL = "redis://localhost/0" 22 | BROKER_POOL_LIMIT = 250 23 | BROKER_TRANSPORT_OPTIONS = { 24 | "visibility_timeout": 24000, 25 | "fanout_prefix": True, 26 | "fanout_patterns": True 27 | } 28 | CELERYD_PREFETCH_MULTIPLIER = 8 29 | # Use custom json encoder. 30 | CELERY_ACCEPT_CONTENT = ["kjson"] 31 | CELERY_RESULT_SERIALIZER = "kjson" 32 | CELERY_TASK_SERIALIZER = "kjson" 33 | CELERY_TASK_RESULT_EXPIRES = 300 34 | CELERY_TIMEZONE = "UTC" 35 | CELERY_ENABLE_UTC = True 36 | CELERY_IGNORE_RESULT = False 37 | CELERY_DISABLE_RATE_LIMITS = True 38 | CELERY_RESULT_BACKEND = "redis://localhost/0" 39 | CELERY_REDIS_MAX_CONNECTIONS = 250 40 | # Custom log format. 41 | CELERYD_LOG_FORMAT = '[%(levelname)8s/%(threadName)10s] %(message)s' 42 | CELERYD_TASK_LOG_FORMAT = ( 43 | '[%(levelname)8s/%(processName)10s] ' 44 | '[%(task_name)s(%(task_id)s)] %(message)s' 45 | ) 46 | # process 20 tasks per child before it's replaced 47 | CELERYD_MAX_TASKS_PER_CHILD = 20 48 | # kill the process if the task takes longer than 60 seconds 49 | CELERYD_TASK_TIME_LIMIT = 90 50 | -------------------------------------------------------------------------------- /app/taskqueue/serializer.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2015,2016 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Custom kernel-ci JSON serializer/deserializer to be used with Celery. 19 | 20 | The following module defines two custom functions to serialize and deserialize 21 | JSON objects that use BSON notation. These functions are intended to be used 22 | as the defual encoder/decoder functions that Celery uses to send messages. 23 | """ 24 | 25 | try: 26 | import simplejson as json 27 | except ImportError: 28 | import json 29 | 30 | from bson.json_util import ( 31 | default, 32 | object_hook 33 | ) 34 | 35 | 36 | def kernelci_json_encoder(obj): 37 | """Custom JSON serialization function. 38 | 39 | :param obj: The object to serialize. 40 | :type obj: dict 41 | :return A unicode string. 42 | """ 43 | return json.dumps( 44 | obj, default=default, ensure_ascii=False, separators=(",", ":")) 45 | 46 | 47 | def kernelci_json_decoder(obj): 48 | """Custome JSON deserialization function. 49 | 50 | :param obj: The object to deserialize. 51 | :type obj: string or unicode 52 | :return A JSON object. 53 | """ 54 | return json.loads(obj, object_hook=object_hook) 55 | -------------------------------------------------------------------------------- /app/taskqueue/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Collabora Limited 2017 2 | # Author: Guillaume Tucker 3 | # 4 | # Copyright (C) Linaro Limited 2015 5 | # Author: Milo Casagrande 6 | # 7 | # This program is free software; you can redistribute it and/or modify it under 8 | # the terms of the GNU Lesser General Public License as published by the Free 9 | # Software Foundation; either version 2.1 of the License, or (at your option) 10 | # any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, but WITHOUT 13 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this library; if not, write to the Free Software Foundation, Inc., 19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 20 | 21 | import celery.result 22 | import taskqueue.celery as taskc 23 | import utils 24 | 25 | 26 | @taskc.app.task(name="error-handler") 27 | def error_handler(uuid): 28 | # ToDo: propagate error if the HTTP response hasn't been sent already 29 | with celery.result.allow_join_result(): 30 | result = celery.result.AsyncResult(uuid) 31 | ex = result.get(propagate=False) 32 | utils.LOG.error("Task failed, UUID: {}, error: {}".format(uuid, ex)) 33 | -------------------------------------------------------------------------------- /app/taskqueue/tasks/bisect.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Collabora Limited 2018, 2019 2 | # Author: Guillaume Tucker 3 | # 4 | # Copyright (C) Linaro Limited 2015 5 | # Author: Milo Casagrande 6 | # 7 | # This program is free software; you can redistribute it and/or modify it under 8 | # the terms of the GNU Lesser General Public License as published by the Free 9 | # Software Foundation; either version 2.1 of the License, or (at your option) 10 | # any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, but WITHOUT 13 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this library; if not, write to the Free Software Foundation, Inc., 19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 20 | 21 | """All bisect related celery tasks.""" 22 | 23 | import taskqueue.celery as taskc 24 | import utils.bisect.defconfig as defconfigb 25 | import utils.bisect.test 26 | 27 | 28 | @taskc.app.task(name="trigger-test-bisections") 29 | def trigger_test_bisections(status, job, branch, kernel, plan): 30 | report_id = "-".join([job, branch, kernel, plan]) 31 | utils.LOG.info("Triggering bisections for '{}'".format(report_id)) 32 | return utils.bisect.test.trigger_bisections( 33 | job, branch, kernel, plan, 34 | taskc.app.conf.get("db_options", {}), 35 | taskc.app.conf.get("jenkins_options", None)) 36 | 37 | 38 | @taskc.app.task(name="update-test-bisect") 39 | def update_test_bisect(data): 40 | """Just a wrapper around the real test bisect update function. 41 | 42 | :param data: Bisection results from the JSON data. 43 | :type data: dictionary 44 | :return Status code with result of the operation. 45 | """ 46 | return utils.bisect.test.update_results(data, taskc.app.conf.db_options) 47 | 48 | 49 | @taskc.app.task(name="build-bisect", ignore_result=False) 50 | def defconfig_bisect(doc_id, db_options, fields=None): 51 | """Run a build bisect operation on the passed build document id. 52 | 53 | :param doc_id: The build document ID. 54 | :type doc_id: string 55 | :param db_options: The database connection parameters. 56 | :type db_options: dictionary 57 | :param fields: A `fields` data structure with the fields to return or 58 | exclude. Default to None. 59 | :type fields: list or dictionary 60 | :return The result of the build bisect operation. 61 | """ 62 | return defconfigb.execute_build_bisection( 63 | doc_id, db_options, fields=fields) 64 | 65 | 66 | @taskc.app.task(name="build-bisect-compared-to", ignore_result=False) 67 | def defconfig_bisect_compared_to(doc_id, compare_to, db_options, fields=None): 68 | """Run a build bisect operation compared to the provided tree name. 69 | 70 | :param doc_id: The build document ID. 71 | :type doc_id: string 72 | :param compare_to: The name of the tree to compare to. 73 | :type compare_to: string 74 | :param db_options: The database connection parameters. 75 | :type db_options: dictionary 76 | :param fields: A `fields` data structure with the fields to return or 77 | exclude. Default to None. 78 | :type fields: list or dictionary 79 | :return The result of the build bisect operation. 80 | """ 81 | return defconfigb.execute_build_bisection_compared_to( 82 | doc_id, compare_to, db_options, fields=fields) 83 | -------------------------------------------------------------------------------- /app/taskqueue/tasks/callback.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Collabora Limited 2017,2019 2 | # Author: Guillaume Tucker 3 | # 4 | # Copyright (C) Baylibre 2017 5 | # Author: Loys Ollivier 6 | # 7 | # This program is free software; you can redistribute it and/or modify it under 8 | # the terms of the GNU Lesser General Public License as published by the Free 9 | # Software Foundation; either version 2.1 of the License, or (at your option) 10 | # any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, but WITHOUT 13 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this library; if not, write to the Free Software Foundation, Inc., 19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 20 | 21 | """All callback related celery tasks.""" 22 | 23 | import taskqueue.celery as taskc 24 | import utils 25 | import utils.callback 26 | 27 | 28 | @taskc.app.task(name="lava-test") 29 | def lava_test(json_obj, job_meta, lab_name): 30 | """Add test data from a LAVA v2 test job callback 31 | 32 | This is a wrapper around the actual function which runs in a Celery task. 33 | 34 | :param json_obj: The JSON object with the values necessary to import the 35 | LAVA test data. 36 | :type json_obj: dictionary 37 | :param lab_name: The name of the LAVA lab that posted the callback. 38 | :type lab_name: string 39 | :return ObjectId The test document object id. 40 | """ 41 | return utils.callback.lava.add_tests(json_obj, job_meta, lab_name, 42 | taskc.app.conf.db_options) 43 | -------------------------------------------------------------------------------- /app/taskqueue/tasks/common.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2015 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """General use celery tasks or functions.""" 19 | 20 | from __future__ import absolute_import 21 | 22 | import celery 23 | 24 | import taskqueue.celery as taskc 25 | import utils.batch.common 26 | 27 | 28 | @taskc.app.task(name="batch-executor", ignore_result=False) 29 | def execute_batch(json_obj, db_options): 30 | """Run batch operations based on the passed JSON object. 31 | 32 | :param json_obj: The JSON object with the operations to perform. 33 | :type json_obj: dict 34 | :param db_options: The database connection parameters. 35 | :type db_options: dict 36 | :return The result of the batch operations. 37 | """ 38 | return utils.batch.common.execute_batch_operation(json_obj, db_options) 39 | 40 | 41 | def run_batch_group(batch_op_list, db_options): 42 | """Execute a list of batch operations. 43 | 44 | :param batch_op_list: List of JSON object used to build the batch 45 | operation. 46 | :type batch_op_list: list 47 | :param db_options: The database connection parameters. 48 | :type db_options: dict 49 | :return A list with all the results. 50 | """ 51 | job = celery.group( 52 | execute_batch.s(batch_op, db_options) 53 | for batch_op in batch_op_list 54 | ) 55 | result = job.apply_async() 56 | while not result.ready(): 57 | pass 58 | # Use the result backend optimezed function to retrieve the results. 59 | # We are using redis. 60 | return result.join_native() 61 | -------------------------------------------------------------------------------- /app/taskqueue/tasks/kcidb.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Collabora Limited 2019 2 | # Author: Guillaume Tucker 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """All kcidb related tasks.""" 19 | 20 | import os 21 | import taskqueue.celery as taskc 22 | import utils 23 | import utils.kcidb 24 | 25 | 26 | @taskc.app.task(name="kcidb-build") 27 | def push_build(args): 28 | build_id, job_id, first = args 29 | kcidb_options = taskc.app.conf.get("kcidb_options") 30 | if kcidb_options: 31 | pid = os.getpid() 32 | kcidb_submit = taskc.app.kcidb_pool.get(pid) 33 | try: 34 | utils.kcidb.push_build(build_id, first, kcidb_options, 35 | kcidb_submit, 36 | taskc.app.conf.db_options) 37 | except Exception, e: 38 | utils.LOG.exception(e) 39 | 40 | return build_id, job_id 41 | 42 | 43 | @taskc.app.task(name="kcidb-tests") 44 | def push_tests(group_id): 45 | kcidb_options = taskc.app.conf.get("kcidb_options") 46 | if kcidb_options: 47 | pid = os.getpid() 48 | kcidb_submit = taskc.app.kcidb_pool[pid] 49 | try: 50 | utils.kcidb.push_tests(group_id, kcidb_options, 51 | kcidb_submit, 52 | taskc.app.conf.db_options) 53 | except Exception, e: 54 | utils.LOG.exception(e) 55 | 56 | return group_id 57 | -------------------------------------------------------------------------------- /app/taskqueue/tasks/stats.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2015,2016,2017 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Tasks to calculate statistics.""" 19 | 20 | import taskqueue.celery as taskc 21 | 22 | import utils 23 | import utils.db 24 | import utils.stats.daily 25 | 26 | 27 | @taskc.app.task(name="calculate-daily-statistics") 28 | def calculate_daily_statistics(): 29 | """Collect daily statistics on the data stored.""" 30 | db_options = taskc.app.conf.db_options 31 | daily_stats = utils.stats.daily.calculate_daily_stats(db_options) 32 | 33 | database = utils.db.get_db_connection(db_options) 34 | ret_val, doc_id = utils.db.save(database, daily_stats) 35 | 36 | return ret_val, doc_id 37 | -------------------------------------------------------------------------------- /app/taskqueue/tasks/test.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Collabora Limited 2017, 2018, 2019, 2020 2 | # Author: Guillaume Tucker 3 | # Author: Ana Guerrero Lopez 4 | # 5 | # Copyright (C) Baylibre 2017 6 | # Author: lollivier 7 | # 8 | # Copyright (C) Linaro Limited 2015,2016 9 | # Author: Milo Casagrande 10 | # 11 | # This program is free software; you can redistribute it and/or modify it under 12 | # the terms of the GNU Lesser General Public License as published by the Free 13 | # Software Foundation; either version 2.1 of the License, or (at your option) 14 | # any later version. 15 | # 16 | # This program is distributed in the hope that it will be useful, but WITHOUT 17 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 18 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 19 | # details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this library; if not, write to the Free Software Foundation, Inc., 23 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 24 | 25 | """All test related celery tasks.""" 26 | 27 | import taskqueue.celery as taskc 28 | import utils 29 | import utils.kci_test.regressions 30 | 31 | 32 | @taskc.app.task(name="test-regressions") 33 | def find_regression(group_id): 34 | """Find test case regressions in the given test group. 35 | 36 | Run this function in a Celery task to find any test case regressions for 37 | the given test group document ID, recursively through all sub-groups. 38 | 39 | :param group_id: Test group document object ID. 40 | :return tuple 200 if OK, 500 in case of errors; a list with created test 41 | regression document ids 42 | """ 43 | return utils.kci_test.regressions.find(group_id, taskc.app.conf.db_options) 44 | -------------------------------------------------------------------------------- /app/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Collabora Limited 2018 2 | # Author: Ana Guerrero Lopez 3 | # 4 | # Copyright (C) Linaro Limited -0700,2014,2015,2016,2017 5 | # Author: Milo Casagrande 6 | # 7 | # This program is free software; you can redistribute it and/or modify it under 8 | # the terms of the GNU Lesser General Public License as published by the Free 9 | # Software Foundation; either version 2.1 of the License, or (at your option) 10 | # any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, but WITHOUT 13 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this library; if not, write to the Free Software Foundation, Inc., 19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 20 | 21 | """Unit tests suite for kernel-ci-backend.""" 22 | 23 | import unittest 24 | 25 | 26 | def test_modules(): 27 | return [ 28 | "handlers.common.tests.test_lab", 29 | "handlers.common.tests.test_query", 30 | "handlers.common.tests.test_token", 31 | "handlers.tests.test_batch_handler", 32 | "handlers.tests.test_bisect_handler", 33 | "handlers.tests.test_build_handler", 34 | "handlers.tests.test_build_logs_handler", 35 | "handlers.tests.test_callback_handler", 36 | "handlers.tests.test_count_handler", 37 | "handlers.tests.test_handler_response", 38 | "handlers.tests.test_job_handler", 39 | "handlers.tests.test_job_logs_handler", 40 | "handlers.tests.test_lab_handler", 41 | "handlers.tests.test_report_handler", 42 | "handlers.tests.test_send_handler", 43 | "handlers.tests.test_stats_handler", 44 | "handlers.tests.test_test_case_handler", 45 | "handlers.tests.test_test_suite_handler", 46 | "handlers.tests.test_token_handler", 47 | "handlers.tests.test_upload_handler", 48 | "handlers.tests.test_version_handler", 49 | "models.tests.test_bisect_model", 50 | "models.tests.test_build_model", 51 | "models.tests.test_error_log_model", 52 | "models.tests.test_error_summary_model", 53 | "models.tests.test_job_model", 54 | "models.tests.test_lab_model", 55 | "models.tests.test_report_model", 56 | "models.tests.test_stats_model", 57 | "models.tests.test_test_case_model", 58 | "models.tests.test_test_suite_model", 59 | "models.tests.test_token_model", 60 | "utils.batch.tests.test_batch_common", 61 | "utils.bisect.tests.test_bisect", 62 | "utils.build.tests.test_build_import", 63 | "utils.report.tests.test_build_report", 64 | "utils.report.tests.test_report_common", 65 | "utils.stats.tests.test_daily_stats", 66 | "utils.tests.test_base", 67 | "utils.tests.test_emails", 68 | "utils.tests.test_log_parser", 69 | "utils.tests.test_upload", 70 | "utils.tests.test_validator" 71 | ] 72 | 73 | 74 | def test_suite(): 75 | """Create a unittest.TestSuite object.""" 76 | modules = test_modules() 77 | suite = unittest.TestSuite() 78 | test_loader = unittest.TestLoader() 79 | 80 | for name in modules: 81 | unit_suite = test_loader.loadTestsFromName(name) 82 | suite.addTests(unit_suite) 83 | return suite 84 | -------------------------------------------------------------------------------- /app/utils/batch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/utils/batch/__init__.py -------------------------------------------------------------------------------- /app/utils/batch/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/utils/batch/tests/__init__.py -------------------------------------------------------------------------------- /app/utils/bisect/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/utils/bisect/__init__.py -------------------------------------------------------------------------------- /app/utils/bisect/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/utils/bisect/tests/__init__.py -------------------------------------------------------------------------------- /app/utils/callback/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Collabora Limited 2017 2 | # Author: Guillaume Tucker 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Functions to process callbacks.""" 19 | 20 | import lava 21 | import lava_filters 22 | -------------------------------------------------------------------------------- /app/utils/callback/lava_filters.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Collabora Limited 2020 2 | # Author: Michal Galka 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """ 19 | This module is meant to contain the lava log filtering functions. 20 | These functions are used to filter out the content of the LAVA log when LAVA 21 | callback is being processed. 22 | 23 | Filtering functions should follow some rules: 24 | 25 | - Name of the filtering function must start with `filter_` 26 | - It can take only one argument which is a log line text to process 27 | - It should return boolean value: 28 | - True - log line stays in the log 29 | - False - log line will be deleted 30 | 31 | `LAVA_FILTERS` is a list of filtering functions (function objects) 32 | and it gets automatically populated during module import. 33 | """ 34 | 35 | import re 36 | import inspect 37 | 38 | 39 | LAVA_SIGNAL_PATTERN = re.compile( 40 | r'^') 41 | 42 | 43 | def filter_log_levels(log_line): 44 | return log_line['lvl'] == 'target' 45 | 46 | 47 | def filter_lava_signal(log_line): 48 | return not LAVA_SIGNAL_PATTERN.match(log_line['msg']) 49 | 50 | 51 | def _get_lava_filters(): 52 | filters = [] 53 | for name, obj in globals().items(): 54 | if name.startswith('filter') and inspect.isfunction(obj): 55 | filters.append(obj) 56 | return filters 57 | 58 | 59 | LAVA_FILTERS = _get_lava_filters() 60 | -------------------------------------------------------------------------------- /app/utils/callback/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/utils/callback/tests/__init__.py -------------------------------------------------------------------------------- /app/utils/database/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/utils/database/__init__.py -------------------------------------------------------------------------------- /app/utils/database/redisdb.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2015,2016 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Perform basic Redis operations.""" 19 | 20 | import redis 21 | 22 | 23 | REDIS_CONNECTION = None 24 | 25 | 26 | def get_db_connection(db_options, db_num=0): 27 | """Get a Redis DB Connection. 28 | 29 | :param db_options: The database connection options. 30 | :type db_options: dict 31 | :return A Redis connection. 32 | """ 33 | global REDIS_CONNECTION 34 | 35 | if not REDIS_CONNECTION: 36 | db_options_get = db_options.get 37 | 38 | redis_host = db_options_get("redis_host", "localhost") 39 | redis_port = db_options_get("redis_port", 6379) 40 | redis_pwd = db_options_get("redis_password", "") 41 | redis_db = db_options_get("redis_db", db_num) 42 | 43 | if redis_pwd: 44 | REDIS_CONNECTION = redis.StrictRedis( 45 | host=redis_host, 46 | port=redis_port, db=redis_db, password=redis_pwd) 47 | else: 48 | REDIS_CONNECTION = redis.StrictRedis( 49 | host=redis_host, port=redis_port, db=redis_db) 50 | 51 | return REDIS_CONNECTION 52 | -------------------------------------------------------------------------------- /app/utils/errors.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Collabora Limited 2017 2 | # Author: Guillaume Tucker 3 | # 4 | # Copyright (C) Linaro Limited 2015 5 | # Author: Milo Casagrande 6 | # 7 | # This program is free software; you can redistribute it and/or modify it under 8 | # the terms of the GNU Lesser General Public License as published by the Free 9 | # Software Foundation; either version 2.1 of the License, or (at your option) 10 | # any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, but WITHOUT 13 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this library; if not, write to the Free Software Foundation, Inc., 19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 20 | 21 | """Functions and classes to handle errors data structure.""" 22 | 23 | import utils 24 | 25 | LOG = utils.log.get_log() 26 | 27 | 28 | class BackendError(Exception): 29 | 30 | def __init__(self, errors): 31 | self.errors = errors 32 | 33 | def __str__(self): 34 | return ", ".join([" ".join([str(k)] + v) 35 | for k, v in self.errors.iteritems()]) 36 | 37 | 38 | def update_errors(to_update, errors): 39 | """Update the errors dictionary from another one. 40 | 41 | :param to_update: The dictionary to be updated. 42 | :type to_update: dictionary 43 | :param errors: The dictionary whose elements will be added. 44 | :type errors: dictionary 45 | """ 46 | if errors: 47 | to_update_keys = to_update.viewkeys() 48 | for k, v in errors.iteritems(): 49 | if k in to_update_keys: 50 | to_update[k].extend(v) 51 | else: 52 | to_update[k] = v 53 | 54 | 55 | def add_error(errors, err_code, err_msg): 56 | """Add error code and message to the provided dictionary. 57 | 58 | :param errors: The dictionary that will store the error codes and messages. 59 | :type errors: dictionary 60 | :param err_code: The error code that will be used as a key. 61 | :type err_code: int 62 | :param err_msg: The message to store. 63 | :type err_msg: string 64 | """ 65 | if all([err_code, err_msg]): 66 | if err_code in errors.viewkeys(): 67 | errors[err_code].append(err_msg) 68 | else: 69 | errors[err_code] = [] 70 | errors[err_code].append(err_msg) 71 | 72 | 73 | def handle_errors(ex=None, msg=None, errors=None): 74 | """Handles data processing errors 75 | 76 | Handles data processing errors by logging issues 77 | and/or raising exceptions appropriate exceptions. 78 | 79 | :param ex: exception to log 80 | :type ex: Exception 81 | :param msg: Message to log 82 | :type msg: str 83 | :param errors: errors to raise as BackendError 84 | :type errors: dict 85 | """ 86 | if ex is not None: 87 | LOG.exception(ex) 88 | if msg is not None: 89 | LOG.error(msg) 90 | if errors: 91 | raise BackendError(errors) 92 | -------------------------------------------------------------------------------- /app/utils/kci_test/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # TODO write python unit tests 2 | -------------------------------------------------------------------------------- /app/utils/log.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2014,2016 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Logging facilities.""" 19 | 20 | import logging 21 | 22 | LOG = None 23 | 24 | 25 | def get_log(debug=False): 26 | """Retrieve a logger. 27 | 28 | :param debug: If debug level should be turned on. 29 | :return: A logger instance. 30 | """ 31 | global LOG 32 | 33 | if LOG is None: 34 | LOG = logging.getLogger() 35 | log_handler = logging.StreamHandler() 36 | 37 | formatter = logging.Formatter( 38 | '[%(levelname)8s/%(threadName)10s] %(message)s') 39 | log_handler.setFormatter(formatter) 40 | 41 | if debug: 42 | log_handler.setLevel(logging.DEBUG) 43 | LOG.setLevel(logging.DEBUG) 44 | else: 45 | log_handler.setLevel(logging.INFO) 46 | LOG.setLevel(logging.INFO) 47 | 48 | LOG.addHandler(log_handler) 49 | 50 | return LOG 51 | -------------------------------------------------------------------------------- /app/utils/logs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/utils/logs/__init__.py -------------------------------------------------------------------------------- /app/utils/logs/build.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2016,2017 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | import io 19 | import itertools 20 | import os 21 | 22 | import models 23 | import utils 24 | import utils.db 25 | 26 | 27 | def create_build_logs_summary(job, kernel, git_branch, db_options): 28 | """Create a summary TXT file with the errors/warnings found in the log. 29 | 30 | :param job: The tree value. 31 | :type job: str 32 | :param kernel: The kernel value. 33 | :type kernel: str 34 | :param git_branch: The branch name. 35 | :type git_branch: str 36 | :param db_options: The database connection parameters. 37 | :type db_options: dict 38 | """ 39 | ret_val = 200 40 | error = "" 41 | 42 | db_conn = utils.db.get_db_connection(db_options) 43 | 44 | result = utils.db.find_one2( 45 | db_conn[models.ERRORS_SUMMARY_COLLECTION], 46 | { 47 | models.GIT_BRANCH_KEY: git_branch, 48 | models.JOB_KEY: job, 49 | models.KERNEL_KEY: kernel 50 | }) 51 | 52 | if result: 53 | errors = result.get(models.ERRORS_KEY) 54 | warnings = result.get(models.WARNINGS_KEY) 55 | mismatches = result.get(models.MISMATCHES_KEY) 56 | 57 | if errors or warnings or mismatches: 58 | file_path = os.path.join( 59 | utils.BASE_PATH, 60 | job, git_branch, kernel, "build-logs-summary.txt") 61 | 62 | try: 63 | with io.open(file_path, mode="w") as to_write: 64 | for line in itertools.chain(errors, warnings, mismatches): 65 | to_write.write( 66 | u"{:>4d} {:s}\n".format(line[0], line[1])) 67 | except IOError, ex: 68 | ret_val = 500 69 | error = ( 70 | "Error writing logs summary for {}-{}-{}: {}".format( 71 | job, git_branch, kernel, ex.strerror)) 72 | 73 | utils.LOG.error(error) 74 | else: 75 | utils.LOG.info( 76 | "No errors/warnings found for %s-%s-%s", 77 | job, git_branch, kernel) 78 | else: 79 | utils.LOG.info( 80 | "No errors summary found for %s-%s-%s", job, git_branch, kernel) 81 | 82 | return ret_val, error 83 | -------------------------------------------------------------------------------- /app/utils/report/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/utils/report/__init__.py -------------------------------------------------------------------------------- /app/utils/report/error.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Linaro Limited 2017 2 | # Author: Milo Casagrande 3 | # 4 | # This program is free software; you can redistribute it and/or modify it under 5 | # the terms of the GNU Lesser General Public License as published by the Free 6 | # Software Foundation; either version 2.1 of the License, or (at your option) 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, but WITHOUT 10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 11 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 12 | # details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this library; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Create the error reports.""" 19 | 20 | 21 | import utils.report.common as rcommon 22 | 23 | 24 | MULTIPLE_EMAILS_SUBJECT = \ 25 | u"Duplicate email trigger received for {job} - {git_branch} - {kernel}" 26 | 27 | 28 | def create_duplicate_email_report(data): 29 | """Create the email report for duplicated trigger emails. 30 | 31 | When we receive multiple triggers for the same job-kernel, send an error 32 | email. 33 | Only a TXT one will be created. 34 | 35 | :param data: The data for the email. 36 | :type dict 37 | :return The txt and html body, and the subject string. 38 | """ 39 | subject_str = MULTIPLE_EMAILS_SUBJECT.format(**data) 40 | txt_body = rcommon.create_txt_email("multiple_emails.txt", **data) 41 | 42 | return txt_body, None, subject_str 43 | -------------------------------------------------------------------------------- /app/utils/report/templates/bisect.txt: -------------------------------------------------------------------------------- 1 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * This automated bisection report was sent to you on the basis * 3 | * that you may be involved with the breaking commit it has * 4 | * found. No manual investigation has been done to verify it, * 5 | * and the root cause of the problem may be somewhere else. * 6 | * * 7 | * If you do send a fix, please include this trailer: * 8 | * Reported-by: "kernelci.org bot" * 9 | * * 10 | * Hope this helps! * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 12 | 13 | {{ subject_str }} 14 | 15 | Summary: 16 | Start: {{ bad }}{% if bad_details_url %} 17 | Details: {{ bad_details_url }}{% endif %} 18 | Plain log: {{ log_url_txt }} 19 | HTML log: {{ log_url_html }} 20 | Result: {{ found }} 21 | 22 | Checks: 23 | {%- for check, result in checks|dictsort %} 24 | {{ "%-11s" | format(check + ":",) }} {{ result }} 25 | {%- endfor %} 26 | 27 | Parameters: 28 | Tree: {{ tree }} 29 | URL: {{ git_url }} 30 | Branch: {{ branch }} 31 | Target: {{ target }} 32 | CPU arch: {{ arch }} 33 | Lab: {{ lab_name }} 34 | Compiler: {{ compiler }} 35 | Config: {{ defconfig }} 36 | Test case: {{ test_case_path }} 37 | 38 | Breaking commit found: 39 | 40 | ------------------------------------------------------------------------------- 41 | {{ show }} 42 | ------------------------------------------------------------------------------- 43 | 44 | 45 | Git bisection log: 46 | 47 | ------------------------------------------------------------------------------- 48 | {%- for line in log %} 49 | {{ line -}} 50 | {% endfor %} 51 | ------------------------------------------------------------------------------- 52 | -------------------------------------------------------------------------------- /app/utils/report/templates/build.txt: -------------------------------------------------------------------------------- 1 | {{ subject_str }} 2 | {# leave an empty space #} 3 | This legacy KernelCI providing this report will shutdown sometime 4 | soon in favor of our new KernelCI infra. Not all tests are being 5 | migrated. 6 | If you are still using this report and want us to prioritize your 7 | usecase in the new system, please let us know at 8 | kernelci@lists.linux.dev 9 | 10 | {{ full_build_summary }} 11 | {# leave an empty space #} 12 | {{ tree_string }} 13 | {{ branch_string }} 14 | {{ git_describe_string }} 15 | {{ git_commit_string }} 16 | {{ git_url_string[0] }} 17 | {%- if built_unique_string %} 18 | {{ built_unique_string }} 19 | {% endif %} 20 | {%- if platforms %} 21 | {%- if platforms.failed_data %} 22 | {%- for summary in platforms.failed_data.summary.txt %} 23 | {{ summary }} 24 | {%- endfor %} 25 | {% for arch, arch_data in platforms.failed_data.data|dictsort %} 26 | {{ arch }} 27 | {%- for defconfig_data in arch_data %} 28 | {{ defconfig_data[0] }} 29 | {%- endfor %}{# defconfig #} 30 | {% endfor %}{# arch #} 31 | {%- endif %}{# end failed_data #} 32 | {%- if platforms.error_data %} 33 | {%- for summary in platforms.error_data.summary.txt %} 34 | {{ summary }} 35 | {%- endfor %} 36 | {% for arch, arch_data in platforms.error_data.data|dictsort %} 37 | {{ arch }} 38 | {%- for defconfig in arch_data %} 39 | {{ defconfig[0] }} 40 | {%- endfor %}{# defconfig #} 41 | {% endfor %}{# arch #} 42 | {%- endif %}{# end error_data #} 43 | {%- endif %}{# end platforms #} 44 | {%- if errors_summary %} 45 | {%- if errors_summary.errors %} 46 | Errors summary: 47 | {% for err in errors_summary.errors %} 48 | {{ "%-3d %s" % (err[0], err[1]) -}} 49 | {% endfor %} 50 | {%- endif %}{# end errors #} 51 | {% if errors_summary.warnings %} 52 | Warnings summary: 53 | {% for warn in errors_summary.warnings %} 54 | {{ "%-3d %s" % (warn[0], warn[1]) -}} 55 | {% endfor %} 56 | {%- endif %}{# end warnings #} 57 | {% if errors_summary.mismatches %} 58 | Section mismatches summary: 59 | {% for mism in errors_summary.mismatches %} 60 | {{ "%-3d %s" % (mism[0], mism[1]) -}} 61 | {% endfor %} 62 | {% endif %}{# end mismatches #} 63 | {%- endif %}{# and errors summary #} 64 | {%- if error_details %} 65 | {{ "{:=^80}".format("") }} 66 | 67 | Detailed per-defconfig build reports: 68 | {% for d in error_details %} 69 | {% set errs = P_("{:d} error", "{:d} errors", d.errors_count).format(d.errors_count) -%} 70 | {%- set warns = P_("{:d} warning", "{:d} warnings", d.warnings_count).format(d.warnings_count) -%} 71 | {%- set mism = P_("{:d} section mismatch", "{:d} section mismatches", d.mismatches_count).format(d.mismatches_count) -%} 72 | 73 | {{ "{:-^80}".format("") }} 74 | {{ "{} ({}, {}) \u2014 {}, {}, {}, {}".format(d.defconfig_full, d.arch, d.build_environment, d.status, errs, warns, mism) }} 75 | {%- if d.errors %} 76 | 77 | Errors: 78 | {%- for line in d.errors %} 79 | {{ line -}} 80 | {%- endfor %} 81 | {%- endif %}{# end error lines #} 82 | {%- if d.warnings %} 83 | 84 | Warnings: 85 | {%- for line in d.warnings %} 86 | {{ line -}} 87 | {%- endfor %} 88 | {%- endif %}{# end warning lines #} 89 | {%- if d.mismatches %} 90 | 91 | Section mismatches: 92 | {%- for line in d.mismatches %} 93 | {{ line -}} 94 | {%- endfor %} 95 | {%- endif %}{# end mismatch lines #} 96 | {% endfor %} 97 | {%- endif %}{# end error_details #} 98 | {%- if info_email %} 99 | --- 100 | For more info write to <{{ info_email }}> 101 | {%- endif %} 102 | -------------------------------------------------------------------------------- /app/utils/report/templates/multiple_emails.txt: -------------------------------------------------------------------------------- 1 | Duplicate email trigger received for: 2 | 3 | Tree: {{ job }} 4 | Branch: {{ git_branch }} 5 | Kernel: {{ kernel }} 6 | {%- if kernel_version %} 7 | Kernel version: {{ kernel_version }} 8 | {%- endif %} 9 | {%- if git_describe_v %} 10 | Git describe ext.: {{ git_describe_v }} 11 | {%- endif %} 12 | {%- if git_commit %} 13 | Commit: {{ git_commit }} 14 | {%- endif %} 15 | {%- if git_url %} 16 | Git URL: {{ git_url }} 17 | {%- endif %} 18 | 19 | at: 20 | 21 | {{ trigger_time }} 22 | 23 | Details 24 | {{ "{:-^80}".format("") }} 25 | 26 | Report type: 27 | * {{ email_type }} 28 | 29 | Email format: 30 | {%- for format in email_format %} 31 | * {{ format }} 32 | {%- endfor %} 33 | {%- if subject %} 34 | 35 | Subject: {{ subject }} 36 | {%- endif %} 37 | {%- if in_reply_to %} 38 | 39 | In reply to: {{ in_reply_to }} 40 | {%- endif %} 41 | {%- if to_addrs %} 42 | 43 | To addresses: 44 | {%- for addr in to_addrs %} 45 | * {{ addr }} 46 | {%- endfor %} 47 | {%- endif %} 48 | {%- if cc_addrs %} 49 | 50 | Cc/Bcc addresses: 51 | {%- for addr in cc_addrs %} 52 | * {{ addr }} 53 | {%- endfor %} 54 | {%- endif %} 55 | -------------------------------------------------------------------------------- /app/utils/report/templates/templates.yaml: -------------------------------------------------------------------------------- 1 | # FIXME: get this from test-configs.yaml or email-configs.yaml 2 | 3 | templates: 4 | v4l2-compliance-uvc: 5 | file: "v4l2-compliance.txt" 6 | subject: "v4l2-compliance on uvcvideo" 7 | params: 8 | driver_name: "uvcvideo" 9 | 10 | v4l2-compliance-vivid: 11 | file: "v4l2-compliance.txt" 12 | subject: "v4l2-compliance on vivid" 13 | params: 14 | driver_name: "vivid" 15 | -------------------------------------------------------------------------------- /app/utils/report/templates/test.txt: -------------------------------------------------------------------------------- 1 | {{ subject_str }} 2 | 3 | This legacy KernelCI providing this report will shutdown sometime 4 | soon in favor of our new KernelCI infra. Not all tests are being 5 | migrated. 6 | If you are still using this report and want us to prioritize your 7 | usecase in the new system, please let us know at 8 | kernelci@lists.linux.dev 9 | 10 | Regressions Summary 11 | ------------------- 12 | 13 | {{ summary_headers }} 14 | {%- for group in test_groups %} 15 | {{ group.summary }} 16 | {%- endfor %} 17 | 18 | Details: {{ base_url }}/test/job/{{ tree }}/branch/{{ branch_uri }}/kernel/{{ kernel }}/plan/{{ plan }}/ 19 | {% block plan_description %} 20 | Test: {{ plan }}{% endblock %} 21 | Tree: {{ tree }} 22 | Branch: {{ branch }} 23 | Describe: {{ kernel }} 24 | URL: {{ git_url }} 25 | SHA: {{ git_commit }} 26 | 27 | {%- if test_suites %} 28 | 29 | Test suite revisions: 30 | {%- for suite in test_suites|sort(attribute='name') %} 31 | {{ suite.name }} 32 | URL: {{ suite.git_url }} 33 | SHA: {{ suite.git_commit }} 34 | {%- endfor %} 35 | {%- endif %} 36 | 37 | {%- if totals.FAIL != 0 %} {# total fail #} 38 | 39 | 40 | Test Regressions 41 | ---------------- 42 | {%- for group in test_groups %} {# test_groups #} 43 | 44 | 45 | {{ summary_headers }} 46 | {{ group.summary }} 47 | 48 | Details: {{ base_url }}/test/plan/id/{{ group._id }} 49 | 50 | Results: {{ group.total_results.PASS }} PASS, {{ group.total_results.FAIL }} FAIL, {{ group.total_results.SKIP }} SKIP 51 | Full config: {{ group.defconfig_full }} 52 | Compiler: {{ group.build_environment }}{% if group.compiler_version_full %} ({{ group.compiler_version_full }}){% endif %} 53 | Plain log: {{ storage_url }}/{{ group.file_server_resource }}/{{ group.lab_name }}/{{ group.boot_log }} 54 | HTML log: {{ storage_url }}/{{ group.file_server_resource }}/{{ group.lab_name }}/{{ group.boot_log_html }} 55 | {%- if group.initrd %} 56 | Rootfs: {{ group.initrd }} 57 | {%- endif %} 58 | {%- if not test_suites and group.initrd_info.tests_suites %} {# suites_info #} 59 | 60 | Test suite revisions: 61 | {%- for suite in group.initrd_info.tests_suites|sort(attribute='name') %} 62 | {{ suite.name }} 63 | URL: {{ suite.git_url }} 64 | SHA: {{ suite.git_commit }} 65 | {%- endfor %} 66 | {%- endif %} {# suites_info #} 67 | 68 | {% for tc in group.regressions %} 69 | * {{ tc.test_case_path }}: {{ base_url }}/test/case/id/{{ tc._id }} 70 | {{ tc.failure_message }} 71 | {%- if tc.measurements -%} 72 | {% for measurement in tc.measurements %} 73 | {{measurement.value}} {{measurement.unit}} 74 | {%- endfor -%} 75 | {%- endif %} 76 | {%- if tc.log_lines_short %} 77 | {% for log_line in tc.log_lines_short %} 78 | {{ log_line.dt }} {{ log_line.msg }} 79 | {%- endfor %} {# log_lines_short #} 80 | {%- if tc.log_lines_removed %} 81 | ... ({{ tc.log_lines_removed }} line(s) more) 82 | {%- endif %} {# log_lines_removed #} 83 | {%- endif %} {# log_lines_short #} 84 | {% endfor -%} 85 | {%- endfor %} {# test_groups #} 86 | {%- endif %} {# total fail #} -------------------------------------------------------------------------------- /app/utils/report/templates/v4l2-compliance.txt: -------------------------------------------------------------------------------- 1 | {% extends 'test.txt' %} 2 | 3 | {% block plan_description %} 4 | V4L2 Compliance on the {{ driver_name }} driver. 5 | 6 | This test ran "v4l2-compliance -s" from v4l-utils: 7 | 8 | https://www.linuxtv.org/wiki/index.php/V4l2-utils 9 | 10 | See each detailed section in the report below to find out the git URL and 11 | particular revision that was used to build the test binaries. 12 | 13 | {% endblock %} 14 | -------------------------------------------------------------------------------- /app/utils/report/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/utils/report/tests/__init__.py -------------------------------------------------------------------------------- /app/utils/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/utils/scripts/__init__.py -------------------------------------------------------------------------------- /app/utils/scripts/update-defconfig.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (C) Collabora Limited 2018 3 | # Author: Guillaume Tucker 4 | # 5 | # Copyright (C) Linaro Limited 2015 6 | # Author: Milo Casagrande 7 | # 8 | # This program is free software; you can redistribute it and/or modify it under 9 | # the terms of the GNU Lesser General Public License as published by the Free 10 | # Software Foundation; either version 2.1 of the License, or (at your option) 11 | # any later version. 12 | # 13 | # This program is distributed in the hope that it will be useful, but WITHOUT 14 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 | # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 16 | # details. 17 | # 18 | # You should have received a copy of the GNU Lesser General Public License 19 | # along with this library; if not, write to the Free Software Foundation, Inc., 20 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 21 | 22 | """Convert the defconfig_id fields into the new build_id one.""" 23 | 24 | import multiprocessing 25 | 26 | import models 27 | import utils 28 | import utils.db 29 | 30 | 31 | database = utils.db.get_db_connection({}) 32 | 33 | 34 | def add_build_type_field(collection_name): 35 | utils.LOG.info("Updating build collection") 36 | collection = database[collection_name] 37 | 38 | for doc in collection.find(): 39 | ret_val = utils.db.update2( 40 | collection, 41 | {models.ID_KEY: doc[models.ID_KEY]}, 42 | {"$set": {models.BUILD_TYPE_KEY: models.KERNEL_BUILD_TYPE}} 43 | ) 44 | 45 | if ret_val == 500: 46 | utils.LOG.error( 47 | "Error updating build document %s", str(doc[models.ID_KEY])) 48 | 49 | 50 | def convert_defconfig_id(collection_name): 51 | utils.LOG.info("Converting collection %s", collection_name) 52 | 53 | collection = database[collection_name] 54 | update_doc = { 55 | "$set": None, 56 | "$unset": {"defconfig_id": ""} 57 | } 58 | 59 | for doc in collection.find(): 60 | update_doc["$set"] = { 61 | models.BUILD_ID_KEY: doc.get("defconfig_id", None) 62 | } 63 | 64 | ret_val = utils.db.update2( 65 | collection, {models.ID_KEY: doc[models.ID_KEY]}, update_doc) 66 | 67 | if ret_val == 500: 68 | utils.LOG.error( 69 | "Error updating document %s", str(doc[models.ID_KEY])) 70 | 71 | 72 | if __name__ == "__main__": 73 | process_pool = multiprocessing.Pool(4) 74 | process_pool.map( 75 | convert_defconfig_id, 76 | [ 77 | models.TEST_GROUP_COLLECTION, 78 | models.ERROR_LOGS_COLLECTION 79 | ] 80 | ) 81 | process_pool.apply(add_build_type_field, (models.BUILD_COLLECTION,)) 82 | 83 | process_pool.close() 84 | process_pool.join() 85 | -------------------------------------------------------------------------------- /app/utils/stats/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/utils/stats/__init__.py -------------------------------------------------------------------------------- /app/utils/stats/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/utils/stats/tests/__init__.py -------------------------------------------------------------------------------- /app/utils/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kernelci/kernelci-backend/7e7bff2703c73b2a3a3734b7bcd0593cbe657705/app/utils/tests/__init__.py -------------------------------------------------------------------------------- /app/utils/tests/assets/upload_file.txt: -------------------------------------------------------------------------------- 1 | Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Typi non habent claritatem insitam; est usus legentis in iis qui facit eorum claritatem. Investigationes demonstraverunt lectores legere me lius quod ii legunt saepius. Claritas est etiam processus dynamicus, qui sequitur mutationem consuetudium lectorum. Mirum est notare quam littera gothica, quam nunc putamus parum claram, anteposuerit litterarum formas humanitatis per seacula quarta decima et quinta decima. Eodem modo typi, qui nunc nobis videntur parum clari, fiant sollemnes in futurum. -------------------------------------------------------------------------------- /doc/.gitignore: -------------------------------------------------------------------------------- 1 | _build 2 | build 3 | _static 4 | _templates 5 | 6 | -------------------------------------------------------------------------------- /doc/collection-report.rst: -------------------------------------------------------------------------------- 1 | .. _collection_report: 2 | 3 | report 4 | ------ 5 | 6 | A **report** is the result of sending a boot or build report. The actual report sent is not stored here, only information on the report status are. 7 | 8 | More info about the report schema can be found :ref:`here `. 9 | 10 | .. note:: 11 | 12 | This resource can only be accessed with an admin or super user token. 13 | 14 | .. note:: 15 | 16 | This resource can also be accessed using the plural form ``reports``. 17 | 18 | GET 19 | *** 20 | 21 | .. http:get:: /report/(string:id) 22 | 23 | Get all the available reports or a single one if ``id`` is provided. 24 | 25 | :param id: The ID of the report to retrieve. 26 | :type id: string 27 | 28 | :reqheader Authorization: The token necessary to authorize the request. 29 | :reqheader Accept-Encoding: Accept the ``gzip`` coding. 30 | 31 | :resheader Content-Type: Will be ``application/json; charset=UTF-8``. 32 | 33 | :query int limit: Number of results to return. Default 0 (all results). 34 | :query int skip: Number of results to skip. Default 0 (none). 35 | :query string sort: Field to sort the results on. Can be repeated multiple times. 36 | :query int sort_order: The sort order of the results: -1 (descending), 1 37 | (ascending). This will be applied only to the first ``sort`` 38 | parameter passed. Default -1. 39 | :query int date_range: Number of days to consider, starting from today 40 | (:ref:`more info `). By default consider all results. 41 | :query string field: The field that should be returned in the response. Can be 42 | repeated multiple times. 43 | :query string nfield: The field that should *not* be returned in the response. Can be repeated multiple times. 44 | :query string _id: The internal ID of the report. 45 | :query string created_on: The creation date: accepted formats are ``YYYY-MM-DD`` and ``YYYYMMDD``. 46 | :query string job: A job name. 47 | :query string kernel: A kernel name. 48 | :query string name: The name of the report. 49 | :query string status: The status of the report report. 50 | :query string type: The type of the report. 51 | 52 | :status 200: Results found. 53 | :status 403: Not authorized to perform the operation. 54 | :status 404: The provided resource has not been found. 55 | :status 500: Internal database error. 56 | 57 | **Example Requests** 58 | 59 | .. sourcecode:: http 60 | 61 | GET /report/ HTTP/1.1 62 | Host: api.kernelci.org 63 | Accept: */* 64 | Authorization: token 65 | 66 | .. sourcecode:: http 67 | 68 | GET /report/?job=next&kernel=next-20140731 HTTP/1.1 69 | Host: api.kernelci.org 70 | Accept: */* 71 | Authorization: token 72 | 73 | **Example Responses** 74 | 75 | .. sourcecode:: http 76 | 77 | HTTP/1.1 200 OK 78 | Vary: Accept-Encoding 79 | Date: Mon, 11 Aug 2014 15:12:50 GMT 80 | Content-Type: application/json; charset=UTF-8 81 | 82 | { 83 | "code": 200, 84 | "count:" 1, 85 | "limit": 0, 86 | "result": [ 87 | { 88 | "status": "SENT", 89 | "job": "next", 90 | "kernel": "next-20140731", 91 | }, 92 | ], 93 | } 94 | 95 | .. note:: 96 | Results shown here do not include the full JSON response. 97 | 98 | POST 99 | **** 100 | 101 | .. caution:: 102 | Not implemented. Will return a :ref:`status code ` 103 | of ``501``. 104 | 105 | 106 | DELETE 107 | ****** 108 | 109 | .. caution:: 110 | Not implemented. Will return a :ref:`status code ` 111 | of ``501``. 112 | 113 | More Info 114 | ********* 115 | 116 | * :ref:`Report schema ` 117 | * :ref:`API results ` 118 | * :ref:`Schema time and date ` 119 | -------------------------------------------------------------------------------- /doc/collection-test-case.rst: -------------------------------------------------------------------------------- 1 | .. _collection_test_case: 2 | 3 | case 4 | ---- 5 | 6 | More info about the test case schema can be found :ref:`here `. 7 | 8 | .. note:: 9 | 10 | This resource can also be accessed using the plural form ``cases``. 11 | 12 | GET 13 | *** 14 | 15 | .. http:get:: /test/case/(string:id)/ 16 | 17 | Get all the available test cases or a single one if ``id`` is provided. 18 | 19 | :param id: The :ref:`ID ` of the test case to retrieve. 20 | :type id: string 21 | 22 | :reqheader Authorization: The token necessary to authorize the request. 23 | :reqheader Accept-Encoding: Accept the ``gzip`` coding. 24 | 25 | :resheader Content-Type: Will be ``application/json; charset=UTF-8``. 26 | 27 | :query int limit: Number of results to return. Default 0 (all results). 28 | :query int skip: Number of results to skip. Default 0 (none). 29 | :query string sort: Field to sort the results on. Can be repeated multiple times. 30 | :query int sort_order: The sort order of the results: -1 (descending), 1 31 | (ascending). This will be applied only to the first ``sort`` 32 | parameter passed. Default -1. 33 | :query int date_range: Number of days to consider, starting from today 34 | (:ref:`more info `). By default consider all results. 35 | :query string field: The field that should be returned in the response. Can be 36 | repeated multiple times. 37 | :query string nfield: The field that should *not* be returned in the response. Can be repeated multiple times. 38 | :query string _id: The internal ID of the test case report. 39 | :query string created_on: The creation date: accepted formats are ``YYYY-MM-DD`` and ``YYYYMMDD``. 40 | :query string kvm_guest: The name of the KVM guest the test was executed on. 41 | :query int maximum: The maximum measurement registered. 42 | :query int minimum: The minimum measurement registered. 43 | :query string name: The name of a test case. 44 | :query int samples: Number of registered measurements. 45 | :query string status: The status of the test execution. 46 | :query string test_suite_id: The ID of the test suite associated with the test case. 47 | :query string test_suite_name: The name of the test suite associated with the test case. 48 | :query string time: The time it took to execute the test case. 49 | :query string vcs_commit: The VCS commit value. 50 | 51 | :status 200: Results found. 52 | :status 403: Not authorized to perform the operation. 53 | :status 404: The provided resource has not been found. 54 | :status 500: Internal server error. 55 | :status 503: Service maintenance. 56 | 57 | **Example Requests** 58 | 59 | .. sourcecode:: http 60 | 61 | GET /test/case HTTP/1.1 62 | Host: api.kernelci.org 63 | Accept: */* 64 | Authorization: token 65 | 66 | .. sourcecode:: http 67 | 68 | GET /tests/cases HTTP/1.1 69 | Host: api.kernelci.org 70 | Accept: */* 71 | Authorization: token 72 | 73 | **Example Responses** 74 | 75 | .. sourcecode:: http 76 | 77 | HTTP/1.1 200 OK 78 | Vary: Accept-Encoding 79 | Date: Mon, 16 Mar 2015 14:03:19 GMT 80 | Content-Type: application/json; charset=UTF-8 81 | 82 | { 83 | "code": 200, 84 | "count": 1, 85 | "result": [ 86 | { 87 | "_id": { 88 | "$oid": "123456789", 89 | "name": "Test case 0" 90 | } 91 | } 92 | ] 93 | } 94 | 95 | .. note:: 96 | Results shown here do not include the full JSON response. 97 | 98 | More Info 99 | ********* 100 | 101 | * :ref:`Test suite schema ` 102 | * :ref:`Test case schema ` 103 | * :ref:`Test schemas ` 104 | * :ref:`API results ` 105 | * :ref:`Schema time and date ` 106 | -------------------------------------------------------------------------------- /doc/collection-test.rst: -------------------------------------------------------------------------------- 1 | .. _collection_test: 2 | 3 | test 4 | ---- 5 | 6 | The ``test`` resource is a collection of multiple resources. 7 | 8 | .. note:: All the resources listed here can be accessed using either the singular form (**test**) or the plural form (**tests**). 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | 13 | collection-test-group 14 | collection-test-case 15 | 16 | How to Use the Test Resources 17 | +++++++++++++++++++++++++++++ 18 | 19 | In order to register tests and to use the test APIs at the best, the following 20 | user cases describe how to interact with the resources available. 21 | 22 | The following are just ideas on how to implement the data flow between the tests 23 | runner and the API. They all have pros and cons. 24 | 25 | .. note:: 26 | If you send (POST) multiple times the same request to register a test group 27 | with the same values, multiple test groups will be registered. To update a test 28 | group perform a PUT request. The same principle applies to the other test 29 | resources. 30 | 31 | Register Test Cases One at the Time 32 | *********************************** 33 | 34 | In the following example, the user first registers the test group obtaining its 35 | ``test_group_id``. 36 | 37 | She then runs all her test cases, and as soon as the first result comes back, 38 | she registers the test case using the ``test_group_id`` value. The last step is 39 | repeated for all the **N** test cases she runs. 40 | 41 | .. image:: images/test-user-case0.svg 42 | :align: center 43 | :height: 600px 44 | :width: 600px 45 | 46 | The following are examples of the JSON data snippet that might be used in this 47 | scenario. The order is: test group data, test case data (for each test cases): 48 | 49 | .. sourcecode:: json 50 | 51 | { 52 | "name": "A test group" 53 | } 54 | 55 | .. sourcecode:: json 56 | 57 | { 58 | "name": "A test case 1…N", 59 | "test_group_id": "a-test-group-id", 60 | } 61 | 62 | 63 | Pros 64 | ~~~~ 65 | 66 | * Small amount of data are sent with each HTTP request. 67 | * Easy to see the flow of the data, and in case interrupt it. 68 | * If a test case run is damaged or does not return any results, it can be 69 | retried before being sent. 70 | * The reference for each registered resource is obtained with the server 71 | response. 72 | 73 | Cons 74 | ~~~~ 75 | 76 | * Necessary to perform an HTTP POST request for each step: one for the test 77 | group and one for each test case. 78 | 79 | Register Test Group and Cases Together 80 | ************************************** 81 | 82 | In the following example, the user registers everything with a single request 83 | (POST) with all the data embedded in the same JSON document. 84 | 85 | This means that the results of the test cases must be available when sending 86 | the data 87 | 88 | .. image:: images/test-user-case1.svg 89 | :align: center 90 | :height: 600px 91 | :width: 600px 92 | 93 | The following is an example of the JSON data snippet that might be used in this 94 | scenario: 95 | 96 | .. sourcecode:: json 97 | 98 | { 99 | "name": "A test group" 100 | "test_cases": [ 101 | { 102 | "name": "A test case 1…N" 103 | } 104 | ] 105 | } 106 | 107 | Note that even if the ``test_group_id`` field is mandatory for test cases, as specified 108 | in their JSON schema, in this case it is not necessary: 109 | it will be injected into the provided data after the test group has been saved. 110 | 111 | Pros 112 | ~~~~ 113 | 114 | * Less number of HTTP POST requests to register all the resources. 115 | * Tests can be re-run if errors arise during the execution. 116 | 117 | Cons 118 | ~~~~ 119 | 120 | * The data sent might be big, depending on the number of test cases. 121 | * Necessary to perform an extra HTTP GET request to obtain all the test references. 122 | -------------------------------------------------------------------------------- /doc/collection-version.rst: -------------------------------------------------------------------------------- 1 | version 2 | ------- 3 | 4 | GET 5 | *** 6 | 7 | .. http:get:: /version 8 | 9 | Provide the version number of the software running. 10 | 11 | :reqheader Accept-Encoding: Accept the ``gzip`` coding. 12 | 13 | :resheader Content-Type: Will be ``application/json; charset=UTF-8``. 14 | 15 | :status 200: Resuslts found. 16 | 17 | .. note:: 18 | 19 | This resource does not require authentication. 20 | 21 | **Example Requests** 22 | 23 | .. sourcecode:: http 24 | 25 | GET /version HTTP/1.1 26 | Host: api.kernelci.org 27 | Accept: */* 28 | 29 | **Example Responses** 30 | 31 | .. sourcecode:: http 32 | 33 | HTTP/1.1 200 OK 34 | Vary: Accept-Encoding 35 | Date: Mon, 24 Nov 2014 18:08:12 GMT 36 | Content-Type: application/json; charset=UTF-8 37 | 38 | { 39 | "code": 200, 40 | "result": 41 | [ 42 | { 43 | "version": "2014.11", 44 | "full_version": "2014.11" 45 | } 46 | ] 47 | } 48 | 49 | POST 50 | **** 51 | 52 | .. caution:: 53 | Not implemented. Will return a :ref:`status code ` 54 | of ``501``. 55 | 56 | 57 | DELETE 58 | ****** 59 | 60 | .. caution:: 61 | Not implemented. Will return a :ref:`status code ` 62 | of ``501``. 63 | -------------------------------------------------------------------------------- /doc/collections.rst: -------------------------------------------------------------------------------- 1 | .. _collections: 2 | 3 | Resources 4 | ========= 5 | 6 | .. note:: 7 | 8 | Some of the following resources can be accessed also using their plural form. For more info refer to each resource documentation. 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | 13 | collection-batch 14 | collection-boot 15 | collection-count 16 | collection-build 17 | collection-job 18 | collection-lab 19 | collection-report 20 | collection-send 21 | collection-token 22 | collection-upload 23 | collection-version 24 | collection-test 25 | -------------------------------------------------------------------------------- /doc/examples.rst: -------------------------------------------------------------------------------- 1 | .. _code_examples: 2 | 3 | Code Examples 4 | ============= 5 | 6 | All the following code examples are Python based and rely on the 7 | `requests `_ module. 8 | 9 | Retrieving boot reports 10 | ----------------------- 11 | 12 | .. literalinclude:: examples/get-all-boot-reports.py 13 | :language: python 14 | 15 | .. literalinclude:: examples/get-all-failed-boot-reports.py 16 | :language: python 17 | 18 | .. literalinclude:: examples/get-all-boot-reports-with-jobid.py 19 | :language: python 20 | 21 | Handling compressed data 22 | ------------------------ 23 | 24 | If you need to directly handle the compressed data as returned by the server, 25 | you can access it from the response object. 26 | 27 | Keep in mind though that the `requests `_ 28 | module automatically handles ``gzip`` and ``deflate`` compressions. 29 | 30 | .. literalinclude:: examples/handling-compressed-data.py 31 | :language: python 32 | 33 | Creating a new lab 34 | ------------------ 35 | 36 | .. note:: 37 | 38 | Creation of new lab that can send boot reports is permitted only with an 39 | administrative token. 40 | 41 | The response object will contain: 42 | 43 | * The ``token`` that should be used to send boot lab reports. 44 | 45 | * The ``name`` of the lab that should be used to send boot lab reports. 46 | 47 | * The lab internal ``_id`` value. 48 | 49 | .. literalinclude:: examples/create-new-lab.py 50 | :language: python 51 | 52 | Sending a boot report 53 | --------------------- 54 | 55 | .. literalinclude:: examples/post-boot-report.py 56 | :language: python 57 | 58 | Sending a boot email report 59 | --------------------------- 60 | 61 | .. literalinclude:: examples/trigger-boot-email-report.py 62 | :language: python 63 | 64 | Uploading files 65 | --------------- 66 | 67 | Upload a single file 68 | ******************** 69 | 70 | .. literalinclude:: examples/upload-single-file.py 71 | :language: python 72 | 73 | Upload multiple files 74 | ********************* 75 | 76 | The following example, before sending the files, will load them in memory. 77 | With big files this might not be convenient. 78 | 79 | .. literalinclude:: examples/upload-multiple-files.py 80 | :language: python 81 | 82 | Upload multiple files - 2 83 | ************************* 84 | 85 | The following example does not load the files in memory, but it relies on an 86 | external library: `requests-toolbelt `_. 87 | 88 | .. literalinclude:: examples/upload-multiple-files-2.py 89 | :language: python 90 | 91 | Uploading tests 92 | --------------- 93 | 94 | Upload each single test schema 95 | ****************************** 96 | 97 | In this example each test schema - test group and test case - will be 98 | uploaded one at the time and the results of each API request will be used to create 99 | the next data to be sent. 100 | 101 | Notes: 102 | 103 | * The test group data sent does not contain any test cases: in this case the response status code will be 201. 104 | 105 | .. literalinclude:: examples/single-tests-registration.py 106 | :language: python 107 | 108 | Upload test group and all test case embedded 109 | ******************************************** 110 | 111 | In this example, the test group data sent contains all the case data. 112 | 113 | Notes: 114 | 115 | * The test group data contains all the necessary data: in this case the response status code will be 202 (the test cases will be imported asynchronously). 116 | * Test tests case do not need the mandatory ``test_group_id`` key: it will be automatically added when being imported. 117 | 118 | .. literalinclude:: examples/import-tests-all-embedded.py 119 | :language: python 120 | -------------------------------------------------------------------------------- /doc/examples/create-new-lab.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | """Create a new lab entry.""" 4 | 5 | try: 6 | import simplejson as json 7 | except ImportError: 8 | import json 9 | 10 | import requests 11 | import urlparse 12 | 13 | BACKEND_URL = "https://api.kernelci.org" 14 | AUTHORIZATION_TOKEN = "foo" 15 | 16 | 17 | def main(): 18 | headers = { 19 | "Authorization": AUTHORIZATION_TOKEN, 20 | "Content-Type": "application/json" 21 | } 22 | 23 | payload = { 24 | "name": "lab-enymton", 25 | "contact": { 26 | "name": "Ema", 27 | "surname": "Nymton", 28 | "email": "ema.nymton@example.org" 29 | } 30 | } 31 | 32 | url = urlparse.urljoin(BACKEND_URL, "/lab") 33 | response = requests.post(url, data=json.dumps(payload), headers=headers) 34 | 35 | print response.content 36 | 37 | 38 | if __name__ == "__main__": 39 | main() 40 | -------------------------------------------------------------------------------- /doc/examples/get-all-boot-reports-with-jobid.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | """Get all boot reports with a specified job ID. 4 | 5 | The results will include only the 'board', 'status' and 'defconfig' fields. 6 | The '_id' field is explicitly excluded. 7 | """ 8 | 9 | import requests 10 | 11 | from urlparse import urljoin 12 | 13 | BACKEND_URL = "https://api.kernelci.org" 14 | AUTHORIZATION_TOKEN = "foo" 15 | 16 | 17 | def main(): 18 | headers = { 19 | "Authorization": AUTHORIZATION_TOKEN 20 | } 21 | 22 | params = { 23 | "job_id": "123456789012345678901234", 24 | "field": ["board", "status", "defconfig"], 25 | "nfield": ["_id"] 26 | } 27 | 28 | url = urljoin(BACKEND_URL, "/boot") 29 | response = requests.get(url, params=params, headers=headers) 30 | 31 | print response.content 32 | 33 | 34 | if __name__ == "__main__": 35 | main() 36 | -------------------------------------------------------------------------------- /doc/examples/get-all-boot-reports.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | """Get all boot reports for a job and a specified kernel.""" 4 | 5 | import requests 6 | 7 | from urlparse import urljoin 8 | 9 | BACKEND_URL = "https://api.kernelci.org" 10 | AUTHORIZATION_TOKEN = "foo" 11 | 12 | 13 | def main(): 14 | headers = { 15 | "Authorization": AUTHORIZATION_TOKEN 16 | } 17 | 18 | params = { 19 | "job": "next", 20 | "kernel": "next-20150717" 21 | } 22 | 23 | url = urljoin(BACKEND_URL, "/boot") 24 | response = requests.get(url, params=params, headers=headers) 25 | 26 | print response.content 27 | 28 | 29 | if __name__ == "__main__": 30 | main() 31 | -------------------------------------------------------------------------------- /doc/examples/get-all-failed-boot-reports.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | """Get all failed boot reports of a job. 4 | 5 | The results will include the 'job', 'kernel' and 'board' fields. By default 6 | they will contain also the '_id' field. 7 | """ 8 | 9 | import requests 10 | 11 | from urlparse import urljoin 12 | 13 | BACKEND_URL = "https://api.kernelci.org" 14 | AUTHORIZATION_TOKEN = "foo" 15 | 16 | 17 | def main(): 18 | headers = { 19 | "Authorization": AUTHORIZATION_TOKEN 20 | } 21 | 22 | params = { 23 | "job": "next", 24 | "status": "FAIL", 25 | "field": ["job", "kernel", "board"] 26 | } 27 | 28 | url = urljoin(BACKEND_URL, "/boot") 29 | response = requests.get(url, params=params, headers=headers) 30 | 31 | print response.content 32 | 33 | 34 | if __name__ == "__main__": 35 | main() 36 | -------------------------------------------------------------------------------- /doc/examples/handling-compressed-data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | """Get all build reports with a specified job ID. 4 | 5 | Explicitly defines the Accept-Encoding header and manually handle the 6 | compressed data. 7 | """ 8 | 9 | import gzip 10 | import requests 11 | 12 | from cStringIO import StringIO 13 | from urlparse import urljoin 14 | 15 | BACKEND_URL = "https://api.kernelci.org" 16 | AUTHORIZATION_TOKEN = "foo" 17 | 18 | 19 | def main(): 20 | headers = { 21 | "Authorization": AUTHORIZATION_TOKEN, 22 | "Accept-Encoding": "gzip" 23 | } 24 | 25 | params = { 26 | "job_id": "123456789012345678901234", 27 | } 28 | 29 | url = urljoin(BACKEND_URL, "/build") 30 | response = requests.get(url, params=params, headers=headers, stream=True) 31 | 32 | in_buffer = StringIO(response.raw.data) 33 | json_str = "" 34 | 35 | with gzip.GzipFile(mode="rb", fileobj=in_buffer) as g_data: 36 | json_str = g_data.read() 37 | 38 | print json_str 39 | 40 | 41 | if __name__ == "__main__": 42 | main() 43 | -------------------------------------------------------------------------------- /doc/examples/import-tests-all-embedded.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | try: 4 | import simplejson as json 5 | except ImportError: 6 | import json 7 | 8 | import requests 9 | 10 | from urlparse import urljoin 11 | 12 | BACKEND_URL = "https://api.kernelci.org" 13 | AUTHORIZATION_TOKEN = "foo" 14 | 15 | 16 | def main(): 17 | headers = { 18 | "Authorization": AUTHORIZATION_TOKEN, 19 | "Content-Type": "application/json" 20 | } 21 | 22 | test_group = { 23 | "name": "A test group", 24 | "build_id": "123456789012345678901234", 25 | "lab_name": "lab-test-00", 26 | "test_cases": [ 27 | { 28 | "name": "A test case - 0" 29 | }, 30 | { 31 | "name": "A test case - 1" 32 | } 33 | ] 34 | } 35 | 36 | url = urljoin(BACKEND_URL, "/test/group") 37 | response = requests.post(url, data=json.dumps(test_group), headers=headers) 38 | 39 | print response.content 40 | 41 | 42 | if __name__ == "__main__": 43 | main() 44 | -------------------------------------------------------------------------------- /doc/examples/post-boot-report.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | try: 4 | import simplejson as json 5 | except ImportError: 6 | import json 7 | 8 | import requests 9 | import urlparse 10 | 11 | BACKEND_URL = "https://api.kernelci.org" 12 | AUTHORIZATION_TOKEN = "foo" 13 | 14 | 15 | def main(): 16 | headers = { 17 | "Authorization": AUTHORIZATION_TOKEN, 18 | "Content-Type": "application/json" 19 | } 20 | 21 | payload = { 22 | "version": "1.0", 23 | "lab_name": "lab-name-00", 24 | "kernel": "next-20141118", 25 | "job": "next", 26 | "defconfig": "arm-omap2plus_defconfig", 27 | "board": "omap4-panda", 28 | "boot_result": "PASS", 29 | "boot_time": 10.4, 30 | "boot_warnings": 1, 31 | "endian": "little", 32 | "git_branch": "local/master", 33 | "git_commit": "fad15b648058ee5ea4b352888afa9030e0092f1b", 34 | "git_describe": "next-20141118" 35 | } 36 | 37 | url = urlparse.urljoin(BACKEND_URL, "/boot") 38 | response = requests.post(url, data=json.dumps(payload), headers=headers) 39 | 40 | print response.content 41 | 42 | 43 | if __name__ == "__main__": 44 | main() 45 | -------------------------------------------------------------------------------- /doc/examples/single-tests-registration.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | try: 4 | import simplejson as json 5 | except ImportError: 6 | import json 7 | 8 | import requests 9 | 10 | from urlparse import urljoin 11 | 12 | BACKEND_URL = "https://api.kernelci.org" 13 | AUTHORIZATION_TOKEN = "foo" 14 | 15 | 16 | def main(): 17 | headers = { 18 | "Authorization": AUTHORIZATION_TOKEN, 19 | "Content-Type": "application/json" 20 | } 21 | 22 | test_group = { 23 | "name": "A test group", 24 | "build_id": "123456789012345678901234", 25 | "lab_name": "lab-test-00" 26 | } 27 | 28 | url = urljoin(BACKEND_URL, "/test/group") 29 | response = requests.post(url, data=json.dumps(test_group), headers=headers) 30 | 31 | if response.status_code == 201: 32 | test_group_result = response.json()["result"][0] 33 | test_group_id = test_group_result["_id"]["$oid"] 34 | 35 | test_case = { 36 | "name": "A test case", 37 | "test_group_id": test_group_id 38 | } 39 | 40 | url = urljoin(BACKEND_URL, "/test/case") 41 | response = requests.post( 42 | url, data=json.dumps(test_case), headers=headers) 43 | 44 | print response.content 45 | 46 | 47 | if __name__ == "__main__": 48 | main() 49 | -------------------------------------------------------------------------------- /doc/examples/trigger-boot-email-report.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | """Send a boot email report in 60 seconds in TXT and HTML formats.""" 4 | 5 | try: 6 | import simplejson as json 7 | except ImportError: 8 | import json 9 | 10 | import requests 11 | 12 | from urlparse import urljoin 13 | 14 | BACKEND_URL = "https://api.kernelci.org" 15 | AUTHORIZATION_TOKEN = "foo" 16 | 17 | 18 | def main(): 19 | headers = { 20 | "Authorization": AUTHORIZATION_TOKEN, 21 | "Content-Type": "application/json" 22 | } 23 | 24 | payload = { 25 | "job": "next", 26 | "kernel": "next-20150105", 27 | "boot_report": 1, 28 | "send_to": ["email1@example.org", "email2@example.org"], 29 | "delay": 60, 30 | "format": ["txt", "html"] 31 | } 32 | 33 | url = urljoin(BACKEND_URL, "/send") 34 | response = requests.post(url, data=json.dumps(payload), headers=headers) 35 | 36 | print response.content 37 | 38 | 39 | if __name__ == "__main__": 40 | main() 41 | -------------------------------------------------------------------------------- /doc/examples/upload-multiple-files-2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import io 4 | import requests 5 | 6 | from requests_toolbelt import MultipartEncoder 7 | from urlparse import urljoin 8 | 9 | BACKEND_URL = "https://api.kernelci.org" 10 | AUTHORIZATION_TOKEN = "foo" 11 | 12 | 13 | def main(): 14 | data = MultipartEncoder( 15 | fields={ 16 | "path": "next/next-20150116/arm64-allnoconfig/", 17 | "file1": ("Image", io.open("/path/to/Image", mode="rb")), 18 | "file2": ("kernel.config", io.open("/path/to/kernel.config", mode="rb")), 19 | "file3": ("build.json", io.open("/path/to/build.json", mode="rb")), 20 | "file4": ("build.log", io.open("/path/to/build.log", mode="rb")), 21 | "file5": ("System.map", io.open("/path/to/System.map", mode="rb")), 22 | } 23 | ) 24 | 25 | headers = { 26 | "Authorization": AUTHORIZATION_TOKEN, 27 | "Content-Type": data.content_type 28 | } 29 | 30 | url = urljoin(BACKEND_URL, "/upload") 31 | response = requests.post(url, headers=headers, data=data) 32 | 33 | print response.content 34 | 35 | 36 | if __name__ == "__main__": 37 | main() 38 | -------------------------------------------------------------------------------- /doc/examples/upload-multiple-files.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import io 4 | import requests 5 | 6 | from urlparse import urljoin 7 | 8 | BACKEND_URL = "https://api.kernelci.org" 9 | AUTHORIZATION_TOKEN = "foo" 10 | 11 | 12 | def main(): 13 | headers = { 14 | "Authorization": AUTHORIZATION_TOKEN 15 | } 16 | 17 | data = { 18 | "path": "next/next-20150116/arm64-allnoconfig/" 19 | } 20 | 21 | files = [ 22 | ("file1", ("Image", io.open("/path/to/Image", mode="rb"))), 23 | ("file2", ("kernel.config", io.open("/path/to/kernel.config", mode="rb"))), 24 | ("file3", ("build.json", io.open("/path/to/build.json", mode="rb"))), 25 | ("file4", ("build.log", io.open("/path/to/build.log", mode="rb"))), 26 | ("file5", ("System.map", io.open("/path/to/System.map", mode="rb"))), 27 | ] 28 | 29 | url = urljoin(BACKEND_URL, "/upload") 30 | response = requests.post(url, data=data, headers=headers, files=files) 31 | 32 | print response.content 33 | 34 | 35 | if __name__ == "__main__": 36 | main() 37 | -------------------------------------------------------------------------------- /doc/examples/upload-single-file.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import io 4 | import requests 5 | 6 | from urlparse import urljoin 7 | 8 | BACKEND_URL = "https://api.kernelci.org" 9 | AUTHORIZATION_TOKEN = "foo" 10 | 11 | 12 | def main(): 13 | headers = { 14 | "Authorization": AUTHORIZATION_TOKEN 15 | } 16 | 17 | url = urljoin( 18 | BACKEND_URL, 19 | "/upload/next/next-20150116/arm64-allnoconfig/lab-name/boot-arch.json") 20 | 21 | with io.open("/path/to/boot-arch.json", mode="rb") as upload_file: 22 | response = requests.put(url, headers=headers, data=upload_file) 23 | 24 | print response.content 25 | 26 | 27 | if __name__ == "__main__": 28 | main() 29 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | API Documentation 2 | ================= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | intro 8 | collections 9 | schema 10 | examples 11 | -------------------------------------------------------------------------------- /doc/schema-attachment.rst: -------------------------------------------------------------------------------- 1 | .. _schema_attachment: 2 | 3 | attachment 4 | ---------- 5 | 6 | An `attachment` object is identified by two elements: 7 | 8 | 1. Its server URI. 9 | 2. Its path on the server URI. 10 | 11 | The server URI is defined as the scheme and the authority in `URI notation `_; the attachment 12 | path concides with the path in `URI notation `_: 13 | 14 | :: 15 | 16 | foo://example.net/I/am/a/file.txt 17 | \_/ \_________/\______________/ 18 | | | | 19 | scheme authority path 20 | 21 | Attachments can be uploaded using the :ref:`upload API `. 22 | 23 | 24 | .. literalinclude:: schema/1.0/attachment.json 25 | :language: json 26 | 27 | Notes 28 | ***** 29 | 30 | * ``server_uri``: If this field is not specified, it will default to ``storage.kernelci.org``. 31 | 32 | More Info 33 | ********* 34 | 35 | * :ref:`Upload API ` 36 | * :ref:`API results ` 37 | * `Uniform Resource Identifier (URI) `_ 38 | -------------------------------------------------------------------------------- /doc/schema-batch.rst: -------------------------------------------------------------------------------- 1 | .. _schema_batch: 2 | 3 | batch 4 | ----- 5 | 6 | A batch request object. 7 | 8 | .. literalinclude:: schema/1.0/post_batch.json 9 | :language: json 10 | -------------------------------------------------------------------------------- /doc/schema-boot.rst: -------------------------------------------------------------------------------- 1 | .. _schema_boot: 2 | 3 | boot 4 | ---- 5 | 6 | .. _schema_boot_get: 7 | 8 | GET 9 | *** 10 | 11 | The following schema covers the data that is available with a GET request. 12 | 13 | .. literalinclude:: schema/1.1/get_boot.json 14 | :language: json 15 | 16 | .. _schema_boot_post: 17 | 18 | POST 19 | **** 20 | 21 | The following schema defines the valid fields that a boot report document should 22 | have when sent to the server. 23 | 24 | .. literalinclude:: schema/1.1/post_boot.json 25 | :language: json 26 | 27 | Notes 28 | +++++ 29 | 30 | * ``defconfig_full``: This field should be used to specify the full defconfig name if config fragments have been used. It should not contain the architecture (``arch``) value. If not defined, the ``defconfig`` value will be used. Its value should conform to: ``defconfig[+fragment[+fragment ... ]]``. 31 | 32 | * ``file_server_url``, ``file_server_resource``: These field should be used to provide the base URL and the actual path where boot related files (i.e. boot logs) are stored. ``file_server_url`` defines the base path, like ``http://storage.kernelci.org/``, ``file_server_resource`` defines the path on the server, like ``kernel-ci/next/``. When both resources are available, they should be joined together with the file names to form the actual URL. Implementation and default values are left to the user or the visualization tool using the data. 33 | 34 | * ``boot_job_path``: The path of the boot test job executor URI *should* contain the job ID. The ``boot_job_id`` field is used for searching and visualization. 35 | 36 | More Info 37 | ********* 38 | 39 | * :ref:`Boot resource ` 40 | * :ref:`Defconfig schema ` 41 | * :ref:`API results ` 42 | * :ref:`Schema time and date ` 43 | -------------------------------------------------------------------------------- /doc/schema-build-logs-summary.rst: -------------------------------------------------------------------------------- 1 | .. _schema_build_logs_summary: 2 | 3 | build_logs_summary 4 | ------------------ 5 | 6 | .. _schema_build_logs_summary_get: 7 | 8 | GET 9 | *** 10 | 11 | The following schema covers the data that is available with a GET request. 12 | 13 | .. literalinclude:: schema/1.1/get_build_logs_summary.json 14 | :language: json 15 | 16 | More Info 17 | ********* 18 | 19 | * :ref:`Job resource ` 20 | * :ref:`Job schema ` 21 | * :ref:`API results ` 22 | * :ref:`Schema time and date ` 23 | -------------------------------------------------------------------------------- /doc/schema-build-logs.rst: -------------------------------------------------------------------------------- 1 | .. _schema_build_logs: 2 | 3 | build_logs 4 | ---------- 5 | 6 | .. _schema_build_logs_get: 7 | 8 | GET 9 | *** 10 | 11 | The following schema covers the data that is available with a GET request. 12 | 13 | .. literalinclude:: schema/1.1/get_build_logs.json 14 | :language: json 15 | 16 | More Info 17 | ********* 18 | 19 | * :ref:`Defconfig resource ` 20 | * :ref:`Defconfig schema ` 21 | * :ref:`Job schema ` 22 | * :ref:`API results ` 23 | * :ref:`Schema time and date ` 24 | -------------------------------------------------------------------------------- /doc/schema-build.rst: -------------------------------------------------------------------------------- 1 | .. _schema_build: 2 | 3 | build 4 | ----- 5 | 6 | .. _schema_build_get: 7 | 8 | GET 9 | *** 10 | 11 | The following schema covers the data that is available with a GET request. 12 | 13 | .. literalinclude:: schema/1.1/get_build.json 14 | :language: json 15 | 16 | .. _schema_build_post: 17 | 18 | POST 19 | **** 20 | 21 | The following schema covers the data that should be available in a build JSON 22 | data file sent to the server. 23 | 24 | .. literalinclude:: schema/1.1/post_build.json 25 | :language: json 26 | 27 | Notes 28 | +++++ 29 | 30 | * ``defconfig_full``: This field should be used to specify the full defconfig name if config fragments have been used. It should not contain the architecture (``arch``) value. If not defined, the ``defconfig`` value will be used. Its value should conform to: ``defconfig[+fragment[+fragment ... ]]``. 31 | 32 | * ``file_server_url``, ``file_server_resource``: These fields should be used to provide the base URL and the actual path where boot related files (i.e. boot logs) are stored. ``file_server_url`` defines the base path, like ``http://storage.kernelci.org/``, ``file_server_resource`` defines the path on the server, like ``kernel-ci/next/``. When both resources are available, they should be joined together with the file names to form the actual URL. Implementation and default values are left to the user or the visualization tool using the data. 33 | 34 | More Info 35 | ********* 36 | 37 | * :ref:`Defconfig resource ` 38 | * :ref:`Job schema ` 39 | * :ref:`API results ` 40 | * :ref:`Schema time and date ` 41 | -------------------------------------------------------------------------------- /doc/schema-job.rst: -------------------------------------------------------------------------------- 1 | .. _schema_job: 2 | 3 | job 4 | --- 5 | 6 | At a lower level, a job is the combination of a tree (or repository) and 7 | the name of the "kernel" that are being built. 8 | 9 | The tree value is an arbitrary name associated with the repository; the kernel value usually is the output of the **git-describe** command: 10 | 11 | * ``git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git`` is the tree called ``mainline``. 12 | * The ``kernel``, for example, could be **v4.0-rc3-194-g5fb0f7fa7f6e**. 13 | 14 | .. _schema_job_get: 15 | 16 | GET 17 | *** 18 | 19 | The following schema covers the data that is available with a GET request. 20 | 21 | .. note:: 22 | Some of the values describe here are not declared in the POST schema. 23 | They are taken from the :ref:`defconfig schema ` at 24 | import time and reported here for easier search. 25 | 26 | .. literalinclude:: schema/1.1/get_job.json 27 | :language: json 28 | 29 | .. _schema_job_post: 30 | 31 | POST 32 | **** 33 | 34 | The following schema covers the data that should be available in the JSON 35 | data sent to the server. 36 | 37 | .. literalinclude:: schema/1.1/post_job.json 38 | :language: json 39 | 40 | 41 | More Info 42 | ********* 43 | 44 | * :ref:`Job resource ` 45 | * :ref:`API results ` 46 | * :ref:`Defconfig schema ` 47 | * :ref:`Schema time and date ` 48 | -------------------------------------------------------------------------------- /doc/schema-lab.rst: -------------------------------------------------------------------------------- 1 | .. _schema_lab: 2 | 3 | lab 4 | --- 5 | 6 | The ``name`` of the lab must be a unique value among all the registered labs. 7 | Use a short but descriptive name to identify the lab, since this value will be 8 | used to perform POST request :ref:`on the boot resource `. 9 | 10 | As a rule of thumbs for creating a lab ``name``: 11 | 12 | * Start the lab name with ``lab-``. 13 | 14 | * Use some of the contact information as the next element (the ``name``, or ``affiliation``). 15 | 16 | * Add a progressive number at the end (``-00``, ``-01``, etc...). 17 | 18 | .. _schema_lab_get: 19 | 20 | GET 21 | *** 22 | 23 | .. literalinclude:: schema/1.0/get_lab.json 24 | :language: json 25 | 26 | .. _schema_lab_post: 27 | 28 | POST 29 | **** 30 | 31 | .. literalinclude:: schema/1.0/post_lab.json 32 | :language: json 33 | 34 | Notes: 35 | 36 | * The mandatory fields for ``POST`` requests are not required for ``PUT`` requests. 37 | 38 | More Info 39 | ********* 40 | 41 | * :ref:`Lab resource ` 42 | * :ref:`Defconfig schema ` 43 | * :ref:`API results ` 44 | * :ref:`Schema time and date ` 45 | -------------------------------------------------------------------------------- /doc/schema-measurement.rst: -------------------------------------------------------------------------------- 1 | .. _schema_measurement: 2 | 3 | measurement 4 | ----------- 5 | 6 | A measurement object is used to store a measure registered by a test. 7 | 8 | .. literalinclude:: schema/1.0/measurement.json 9 | :language: json 10 | 11 | Notes 12 | +++++ 13 | 14 | For the ``measure`` field, the following conversions are applied based on the specified ``unit``: 15 | 16 | * ``number``: Will be treated as a floating point number. 17 | * ``integer``: Will be treated as an integer number; if a floating point number was provided, details/precision of the value will be lost. 18 | * ``time``: Will be treated as an integer describing the total number of seconds. 19 | * ``epoch``: Will be treated as an integer describing the milliseconds from epoch time (1970-01-01). 20 | * ``watt``, ``volt``: Will be treated as floating point numbers. 21 | * ``string``: Will be treated as a normal string. 22 | 23 | More Info 24 | ********* 25 | 26 | * :ref:`Test case schema ` 27 | * :ref:`API results ` 28 | -------------------------------------------------------------------------------- /doc/schema-report.rst: -------------------------------------------------------------------------------- 1 | .. _schema_report: 2 | 3 | report 4 | ------ 5 | 6 | A report is an internally used data structure to save email reports status. It 7 | is used to provide a view on the reports sent when error arises. 8 | 9 | GET 10 | *** 11 | 12 | .. literalinclude:: schema/1.0/get_report.json 13 | :language: json 14 | 15 | More Info 16 | ********* 17 | 18 | * :ref:`API results ` 19 | * :ref:`Schema time and date ` 20 | -------------------------------------------------------------------------------- /doc/schema-send.rst: -------------------------------------------------------------------------------- 1 | .. _schema_send: 2 | 3 | send 4 | ---- 5 | 6 | The send schema is used to trigger email reporting. 7 | 8 | .. _schema_send_post: 9 | 10 | POST 11 | **** 12 | 13 | The following schema covers the data that should be available in the JSON 14 | data sent to the server. 15 | 16 | .. literalinclude:: schema/1.0/post_send.json 17 | :language: json 18 | 19 | More Info 20 | ********* 21 | 22 | * :ref:`Report schema ` 23 | * :ref:`API results ` 24 | * :ref:`Schema time and date ` 25 | -------------------------------------------------------------------------------- /doc/schema-test-case.rst: -------------------------------------------------------------------------------- 1 | .. _schema_test_case: 2 | 3 | test_case 4 | --------- 5 | 6 | A test case is the single unit of test that gets executed. Each test must have 7 | its own ``name`` defined and must be associated with a 8 | :ref:`test group `. 9 | 10 | A test case can be associated with only one 11 | :ref:`test group `. A test case can also register 12 | multiple :ref:`measurements `. 13 | 14 | A test case ``name`` must start and end with an alphanumeric character, and it 15 | must match the following regular expression: ``[a-zA-Z0-9.-_+]+`` 16 | 17 | .. _schema_test_case_get: 18 | 19 | GET 20 | *** 21 | 22 | .. literalinclude:: schema/1.0/get_test_case.json 23 | :language: json 24 | 25 | Notes 26 | +++++ 27 | 28 | * ``attachments``: Each :ref:`attachment ` referenced here, must be available at its specified URI or uploaded using the :ref:`upload API `. 29 | 30 | More Info 31 | ********* 32 | 33 | * :ref:`Test group schema ` 34 | * :ref:`Attachment schema ` 35 | * :ref:`Measurement schema ` 36 | * :ref:`File upload API ` 37 | * :ref:`API results ` 38 | * :ref:`Schema time and date ` 39 | -------------------------------------------------------------------------------- /doc/schema-test-defect.rst: -------------------------------------------------------------------------------- 1 | .. _schema_test_defect: 2 | 3 | test_defect 4 | ----------- 5 | 6 | A test defect is an object, associated with a test, that describes the known 7 | problems. 8 | 9 | 10 | .. literalinclude:: schema/1.0/test_defect.json 11 | :language: json 12 | 13 | 14 | More Info 15 | ********* 16 | 17 | * :ref:`Test schema ` 18 | * :ref:`API results ` 19 | * `Uniform Resource Identifier (URI) `_ 20 | -------------------------------------------------------------------------------- /doc/schema-test-group.rst: -------------------------------------------------------------------------------- 1 | .. _schema_test_group: 2 | 3 | test_group 4 | ---------- 5 | 6 | A test group is the collection of :ref:`test cases `. 7 | 8 | A test group must define its own ``name`` and must be associated with a :ref:`lab ` and with a :ref:`build `. The association with the ``build`` object is performed through the ``build_id`` value, the lab association via the lab ``name`` value. 9 | 10 | A test group ``name`` must start and end with an alphanumeric character, and it 11 | must match the following regular expression: ``[a-zA-Z0-9.-_+]+`` 12 | 13 | .. _schema_test_group_get: 14 | 15 | GET 16 | *** 17 | 18 | .. literalinclude:: schema/1.0/get_test_group.json 19 | :language: json 20 | 21 | Notes 22 | +++++ 23 | 24 | * ``test_cases``: By default, the API will provide the IDs of the ``test_cases`` objects. To expand the selection in order to include the actual objects, please refer to the ``test`` resource query arguments. 25 | 26 | .. _schema_test_group_post: 27 | 28 | POST 29 | **** 30 | 31 | .. literalinclude:: schema/1.2/post_test_group.json 32 | :language: json 33 | 34 | Notes 35 | +++++ 36 | 37 | * ``test_cases``: If not specified, the test cases executed must be registered using the appropriate API call. 38 | 39 | More Info 40 | ********* 41 | 42 | * :ref:`Test case schema ` 43 | * :ref:`Defconfig schema ` 44 | * :ref:`Lab schema ` 45 | * :ref:`API results ` 46 | * :ref:`Schema time and date ` 47 | -------------------------------------------------------------------------------- /doc/schema-test.rst: -------------------------------------------------------------------------------- 1 | .. _schema_test: 2 | 3 | test 4 | ---- 5 | 6 | The test schema to store and retrieve test results is composed of the following schemas. 7 | 8 | .. toctree:: 9 | :maxdepth: 2 10 | 11 | schema-test-group 12 | schema-test-case 13 | 14 | The following schemas are used by the test schemas defined above. 15 | 16 | .. toctree:: 17 | :maxdepth: 1 18 | 19 | schema-attachment 20 | schema-test-defect 21 | schema-measurement 22 | -------------------------------------------------------------------------------- /doc/schema-token.rst: -------------------------------------------------------------------------------- 1 | .. _schema_token: 2 | 3 | token 4 | ----- 5 | 6 | A token object as stored in the database. 7 | 8 | .. _schema_token_get: 9 | 10 | GET 11 | *** 12 | 13 | .. literalinclude:: schema/1.0/get_token.json 14 | :language: json 15 | 16 | .. _schema_token_post: 17 | 18 | POST 19 | **** 20 | 21 | .. literalinclude:: schema/1.0/post_token.json 22 | :language: json 23 | 24 | .. note:: 25 | 26 | In case of a POST request to **update an existing token** the required field is not mandatory. 27 | 28 | Token Properties 29 | **************** 30 | 31 | The following table describes the ``properties`` array of a token: 32 | 33 | +----------+-------------------------------------+ 34 | | Position | Description | 35 | +==========+=====================================+ 36 | | 0 | If the token is an admin token. | 37 | +----------+-------------------------------------+ 38 | | 1 | If the token is a super-user token. | 39 | +----------+-------------------------------------+ 40 | | 2 | If the token can perform GET. | 41 | +----------+-------------------------------------+ 42 | | 3 | If the token can perform POST/PUT. | 43 | +----------+-------------------------------------+ 44 | | 4 | If the token can perform DELETE. | 45 | +----------+-------------------------------------+ 46 | | 5 | If the token is IP restricted. | 47 | +----------+-------------------------------------+ 48 | | 6 | If the token can create new tokens. | 49 | +----------+-------------------------------------+ 50 | | 7 | If the token is a boot lab token. | 51 | +----------+-------------------------------------+ 52 | | 8 | If the token can upload files. | 53 | +----------+-------------------------------------+ 54 | | 9 | If the token is a test lab token. | 55 | +----------+-------------------------------------+ 56 | | 10 - 15 | Not used. | 57 | +----------+-------------------------------------+ 58 | 59 | Other Token Info 60 | **************** 61 | 62 | Administrator & Superuser Tokens 63 | ================================ 64 | 65 | An administrator token can perform all operations: GET, DELETE, POST/PUT and can upload files. 66 | It can also create new tokens and update existing ones. 67 | 68 | A superuser token can perform all operations: GET, DELETE, POST/PUT and can upload files. 69 | It cannot create new tokens nor update existing ones. 70 | 71 | Lab Token 72 | ========= 73 | 74 | Boot Lab 75 | ++++++++ 76 | 77 | A boot lab token is a token that is being used inside a lab for boot testing. 78 | This property is used only internally to describe the token. 79 | 80 | Test Lab 81 | ++++++++ 82 | 83 | A test lab token is a token that is being used to run general tests. 84 | This property is used only internally to describe the token. 85 | 86 | IP Restricted Token 87 | =================== 88 | 89 | A token can be restricted to be used only from one or more IP addresses. 90 | IP addresses can be specified as a single IP address (i.e. 192.168.1.0) or as 91 | a network of IP addresses (i.e. 192.0.3.112/22). 92 | 93 | Both IPv4 and IPv6 are supported. 94 | 95 | More Info 96 | ********* 97 | 98 | * :ref:`API results ` 99 | * :ref:`Schema time and date ` 100 | 101 | -------------------------------------------------------------------------------- /doc/schema.rst: -------------------------------------------------------------------------------- 1 | .. _schema: 2 | 3 | Schema 4 | ====== 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | schema-batch 10 | schema-boot 11 | schema-boot-regressions 12 | schema-build 13 | schema-build-logs 14 | schema-build-logs-summary 15 | schema-job 16 | schema-lab 17 | schema-report 18 | schema-send 19 | schema-test 20 | schema-token 21 | -------------------------------------------------------------------------------- /doc/schema/1.0/attachment.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.0/attachment.json", 3 | "id": "http://api.kernelci.org/json-schema/1.0/attachment.json", 4 | "title": "attachment", 5 | "description": "An attachment/artifact JSON schema", 6 | "type": "object", 7 | "properties": { 8 | "server_uri": { 9 | "type": "string", 10 | "description": "The URI of the server that is hosting this attachment" 11 | }, 12 | "path": { 13 | "type": "string", 14 | "description": "The path on the server that identifies this attachment" 15 | } 16 | }, 17 | "required": ["path"] 18 | } 19 | -------------------------------------------------------------------------------- /doc/schema/1.0/error_log.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.0/error_log.json", 3 | "id": "http://api.kernelci.org/json-schema/1.0/error_log.json", 4 | "title": "error_log", 5 | "description": "A build log errors data structure", 6 | "type": "object", 7 | "properties": { 8 | "version": { 9 | "type": "string", 10 | "description": "The version number of this JSON schema", 11 | "enum": ["1.0"] 12 | }, 13 | "_id": { 14 | "type": "string", 15 | "description": "The ID associated with the object" 16 | }, 17 | "created_on": { 18 | "type": "object", 19 | "description": "Creation date of the object", 20 | "properties": { 21 | "$date": { 22 | "type": "number", 23 | "description": "Milliseconds from epoch time" 24 | } 25 | } 26 | }, 27 | "job": { 28 | "type": "string", 29 | "description": "The job associated with this object" 30 | }, 31 | "job_id": { 32 | "type": "object", 33 | "description": "The ID of the associated job", 34 | "properties": { 35 | "$oid": { 36 | "type": "string", 37 | "description": "The actual ID value" 38 | } 39 | } 40 | }, 41 | "kernel": { 42 | "type": "string", 43 | "description": "The kernel associated with this object" 44 | }, 45 | "defconfig": { 46 | "type": "string", 47 | "description": "The name of the defconfig as reported by the continuous integration system" 48 | }, 49 | "defconfig_full": { 50 | "type": "string", 51 | "description": "The full name of the defconfig, can contain also config fragments information", 52 | "default": "The defconfig value" 53 | }, 54 | "build_id": { 55 | "type": "object", 56 | "description": "The ID of the associated build report", 57 | "properties": { 58 | "$oid": { 59 | "type": "string", 60 | "description": "The actual ID value" 61 | } 62 | } 63 | }, 64 | "arch" : { 65 | "type": "string", 66 | "description": "The architecture type of this board", 67 | "enum": ["arm", "arm64", "x86"], 68 | "default": "arm" 69 | }, 70 | "errors": { 71 | "type": "array", 72 | "description": "The list of error lines found in the build log" 73 | }, 74 | "mismatches": { 75 | "type": "array", 76 | "description": "The list of mismatched lines found in the build log" 77 | }, 78 | "warnings": { 79 | "type": "array", 80 | "description": "The list of warning lines found in the build log" 81 | } 82 | }, 83 | "required": ["job_id", "version"] 84 | } 85 | -------------------------------------------------------------------------------- /doc/schema/1.0/get_build_logs.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.0/get_build_logs.json", 3 | "id": "http://api.kernelci.org/json-schema/1.0/get_build_logs.json", 4 | "title": "build_logs", 5 | "description": "The redacted logs of a build", 6 | "type": "object", 7 | "properties": { 8 | "version": { 9 | "type": "string", 10 | "description": "The version number of this JSON schema", 11 | "enum": ["1.0"] 12 | }, 13 | "_id": { 14 | "type": "string", 15 | "description": "The internal ID associated with the object" 16 | }, 17 | "created_on": { 18 | "type": "object", 19 | "description": "Creation date of the object", 20 | "properties": { 21 | "$date": { 22 | "type": "number", 23 | "description": "Milliseconds from epoch time" 24 | } 25 | } 26 | }, 27 | "job": { 28 | "type": "string", 29 | "description": "The job associated with this object" 30 | }, 31 | "job_id": { 32 | "type": "object", 33 | "description": "The ID of the associated job", 34 | "properties": { 35 | "$oid": { 36 | "type": "string", 37 | "description": "The actual ID value" 38 | } 39 | } 40 | }, 41 | "build_id": { 42 | "type": "object", 43 | "description": "The ID of the associated build", 44 | "properties": { 45 | "$oid": { 46 | "type": "string", 47 | "description": "The actual ID value" 48 | } 49 | } 50 | }, 51 | "kernel": { 52 | "type": "string", 53 | "description": "The kernel associated with this object" 54 | }, 55 | "defconfig": { 56 | "type": "string", 57 | "description": "The name of the defconfig as reported by the continuous integration system" 58 | }, 59 | "defconfig_full": { 60 | "type": "string", 61 | "description": "The full name of the defconfig, can contain also config fragments information", 62 | "default": "The defconfig value" 63 | }, 64 | "status": { 65 | "type": "string", 66 | "description": "The status of the build", 67 | "enum": ["FAIL", "PASS", "UNKNOWN"] 68 | }, 69 | "errors": { 70 | "type": "array", 71 | "description": "The list of error lines" 72 | }, 73 | "errors_count": { 74 | "type": "integer", 75 | "description": "The number of errors", 76 | "default": 0 77 | }, 78 | "warnings": { 79 | "type": "array", 80 | "description": "The list of warning lines" 81 | }, 82 | "warnings_count": { 83 | "type": "integer", 84 | "description": "The number of warnings", 85 | "default": 0 86 | }, 87 | "mismatches": { 88 | "type": "array", 89 | "description": "The list of mismatched lines" 90 | }, 91 | "mismatches_count": { 92 | "type": "integer", 93 | "description": "The number of mismatches", 94 | "default": 0 95 | }, 96 | "arch": { 97 | "type": "string", 98 | "description": "The architecture of the build" 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /doc/schema/1.0/get_job.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.0/get_job.json", 3 | "id": "http://api.kernelci.org/json-schema/1.0/get_job.json", 4 | "title": "job", 5 | "description": "A job as provided by the CI loop", 6 | "type": "object", 7 | "properties": { 8 | "version": { 9 | "type": "string", 10 | "description": "The version number of this JSON schema", 11 | "enum": ["1.0"] 12 | }, 13 | "_id": { 14 | "type": "object", 15 | "description": "The ID of ths object", 16 | "properties": { 17 | "$oid": { 18 | "type": "string", 19 | "description": "The actual ID value" 20 | } 21 | } 22 | }, 23 | "created_on": { 24 | "type": "object", 25 | "description": "Creation date of the object", 26 | "properties": { 27 | "$date": { 28 | "type": "number", 29 | "description": "Milliseconds from epoch time" 30 | } 31 | } 32 | }, 33 | "private": { 34 | "type": "boolean", 35 | "description": "If the job is private or not", 36 | "default": false 37 | }, 38 | "kernel": { 39 | "type": "string", 40 | "description": "The name of the kernel" 41 | }, 42 | "job": { 43 | "type": "string", 44 | "description": "The name of the job" 45 | }, 46 | "status": { 47 | "type": "string", 48 | "description": "The status of the job", 49 | "enum": ["BUILD", "FAIL", "PASS", "UNKNOWN"] 50 | }, 51 | "git_branch": { 52 | "type": "string", 53 | "description": "The name of the branch" 54 | }, 55 | "git_commit": { 56 | "type": "string", 57 | "description": "The git SHA of the commit used for the build" 58 | }, 59 | "git_describe": { 60 | "type": "string", 61 | "description": "The name of the git describe command" 62 | }, 63 | "git_url": { 64 | "type": "string", 65 | "description": "The URL of the git web interface where the code used to build can be found" 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /doc/schema/1.0/get_report.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.0/get_report.json", 3 | "id": "http://api.kernelci.org/json-schema/1.0/get_report.json", 4 | "title": "report", 5 | "description": "A report object", 6 | "type": "object", 7 | "properties": { 8 | "version": { 9 | "type": "string", 10 | "description": "The version of this JSON schema", 11 | "enum": ["1.0"] 12 | }, 13 | "name": { 14 | "type": "string", 15 | "description": "The name associated with the object" 16 | }, 17 | "_id": { 18 | "type": "string", 19 | "description": "The ID associated with the object as provided by the database" 20 | }, 21 | "created_on": { 22 | "type": "object", 23 | "description": "Creation date of the object", 24 | "properties": { 25 | "$date": { 26 | "type": "number", 27 | "description": "Milliseconds from epoch time", 28 | "format": "utc-millisec" 29 | } 30 | } 31 | }, 32 | "updated_on": { 33 | "type": "object", 34 | "description": "Update date of the object", 35 | "properties": { 36 | "$date": { 37 | "type": "number", 38 | "description": "Milliseconds from epoch time", 39 | "format": "utc-millisec" 40 | } 41 | } 42 | }, 43 | "errors": { 44 | "type": "array", 45 | "description": "An array of arrays containing error codes and descriptions from the SMTP server" 46 | }, 47 | "job": { 48 | "type": "string", 49 | "description": "The job name associated with the object" 50 | }, 51 | "kernel": { 52 | "type": "string", 53 | "description": "The kernel name associated with the object" 54 | }, 55 | "type": { 56 | "type": "string", 57 | "description": "The type of the report", 58 | "enum": ["boot", "build"] 59 | }, 60 | "status": { 61 | "type": "string", 62 | "description": "The status of the report", 63 | "enum": ["SENT", "ERROR"] 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /doc/schema/1.0/get_token.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.0/get_token.json", 3 | "id": "http://api.kernelci.org/json-schema/1.0/get_token.json", 4 | "title": "token", 5 | "description": "A token used to interact with the API", 6 | "type": "object", 7 | "properties": { 8 | "version": { 9 | "type": "string", 10 | "description": "The version number of this JSON schema", 11 | "enum": ["1.0"] 12 | }, 13 | "_id": { 14 | "type": "string", 15 | "description": "The ID associated with this object" 16 | }, 17 | "created_on": { 18 | "type": "object", 19 | "description": "Creation date of the object", 20 | "properties": { 21 | "$date": { 22 | "type": "number", 23 | "description": "Milliseconds from epoch time" 24 | } 25 | } 26 | }, 27 | "token": { 28 | "type": "string", 29 | "description": "The token that will be used to interact with the API" 30 | }, 31 | "expires_on": { 32 | "type": "object", 33 | "description": "The date when the token is supposed to expire", 34 | "properties": { 35 | "$date": { 36 | "type": "number", 37 | "description": "Milliseconds from epoch time" 38 | } 39 | } 40 | }, 41 | "expired": { 42 | "type": "boolean", 43 | "description": "If the token has expired" 44 | }, 45 | "username": { 46 | "type": "string", 47 | "description": "The user name associated with the token" 48 | }, 49 | "email": { 50 | "type": "string", 51 | "description": "The email address associated with the token" 52 | }, 53 | "ip_address": { 54 | "type": "array", 55 | "description": "List of IP addresses the token is restricted to" 56 | }, 57 | "properties": { 58 | "type": "array", 59 | "description": "An array of length 16 of integer values; each value defines a properties of the token" 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /doc/schema/1.0/measurement.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.0/measurement.json", 3 | "id": "http://api.kernelci.org/json-schema/1.0/measurement.json", 4 | "title": "measurement", 5 | "description": "A measurement registered by a test case", 6 | "type": "object", 7 | "properties": { 8 | "name": { 9 | "type": "string", 10 | "description": "The name given to this measurement" 11 | }, 12 | "time": { 13 | "type": "number", 14 | "description": "Epoch time when the measurement was registered", 15 | "format": "utc-millisec" 16 | }, 17 | "unit": { 18 | "type": "string", 19 | "description": "The unit of this measurement", 20 | "enum": ["string", "epoch", "time", "watt", "volt", "number", "integer"], 21 | "default": "string" 22 | }, 23 | "measure": { 24 | "type": ["string", "number", "integer"], 25 | "description": "The data measured during the test case execution; the value will be interpreted based on the $unit field" 26 | } 27 | }, 28 | "required": ["measure"] 29 | } 30 | -------------------------------------------------------------------------------- /doc/schema/1.0/post_batch.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.0/post_batch.json", 3 | "id": "http://api.kernelci.org/json-schema/1.0/post_batch.json", 4 | "title": "batch", 5 | "description": "A batch request to perform multiple operations", 6 | "type": "array", 7 | "items": { 8 | "type": "object", 9 | "properties": { 10 | "method": { 11 | "type": "string", 12 | "description": "The HTTP method this batch operation should be matched to" 13 | }, 14 | "operation_id": { 15 | "type": "string", 16 | "description": "A user provided identifier for the operation to perform" 17 | }, 18 | "resource": { 19 | "type": "string", 20 | "description": "The resource where to perform the operation" 21 | }, 22 | "document": { 23 | "type": "string", 24 | "description": "The ID of the document in the specified resource" 25 | }, 26 | "query": { 27 | "type": "string", 28 | "description": "A key=value pairs, separated by the ampersand character, that define the query to perform on the resource" 29 | } 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /doc/schema/1.0/post_job.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.0/post_job.json", 3 | "id": "http://api.kernelci.org/json-schema/1.0/post_job.json", 4 | "title": "job", 5 | "description": "A job data to trigger build import", 6 | "type": "object", 7 | "properties": { 8 | "version": { 9 | "type": "string", 10 | "description": "The version number of this JSON schema", 11 | "enum": ["1.0"] 12 | }, 13 | "job": { 14 | "type": "string", 15 | "description": "The job associated with this object" 16 | }, 17 | "kernel": { 18 | "type": "string", 19 | "description": "The kernel associated with this object" 20 | }, 21 | "status": { 22 | "type": "string", 23 | "description": "The status this object should be set to", 24 | "enum": ["PASS", "FAIL", "BUILD", "UNKNOWN"] 25 | } 26 | }, 27 | "required": ["job", "kernel"] 28 | } 29 | -------------------------------------------------------------------------------- /doc/schema/1.0/post_lab.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.0/post_lab.json", 3 | "id": "http://api.kernelci.org/json-schema/1.0/post_lab.json", 4 | "title": "lab", 5 | "description": "A lab object", 6 | "type": "object", 7 | "properties": { 8 | "version": { 9 | "type": "string", 10 | "description": "The version number of this JSON schema", 11 | "enum": ["1.0"] 12 | }, 13 | "name": { 14 | "type": "string", 15 | "description": "The name associated with the object" 16 | }, 17 | "contact": { 18 | "type": "object", 19 | "description": "The contact details of the object", 20 | "properties": { 21 | "name": { 22 | "type": "string", 23 | "description": "The name of the contact" 24 | }, 25 | "surname": { 26 | "type": "string", 27 | "description": "The surname of the contact" 28 | }, 29 | "email": { 30 | "type": "string", 31 | "description": "The email of the contact" 32 | }, 33 | "telephone": { 34 | "type": "string", 35 | "description": "The land-line phone number" 36 | }, 37 | "mobile": { 38 | "type": "string", 39 | "description": "The mobile phone number" 40 | }, 41 | "affiliation": { 42 | "type": "string", 43 | "description": "The name of the company, or association this contact is part of" 44 | }, 45 | "required": ["name", "surname", "email"] 46 | } 47 | }, 48 | "address": { 49 | "type": "object", 50 | "description": "The address where the lab is located", 51 | "properties": { 52 | "street_1": { 53 | "type": "string", 54 | "description": "First line for the address" 55 | }, 56 | "street_2": { 57 | "type": "string", 58 | "description": "Second line for the address" 59 | }, 60 | "city": { 61 | "type": "string", 62 | "description": "The city name" 63 | }, 64 | "country": { 65 | "type": "string", 66 | "description": "The country name" 67 | }, 68 | "zipcode": { 69 | "type": "string", 70 | "description": "The zip code" 71 | }, 72 | "longitude": { 73 | "type": "number", 74 | "description": "Latitude of the lab location" 75 | }, 76 | "longitude": { 77 | "type": "number", 78 | "description": "Longitude of the lab location" 79 | } 80 | } 81 | }, 82 | "private": { 83 | "type": "boolean", 84 | "description": "If the lab is private or not", 85 | "default": "false" 86 | }, 87 | "token": { 88 | "type": "string", 89 | "description": "The token to associated with this lab" 90 | } 91 | }, 92 | "required": ["version", "name", "contact"] 93 | } 94 | -------------------------------------------------------------------------------- /doc/schema/1.0/post_send.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.0/post_send.json", 3 | "id": "http://api.kernelci.org/json-schema/1.0/post_send.json", 4 | "title": "send", 5 | "description": "Data to trigger the email report", 6 | "type": "object", 7 | "properties": { 8 | "job": { 9 | "type": "string", 10 | "description": "The job name associated with the object" 11 | }, 12 | "kernel": { 13 | "type": "string", 14 | "description": "The kernel name associated with the object" 15 | }, 16 | "lab_name": { 17 | "type": "string", 18 | "description": "The name of the lab to trigger the report for" 19 | }, 20 | "boot_report": { 21 | "type": "boolean", 22 | "description": "Whether the boot report should be created and sent", 23 | "default": 0 24 | }, 25 | "build_report": { 26 | "type": "boolean", 27 | "description": "Whether the build report should be created and sent", 28 | "default": 0 29 | }, 30 | "boot_send_to": { 31 | "type": ["string", "array"], 32 | "description": "The emails to sent the boot report to" 33 | }, 34 | "boot_send_cc": { 35 | "type": ["string", "array"], 36 | "description": "The emails to sent the boot report to in carbon copy" 37 | }, 38 | "boot_send_bcc": { 39 | "type": ["string", "array"], 40 | "description": "The emails to sent the boot report to in blind carbon copy" 41 | }, 42 | "build_send_to": { 43 | "type": ["string", "array"], 44 | "description": "The emails to send the build report to" 45 | }, 46 | "build_send_cc": { 47 | "type": ["string", "array"], 48 | "description": "The emails to send the build report to in carbon copy" 49 | }, 50 | "build_send_bcc": { 51 | "type": ["string", "array"], 52 | "description": "The emails to send the build report to in blind carbon copy" 53 | }, 54 | "send_to": { 55 | "type": ["string", "array"], 56 | "description": "The emails to send the reports to, will be appended to the more specific email control values" 57 | }, 58 | "send_cc": { 59 | "type": ["string", "array"], 60 | "description": "The emails to send the reports to in carbon copy, will be appended to the more specific email control values" 61 | }, 62 | "send_bcc": { 63 | "type": ["string", "array"], 64 | "description": "The emails to send the reports to in blind carbon copy, will be appended to the more specific email control values" 65 | }, 66 | "in_reply_to": { 67 | "type": "string", 68 | "description": "The message ID of the previous message this email report is a reply" 69 | }, 70 | "delay": { 71 | "type": "number", 72 | "description": "The number of seconds after which the email should be sent", 73 | "default": 3600 74 | }, 75 | "format": { 76 | "type": "array", 77 | "description": "The format of the email", 78 | "enum": ["txt", "html"], 79 | "default": ["txt"] 80 | }, 81 | "subject": { 82 | "type": "string", 83 | "description": "The subject for the email: overrides the default one. If sending both boot and build reports, the same subject will be used for both emails. " 84 | } 85 | }, 86 | "required": ["job", "kernel"] 87 | } 88 | -------------------------------------------------------------------------------- /doc/schema/1.0/post_test_case.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.0/post_test_case.json", 3 | "id": "http://api.kernelci.org/json-schema/1.0/post_test_case.json", 4 | "title": "test_case", 5 | "description": "A test case JSON object", 6 | "type": "object", 7 | "properties": { 8 | "version": { 9 | "type": "string", 10 | "description": "The version number of this JSON schema", 11 | "enum": ["1.0"] 12 | }, 13 | "name": { 14 | "type": "string", 15 | "description": "The name given to this test case" 16 | }, 17 | "test_group_id": { 18 | "type": "string", 19 | "description": "The test group ID associated with this test case" 20 | }, 21 | "measurements": { 22 | "type": "array", 23 | "description": "Array of measurement objects registered by this test case", 24 | "items": {"$ref": "http://api.kernelci.org/json-schema/1.0/measurement.json"}, 25 | "additionalItems": true 26 | }, 27 | "minimum": { 28 | "type": ["integer", "number"], 29 | "description": "The minimum measurement registered" 30 | }, 31 | "maximum": { 32 | "type": ["integer", "number"], 33 | "description": "The maximum measurement registered" 34 | }, 35 | "samples": { 36 | "type": "integer", 37 | "description": "Number of registered measurements" 38 | }, 39 | "samples_sum": { 40 | "type": ["integer", "number"], 41 | "description": "Sum of the registered measurements" 42 | }, 43 | "samples_sqr_sum": { 44 | "type": ["integer", "number"], 45 | "description": "Sum of the square of the registered measurements" 46 | }, 47 | "parameters": { 48 | "type": "object", 49 | "description": "Free form object to store key-value pairs describing the parameters used to run the test case" 50 | }, 51 | "status": { 52 | "type": "string", 53 | "description": "The status of the execution of this test case", 54 | "enum": ["PASS", "FAIL", "SKIP", "ERROR"], 55 | "default": "PASS" 56 | }, 57 | "time": { 58 | "type": "number", 59 | "description": "The number of seconds it took to execute this test case", 60 | "default": -1 61 | }, 62 | "definition_uri": { 63 | "type": "string", 64 | "description": "The URI where this test case definition is stored" 65 | }, 66 | "vcs_commit": { 67 | "type": "string", 68 | "description": "The VCS commit value if the $definition_uri field is a VCS URI" 69 | }, 70 | "attachments": { 71 | "type": "array", 72 | "description": "List of attachment objects produced by this test case", 73 | "items": {"$ref": "http://api.kernelci.org/json-schema/1.0/attachment.json"}, 74 | "additionalItems": true 75 | }, 76 | "kvm_guest": { 77 | "type": "string", 78 | "description": "The name of the KVM guest this test case has been executed on" 79 | }, 80 | "metadata": { 81 | "type": "object", 82 | "description": "Free form object where to store accessory test case data" 83 | } 84 | }, 85 | "required": ["name", "test_group_id"] 86 | } 87 | -------------------------------------------------------------------------------- /doc/schema/1.0/post_token.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.0/post_token.json", 3 | "id": "http://api.kernelci.org/json-schema/1.0/post_token.json", 4 | "title": "token", 5 | "description": "The JSON schema to create/update tokens", 6 | "type": "object", 7 | "properties": { 8 | "version": { 9 | "type": "string", 10 | "description": "The version number of this JSON schema", 11 | "enum": ["1.0"] 12 | }, 13 | "email": { 14 | "type": "string", 15 | "description": "The email address associated with the token" 16 | }, 17 | "username": { 18 | "type": "string", 19 | "description": "The user name associated with the token" 20 | }, 21 | "expired": { 22 | "type": "boolean", 23 | "description": "If the token has expired" 24 | }, 25 | "expires_on": { 26 | "type": "string", 27 | "description": "The date when the token is supposed to expire in the format YYYY-MM-DD" 28 | }, 29 | "get": { 30 | "type": "boolean", 31 | "description": "If the token can perform GET operations" 32 | }, 33 | "post": { 34 | "type": "boolean", 35 | "description": "If the token can perform POST/PUT operations" 36 | }, 37 | "delete": { 38 | "type": "boolean", 39 | "description": "If the token can perform DELETE operations" 40 | }, 41 | "upload": { 42 | "type": "boolean", 43 | "description": "If the token can be used to upload files" 44 | }, 45 | "admin": { 46 | "type": "boolean", 47 | "description": "If the token is an administrator token" 48 | }, 49 | "superuser": { 50 | "type": "boolean", 51 | "description": "If the token is a super user one" 52 | }, 53 | "lab": { 54 | "type": "boolean", 55 | "description": "If the token is used by a boot lab" 56 | }, 57 | "test_lab": { 58 | "type": "boolean", 59 | "description": "If the token is used by a test lab" 60 | }, 61 | "ip_restricted": { 62 | "type": "boolean", 63 | "description": "If the token is IP restricted" 64 | }, 65 | "ip_address": { 66 | "type": "array", 67 | "description": "Array of IP addresses the token is restricted to" 68 | } 69 | }, 70 | "required": ["email"] 71 | } 72 | -------------------------------------------------------------------------------- /doc/schema/1.0/test_defect.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.0/test_defect.json", 3 | "id": "http://api.kernelci.org/json-schema/1.0/test_defect.json", 4 | "title": "test_defect", 5 | "description": "A defect/problem associated with a test", 6 | "type": "object", 7 | "properties": { 8 | "defect_url": { 9 | "type": "string", 10 | "description": "The URL where the defect can be found" 11 | }, 12 | "defect_comment": { 13 | "type": "string", 14 | "description": "A free-form description of the defect" 15 | }, 16 | "defect_ack": { 17 | "type": "boolean", 18 | "description": "If the defect is known or not" 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /doc/schema/1.1/get_build_logs.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.1/get_build_logs.json", 3 | "id": "http://api.kernelci.org/json-schema/1.1/get_build_logs.json", 4 | "title": "build_logs", 5 | "description": "The redacted logs of a build", 6 | "type": "object", 7 | "properties": { 8 | "version": { 9 | "type": "string", 10 | "description": "The version number of this JSON schema", 11 | "enum": ["1.0", "1.1"] 12 | }, 13 | "_id": { 14 | "type": "string", 15 | "description": "The internal ID associated with the object" 16 | }, 17 | "created_on": { 18 | "type": "object", 19 | "description": "Creation date of the object", 20 | "properties": { 21 | "$date": { 22 | "type": "number", 23 | "description": "Milliseconds from epoch time" 24 | } 25 | } 26 | }, 27 | "job": { 28 | "type": "string", 29 | "description": "The job associated with this object" 30 | }, 31 | "job_id": { 32 | "type": "object", 33 | "description": "The ID of the associated job", 34 | "properties": { 35 | "$oid": { 36 | "type": "string", 37 | "description": "The actual ID value" 38 | } 39 | } 40 | }, 41 | "build_id": { 42 | "type": "object", 43 | "description": "The ID of the associated build", 44 | "properties": { 45 | "$oid": { 46 | "type": "string", 47 | "description": "The actual ID value" 48 | } 49 | } 50 | }, 51 | "kernel": { 52 | "type": "string", 53 | "description": "The kernel associated with this object" 54 | }, 55 | "defconfig": { 56 | "type": "string", 57 | "description": "The name of the defconfig as reported by the continuous integration system" 58 | }, 59 | "defconfig_full": { 60 | "type": "string", 61 | "description": "The full name of the defconfig, can contain also config fragments information", 62 | "default": "The defconfig value" 63 | }, 64 | "git_branch": { 65 | "type": "string", 66 | "description": "The name of the branch" 67 | }, 68 | "status": { 69 | "type": "string", 70 | "description": "The status of the build", 71 | "enum": ["FAIL", "PASS", "UNKNOWN"] 72 | }, 73 | "errors": { 74 | "type": "array", 75 | "description": "The list of error lines" 76 | }, 77 | "errors_count": { 78 | "type": "integer", 79 | "description": "The number of errors", 80 | "default": 0 81 | }, 82 | "warnings": { 83 | "type": "array", 84 | "description": "The list of warning lines" 85 | }, 86 | "warnings_count": { 87 | "type": "integer", 88 | "description": "The number of warnings", 89 | "default": 0 90 | }, 91 | "mismatches": { 92 | "type": "array", 93 | "description": "The list of mismatched lines" 94 | }, 95 | "mismatches_count": { 96 | "type": "integer", 97 | "description": "The number of mismatches", 98 | "default": 0 99 | }, 100 | "arch": { 101 | "type": "string", 102 | "description": "The architecture of the build" 103 | } 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /doc/schema/1.1/get_job.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.1/get_job.json", 3 | "id": "http://api.kernelci.org/json-schema/1.1/get_job.json", 4 | "title": "job", 5 | "description": "A job as provided by the CI loop", 6 | "type": "object", 7 | "properties": { 8 | "version": { 9 | "type": "string", 10 | "description": "The version number of this JSON schema", 11 | "enum": ["1.0", "1.1"] 12 | }, 13 | "_id": { 14 | "type": "object", 15 | "description": "The ID of ths object", 16 | "properties": { 17 | "$oid": { 18 | "type": "string", 19 | "description": "The actual ID value" 20 | } 21 | } 22 | }, 23 | "created_on": { 24 | "type": "object", 25 | "description": "Creation date of the object", 26 | "properties": { 27 | "$date": { 28 | "type": "number", 29 | "description": "Milliseconds from epoch time" 30 | } 31 | } 32 | }, 33 | "private": { 34 | "type": "boolean", 35 | "description": "If the job is private or not", 36 | "default": false 37 | }, 38 | "kernel": { 39 | "type": "string", 40 | "description": "The name of the kernel" 41 | }, 42 | "job": { 43 | "type": "string", 44 | "description": "The name of the job" 45 | }, 46 | "status": { 47 | "type": "string", 48 | "description": "The status of the job", 49 | "enum": ["BUILD", "FAIL", "PASS", "UNKNOWN"] 50 | }, 51 | "git_branch": { 52 | "type": "string", 53 | "description": "The name of the branch" 54 | }, 55 | "git_commit": { 56 | "type": "string", 57 | "description": "The git SHA of the commit used for the build" 58 | }, 59 | "git_describe": { 60 | "type": "string", 61 | "description": "The name of the git describe command" 62 | }, 63 | "git_url": { 64 | "type": "string", 65 | "description": "The URL of the git web interface where the code used to build can be found" 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /doc/schema/1.1/post_job.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.1/post_job.json", 3 | "id": "http://api.kernelci.org/json-schema/1.1/post_job.json", 4 | "title": "job", 5 | "description": "A job data to trigger build import", 6 | "type": "object", 7 | "properties": { 8 | "version": { 9 | "type": "string", 10 | "description": "The version number of this JSON schema", 11 | "enum": ["1.0", "1.1"], 12 | "default": "1.1" 13 | }, 14 | "job": { 15 | "type": "string", 16 | "description": "The job associated with this object" 17 | }, 18 | "kernel": { 19 | "type": "string", 20 | "description": "The kernel associated with this object" 21 | }, 22 | "git_branch": { 23 | "type": "string", 24 | "description": "The name of the branch" 25 | }, 26 | "git_commit": { 27 | "type": "string", 28 | "description": "The git SHA of the commit used" 29 | }, 30 | "status": { 31 | "type": "string", 32 | "description": "The status this object should be set to", 33 | "enum": ["PASS", "FAIL", "BUILD", "UNKNOWN"] 34 | } 35 | }, 36 | "required": ["job", "kernel", "git_branch"] 37 | } 38 | -------------------------------------------------------------------------------- /doc/schema/1.1/post_send.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.1/post_send.json", 3 | "id": "http://api.kernelci.org/json-schema/1.1/post_send.json", 4 | "title": "send", 5 | "description": "Data to trigger the email report", 6 | "type": "object", 7 | "properties": { 8 | "job": { 9 | "type": "string", 10 | "description": "The job name associated with the object" 11 | }, 12 | "git_branch": { 13 | "type": "string", 14 | "description": "The branch name" 15 | }, 16 | "kernel": { 17 | "type": "string", 18 | "description": "The kernel name associated with the object" 19 | }, 20 | "lab_name": { 21 | "type": "string", 22 | "description": "The name of the lab to trigger the report for" 23 | }, 24 | "boot_report": { 25 | "type": "boolean", 26 | "description": "Whether the boot report should be created and sent", 27 | "default": 0 28 | }, 29 | "build_report": { 30 | "type": "boolean", 31 | "description": "Whether the build report should be created and sent", 32 | "default": 0 33 | }, 34 | "boot_send_to": { 35 | "type": ["string", "array"], 36 | "description": "The emails to sent the boot report to" 37 | }, 38 | "boot_send_cc": { 39 | "type": ["string", "array"], 40 | "description": "The emails to sent the boot report to in carbon copy" 41 | }, 42 | "boot_send_bcc": { 43 | "type": ["string", "array"], 44 | "description": "The emails to sent the boot report to in blind carbon copy" 45 | }, 46 | "build_send_to": { 47 | "type": ["string", "array"], 48 | "description": "The emails to send the build report to" 49 | }, 50 | "build_send_cc": { 51 | "type": ["string", "array"], 52 | "description": "The emails to send the build report to in carbon copy" 53 | }, 54 | "build_send_bcc": { 55 | "type": ["string", "array"], 56 | "description": "The emails to send the build report to in blind carbon copy" 57 | }, 58 | "send_to": { 59 | "type": ["string", "array"], 60 | "description": "The emails to send the reports to, will be appended to the more specific email control values" 61 | }, 62 | "send_cc": { 63 | "type": ["string", "array"], 64 | "description": "The emails to send the reports to in carbon copy, will be appended to the more specific email control values" 65 | }, 66 | "send_bcc": { 67 | "type": ["string", "array"], 68 | "description": "The emails to send the reports to in blind carbon copy, will be appended to the more specific email control values" 69 | }, 70 | "in_reply_to": { 71 | "type": "string", 72 | "description": "The message ID of the previous message this email report is a reply" 73 | }, 74 | "delay": { 75 | "type": "number", 76 | "description": "The number of seconds after which the email should be sent", 77 | "default": 3600 78 | }, 79 | "format": { 80 | "type": "array", 81 | "description": "The format of the email", 82 | "enum": ["txt", "html"], 83 | "default": ["txt"] 84 | }, 85 | "subject": { 86 | "type": "string", 87 | "description": "The subject for the email: overrides the default one. If sending both boot and build reports, the same subject will be used for both emails. " 88 | } 89 | }, 90 | "required": ["job", "kernel", "git_branch"] 91 | } 92 | -------------------------------------------------------------------------------- /doc/schema/1.2/post_test_group.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://api.kernelci.org/json-schema/1.2/post_test_group.json", 3 | "id": "http://api.kernelci.org/json-schema/1.2/post_test_group.json", 4 | "title": "test_group", 5 | "description": "A test group JSON object", 6 | "type": "object", 7 | "properties": { 8 | "version": { 9 | "type": "string", 10 | "description": "The version number of this JSON schema", 11 | "enum": ["1.0", "1.2"] 12 | }, 13 | "name": { 14 | "type": "string", 15 | "description": "The name given to this test group" 16 | }, 17 | "lab_name": { 18 | "type": "string", 19 | "description": "The name of the lab executing this test group" 20 | }, 21 | "time": { 22 | "type": "number", 23 | "description": "The number of seconds it took to execute the entire test group", 24 | "default": -1 25 | }, 26 | "job": { 27 | "type": "string", 28 | "description": "The name of the job (aka the git tree)" 29 | }, 30 | "kernel": { 31 | "type": "string", 32 | "description": "The name of the kernel or the git describe value" 33 | }, 34 | "defconfig": { 35 | "type": "string", 36 | "description": "The name of the defconfig" 37 | }, 38 | "defconfig_full": { 39 | "type": "string", 40 | "description": "The full name of the defconfig, can also contain config fragments information", 41 | "default": "The defconfig value" 42 | }, 43 | "arch": { 44 | "type": "string", 45 | "description": "The architecture type of this board", 46 | "enum": ["arm", "arm64", "x86"], 47 | "default": "arm" 48 | }, 49 | "board": { 50 | "type": "string", 51 | "description": "The name of the board" 52 | }, 53 | "board_instance": { 54 | "type": "string", 55 | "description": "The instance identifier of the board" 56 | }, 57 | "build_environment": { 58 | "type": "string", 59 | "description": "Build environment name as used in the build configuration e.g. gcc-8" 60 | }, 61 | "git_branch": { 62 | "type": "string", 63 | "description": "The branch used for testing" 64 | }, 65 | "metadata": { 66 | "type": "object", 67 | "description": "Free form object where to store accessory test group data" 68 | }, 69 | "test_cases": { 70 | "type": "array", 71 | "description": "The list of test case objects executed by this test group", 72 | "items": {"$ref": "http://api.kernelci.org/json-schema/1.0/test_case_get.json"}, 73 | "additionalItems": true 74 | }, 75 | "sub_groups": { 76 | "type": "array", 77 | "description": "The list of sub-group objects included in this group", 78 | "items": {"$ref": "http://api.kernelci.org/json-schema/1.2/post_test_group.json.json"}, 79 | "additionalItems": true 80 | }, 81 | "definition_uri": { 82 | "type": "string", 83 | "description": "The URI where this test group definition is stored" 84 | }, 85 | "vcs_commit": { 86 | "type": "string", 87 | "description": "The VCS commit value if the $definition_uri field is a VCS URI" 88 | }, 89 | "log": { 90 | "type": "string", 91 | "description": "Test log content, which to be saved in a file" 92 | } 93 | 94 | }, 95 | "required": ["arch", "build_environment", "defconfig", "git_branch", "job", 96 | "kernel", "name", "lab_name"] 97 | } 98 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | lupa 3 | mock>0.7.2 4 | mongomock==3.19.0 5 | fakeredis 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Sphinx==1.4.9 2 | celery[redis]==3.1.25 3 | docutils==0.12 4 | futures==3.0.4 5 | hiredis==0.2.0 6 | jinja2 7 | jsonschema==2.6.0 8 | netaddr==0.7.18 9 | pycares==4.2.0 10 | pymongo==3.9.0 11 | python-jenkins==0.4.11 12 | python-dateutil 13 | pytz 14 | pyyaml==5.4 15 | redis==2.10.6 16 | scandir 17 | simplejson 18 | sphinx-bootstrap-theme 19 | sphinxcontrib-httpdomain==1.3.0 20 | tornado==3.2.2 21 | six==1.13.0 22 | --------------------------------------------------------------------------------