├── dlrn ├── tests │ ├── __init__.py │ ├── samples │ │ ├── commits_remote.yaml │ │ ├── commits_3.yaml │ │ ├── commits_components.yaml │ │ ├── projects.ini.detect │ │ ├── projects_force.ini │ │ └── versions.csv │ ├── base.py │ ├── test_remote.py │ ├── test_driver_local.py │ ├── test_prom_metrics.py │ ├── test_driver_copr.py │ ├── test_config.py │ └── test_notifications.py ├── api │ ├── drivers │ │ ├── __init__.py │ │ ├── dbauthentication.py │ │ └── auth.py │ ├── inputs │ │ ├── __init__.py │ │ ├── recheck_package.py │ │ ├── agg_status.py │ │ ├── remote_import.py │ │ ├── repo_status.py │ │ ├── metrics.py │ │ ├── civotes.py │ │ ├── last_tested_repo.py │ │ ├── report_result.py │ │ └── promotions.py │ ├── responses │ │ ├── __init__.py │ │ ├── health.py │ │ └── metrics.py │ ├── config.py │ ├── static │ │ └── styles.css │ ├── templates │ │ ├── votes_agg.j2 │ │ ├── votes.j2 │ │ ├── report.j2 │ │ ├── votes_general_agg.j2 │ │ └── votes_general.j2 │ ├── __init__.py │ └── prom_metrics.py ├── drivers │ ├── __init__.py │ ├── buildrpm.py │ ├── pkginfo.py │ ├── local.py │ └── mockdriver.py ├── migrations │ ├── README │ ├── script.py.mako │ ├── versions │ │ ├── 6a3d982b967b_add_versions_url_columnt_to_commits.py │ │ ├── b6f658f481f8_add_commit_type.py │ │ ├── 2d503b5034b7_rename_artifacts.py │ │ ├── 837138eb7daa_extend_extended_hash_to_128_chars.py │ │ ├── 00a31f1f39c0_add_component_to_civote.py │ │ ├── 1268c799620f_add_commit_branch_to_db.py │ │ ├── f84aca0549fd_add_component_to_commits.py │ │ ├── 4a5651777e5e_add_promotions_table.py │ │ ├── 47ebe0522809_scheme_change_due_to_scm_support_moving_.py │ │ ├── 7bed5ff86925_add_component_to_promotions.py │ │ ├── ade85b2396bc_add_extended_hash_and_dt_extended_columns_to_.py │ │ ├── 638f980c9169_add_tables_required_by_dlrn_api.py │ │ ├── cab7697f6564_add_user_column_to_civote_and_promotion.py │ │ ├── 7fbd3a18502f_extra_tables_for_votes_on_aggregates.py │ │ ├── 3c62b0d3ec34_initial_creation.py │ │ ├── 2a0313a8a7d6_change_user_usernames_column_length.py │ │ └── f38ba3389b85_set_string_length.py │ └── env.py ├── templates │ ├── notification_email.j2 │ ├── status_report_csv.j2 │ ├── stylesheets │ │ └── styles.css │ ├── queue.j2 │ ├── report.j2 │ └── status_report.j2 ├── stylesheets │ └── styles.css ├── version.py ├── __init__.py └── notifications.py ├── babel.cfg ├── scripts ├── centos9.cfg ├── centos10.cfg ├── centos10-stream.cfg ├── centos9-stream.cfg ├── api.py ├── fedora.cfg ├── get_rdo_review.py ├── db_migrate.py ├── bisect.sh ├── submit_review.sh ├── centos-stream-9.cfg ├── recreate-promotion-symlinks.py ├── redhat.cfg ├── centos-stream-10.cfg └── build_srpm.sh ├── doc ├── requirements.txt └── source │ ├── contributing.rst │ ├── _images │ └── DLRN.png │ ├── intro.rst │ ├── index.rst │ └── troubleshooting.rst ├── playbooks ├── tox_run.yaml ├── README.md ├── python-devel-install.yaml ├── openldap-devel-install.yaml ├── vars │ ├── family-redhat.yml │ ├── family-redhat-9.yml │ └── family-redhat-8.yml ├── krb5-devel-install.yaml ├── tripleo-ci-oooq-getlogs.yaml ├── retrieve-logs.yaml ├── dlrndocbuild.yaml ├── dlrn-api-functional-getlogs.yaml ├── rpmbuild.yaml └── prepare.yaml ├── .gitreview ├── .mailmap ├── MANIFEST.in ├── .coveragerc ├── HACKING.rst ├── bindep.txt ├── test-requirements.txt ├── .testr.conf ├── requirements.txt ├── contrib ├── cccp.yaml ├── import.py ├── Dockerfile ├── run.sh └── README.rst ├── .gitignore ├── setup.py ├── setup.cfg ├── alembic.ini ├── tox.ini ├── projects.ini ├── CONTRIBUTING.rst ├── README.rst └── .zuul.yaml /dlrn/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dlrn/api/drivers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dlrn/api/inputs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dlrn/drivers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /babel.cfg: -------------------------------------------------------------------------------- 1 | [python: **.py] 2 | -------------------------------------------------------------------------------- /dlrn/api/responses/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scripts/centos9.cfg: -------------------------------------------------------------------------------- 1 | centos-stream-9.cfg -------------------------------------------------------------------------------- /scripts/centos10.cfg: -------------------------------------------------------------------------------- 1 | centos-stream-10.cfg -------------------------------------------------------------------------------- /scripts/centos10-stream.cfg: -------------------------------------------------------------------------------- 1 | centos-stream-10.cfg -------------------------------------------------------------------------------- /scripts/centos9-stream.cfg: -------------------------------------------------------------------------------- 1 | centos-stream-9.cfg -------------------------------------------------------------------------------- /dlrn/migrations/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. -------------------------------------------------------------------------------- /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx>=1.1.2 2 | sphinx-rtd-theme 3 | -------------------------------------------------------------------------------- /doc/source/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../CONTRIBUTING.rst -------------------------------------------------------------------------------- /playbooks/tox_run.yaml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | roles: 3 | - revoke-sudo 4 | - tox -------------------------------------------------------------------------------- /playbooks/README.md: -------------------------------------------------------------------------------- 1 | This directory contains Ansible playbooks used by the DLRN CI. 2 | -------------------------------------------------------------------------------- /.gitreview: -------------------------------------------------------------------------------- 1 | [gerrit] 2 | host=softwarefactory-project.io 3 | port=29418 4 | project=DLRN 5 | -------------------------------------------------------------------------------- /scripts/api.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from dlrn.api import app 3 | app.run(debug=True) 4 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | # Format is: 2 | # 3 | # -------------------------------------------------------------------------------- /doc/source/_images/DLRN.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/softwarefactory-project/DLRN/HEAD/doc/source/_images/DLRN.png -------------------------------------------------------------------------------- /dlrn/api/config.py: -------------------------------------------------------------------------------- 1 | DB_PATH = 'sqlite:///commits.sqlite' 2 | REPO_PATH = 'data/repos' 3 | CONFIG_FILE = 'projects.ini' 4 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS 2 | include ChangeLog 3 | exclude .gitignore 4 | exclude .gitreview 5 | 6 | global-exclude *.pyc -------------------------------------------------------------------------------- /doc/source/intro.rst: -------------------------------------------------------------------------------- 1 | Introduction 2 | ============ 3 | 4 | DLRN builds and maintains yum repositories following Openstack's upstream repositories. 5 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = dlrn 4 | omit = 5 | dlrn/tests/* 6 | 7 | [paths] 8 | source = dlrn 9 | 10 | [report] 11 | ignore_errors = True 12 | -------------------------------------------------------------------------------- /HACKING.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | delorean Style Commandments 3 | =========================== 4 | 5 | Read the OpenStack Style Commandments http://docs.openstack.org/developer/hacking/ 6 | -------------------------------------------------------------------------------- /playbooks/python-devel-install.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | tasks: 4 | - name: Install python3-devel 5 | package: 6 | name: 7 | - python3-devel 8 | state: present 9 | become: true -------------------------------------------------------------------------------- /playbooks/openldap-devel-install.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: centos-7 3 | tasks: 4 | - name: Install openldap-devel 5 | package: 6 | name: 7 | - openldap-devel 8 | state: present 9 | become: true -------------------------------------------------------------------------------- /bindep.txt: -------------------------------------------------------------------------------- 1 | # This is a cross-platform list tracking distribution packages needed by tests; 2 | # see http://docs.openstack.org/infra/bindep/ for additional information. 3 | 4 | createrepo 5 | krb5-devel 6 | mock 7 | redhat-rpm-config 8 | rpmdevtools 9 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | setuptools>=17.1 2 | hacking>=0.5.6 3 | coverage>=3.6 4 | fixtures>=0.3.14 5 | mock 6 | python-subunit 7 | requests-mock 8 | testrepository>=0.0.17 9 | testscenarios>=0.4,<0.5 10 | testtools>=0.9.32 11 | gssapi 12 | ipalib 13 | bandit 14 | pip-audit 15 | -------------------------------------------------------------------------------- /playbooks/vars/family-redhat.yml: -------------------------------------------------------------------------------- 1 | system_packages: 2 | - createrepo 3 | - gcc 4 | - git 5 | - libffi-devel 6 | - mock 7 | - openssl-devel 8 | - policycoreutils 9 | - python-setuptools 10 | - python2-devel 11 | - redhat-rpm-config 12 | - rpmdevtools 13 | - yum-utils 14 | -------------------------------------------------------------------------------- /.testr.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ 3 | OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ 4 | OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ 5 | ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION 6 | test_id_option=--load-list $IDFILE 7 | test_list_option=--list -------------------------------------------------------------------------------- /playbooks/vars/family-redhat-9.yml: -------------------------------------------------------------------------------- 1 | system_packages: 2 | - createrepo 3 | - distribution-gpg-keys # epel 4 | - gcc 5 | - git 6 | - libffi-devel 7 | - mock 8 | - openssl-devel 9 | - policycoreutils 10 | - python3-devel 11 | - python3-virtualenv 12 | - redhat-rpm-config 13 | - rpmdevtools 14 | - systemd-container 15 | - tox 16 | - yum 17 | - yum-utils 18 | -------------------------------------------------------------------------------- /playbooks/krb5-devel-install.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | tasks: 4 | - name: Install krb5-devel 5 | package: 6 | name: 7 | - krb5-devel 8 | state: present 9 | become: true 10 | 11 | - name: Install krb5-workstation 12 | package: 13 | name: 14 | - krb5-workstation 15 | state: present 16 | become: true 17 | -------------------------------------------------------------------------------- /playbooks/vars/family-redhat-8.yml: -------------------------------------------------------------------------------- 1 | system_packages: 2 | - createrepo 3 | - distribution-gpg-keys # epel 4 | - gcc 5 | - git 6 | - libffi-devel 7 | - mock 8 | - openssl-devel 9 | - policycoreutils 10 | - python2-devel 11 | - python3-devel 12 | - python3-virtualenv 13 | - redhat-rpm-config 14 | - rpmdevtools 15 | - systemd-container 16 | - yum 17 | - yum-utils -------------------------------------------------------------------------------- /dlrn/templates/notification_email.j2: -------------------------------------------------------------------------------- 1 | A build of the package {{ details.name }} has failed against the current master[1] of 2 | the upstream project, please see log[2] and update the packaging[3]. 3 | 4 | You are receiving this email because you are listed as one of the 5 | maintainers for the {{ details.name }}. 6 | 7 | [1] - {{ details.upstream }} 8 | [2] - {{ details.logurl }} 9 | [3] - {{ details['master-distgit'] }} 10 | -------------------------------------------------------------------------------- /dlrn/templates/status_report_csv.j2: -------------------------------------------------------------------------------- 1 | Project,Source Sha,Dist Sha,Extended Sha,Component,Status,Timestamp,Packages 2 | {% for pkg in pkgs -%} 3 | {{ pkg["name"] }},{{ pkg["last_build"].commit_hash }},{{ pkg["last_build"].distro_hash }},{{ pkg["last_build"].extended_hash }},{{ pkg["last_build"].component }},{{ pkg["last_build"].status }},{{ pkg["last_build"].dt_build }},{{ pkg["last_build"].artifacts | replace(",","|") }} 4 | {% endfor %} 5 | -------------------------------------------------------------------------------- /playbooks/tripleo-ci-oooq-getlogs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | tasks: 4 | - name: Ensure log directory is created 5 | file: 6 | path: '{{ zuul.project.src_dir }}/logs' 7 | state: directory 8 | 9 | - name: Fetch oooq logs 10 | shell: 11 | cmd: | 12 | cp -p /tmp/gating_repo.tar.gz logs 13 | cp -pr data/repos logs 14 | chdir: '{{ zuul.project.src_dir }}' 15 | -------------------------------------------------------------------------------- /playbooks/retrieve-logs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | tasks: 4 | - name: Upload logs 5 | synchronize: 6 | src: '{{ zuul.project.src_dir }}/logs' 7 | dest: '{{ zuul.executor.log_root }}' 8 | mode: pull 9 | copy_links: true 10 | verify_host: true 11 | rsync_opts: 12 | - --include=/logs/** 13 | - --include=*/ 14 | - --exclude=* 15 | - --prune-empty-dirs 16 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pbr>=0.5.6 2 | Jinja2 3 | sh<2.0.0 4 | six 5 | SQLAlchemy<2.0.0 6 | PyMySQL!=1.0.0 7 | PyYAML 8 | rdopkg>=0.45.0 9 | distroinfo 10 | alembic>=0.7.0,!=1.4.2 11 | renderspec 12 | pymod2pkg>=0.5.5 13 | requests 14 | Flask 15 | Flask-HTTPAuth>=4.5.0 16 | passlib>=1.7.0 17 | graphene<3.0 18 | graphene-sqlalchemy>=2.0 19 | flask-sqlalchemy 20 | flask-graphql 21 | flask-container-scaffold>=0.3.1 22 | prometheus-client 23 | cryptography<=37.0.2 24 | stevedore 25 | pydantic<2.0.0 -------------------------------------------------------------------------------- /playbooks/dlrndocbuild.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | vars: 4 | working_dir: '/workspace/{{ zuul.project.src_dir }}' 5 | tasks: 6 | - name: Install test-requirements for building Spinx doc 7 | pip: 8 | requirements: "{{ working_dir }}/doc/requirements.txt" 9 | 10 | - name: "Run sphinx-build" 11 | command: 12 | "sphinx-build -W -b html -d build/doctrees doc/source {{ ansible_user_dir }}/zuul-output/logs/docs-html/" 13 | args: 14 | chdir: "{{ working_dir }}" 15 | -------------------------------------------------------------------------------- /playbooks/dlrn-api-functional-getlogs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | vars: 4 | working_dir: '{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/..' 5 | 6 | tasks: 7 | - name: Ensure log directory is created 8 | file: 9 | path: '{{ zuul.project.src_dir }}/logs' 10 | state: directory 11 | 12 | - name: Copy logs 13 | shell: 14 | cmd: | 15 | rsync -avL {{ working_dir }}/DLRN/data/repos {{ zuul.project.src_dir }}/logs/DLRN 16 | cp -p {{ working_dir }}/DLRN/commits.sqlite {{ zuul.project.src_dir }}/logs 17 | -------------------------------------------------------------------------------- /dlrn/api/static/styles.css: -------------------------------------------------------------------------------- 1 | @import url(https://unpkg.com/@patternfly/patternfly@4.90.5/patternfly.css); 2 | @import url(https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css); 3 | 4 | #dlrn tr.failure td { 5 | color: #000000; 6 | background-color: #f5c6cb; 7 | } 8 | 9 | #dlrn tr.failure:hover td { 10 | background-color: #f1b0b7; 11 | } 12 | 13 | #dlrn th { 14 | color: #ffffff; 15 | background-color: #550000; 16 | } 17 | 18 | h1 { 19 | color: #7F0A0C; 20 | margin-top: 2.25rem; 21 | margin-bottom: 1em; 22 | font-size: 2em; 23 | } 24 | -------------------------------------------------------------------------------- /dlrn/stylesheets/styles.css: -------------------------------------------------------------------------------- 1 | @import url(https://unpkg.com/@patternfly/patternfly/patternfly.css); 2 | @import url(https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css); 3 | 4 | #dlrn tr.failure td { 5 | color: #000000; 6 | background-color: #f5c6cb; 7 | } 8 | 9 | #dlrn tr.failure:hover td { 10 | background-color: #f1b0b7; 11 | } 12 | 13 | #dlrn th { 14 | color: #ffffff; 15 | background-color: #550000; 16 | } 17 | 18 | h1 { 19 | color: #7F0A0C; 20 | margin-top: 2.25rem; 21 | margin-bottom: 1em; 22 | font-size: 2em; 23 | } 24 | 25 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | .. image:: _images/DLRN.png 2 | :width: 794px 3 | :align: center 4 | :height: 187px 5 | :alt: DLRN Logo 6 | 7 | Welcome to DLRN's documentation! 8 | ==================================== 9 | 10 | Contents: 11 | 12 | .. toctree:: 13 | :maxdepth: 2 14 | 15 | intro 16 | installation 17 | repositories 18 | usage 19 | troubleshooting 20 | api 21 | graphql 22 | contributing 23 | internals 24 | 25 | Indices and tables 26 | ================== 27 | 28 | * :ref:`genindex` 29 | * :ref:`modindex` 30 | * :ref:`search` 31 | -------------------------------------------------------------------------------- /contrib/cccp.yaml: -------------------------------------------------------------------------------- 1 | # This file is required for the purpose of building a Docker container on CCCP. 2 | # More information (and a quickstart) found here: https://github.com/CentOS/container-index/pull/265 3 | 4 | # This will be part of the name of the container. It should match the job-id in index entry 5 | job-id: dlrnapi 6 | 7 | # This flag tells the container pipeline to skip user defined tests on their container 8 | test-skip: True 9 | 10 | # This is path of the script that initiates the user defined tests. It must be able to return an exit code. 11 | # test-script: test-script.sh 12 | -------------------------------------------------------------------------------- /dlrn/templates/stylesheets/styles.css: -------------------------------------------------------------------------------- 1 | @import url(https://unpkg.com/@patternfly/patternfly/patternfly.css); 2 | @import url(https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css); 3 | 4 | #dlrn tr.failure td { 5 | color: #000000; 6 | background-color: #f5c6cb; 7 | } 8 | 9 | #dlrn tr.failure:hover td { 10 | background-color: #f1b0b7; 11 | } 12 | 13 | #dlrn th { 14 | color: #ffffff; 15 | background-color: #550000; 16 | } 17 | 18 | h1 { 19 | color: #7F0A0C; 20 | margin-top: 2.25rem; 21 | margin-bottom: 1em; 22 | font-size: 2em; 23 | } 24 | 25 | -------------------------------------------------------------------------------- /dlrn/version.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import pbr.version 14 | 15 | 16 | version_info = pbr.version.VersionInfo('dlrn') 17 | version_string = version_info.version_string 18 | -------------------------------------------------------------------------------- /dlrn/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import pbr.version 16 | 17 | 18 | __version__ = pbr.version.VersionInfo( 19 | 'dlrn').version_string() 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | .tox 27 | nosetests.xml 28 | .testrepository 29 | 30 | # Translations 31 | *.mo 32 | 33 | # Mr Developer 34 | .mr.developer.cfg 35 | .project 36 | .pydevproject 37 | 38 | # Complexity 39 | output/*.html 40 | output/*/index.html 41 | 42 | # Sphinx 43 | doc/build 44 | 45 | # pbr generates these 46 | AUTHORS 47 | ChangeLog 48 | 49 | # Editors 50 | *~ 51 | .*.swp 52 | 53 | # Generated files 54 | /data 55 | commits.sqlite 56 | commits.sqlite-journal 57 | scripts/dlrn.cfg 58 | scripts/dlrn.cfg.new 59 | -------------------------------------------------------------------------------- /dlrn/api/responses/health.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | from pydantic import BaseModel 14 | 15 | 16 | class HealthResponse(BaseModel): 17 | """Response class that builds responses for the health endpoint 18 | 19 | :param str result: API health status string 20 | """ 21 | result: str 22 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | # implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT 18 | import setuptools 19 | 20 | setuptools.setup( 21 | setup_requires=['pbr'], 22 | pbr=True) 23 | -------------------------------------------------------------------------------- /contrib/import.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | from dlrn.db import getSession 14 | from dlrn.utils import loadYAML 15 | import os 16 | 17 | dbpath = os.getenv('DLRNAPI_DBPATH') 18 | if not dbpath: 19 | dbpath = 'sqlite:////data/commits.sqlite' 20 | session = getSession(dbpath) 21 | loadYAML(session, '/DLRN/dlrn/tests/samples/commits_2.yaml') 22 | session.close() 23 | -------------------------------------------------------------------------------- /dlrn/api/inputs/recheck_package.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | from pydantic import BaseModel 14 | from pydantic import StrictStr 15 | 16 | 17 | class RecheckPackageInput(BaseModel): 18 | """Input class that validates request's arguments for recheck_package 19 | 20 | :param str package_name: Package name to be rechecked 21 | """ 22 | package_name: StrictStr 23 | -------------------------------------------------------------------------------- /dlrn/drivers/buildrpm.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | 14 | class BuildRPMDriver(object): 15 | """Abstract base class for Build RPM drivers 16 | 17 | """ 18 | def __init__(self, *args, **kwargs): 19 | self.config_options = kwargs.get('cfg_options') 20 | 21 | def build_package(self): 22 | """Perform package build 23 | 24 | """ 25 | return False 26 | -------------------------------------------------------------------------------- /contrib/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:8 2 | LABEL maintainer="jpena@redhat.com" 3 | 4 | RUN \ 5 | yum -y install epel-release && \ 6 | yum -y update && \ 7 | yum -y install git createrepo mock redhat-rpm-config rpmdevtools yum-utils python3-pip && \ 8 | yum clean all -y && \ 9 | rm -rf /var/cache/yum && \ 10 | git clone https://github.com/softwarefactory-project/DLRN && \ 11 | pushd DLRN && \ 12 | sed -i 's#^REPO_PATH.*#REPO_PATH = "/data/repos"#' dlrn/api/config.py && \ 13 | sed -i 's/^app.run.*/app.run(debug=True, host="0.0.0.0")/' scripts/api.py && \ 14 | pip3 install --upgrade pip && \ 15 | pip3 install -r requirements.txt && \ 16 | pip3 install . && \ 17 | popd && \ 18 | mkdir /data && \ 19 | chgrp -R 0 /data && \ 20 | chmod -R g=u /data && \ 21 | chmod -R g=u /DLRN 22 | 23 | COPY import.py / 24 | COPY run.sh / 25 | 26 | RUN chmod 755 /run.sh 27 | 28 | VOLUME ["/data"] 29 | EXPOSE 5000 30 | 31 | CMD ["/run.sh"] 32 | -------------------------------------------------------------------------------- /dlrn/api/responses/metrics.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | from pydantic import BaseModel 14 | 15 | 16 | class MetricsResponse(BaseModel): 17 | """Response class that builds responses for the metrics endpoint 18 | 19 | :param int succeeded: Number of successful builds 20 | :param int failed: Number of failed builds 21 | :param int total: Total number of builds 22 | """ 23 | succeeded: int 24 | failed: int 25 | total: int 26 | -------------------------------------------------------------------------------- /dlrn/api/inputs/agg_status.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | from pydantic import BaseModel 13 | from pydantic import StrictStr 14 | from typing import Optional 15 | 16 | 17 | class AggStatusInput(BaseModel): 18 | """Input class that validates request's arguments for agg_status endpoint 19 | 20 | :param str aggregate_hash: A reference to the aggregate hash 21 | :param bool success(optional): Only report successful/unsuccessful votes 22 | """ 23 | aggregate_hash: StrictStr 24 | success: Optional[bool] 25 | -------------------------------------------------------------------------------- /dlrn/api/inputs/remote_import.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | from pydantic import AnyHttpUrl 14 | from pydantic import BaseModel 15 | 16 | 17 | class RemoteImportInput(BaseModel): 18 | """Input class that validates request's arguments for remote_import 19 | 20 | :param str repo_url: Other DLRN instance repo with hash 21 | to import from by using HTTP or HTTPS 22 | protocols. For example https://$instance_FQDN/\ 23 | $builder/$repo/$component/cd/88/cd88... 24 | """ 25 | repo_url: AnyHttpUrl 26 | -------------------------------------------------------------------------------- /contrib/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 3 | # not use this file except in compliance with the License. You may obtain 4 | # a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 10 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 11 | # License for the specific language governing permissions and limitations 12 | # under the License. 13 | 14 | if [ -n "$DLRNAPI_DBPATH" ]; 15 | then 16 | export CONFIG_FILE='/data/custom_config.py' 17 | echo "DB_PATH = \"$DLRNAPI_DBPATH\"" > /data/custom_config.py 18 | fi 19 | 20 | if [ -n "$DLRNAPI_USE_SAMPLE_DATA" ]; 21 | then 22 | python3 /import.py 23 | mkdir -p /data/repos/17/23/17234e9ab9dfab4cf5600f67f1d24db5064f1025_024e24f0 24 | mkdir -p /data/repos/1c/67/1c67b1ab8c6fe273d4e175a14f0df5d3cbbd0edc_8170b868 25 | mkdir -p /data/repos/1c/67/1c67b1ab8c6fe273d4e175a14f0df5d3cbbd0e77_8170b868 26 | fi 27 | 28 | python3 /DLRN/scripts/api.py 29 | -------------------------------------------------------------------------------- /playbooks/rpmbuild.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | tasks: 4 | - name: Build package 5 | shell: 6 | cmd: | 7 | export TAG="{{ tag | default('') }}" 8 | export CENTOS_VERS="{{ centos_version | default('centos9') }}" 9 | export REPO_SERVER="https://trunk.rdoproject.org" 10 | export PYTHON_VERSION="{{ python_version | default('py39') }}" 11 | export REQUESTS_CA_BUNDLE=/etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt 12 | if [ ${CENTOS_VERS} = "centos9" ]; then 13 | export TARGET="centos9" 14 | export USE_COMPONENTS="True" 15 | fi 16 | if [ -z "$TAG" ];then 17 | ARGS="$TARGET ${REPO_SERVER}/${CENTOS_VERS}/" 18 | export ZUUL_BRANCH="rpm-master" 19 | else 20 | ARGS="$TARGET ${REPO_SERVER}/${CENTOS_VERS}/ $TAG" 21 | export ZUUL_BRANCH="$TAG-rdo" 22 | fi 23 | timeout --signal=SIGKILL 3600 ./scripts/run_tests.sh http://review.rdoproject.org/r/rdoinfo.git $ARGS 24 | args: 25 | chdir: "{{ ansible_user_dir }}/{{ zuul.projects['softwarefactory-project.io/DLRN'].src_dir }}" 26 | -------------------------------------------------------------------------------- /doc/source/troubleshooting.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | Troubleshooting 3 | =============== 4 | 5 | If you interrupt dlrn during mock build you might get an error 6 | 7 | .. code-block:: bash 8 | 9 | OSError: [Errno 16] Device or resource busy: '/var/lib/mock/dlrn-centos-x86_64/root/var/cache/yum' 10 | 11 | Solution is to clear left-over bind mount as root: 12 | 13 | .. code-block:: shell-session 14 | 15 | # umount /var/lib/mock/dlrn-centos-x86_64/root/var/cache/yum 16 | 17 | Other requirements 18 | ================== 19 | 20 | If the git clone operation fails for a package, DLRN will try to remove the 21 | source directory using sudo. Please make sure the user running DLRN can run 22 | ``rm -rf /path/to/dlrn/data/*`` without being asked for a password, otherwise 23 | DLRN will fail to process new commits. 24 | 25 | API issues 26 | ========== 27 | 28 | If you want to quickly check the API status, you can use the /api/health 29 | endpoint. It will allow you to test API connectivity, database access and 30 | authentication: 31 | 32 | .. code-block:: bash 33 | 34 | # curl http://localhost:5000/api/health 35 | # curl -d test=test --user user:password http://localhost:5000/api/health 36 | -------------------------------------------------------------------------------- /scripts/fedora.cfg: -------------------------------------------------------------------------------- 1 | config_opts['root'] = 'dlrn-fedora-rawhide-x86_64' 2 | config_opts['target_arch'] = 'x86_64' 3 | config_opts['legal_host_arches'] = ('x86_64',) 4 | config_opts['chroot_setup_cmd'] = 'install basesystem rpm-build python2-devel gcc make python-sqlalchemy python-webob ghostscript graphviz python-sphinx python-eventlet python-six python-pbr python3-pbr git-core yum-plugin-priorities rubygems coreutils glibc-langpack-en python2-setuptools_scm' 5 | config_opts['dist'] = 'rawhide' # only useful for --resultdir variable subst 6 | config_opts['extra_chroot_dirs'] = [ '/run/lock', ] 7 | config_opts['releasever'] = '28' 8 | config_opts['package_manager'] = 'dnf' 9 | config_opts['priorities.conf'] = """ 10 | [main] 11 | enabled = 1 12 | check_obsoletes = 1 13 | """ 14 | 15 | config_opts['yum.conf'] = """ 16 | [main] 17 | keepcache=1 18 | debuglevel=2 19 | reposdir=/dev/null 20 | logfile=/var/log/yum.log 21 | retries=20 22 | obsoletes=1 23 | gpgcheck=0 24 | assumeyes=1 25 | plugins=1 26 | syslog_ident=mock 27 | syslog_device= 28 | 29 | # repos 30 | 31 | [fedora] 32 | name=fedora 33 | metalink=https://mirrors.fedoraproject.org/metalink?repo=rawhide&arch=$basearch 34 | failovermethod=priority 35 | """ 36 | -------------------------------------------------------------------------------- /dlrn/migrations/script.py.mako: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """${message} 14 | 15 | Revision ID: ${up_revision} 16 | Revises: ${down_revision | comma,n} 17 | Create Date: ${create_date} 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | ${imports if imports else ""} 24 | 25 | 26 | # revision identifiers, used by Alembic. 27 | revision = ${repr(up_revision)} 28 | down_revision = ${repr(down_revision)} 29 | branch_labels = ${repr(branch_labels)} 30 | depends_on = ${repr(depends_on)} 31 | 32 | 33 | def upgrade(): 34 | ${upgrades if upgrades else "pass"} 35 | 36 | 37 | def downgrade(): 38 | ${downgrades if downgrades else "pass"} 39 | -------------------------------------------------------------------------------- /dlrn/tests/samples/commits_remote.yaml: -------------------------------------------------------------------------------- 1 | commits: 2 | - commit_hash: 3a9326f251b9a4162eb0dfa9f1c924ef47c2c55a 3 | distro_hash: 024e24f0cf4366c2290c22f24e42de714d1addd1 4 | dt_build: '1444139517' 5 | dt_commit: '1444139038' 6 | dt_distro: '1431949433' 7 | flags: '0' 8 | id: '7835' 9 | notes: OK 10 | project_name: python-pysaml2 11 | repo_dir: /home/centos-master/data/python-pysaml2 12 | artifacts: repos/3a/93/3a9326f251b9a4162eb0dfa9f1c924ef47c2c55a_024e24f0/python-pysaml2-3.0-1a.el7.centos.noarch.rpm,repos/3a/93/3a9326f251b9a4162eb0dfa9f1c924ef47c2c55a_024e24f0/python-pysaml2-3.0-1a.el7.centos.src.rpm 13 | status: SUCCESS 14 | - commit_hash: 3a9326f251b9a4162eb0dfa9f1c924ef47c2c55a 15 | distro_hash: abce24f0cf4366c2290c22f24e42de714d1addd1 16 | dt_build: '1444139517' 17 | dt_commit: '1444139038' 18 | dt_distro: '1441949433' 19 | flags: '0' 20 | id: '7836' 21 | notes: OK 22 | project_name: python-pysaml2 23 | repo_dir: /home/centos-master/data/python-pysaml2 24 | artifacts: repos/3a/93/3a9326f251b9a4162eb0dfa9f1c924ef47c2c55a_abce24f0/python-pysaml2-3.0-1a.el7.centos.noarch.rpm,repos/3a/93/3a9326f251b9a4162eb0dfa9f1c924ef47c2c55a_abce24f0/python-pysaml2-3.0-1a.el7.centos.src.rpm 25 | status: SUCCESS 26 | -------------------------------------------------------------------------------- /dlrn/api/inputs/repo_status.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | from typing import Optional 13 | 14 | from pydantic import BaseModel 15 | from pydantic import StrictStr 16 | 17 | 18 | class RepoStatusInput(BaseModel): 19 | """Input class that validates request's arguments for repo_status endpoint 20 | 21 | :param str commit_hash: A reference to the commit 22 | :param str distro_hash: A reference to the distro 23 | :param str extended_hash(optional): A reference to the extended commit 24 | :param bool success(optional): Only report successful/unsuccessful votes 25 | """ 26 | commit_hash: StrictStr 27 | distro_hash: StrictStr 28 | extended_hash: Optional[StrictStr] 29 | success: Optional[bool] 30 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/6a3d982b967b_add_versions_url_columnt_to_commits.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Add versions_url columnt to commits 14 | 15 | Revision ID: 6a3d982b967b 16 | Revises: 837138eb7daa 17 | Create Date: 2023-01-18 14:05:54.175709 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | 24 | # revision identifiers, used by Alembic. 25 | revision = '6a3d982b967b' 26 | down_revision = '837138eb7daa' 27 | branch_labels = None 28 | depends_on = None 29 | 30 | 31 | def upgrade(): 32 | op.add_column('commits', sa.Column('versions_csv', sa.String(256))) 33 | 34 | 35 | def downgrade(): 36 | with op.batch_alter_table('commits') as batch_op: 37 | batch_op.drop_column('versions_csv') 38 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = DLRN 3 | summary = Build packages 4 | description-file = 5 | README.rst 6 | author = OpenStack 7 | author-email = openstack-dev@lists.openstack.org 8 | home-page = http://www.openstack.org/ 9 | classifier = 10 | Environment :: OpenStack 11 | Intended Audience :: Information Technology 12 | Intended Audience :: System Administrators 13 | License :: OSI Approved :: Apache Software License 14 | Operating System :: POSIX :: Linux 15 | Programming Language :: Python 16 | Programming Language :: Python :: 3 17 | Programming Language :: Python :: 3.6 18 | Programming Language :: Python :: 3.9 19 | 20 | [files] 21 | packages = 22 | dlrn 23 | 24 | data_files = 25 | share/dlrn/scripts = scripts/* 26 | 27 | [options.extras_require] 28 | kerberos = 29 | gssapi 30 | ipalib 31 | 32 | [upload_sphinx] 33 | upload-dir = doc/build/html 34 | 35 | [entry_points] 36 | console_scripts = 37 | dlrn = dlrn.shell:main 38 | delorean = dlrn.shell:deprecation 39 | dlrn-purge = dlrn.purge:purge 40 | dlrn-remote = dlrn.remote:remote 41 | dlrn-user = dlrn.user:user_manager 42 | dlrn.api.drivers = 43 | DBAuthentication = dlrn.api.drivers.dbauthentication:DBAuthentication 44 | KrbAuthentication = dlrn.api.drivers.krbauthentication:KrbAuthentication 45 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/b6f658f481f8_add_commit_type.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Add type column 14 | 15 | Revision ID: b6f658f481f8 16 | Revises: 2d503b5034b7 17 | Create Date: 2019-04-26 07:29:09.071710 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | from sqlalchemy.sql import table, column 24 | 25 | 26 | # revision identifiers, used by Alembic. 27 | revision = 'b6f658f481f8' 28 | down_revision = '2d503b5034b7' 29 | branch_labels = None 30 | depends_on = None 31 | 32 | 33 | def upgrade(): 34 | op.add_column("commits", sa.Column("type", sa.String(18))) 35 | commits = table('commits', column('type', sa.String(18))) 36 | op.execute(commits.update().values(type="rpm")) 37 | 38 | 39 | def downgrade(): 40 | with op.batch_alter_table('commits') as batch_op: 41 | batch_op.drop_column('type') 42 | -------------------------------------------------------------------------------- /dlrn/api/templates/votes_agg.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | CI Votes for Aggregates 5 | 6 | 7 | 8 | 9 | 10 | 11 |

12 | 13 | {{ target | capitalize }} 14 |

15 | 16 |
17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 |
CI NameAggregate hashResults URLVoteIn Progress?Timestampnotes
30 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /dlrn/tests/samples/commits_3.yaml: -------------------------------------------------------------------------------- 1 | commits: 2 | - commit_hash: 1c67b1ab8c6fe273d4e175a14f0df5d3cbbd0edd 3 | distro_hash: 885e80778edb6cbb8ee4d8909623be8062369a04 4 | extended_hash: 5 | commit_branch: 'master' 6 | dt_build: '1441675005' 7 | dt_commit: '1441245153' 8 | dt_distro: '1441674764' 9 | flags: '0' 10 | id: '5662' 11 | notes: "\n\n RAN: '/bin/docker run -t --volume=/home/centos-master/data:/data --volume=/home/centos-master/de" 12 | project_name: python-stevedore 13 | repo_dir: /home/centos-master/data/python-stevedore 14 | artifacts: None 15 | status: FAILED 16 | - commit_hash: 17234e9ab9dfab4cf5600f67f1d24db5064f1025 17 | distro_hash: 024e24f0cf4366c2290c22f24e42de714d1addd1 18 | component: tripleo 19 | dt_build: '1441635089' 20 | dt_commit: '1441634092' 21 | dt_distro: '1431949433' 22 | flags: '0' 23 | id: '5627' 24 | notes: OK 25 | project_name: python-pysaml2 26 | repo_dir: /home/centos-master/data/python-pysaml2 27 | artifacts: repos/17/23/17234e9ab9dfab4cf5600f67f1d24db5064f1025_024e24f0/python-pysaml2-3.0-1a.el7.centos.noarch.rpm,repos/17/23/17234e9ab9dfab4cf5600f67f1d24db5064f1025_024e24f0/python-pysaml2-3.0-1a.el7.centos.src.rpm 28 | status: SUCCESS 29 | users: 30 | - username: 'foo' 31 | # passlib.hash.sha512_crypt.encrypt("bar") 32 | password: '$6$rounds=656000$T678cDhCoMtxLlnq$yXrpe.kJgiP8Y1JZpAYQ0.yCE12Qr6zv6nZY1VY7OP6bxhoZezlzWSitvoqkx26Z30pUBKM4kX/JZJPdMIv1a.' 33 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/2d503b5034b7_rename_artifacts.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Rename rpms to artifacts 14 | 15 | Revision ID: 2d503b5034b7 16 | Revises: 2a0313a8a7d6 17 | Create Date: 2019-04-26 01:06:50.462042 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | 24 | 25 | # revision identifiers, used by Alembic. 26 | revision = '2d503b5034b7' 27 | down_revision = '2a0313a8a7d6' 28 | branch_labels = None 29 | depends_on = None 30 | 31 | 32 | def upgrade(): 33 | with op.batch_alter_table("commits") as batch_op: 34 | batch_op.alter_column('rpms', existing_type=sa.Text(), 35 | new_column_name='artifacts') 36 | 37 | 38 | def downgrade(): 39 | with op.batch_alter_table("commits") as batch_op: 40 | batch_op.alter_column('artifacts', existing_type=sa.Text(), 41 | new_column_name='rpms') 42 | -------------------------------------------------------------------------------- /dlrn/api/templates/votes.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | CI Votes 5 | 6 | 7 | 8 | 9 | 10 | 11 |

12 | 13 | {{ target | capitalize }} 14 |

15 | 16 |
17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 |
CI NameRepo hash (commit_distro)ComponentResults URLVoteIn Progress?Timestampnotes
31 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/837138eb7daa_extend_extended_hash_to_128_chars.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Extend extended_hash to 128 chars 14 | 15 | Revision ID: 837138eb7daa 16 | Revises: 7fbd3a18502f 17 | Create Date: 2020-11-10 14:51:30.395443 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | 24 | 25 | # revision identifiers, used by Alembic. 26 | revision = '837138eb7daa' 27 | down_revision = '7fbd3a18502f' 28 | branch_labels = None 29 | depends_on = None 30 | 31 | 32 | def upgrade(): 33 | with op.batch_alter_table("commits") as batch_op: 34 | batch_op.alter_column('extended_hash', existing_type=sa.String(64), 35 | type_=sa.String(128)) 36 | 37 | 38 | def downgrade(): 39 | with op.batch_alter_table("commits") as batch_op: 40 | batch_op.alter_column('extended_hash', existing_type=sa.String(128), 41 | type_=sa.String(64)) 42 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/00a31f1f39c0_add_component_to_civote.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Add component to CIVote 14 | 15 | Revision ID: 00a31f1f39c0 16 | Revises: 7bed5ff86925 17 | Create Date: 2019-10-07 10:47:29.032909 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | from sqlalchemy.sql import table, column 24 | 25 | 26 | # revision identifiers, used by Alembic. 27 | revision = '00a31f1f39c0' 28 | down_revision = '7bed5ff86925' 29 | branch_labels = None 30 | depends_on = None 31 | 32 | 33 | def upgrade(): 34 | op.add_column('civotes', sa.Column('component', sa.String(64))) 35 | civotes = table('civotes', 36 | column('id', sa.Integer), 37 | column('component', sa.String)) 38 | # For existing civotes, set component to None 39 | op.execute(civotes.update() 40 | .where(civotes.c.id == civotes.c.id) 41 | .values(component=None)) 42 | 43 | 44 | def downgrade(): 45 | with op.batch_alter_table('civotes') as batch_op: 46 | batch_op.drop_column('component') 47 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/1268c799620f_add_commit_branch_to_db.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Add commit_branch to DB 14 | 15 | Revision ID: 1268c799620f 16 | Revises: 47ebe0522809 17 | Create Date: 2016-08-17 12:08:54.246311 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | from sqlalchemy.sql import table, column 24 | 25 | # revision identifiers, used by Alembic. 26 | revision = '1268c799620f' 27 | down_revision = '47ebe0522809' 28 | branch_labels = None 29 | depends_on = None 30 | 31 | 32 | def upgrade(): 33 | op.add_column('commits', sa.Column('commit_branch', sa.String())) 34 | commits = table('commits', 35 | column('id', sa.Integer), 36 | column('commit_branch', sa.String)) 37 | # We cannot retrieve the previous commit_branch in any way, let's 38 | # set it to master 39 | op.execute(commits.update() 40 | .where(commits.c.id == commits.c.id) 41 | .values(commit_branch="master")) 42 | 43 | 44 | def downgrade(): 45 | op.drop_column('commits', 'commit_branch') 46 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/f84aca0549fd_add_component_to_commits.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Add component to commits 14 | 15 | Revision ID: f84aca0549fd 16 | Revises: b6f658f481f8 17 | Create Date: 2019-09-24 11:46:13.682176 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | from sqlalchemy.sql import table, column 24 | 25 | 26 | # revision identifiers, used by Alembic. 27 | revision = 'f84aca0549fd' 28 | down_revision = 'b6f658f481f8' 29 | branch_labels = None 30 | depends_on = None 31 | 32 | 33 | def upgrade(): 34 | op.add_column('commits', sa.Column('component', sa.String(64))) 35 | commits = table('commits', 36 | column('id', sa.Integer), 37 | column('component', sa.String)) 38 | # For existing commits, set component to None 39 | op.execute(commits.update() 40 | .where(commits.c.id == commits.c.id) 41 | .values(component=None)) 42 | 43 | 44 | def downgrade(): 45 | with op.batch_alter_table('commits') as batch_op: 46 | batch_op.drop_column('component') 47 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/4a5651777e5e_add_promotions_table.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Add promotions table 14 | 15 | Revision ID: 4a5651777e5e 16 | Revises: 638f980c9169 17 | Create Date: 2017-08-21 11:09:46.349973 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | 24 | 25 | # revision identifiers, used by Alembic. 26 | revision = '4a5651777e5e' 27 | down_revision = '638f980c9169' 28 | branch_labels = None 29 | depends_on = None 30 | 31 | 32 | def upgrade(): 33 | op.create_table('promotions', 34 | sa.Column('id', sa.Integer(), nullable=False), 35 | sa.Column('commit_id', sa.Integer(), 36 | sa.ForeignKey('commits.id'), 37 | nullable=False), 38 | sa.Column('promotion_name', sa.String(256), 39 | nullable=False), 40 | sa.Column('timestamp', sa.Integer(), nullable=False), 41 | sa.PrimaryKeyConstraint('id')) 42 | 43 | 44 | def downgrade(): 45 | op.drop_table('promotions') 46 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/47ebe0522809_scheme_change_due_to_scm_support_moving_.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Scheme change due to SCM support moving to a plugin 14 | 15 | Revision ID: 47ebe0522809 16 | Revises: o3c62b0d3ec34 17 | Create Date: 2016-06-08 11:29:16.460075 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | from sqlalchemy.sql import table, column 24 | 25 | 26 | # revision identifiers, used by Alembic. 27 | revision = '47ebe0522809' 28 | down_revision = '3c62b0d3ec34' 29 | branch_labels = None 30 | depends_on = None 31 | 32 | 33 | def upgrade(): 34 | op.add_column('commits', sa.Column('distgit_dir', sa.String())) 35 | commits = table('commits', 36 | column('id', sa.Integer), 37 | column('repo_dir', sa.String), 38 | column('distgit_dir', sa.String)) 39 | op.execute(commits.update() 40 | .where(commits.c.id == commits.c.id) 41 | .values(distgit_dir=(commits.c.repo_dir + "_distro"))) 42 | 43 | 44 | def downgrade(): 45 | op.drop_column('commits', 'distgit_dir') 46 | -------------------------------------------------------------------------------- /dlrn/api/templates/report.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {{ project_name }} Packaging By DLRN 5 | 6 | 7 | 8 | 9 | 10 | 11 |

12 | 13 | {{ reponame | capitalize }} - {{ target | capitalize }} ({{ src }}) 14 |

15 | 16 |
17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 |
Build Date TimeCommit Date TimeProject NameVersionReleaseCommit HashComponentStatusRepositoryBuild Log
33 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /scripts/get_rdo_review.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2016 Red Hat, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. You may obtain 7 | # a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations 15 | # under the License. 16 | 17 | ''' 18 | ''' 19 | 20 | from __future__ import print_function 21 | import json 22 | import requests 23 | import sys 24 | 25 | if len(sys.argv) != 3: 26 | sys.stderr.write('Usage: %s ' 27 | '\n' % sys.argv[0]) 28 | sys.exit(1) 29 | 30 | r = requests.get('http://review.rdoproject.org/r' 31 | "/changes/?q=status:open+project:%s&o=CURRENT_COMMIT&" 32 | "o=CURRENT_REVISION" % sys.argv[1]) 33 | 34 | changes = json.loads('\n'.join(r.text.split('\n')[1:])) 35 | 36 | for change in changes: 37 | for rev in change['revisions'].keys(): 38 | lines = change['revisions'][rev]['commit']['message'].split('\n') 39 | for line in lines: 40 | parts = line.strip().split(': ') 41 | if (len(parts) == 2 and parts[0] == 'Upstream-Id' and 42 | parts[1] == sys.argv[2]): 43 | print(change['change_id']) 44 | sys.exit(0) 45 | sys.exit(1) 46 | 47 | # get_rdo_review.py ends here 48 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/7bed5ff86925_add_component_to_promotions.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Add component to promotions 14 | 15 | Revision ID: 7bed5ff86925 16 | Revises: f84aca0549fd 17 | Create Date: 2019-09-27 14:33:21.479735 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | from sqlalchemy.sql import table, column 24 | 25 | 26 | # revision identifiers, used by Alembic. 27 | revision = '7bed5ff86925' 28 | down_revision = 'f84aca0549fd' 29 | branch_labels = None 30 | depends_on = None 31 | 32 | 33 | def upgrade(): 34 | with op.batch_alter_table('promotions') as batch_op: 35 | batch_op.add_column(sa.Column('component', sa.String(64))) 36 | 37 | promotions = table('promotions', 38 | column('id', sa.Integer), 39 | column('component', sa.String)) 40 | # For existing promotions, set component to None 41 | op.execute(promotions.update() 42 | .where(promotions.c.id == promotions.c.id) 43 | .values(component=None)) 44 | 45 | 46 | def downgrade(): 47 | with op.batch_alter_table('promotions') as batch_op: 48 | batch_op.drop_column('component') 49 | -------------------------------------------------------------------------------- /alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # path to migration scripts 5 | script_location = dlrn/migrations 6 | 7 | # template used to generate migration files 8 | # file_template = %%(rev)s_%%(slug)s 9 | 10 | # max length of characters to apply to the 11 | # "slug" field 12 | #truncate_slug_length = 40 13 | 14 | # set to 'true' to run the environment during 15 | # the 'revision' command, regardless of autogenerate 16 | # revision_environment = false 17 | 18 | # set to 'true' to allow .pyc and .pyo files without 19 | # a source .py file to be detected as revisions in the 20 | # versions/ directory 21 | # sourceless = false 22 | 23 | # version location specification; this defaults 24 | # to migrations/versions. When using multiple version 25 | # directories, initial revisions must be specified with --version-path 26 | # version_locations = %(here)s/bar %(here)s/bat migrations/versions 27 | 28 | # the output encoding used when revision files 29 | # are written from script.py.mako 30 | # output_encoding = utf-8 31 | 32 | sqlalchemy.url = sqlite:///commits.sqlite 33 | 34 | 35 | # Logging configuration 36 | [loggers] 37 | keys = root,sqlalchemy,alembic 38 | 39 | [handlers] 40 | keys = console 41 | 42 | [formatters] 43 | keys = generic 44 | 45 | [logger_root] 46 | level = WARN 47 | handlers = console 48 | qualname = 49 | 50 | [logger_sqlalchemy] 51 | level = WARN 52 | handlers = 53 | qualname = sqlalchemy.engine 54 | 55 | [logger_alembic] 56 | level = INFO 57 | handlers = 58 | qualname = alembic 59 | 60 | [handler_console] 61 | class = StreamHandler 62 | args = (sys.stderr,) 63 | level = NOTSET 64 | formatter = generic 65 | 66 | [formatter_generic] 67 | format = %(levelname)-5.5s [%(name)s] %(message)s 68 | datefmt = %H:%M:%S 69 | -------------------------------------------------------------------------------- /dlrn/api/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | import logging 13 | import logging.config as logging_config 14 | import os 15 | 16 | from flask import Flask 17 | 18 | from dlrn.api.api_logging import setup_dict_config 19 | from dlrn.api.utils import ConfigurationValidator 20 | 21 | app = Flask(__name__) 22 | app.config.from_object('dlrn.api.config') 23 | try: 24 | app.config.from_pyfile(os.environ['CONFIG_FILE'], silent=True) 25 | except KeyError: 26 | pass 27 | 28 | 29 | def setup_api_logging(config): 30 | log_dictConfig = setup_dict_config(app.config) 31 | logging_config.dictConfig(log_dictConfig) 32 | 33 | 34 | setup_api_logging(app.config) 35 | log_api = logging.getLogger("dlrn") 36 | configuration_validation = ConfigurationValidator(app.config) 37 | if configuration_validation.is_valid(): 38 | log_api.debug(configuration_validation) 39 | # TODO(evallesp): Change how we initialize the API to move 40 | # imports to the top. If the config is valid, then we start 41 | # the whole API 42 | from dlrn.api import dlrn_api # noqa 43 | from dlrn.api import graphql # noqa 44 | from dlrn.api import prom_metrics # noqa 45 | else: 46 | log_api.error(configuration_validation) 47 | raise Exception 48 | -------------------------------------------------------------------------------- /contrib/README.rst: -------------------------------------------------------------------------------- 1 | ================== 2 | DLRN API container 3 | ================== 4 | 5 | This directory contains a set of files used to build a ready to use DLRN API 6 | container image, and the YAML file needed to publish it in the `CentOS 7 | Container Registry `_. 8 | 9 | Creating the image 10 | ------------------ 11 | 12 | You can build the image using Buildah: 13 | 14 | .. code-block:: shell-session 15 | 16 | # buildah build-using-dockerfile -t dlrnapi:latest -f Dockerfile . 17 | 18 | and also Docker: 19 | 20 | .. code-block:: shell-session 21 | 22 | # docker build -t dlrnapi:latest -f Dockerfile . 23 | 24 | Running 25 | ------- 26 | 27 | The container image exposes the DLRN API on port 5000, using HTTP. It supports 28 | the following environment variables: 29 | 30 | - ``DLRNAPI_USE_SAMPLE_DATA``: if set to any value, the container will 31 | pre-create some basic data (commits, CI votes and a user ``foo``, with 32 | password ``bar``). This can be useful for tests. 33 | 34 | - ``DLRNAPI_DBPATH``: if set to any value, the container will use this 35 | connection string to connect to a database supported by SQLAlchemy. If not set, 36 | it will default to a local SQLite3 database. 37 | 38 | An example command-line using podman, where we use the sample data: 39 | 40 | .. code-block:: shell-session 41 | 42 | # podman run -p 5000:5000 -e DLRNAPI_USE_SAMPLE_DATA=yes dlrnapi:latest 43 | 44 | For Docker, with no sample data: 45 | 46 | .. code-block:: shell-session 47 | 48 | # docker run -p 5000:5000 dlrnapi:latest 49 | 50 | Data is stored in that /data directory inside the container, so you can use 51 | a volume to persist it across multiple executions: 52 | 53 | .. code-block:: shell-session 54 | 55 | # podman run -p 5000:5000 --volume dlrn-data:/data dlrnapi:latest 56 | -------------------------------------------------------------------------------- /scripts/db_migrate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 3 | # not use this file except in compliance with the License. You may obtain 4 | # a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 10 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 11 | # License for the specific language governing permissions and limitations 12 | # under the License. 13 | 14 | import argparse 15 | import os 16 | import sys 17 | 18 | from tempfile import mkstemp 19 | 20 | from dlrn.db import getSession 21 | from dlrn.utils import loadYAML 22 | from dlrn.utils import saveYAML 23 | 24 | 25 | def migrate_db(source_string, dest_string): 26 | osfd, tmpfilename = mkstemp() 27 | session = getSession(source_string) 28 | saveYAML(session, tmpfilename) 29 | session.close() 30 | session2 = getSession(dest_string) 31 | loadYAML(session2, tmpfilename) 32 | os.remove(tmpfilename) 33 | 34 | 35 | def main(): 36 | parser = argparse.ArgumentParser() 37 | parser.add_argument('--source', 38 | help="SQLAlchemy connection string for the source" 39 | "database.", 40 | required=True) 41 | parser.add_argument('--dest', 42 | help="SQLAlchemy connection string for the" 43 | "destination database.", 44 | required=True) 45 | 46 | options, args = parser.parse_known_args(sys.argv[1:]) 47 | 48 | migrate_db(options.source, options.dest) 49 | 50 | 51 | if __name__ == '__main__': 52 | main() 53 | exit(0) 54 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/ade85b2396bc_add_extended_hash_and_dt_extended_columns_to_.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Add extended_hash and dt_extended columns to commit table 14 | 15 | Revision ID: ade85b2396bc 16 | Revises: cab7697f6564 17 | Create Date: 2018-05-23 12:20:39.199017 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | from sqlalchemy.sql import table, column 24 | 25 | # revision identifiers, used by Alembic. 26 | revision = 'ade85b2396bc' 27 | down_revision = 'cab7697f6564' 28 | branch_labels = None 29 | depends_on = None 30 | 31 | 32 | def upgrade(): 33 | op.add_column('commits', sa.Column('extended_hash', sa.String(64))) 34 | op.add_column('commits', sa.Column('dt_extended', sa.Integer)) 35 | commits = table('commits', 36 | column('id', sa.Integer), 37 | column('dt_extended', sa.Integer)) 38 | # For existing commits, set dt_extended to 0 (extended_hash is None) 39 | op.execute(commits.update() 40 | .where(commits.c.id == commits.c.id) 41 | .values(dt_extended=0)) 42 | 43 | 44 | def downgrade(): 45 | with op.batch_alter_table('commits') as batch_op: 46 | batch_op.drop_column('extended_hash') 47 | batch_op.drop_column('dt_extended') 48 | -------------------------------------------------------------------------------- /dlrn/api/inputs/metrics.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | import calendar 13 | import time 14 | 15 | from pydantic import BaseModel 16 | from pydantic import StrictStr 17 | from pydantic import validator 18 | from typing import Optional 19 | 20 | from dlrn.api.utils import InvalidUsage 21 | 22 | 23 | class MetricsInput(BaseModel): 24 | """Input class that validates request's arguments for metrics endpoint 25 | 26 | :param str start_date: Start date for period, in YYYY-mm-dd format (UTC) 27 | :param str end_date: End date for period, in YYYY-mm-dd format (UTC) 28 | :param package_name(optional): Return metrics for package_name 29 | """ 30 | start_date: str 31 | end_date: str 32 | package_name: Optional[StrictStr] = None 33 | 34 | @validator('start_date', 'end_date') 35 | @classmethod 36 | def validate_datetime(cls, date_value): 37 | # Convert dates to timestamp 38 | fmt = '%Y-%m-%d' 39 | try: 40 | date_timestamp = int(calendar.timegm(time.strptime(date_value, 41 | fmt))) 42 | except ValueError: 43 | raise InvalidUsage('Invalid date format, it must be YYYY-mm-dd', 44 | status_code=400) 45 | return date_timestamp 46 | -------------------------------------------------------------------------------- /dlrn/templates/queue.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Pending commits for the current run 6 | 7 | 8 | 9 | 10 | 11 |

12 | 13 | Pending commits for the current run: {{ target | capitalize }} ({{ src }}) 14 |

15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | {% for commit in commits %} 25 | 26 | 27 | 28 | 34 | 35 | {% endfor %} 36 | 37 |
Commit Date TimeProject NameCommit Hash
{{ commit.dt_commit | strftime }}{{ commit.project_name }} 29 | 30 | 31 | {{ commit.commit_hash }} 32 | 33 |
38 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /dlrn/tests/samples/commits_components.yaml: -------------------------------------------------------------------------------- 1 | civotes: [] 2 | commits: 3 | - artifacts: repos/component/tripleo/c9/37/c937de75c28e63fba8d8738ad6a5f2ede517e53d_b60c7f2f/puppet-vlan-0.1.0-0.20200110140419.c937de7.el7.src.rpm,repos/component/tripleo/c9/37/c937de75c28e63fba8d8738ad6a5f2ede517e53d_b60c7f2f/puppet-vlan-0.1.0-0.20200110140419.c937de7.el7.noarch.rpm 4 | commit_branch: master 5 | commit_hash: c937de75c28e63fba8d8738ad6a5f2ede517e53d 6 | component: tripleo 7 | distgit_dir: ./data/puppet-vlan_distro/ 8 | distro_hash: b60c7f2fd9a64e968e4e3139c8f1f5aff4aa07a7 9 | dt_build: '1578664973' 10 | dt_commit: '1354617692' 11 | dt_distro: '1489056715' 12 | dt_extended: '0' 13 | extended_hash: None 14 | flags: '0' 15 | id: '1' 16 | notes: OK 17 | project_name: puppet-vlan 18 | repo_dir: ./data/puppet-vlan 19 | status: SUCCESS 20 | type: rpm 21 | - artifacts: repos/component/tripleo/eb/fb/ebfb108bf1a19d70aec6c52cdd1952c0e86593fa_642db7af/puppet-apache-5.3.1-0.20200110141918.ebfb108.el7.src.rpm,repos/component/tripleo/eb/fb/ebfb108bf1a19d70aec6c52cdd1952c0e86593fa_642db7af/puppet-apache-5.3.1-0.20200110141918.ebfb108.el7.noarch.rpm 22 | commit_branch: master 23 | commit_hash: ebfb108bf1a19d70aec6c52cdd1952c0e86593fa 24 | component: tripleo 25 | distgit_dir: ./data/puppet-apache_distro/ 26 | distro_hash: 642db7af784690c3f17d85ff85b0424001c97b82 27 | dt_build: '1578665875' 28 | dt_commit: '1576575518' 29 | dt_distro: '1489056308' 30 | dt_extended: '0' 31 | extended_hash: None 32 | flags: '0' 33 | id: '2' 34 | notes: OK 35 | project_name: puppet-apache 36 | repo_dir: ./data/puppet-apache 37 | status: SUCCESS 38 | type: rpm 39 | projects: 40 | - id: '1' 41 | last_email: '1578665751' 42 | project_name: puppet-apache 43 | promotions: [] 44 | users: 45 | # passlib.hash.sha512_crypt.encrypt("test") 46 | - password: $6$rounds=656000$PA43St1i5YORkfco$tdLlOIsMDBKBoRJCfBWHxkAJt5jdAJjeK/8PlxT4HTyeut4S/hVlRI6s7el5pOFDZdadm3aEtJFqYT0bfy4b21 47 | username: test 48 | 49 | -------------------------------------------------------------------------------- /dlrn/drivers/pkginfo.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | # PkgInfoDriver derived classes expose the following functions: 14 | # 15 | # getpackages(). This function will return an array of hashes. Each individual 16 | # hash must contain the following mandatory parameters (others are optional): 17 | # - 'name' : package name 18 | # - 'upstream': URL for upstream repo 19 | # - 'master-distgit': URL for distgit repo 20 | # - 'maintainers': list of e-mail addresses for package maintainers 21 | # 22 | # getinfo(). This function will return a list of commits to be processed for a 23 | # specific package, and True if the package was skipped due to any 24 | # git clone error, False if not. 25 | # 26 | # preprocess(). This function will run any required pre-processing for the spec 27 | # files. 28 | # 29 | # distgit_dir(). This function will return the distgit repo directory for a 30 | # given package name. 31 | 32 | from collections import namedtuple 33 | 34 | 35 | class PkgInfoDriver(object): 36 | Info = namedtuple('Info', ['commits', 'skipped']) 37 | 38 | def __init__(self, *args, **kwargs): 39 | self.packages = [] 40 | self.config_options = kwargs.get('cfg_options') 41 | 42 | def getpackages(self): 43 | return self.packages 44 | 45 | def getinfo(self): 46 | return Info(None, False) 47 | 48 | def preprocess(self): 49 | return 50 | 51 | def distgit_dir(self, package_name): 52 | return '' 53 | -------------------------------------------------------------------------------- /dlrn/tests/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2010-2011 OpenStack Foundation 4 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import os 19 | 20 | import fixtures 21 | import testtools 22 | 23 | _TRUE_VALUES = ('True', 'true', '1', 'yes') 24 | 25 | 26 | class TestCase(testtools.TestCase): 27 | 28 | "Test case base class for all unit tests." 29 | 30 | def setUp(self): 31 | "Run before each test method to initialize test environment." 32 | 33 | super(TestCase, self).setUp() 34 | test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0) 35 | try: 36 | test_timeout = int(test_timeout) 37 | except ValueError: 38 | # If timeout value is invalid do not set a timeout. 39 | test_timeout = 0 40 | if test_timeout > 0: 41 | self.useFixture(fixtures.Timeout(test_timeout, gentle=True)) 42 | 43 | self.useFixture(fixtures.NestedTempfile()) 44 | self.useFixture(fixtures.TempHomeDir()) 45 | 46 | if os.environ.get('OS_STDOUT_CAPTURE') in _TRUE_VALUES: 47 | stdout = self.useFixture(fixtures.StringStream('stdout')).stream 48 | self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout)) 49 | if os.environ.get('OS_STDERR_CAPTURE') in _TRUE_VALUES: 50 | stderr = self.useFixture(fixtures.StringStream('stderr')).stream 51 | self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr)) 52 | 53 | self.log_fixture = self.useFixture(fixtures.FakeLogger()) 54 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/638f980c9169_add_tables_required_by_dlrn_api.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Add tables required by DLRN API 14 | 15 | Revision ID: 638f980c9169 16 | Revises: f38ba3389b85 17 | Create Date: 2016-11-22 12:43:21.363066 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | 24 | 25 | # revision identifiers, used by Alembic. 26 | revision = '638f980c9169' 27 | down_revision = 'f38ba3389b85' 28 | branch_labels = None 29 | depends_on = None 30 | 31 | 32 | def upgrade(): 33 | op.create_table('users', 34 | sa.Column('username', sa.String(256), nullable=False), 35 | sa.Column('password', sa.String(256), nullable=False), 36 | sa.PrimaryKeyConstraint('username')) 37 | 38 | op.create_table('civotes', 39 | sa.Column('id', sa.Integer(), nullable=False), 40 | sa.Column('commit_id', sa.Integer(), 41 | sa.ForeignKey('commits.id'), 42 | nullable=False), 43 | sa.Column('ci_name', sa.String(256), nullable=True), 44 | sa.Column('ci_url', sa.String(1024), nullable=True), 45 | sa.Column('ci_vote', sa.Boolean(), nullable=True), 46 | sa.Column('ci_in_progress', sa.Boolean(), nullable=True), 47 | sa.Column('timestamp', sa.Integer(), nullable=True), 48 | sa.Column('notes', sa.Text(), nullable=True), 49 | sa.PrimaryKeyConstraint('id')) 50 | 51 | 52 | def downgrade(): 53 | op.drop_table('users') 54 | op.drop_table('civotes') 55 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/cab7697f6564_add_user_column_to_civote_and_promotion.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Add user column to civote and promotion 14 | 15 | Revision ID: cab7697f6564 16 | Revises: 4a5651777e5e 17 | Create Date: 2017-11-24 13:14:05.750269 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | 24 | # revision identifiers, used by Alembic. 25 | revision = 'cab7697f6564' 26 | down_revision = '4a5651777e5e' 27 | branch_labels = None 28 | depends_on = None 29 | 30 | 31 | def upgrade(): 32 | # Note that existing votes and promotions will have "null" as the user 33 | # Since SQLite3 does not allow ALTER TABLE statements, we need to do a 34 | # batch operation: http://alembic.zzzcomputing.com/en/latest/batch.html 35 | with op.batch_alter_table('civotes') as batch_op: 36 | batch_op.add_column(sa.Column('user', 37 | sa.String(256), 38 | sa.ForeignKey('users.username', 39 | name='civ_user_fk'))) 40 | with op.batch_alter_table('promotions') as batch_op: 41 | batch_op.add_column(sa.Column('user', sa.String(256), 42 | sa.ForeignKey('users.username', 43 | name='prom_user_fk'))) 44 | 45 | 46 | def downgrade(): 47 | with op.batch_alter_table('civotes') as batch_op: 48 | batch_op.drop_constraint('civ_user_fk', type_='foreignkey') 49 | batch_op.drop_column('user') 50 | 51 | with op.batch_alter_table('promotions') as batch_op: 52 | batch_op.drop_constraint('prom_user_fk', type_='foreignkey') 53 | batch_op.drop_column('user') 54 | -------------------------------------------------------------------------------- /scripts/bisect.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright (C) 2017 Red Hat, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. You may obtain 7 | # a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations 15 | # under the License. 16 | 17 | # script to run in a git bisect for compilation issues with dlrn 18 | 19 | set -e 20 | 21 | if [ $# -lt 4 ]; then 22 | echo "Usage: $0 []" 1>&2 23 | exit 1 24 | fi 25 | 26 | conf=$1 27 | proj=$2 28 | good=$3 29 | bad=$4 30 | shift 4 31 | args="$@" 32 | 33 | if [ ! -r $conf ]; then 34 | echo "Unable to read dlrn configuration file $conf" 1>&2 35 | exit 1 36 | fi 37 | 38 | if ! type -p dlrn; then 39 | echo "dlrn command not found in path" 1>&2 40 | exit 1 41 | fi 42 | 43 | # verify that the project is managed by dlrn 44 | 45 | eval $(grep '^datadir=' $conf) 46 | 47 | if [ -z "$datadir" ]; then 48 | echo "Unable to read datadir from $conf" 1>&2 49 | exit 1 50 | fi 51 | 52 | echo "Checking $proj" 53 | dlrn --config $conf --package-name $proj $args --run /bin/true 54 | 55 | cd $datadir/$proj 56 | 57 | # create the compilation script 58 | 59 | executor=$(mktemp) 60 | chmod +x $executor 61 | 62 | logdir=$(mktemp -d) 63 | 64 | [ -d $logdir -a -w $logdir ] 65 | 66 | echo "dlrn log in $logdir" 67 | 68 | cleanup(){ 69 | rm -f $executor 70 | } 71 | 72 | trap cleanup 0 73 | 74 | cat > $executor < $logdir/\$(git rev-parse HEAD) 2>&1 78 | 79 | # we are in repos/ 80 | cd ../.. 81 | 82 | dlrn --config $conf --dev --local --package-name $proj $args 83 | 84 | exit $? 85 | EOF 86 | 87 | # launch the git bisect process 88 | 89 | git bisect reset || : 90 | git bisect start 91 | git bisect good $good 92 | git bisect bad $bad 93 | git bisect run $executor 94 | git bisect visualize 95 | 96 | # bisect.sh ends here 97 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | minversion = 1.6 3 | envlist = py{3,312},pep8,bandit 4 | 5 | [testenv] 6 | usedevelop = True 7 | install_command = pip install -U {opts} {packages} 8 | setenv = 9 | VIRTUAL_ENV={envdir} 10 | VIRTUALENV_NO_DOWNLOAD=1 11 | deps = -r{toxinidir}/requirements.txt 12 | -r{toxinidir}/test-requirements.txt 13 | commands = 14 | coverage run --branch --include 'dlrn*' -m unittest {posargs:discover dlrn.tests} 15 | coverage report -m 16 | passenv = 17 | TERM 18 | 19 | [testenv:bindep] 20 | # Do not install any requirements. We want this to be fast and work even if 21 | # system dependencies are missing, since it's used to tell you what system 22 | # dependencies are missing! This also means that bindep must be installed 23 | # separately, outside of the requirements files. 24 | deps = bindep 25 | commands = bindep test 26 | 27 | [testenv:docs] 28 | deps = -r{toxinidir}/doc/requirements.txt 29 | commands = sphinx-build -W -b html -d build/doctrees doc/source doc/build/html 30 | 31 | [testenv:pep8] 32 | commands = flake8 33 | 34 | [testenv:venv] 35 | commands = {posargs} 36 | 37 | [testenv:cover] 38 | commands = python setup.py testr --coverage --testr-args='{posargs}' 39 | 40 | [flake8] 41 | # H803 skipped on purpose per list discussion. 42 | # E123, E125 skipped as they are invalid PEP-8. 43 | # F821 skipped for now, as flake8 is not recognizing session global. 44 | # It would might be better to pass a session around rather than the global. 45 | # rdoinfo skipped as we are changing to rdopkg 46 | # https://review.gerrithub.io/#/c/214249/ 47 | 48 | show-source = True 49 | extend-ignore = E123,E125,H803,F821,H216 50 | builtins = _ 51 | exclude=.venv,venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,rdoinfo.py,*rdoinfo/*,verify.py,data 52 | 53 | [bandit] 54 | exclude = dlrn/tests,dlrn/dist,dlrn/build,dlrn/static,dlrn/DLRN.egg-info,dlrn/__pycache__,dlrn/UNKNOWN.egg-info,dlrn/lib,dlrn/bin,dlrn/migrations 55 | 56 | [testenv:bandit] 57 | basepython = python3 58 | deps = -r{toxinidir}/test-requirements.txt 59 | bandit 60 | # Execute bandit on the dlrn directory with 5 lines of context, skipping low severity issues 61 | commands = bandit --ini tox.ini -n 5 -r dlrn -ll 62 | 63 | [testenv:pip-audit] 64 | basepython = python3 65 | deps = -r{toxinidir}/test-requirements.txt 66 | pip-audit 67 | commands = pip-audit --requirement requirements.txt 68 | 69 | -------------------------------------------------------------------------------- /scripts/submit_review.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Copyright (C) 2016 Red Hat, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. You may obtain 7 | # a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations 15 | # under the License. 16 | 17 | set -o pipefail 18 | 19 | source $(dirname $0)/common-functions 20 | 21 | exec > $OUTPUT_DIRECTORY/review.log 2>&1 22 | 23 | set -x 24 | 25 | if [ -n "$GERRIT_URL" -a -n "$GERRIT_LOG" -a -n "$GERRIT_MAINTAINERS" -a -n "$GERRIT_TOPIC" ]; then 26 | cd ${DATA_DIR}/${PROJECT_NAME} 27 | LONGSHA1=$(git rev-parse HEAD) 28 | SHORTSHA1=$(git rev-parse --short HEAD) 29 | cd ${DISTGIT_DIR} 30 | CURBRANCH=$(git rev-parse --abbrev-ref HEAD) 31 | git branch -D branch-$SHORTSHA1 || true 32 | git checkout -b branch-$SHORTSHA1 33 | git review -s 34 | # we need to inject a pseudo-modification to the spec file to have a 35 | # change to commit 36 | sed -i -e "\$a\\# REMOVEME: error caused by commit ${GERRIT_URL}\\" *.spec 37 | echo -e "${PROJECT_NAME}: failed to build ${SHORTSHA1}\n\nNeed to fix build error caused by ${GERRIT_URL}\nSee log at ${GERRIT_LOG}"|git commit -F- *.spec 38 | CHID=$(git log -1|grep -F Change-Id: |cut -d':' -f2) 39 | MAINTAINERS="${GERRIT_MAINTAINERS//,/ -a }" 40 | REMOTE=$(git remote show -n gerrit|grep -F 'Fetch URL'|sed 's/.*: //') 41 | # extract the component from the remote git repository 42 | # the url should be in this form: ssh://@:/ 43 | REMOTE_HOST=$(echo $REMOTE|cut -d/ -f3|cut -d: -f1) 44 | REMOTE_PORT=$(echo $REMOTE|cut -d/ -f3|cut -d: -f2) 45 | PROJECT=$(echo $REMOTE|sed 's@.*://@@'|sed 's@[^/]*/\(.*\)$@\1@') 46 | git review -t ${GERRIT_TOPIC} < /dev/null 47 | ssh -p $REMOTE_PORT $REMOTE_HOST gerrit set-reviewers --project $PROJECT -a $MAINTAINERS -- $CHID 48 | git checkout ${CURBRANCH:-master} 49 | else 50 | echo "No gerrit review to create" 51 | fi 52 | 53 | # submit_review.sh ends here 54 | -------------------------------------------------------------------------------- /dlrn/tests/test_remote.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import mock 16 | import sh 17 | import sys 18 | import tempfile 19 | 20 | from dlrn.tests import base 21 | 22 | from dlrn import db 23 | from dlrn import remote 24 | from dlrn import utils 25 | 26 | 27 | def mocked_session(url): 28 | db_fd, filepath = tempfile.mkstemp() 29 | session = db.getSession("sqlite:///%s" % filepath) 30 | utils.loadYAML(session, './dlrn/tests/samples/commits_1.yaml') 31 | return session 32 | 33 | 34 | def mocked_get(url, timeout=None): 35 | mock_resp = mock.Mock() 36 | with open('./dlrn/tests/samples/commits_remote.yaml', 'rb') as fp: 37 | mock_resp.status_code = 200 38 | mock_resp.content = fp.read() 39 | mock_resp.text = mock_resp.content.decode('utf-8') 40 | return mock_resp 41 | 42 | 43 | @mock.patch('os.rename') 44 | @mock.patch('os.symlink') 45 | @mock.patch('dlrn.drivers.rdoinfo.RdoInfoDriver.getpackages') 46 | @mock.patch.object(sh.Command, '__call__', autospec=True) 47 | @mock.patch('dlrn.remote.post_build') 48 | @mock.patch('dlrn.remote.getSession', side_effect=mocked_session) 49 | @mock.patch('dlrn.remote.requests.get', side_effect=mocked_get) 50 | class TestRemote(base.TestCase): 51 | def test_remote(self, req_mock, db_mock, build_mock, sh_mock, gp_mock, 52 | sl_mock, rn_mock): 53 | testargs = ["dlrn-remote", "--config-file", "projects.ini", 54 | "--repo-url", "http://example.com/1/"] 55 | # There should be only one call to post_build(), even though there are 56 | # 2 commits in the commits_remote.yaml file. This is because the first 57 | # one is already in the database 58 | with mock.patch.object(sys, 'argv', testargs): 59 | remote.remote() 60 | build_mock.assert_called_once() 61 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/7fbd3a18502f_extra_tables_for_votes_on_aggregates.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Extra tables for votes on aggregates 14 | 15 | Revision ID: 7fbd3a18502f 16 | Revises: 00a31f1f39c0 17 | Create Date: 2020-01-16 15:22:32.090726 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | 24 | 25 | # revision identifiers, used by Alembic. 26 | revision = '7fbd3a18502f' 27 | down_revision = '00a31f1f39c0' 28 | branch_labels = None 29 | depends_on = None 30 | 31 | 32 | def upgrade(): 33 | # Since SQLite3 does not allow ALTER TABLE statements, we need to do a 34 | # batch operation: http://alembic.zzzcomputing.com/en/latest/batch.html 35 | with op.batch_alter_table('promotions') as batch_op: 36 | batch_op.add_column(sa.Column('aggregate_hash', sa.String(64), 37 | nullable=True)) 38 | 39 | op.create_table('civotes_agg', 40 | sa.Column('id', sa.Integer(), nullable=False), 41 | sa.Column('ref_hash', sa.String(64), nullable=False), 42 | sa.Column('ci_name', sa.String(256), nullable=True), 43 | sa.Column('ci_url', sa.String(1024), nullable=True), 44 | sa.Column('ci_vote', sa.Boolean(), nullable=True), 45 | sa.Column('ci_in_progress', sa.Boolean(), nullable=True), 46 | sa.Column('timestamp', sa.Integer(), nullable=True), 47 | sa.Column('notes', sa.Text(), nullable=True), 48 | sa.Column('user', sa.String(255), 49 | sa.ForeignKey('users.username')), 50 | sa.PrimaryKeyConstraint('id')) 51 | 52 | 53 | def downgrade(): 54 | op.drop_table('civotes_agg') 55 | with op.batch_alter_table('promotions') as batch_op: 56 | batch_op.drop_column('aggregate_hash') 57 | -------------------------------------------------------------------------------- /dlrn/api/drivers/dbauthentication.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | import logging 13 | 14 | from flask import request 15 | from flask_httpauth import HTTPBasicAuth 16 | import passlib.hash 17 | 18 | from dlrn.api import app 19 | from dlrn.db import getSession 20 | from dlrn.db import User 21 | 22 | 23 | log_auth = logging.getLogger("auth") 24 | log_api = logging.getLogger("dlrn") 25 | 26 | 27 | class DBAuthentication(HTTPBasicAuth): 28 | def __init__(self, scheme='Basic', realm=None, header=None): 29 | super(HTTPBasicAuth, self).__init__(scheme=scheme, realm=realm, 30 | header=header) 31 | self.verbose_build = False 32 | self.verify_password_callback = self.verify_pw 33 | 34 | # TODO(evallesp): Implement authorization 35 | def authorize(self, role, user, auth): 36 | return True 37 | 38 | def verify_pw(self, username, password): 39 | allowed = False 40 | session = getSession(app.config['DB_PATH']) 41 | if not username or not password: 42 | log_api.error("No user or password in the request headers.") 43 | elif session is not None: 44 | user = session.query(User).filter(User.username == username) \ 45 | .first() 46 | if user is not None: 47 | if passlib.hash.sha512_crypt.verify(password, user.password): 48 | log_auth.info('"User": %s, "event": login, "success": ' 49 | 'true, "path": %s, "method": %s', username, 50 | request.path, request.method) 51 | allowed = True 52 | else: 53 | log_auth.info('"User": %s, "event": login, "success": ' 54 | 'false, "path": %s, "method": %s', username, 55 | request.path, request.method) 56 | return allowed 57 | -------------------------------------------------------------------------------- /dlrn/tests/samples/projects.ini.detect: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | #datadir=./data 3 | #scriptsdir=./scripts 4 | configdir= 5 | baseurl=http://trunk.rdoproject.org/centos7/ 6 | distro=rpm-master 7 | source=master 8 | target=centos 9 | smtpserver= 10 | reponame=delorean 11 | #templatedir=./dlrn/templates 12 | project_name=RDO 13 | maxretries=3 14 | pkginfo_driver=dlrn.drivers.rdoinfo.RdoInfoDriver 15 | build_driver=dlrn.drivers.mockdriver.MockBuildDriver 16 | tags= 17 | #tags=mitaka 18 | rsyncdest= 19 | rsyncport=22 20 | workers=1 21 | gerrit_topic=rdo-FTBFS 22 | database_connection=sqlite:///commits.sqlite 23 | fallback_to_master=1 24 | release_numbering=0.date.hash 25 | custom_preprocess= 26 | include_srpm_in_repo=true 27 | 28 | [gitrepo_driver] 29 | # options to be specified if pkginfo_driver is set to 30 | # dlrn.drivers.gitrepo.GitRepoDriver 31 | # 32 | #repo=http://github.com/openstack/rpm-packaging 33 | #directory=/openstack 34 | #skip=openstack-macros,keystoneauth1 35 | #use_version_from_spec=1 36 | #keep_tarball=0 37 | 38 | [rdoinfo_driver] 39 | # options to be specified if pkginfo_driver is set to 40 | # dlrn.drivers.rdoinfo.RdoInfoDriver 41 | # 42 | #repo=http://github.com/org/rdoinfo-fork 43 | 44 | [downstream_driver] 45 | # options to be specified if pkginfo_driver is set to 46 | # dlrn.drivers.downstream.DownstreamInfoDriver 47 | # 48 | #repo=http://github.com/org/fooinfo 49 | #info_files=foo.yml 50 | #versions_url=https://trunk.rdoproject.org/centos7-master/current/versions.csv 51 | #downstream_distro_branch=foo-rocky 52 | #downstream_tag=foo- 53 | #downstream_distgit_key=foo-distgit 54 | #use_upstream_spec=False 55 | #downstream_spec_replace_list=^foo/bar,string1/string2 56 | 57 | [mockbuild_driver] 58 | # options to be specified if build_driver is set to 59 | # dlrn.drivers.mockdriver.MockBuildDriver 60 | #install_after_build=1 61 | 62 | [kojibuild_driver] 63 | # options to be specified if build_driver is set to 64 | # dlrn.drivers.kojidriver.KojiBuildDriver 65 | #koji_exe=koji 66 | #krb_principal=user@EXAMPLE.COM 67 | #krb_keytab=/home/user/user.keytab 68 | #scratch_build=True 69 | #build_target=koji-target-build 70 | #arch=x86_64 71 | #use_rhpkg=False 72 | #fetch_mock_config=False 73 | #mock_base_packages=basesystem rpm-build python2-devel gcc make python-sqlalchemy python-webob ghostscript graphviz python-sphinx python-eventlet python-six python-pbr openstack-macros git yum-plugin-priorities rubygems python-setuptools_scm 74 | 75 | [coprbuild_driver] 76 | # options to be specified if build_driver is set to 77 | # dlrn.drivers.coprdriver.CoprBuildDriver 78 | #coprid=account/repo 79 | -------------------------------------------------------------------------------- /dlrn/api/drivers/auth.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | import logging 13 | 14 | from flask_httpauth import MultiAuth 15 | import stevedore 16 | 17 | from dlrn.utils import import_class 18 | 19 | 20 | DEFAULT_DRIVER = "dlrn.api.drivers.dbauthentication.DBAuthentication" 21 | NAMESPACE = 'dlrn.api.drivers' 22 | logger = logging.getLogger("dlrn") 23 | 24 | 25 | class Auth: 26 | 27 | def __init__(self, config): 28 | self.auth_multi = None 29 | self._setup_api_auth(config) 30 | 31 | def _get_default_driver(self): 32 | return DEFAULT_DRIVER 33 | 34 | def _setup_api_auth(self, config): 35 | auth_drivers = [] 36 | if 'AUTHENTICATION_DRIVERS' not in config.keys(): 37 | config['AUTHENTICATION_DRIVERS'] = [] 38 | 39 | self._ext_mgr = stevedore.ExtensionManager( 40 | NAMESPACE, 41 | invoke_on_load=True, 42 | on_load_failure_callback=lambda _, entry, exception: 43 | logger.error(f"Error while initializing driver {entry.value} " 44 | f"with exception: {exception}")) 45 | 46 | for driver_name in config['AUTHENTICATION_DRIVERS']: 47 | try: 48 | auth_drivers.append(self._ext_mgr[driver_name].obj) 49 | logger.info("Added auth driver: %s", driver_name) 50 | except KeyError as faulty_driver: 51 | logger.error("Driver not found: %s", faulty_driver) 52 | if len(auth_drivers) == 0: 53 | try: 54 | default_driver = self._get_default_driver() 55 | logger.info("Trying to load default auth driver: %s", 56 | default_driver) 57 | auth_drivers.append(import_class(default_driver)()) 58 | logger.info("Default auth driver loaded.") 59 | except (ModuleNotFoundError, ImportError): 60 | logger.error("Error, driver not found. No auth driver loaded") 61 | exit(1) 62 | self.auth_multi = MultiAuth(*auth_drivers) 63 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/3c62b0d3ec34_initial_creation.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Initial creation 14 | 15 | Revision ID: 3c62b0d3ec34 16 | Revises: 17 | Create Date: 2016-06-17 10:35:54.703517 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | 24 | 25 | # revision identifiers, used by Alembic. 26 | revision = '3c62b0d3ec34' 27 | down_revision = None 28 | branch_labels = None 29 | depends_on = None 30 | 31 | 32 | def upgrade(): 33 | try: 34 | op.create_table('commits', 35 | sa.Column('id', sa.Integer(), nullable=False), 36 | sa.Column('dt_commit', sa.Integer(), nullable=True), 37 | sa.Column('dt_distro', sa.Integer(), nullable=True), 38 | sa.Column('dt_build', sa.Integer(), nullable=True), 39 | sa.Column('project_name', sa.String(), nullable=True), 40 | sa.Column('repo_dir', sa.String(), nullable=True), 41 | sa.Column('commit_hash', sa.String(), nullable=True), 42 | sa.Column('distro_hash', sa.String(), nullable=True), 43 | sa.Column('status', sa.String(), nullable=True), 44 | sa.Column('rpms', sa.String(), nullable=True), 45 | sa.Column('notes', sa.String(), nullable=True), 46 | sa.Column('flags', sa.Integer(), nullable=True), 47 | sa.PrimaryKeyConstraint('id')) 48 | op.create_table('projects', 49 | sa.Column('id', sa.Integer(), nullable=False), 50 | sa.Column('project_name', sa.String(), nullable=True), 51 | sa.Column('last_email', sa.Integer(), nullable=True), 52 | sa.PrimaryKeyConstraint('id')) 53 | except Exception: 54 | # this means that's a legacy table unversioned by alembic 55 | pass 56 | 57 | 58 | def downgrade(): 59 | op.drop_table('projects') 60 | op.drop_table('commits') 61 | -------------------------------------------------------------------------------- /dlrn/tests/samples/projects_force.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | datadir=./data 3 | scriptsdir=./scripts 4 | configdir= 5 | baseurl=http://trunk.rdoproject.org/centos7/ 6 | distro=rpm-master 7 | source=master 8 | target=centos 9 | smtpserver= 10 | reponame=delorean 11 | templatedir=./dlrn/templates 12 | project_name=RDO 13 | maxretries=3 14 | pkginfo_driver=dlrn.drivers.rdoinfo.RdoInfoDriver 15 | build_driver=dlrn.drivers.mockdriver.MockBuildDriver 16 | tags= 17 | #tags=mitaka 18 | rsyncdest= 19 | rsyncport=22 20 | workers=1 21 | gerrit_topic=rdo-FTBFS 22 | database_connection=sqlite:///commits.sqlite 23 | fallback_to_master=1 24 | nonfallback_branches=^master$,^rpm-master$ 25 | release_numbering=0.date.hash 26 | custom_preprocess= 27 | include_srpm_in_repo=true 28 | keep_changelog=false 29 | allow_force_rechecks=true 30 | 31 | [gitrepo_driver] 32 | # options to be specified if pkginfo_driver is set to 33 | # dlrn.drivers.gitrepo.GitRepoDriver 34 | # 35 | #repo=http://github.com/openstack/rpm-packaging 36 | #directory=/openstack 37 | #skip=openstack-macros,keystoneauth1 38 | #use_version_from_spec=1 39 | #keep_tarball=0 40 | 41 | [rdoinfo_driver] 42 | # options to be specified if pkginfo_driver is set to 43 | # dlrn.drivers.rdoinfo.RdoInfoDriver 44 | # 45 | #repo=http://github.com/org/rdoinfo-fork 46 | #info_files=rdo.yml 47 | #cache_dir= 48 | 49 | [downstream_driver] 50 | # options to be specified if pkginfo_driver is set to 51 | # dlrn.drivers.downstream.DownstreamInfoDriver 52 | # 53 | #repo=http://github.com/org/fooinfo 54 | #info_files=foo.yml 55 | #versions_url=https://trunk.rdoproject.org/centos7-master/current/versions.csv 56 | #downstream_distro_branch=foo-rocky 57 | #downstream_tag=foo- 58 | #downstream_distgit_key=foo-distgit 59 | #use_upstream_spec=False 60 | #downstream_spec_replace_list=^foo/bar,string1/string2 61 | #cache_dir= 62 | 63 | [mockbuild_driver] 64 | # options to be specified if build_driver is set to 65 | # dlrn.drivers.mockdriver.MockBuildDriver 66 | #install_after_build=1 67 | 68 | [kojibuild_driver] 69 | # options to be specified if build_driver is set to 70 | # dlrn.drivers.kojidriver.KojiBuildDriver 71 | #koji_exe=koji 72 | #krb_principal=user@EXAMPLE.COM 73 | #krb_keytab=/home/user/user.keytab 74 | #scratch_build=True 75 | #build_target=koji-target-build 76 | #arch=x86_64 77 | #use_rhpkg=False 78 | #fetch_mock_config=False 79 | #mock_base_packages=basesystem rpm-build python2-devel gcc make python-sqlalchemy python-webob ghostscript graphviz python-sphinx python-eventlet python-six python-pbr openstack-macros git yum-plugin-priorities rubygems python-setuptools_scm 80 | 81 | [coprbuild_driver] 82 | # options to be specified if build_driver is set to 83 | # dlrn.drivers.coprdriver.CoprBuildDriver 84 | #coprid=account/repo 85 | -------------------------------------------------------------------------------- /dlrn/api/templates/votes_general_agg.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | CI Votes for Aggregates 5 | 6 | 7 | 8 | 9 | 10 | 11 |

12 | 13 | {{ target | capitalize }} 14 |

15 | 16 |
17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | {% for repo in aggdetail %} 28 | 29 | 35 | 36 | 43 | 50 | 51 | {% endfor %} 52 | 53 |
Aggregate hashLast timestampSuccessful jobsFailed jobs
30 |
31 | 32 | 33 |
34 | 		{{ repo.timestamp | strftime }} 37 |
38 | 39 | 40 | 41 |
42 | 		44 |
45 | 46 | 47 | 48 |
49 |
54 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /dlrn/api/inputs/civotes.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | from typing import Optional 13 | 14 | from pydantic import BaseModel 15 | from pydantic import NonNegativeInt 16 | from pydantic import root_validator 17 | from pydantic import StrictStr 18 | 19 | from dlrn.api.utils import InvalidUsage 20 | 21 | 22 | class CIVotesInput(BaseModel): 23 | """Input class that validates request's arguments for civotes get endpoint 24 | 25 | :param int offset: Number of votes to skip (must be positive) 26 | """ 27 | offset: Optional[NonNegativeInt] = None 28 | 29 | 30 | class CIVotesDetailInput(BaseModel): 31 | """Input class that validates request's arguments for civotes_detail 32 | 33 | :param str commit_hash: Commit hash for filtering 34 | :param str distro_hash: Distro hash for filtering 35 | :param str component: Component name used for web page template query 36 | :param str ci_name: CI name used for web page template query 37 | """ 38 | commit_hash: Optional[StrictStr] = None 39 | distro_hash: Optional[StrictStr] = None 40 | component: Optional[StrictStr] = None 41 | ci_name: Optional[StrictStr] = None 42 | 43 | @root_validator 44 | def validate_arguments_logic(cls, values): 45 | if ((not values.get('commit_hash') or not values.get('distro_hash')) 46 | and not values.get('component') and not values.get("ci_name")): 47 | raise InvalidUsage("Please specify either commit_hash+distro_hash," 48 | " component or ci_name as parameters.", 49 | status_code=400) 50 | return values 51 | 52 | 53 | class CIVotesAggDetailInput(BaseModel): 54 | """Input class that validates request's arguments for civotes_agg_detail 55 | 56 | :param str ref_hash: ref_hash used for web page templating 57 | :param str ci_name: CI name used for web page templating 58 | """ 59 | ref_hash: Optional[StrictStr] = None 60 | ci_name: Optional[StrictStr] = None 61 | 62 | @root_validator 63 | def validate_arguments_logic(cls, values): 64 | if not values.get('ref_hash') and not values.get('ci_name'): 65 | raise InvalidUsage("Please specify either ref_hash or " 66 | "ci_name as parameters.", status_code=400) 67 | return values 68 | -------------------------------------------------------------------------------- /dlrn/api/inputs/last_tested_repo.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | from pydantic import BaseModel 14 | from pydantic import root_validator 15 | from pydantic import StrictStr 16 | from pydantic import validator 17 | from typing import Optional 18 | 19 | from dlrn.api.utils import InvalidUsage 20 | 21 | 22 | class LastTestedRepoInput(BaseModel): 23 | """Input class that validates request's arguments for last_tested_repo 24 | 25 | :param str max_age: Maximum age in hours, used as base for the search 26 | :param bool success(optional): Only report successful/unsuccessful votes 27 | :param str job_id(optional): Name of the CI that sent the vote 28 | :param bool sequential_mode(optional): If set to true, change the search 29 | algorithm to only use 30 | previous_job_id as CI name to 31 | search for. Defaults to false 32 | :param str previous_job_id(optional): CI name to search for, 33 | if sequential_mode is True 34 | :param str component(optional): Only get votes for this component 35 | 36 | """ 37 | max_age: int 38 | success: Optional[bool] 39 | job_id: Optional[StrictStr] 40 | sequential_mode: Optional[bool] 41 | previous_job_id: Optional[StrictStr] = None 42 | component: Optional[StrictStr] 43 | 44 | @validator('max_age') 45 | @classmethod 46 | def validate_max_age(cls, max_age): 47 | if int(max_age) < 0: 48 | raise InvalidUsage('Max age parameter must be greater than or ' 49 | 'equal to 0', status_code=400) 50 | return int(max_age) 51 | 52 | @root_validator 53 | def validate_previous_with_sequential(cls, values): 54 | if (values.get('sequential_mode') and 55 | values.get('previous_job_id') is None): 56 | raise InvalidUsage('Missing parameter previous_job_id', 57 | status_code=400) 58 | return values 59 | 60 | 61 | class LastTestedRepoInputPost(LastTestedRepoInput): 62 | """Input class that validates request's arguments for last_tested_repo 63 | 64 | :param str reporting_job_id: Name of the CI that will test this repo 65 | 66 | """ 67 | reporting_job_id: StrictStr 68 | -------------------------------------------------------------------------------- /projects.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | datadir=./data 3 | scriptsdir=./scripts 4 | configdir= 5 | baseurl=http://trunk.rdoproject.org/centos9/ 6 | distro=rpm-master 7 | source=master 8 | target=centos9 9 | smtpserver= 10 | reponame=delorean 11 | templatedir=./dlrn/templates 12 | project_name=RDO 13 | maxretries=3 14 | pkginfo_driver=dlrn.drivers.rdoinfo.RdoInfoDriver 15 | build_driver=dlrn.drivers.mockdriver.MockBuildDriver 16 | tags= 17 | #tags=mitaka 18 | rsyncdest= 19 | rsyncport=22 20 | workers=1 21 | gerrit_topic=rdo-FTBFS 22 | database_connection=sqlite:///commits.sqlite 23 | fallback_to_master=1 24 | nonfallback_branches=^master$,^main$,^rpm-master$ 25 | release_numbering=0.date.hash 26 | release_minor=0 27 | custom_preprocess= 28 | include_srpm_in_repo=true 29 | keep_changelog=false 30 | allow_force_rechecks=false 31 | use_components=false 32 | deps_url= 33 | 34 | [gitrepo_driver] 35 | # options to be specified if pkginfo_driver is set to 36 | # dlrn.drivers.gitrepo.GitRepoDriver 37 | # 38 | #repo=http://github.com/openstack/rpm-packaging 39 | #directory=/openstack 40 | #skip=openstack-macros,keystoneauth1 41 | #use_version_from_spec=1 42 | #keep_tarball=0 43 | 44 | [rdoinfo_driver] 45 | # options to be specified if pkginfo_driver is set to 46 | # dlrn.drivers.rdoinfo.RdoInfoDriver 47 | # 48 | #repo=http://github.com/org/rdoinfo-fork 49 | #info_files=rdo.yml 50 | #cache_dir= 51 | 52 | [downstream_driver] 53 | # options to be specified if pkginfo_driver is set to 54 | # dlrn.drivers.downstream.DownstreamInfoDriver 55 | # 56 | #repo=http://github.com/org/fooinfo 57 | #info_files=foo.yml 58 | #versions_url=https://trunk.rdoproject.org/centos9-master/current/versions.csv 59 | #downstream_distro_branch=foo-rocky 60 | #downstream_tag=foo- 61 | #downstream_distgit_key=foo-distgit 62 | #use_upstream_spec=False 63 | #downstream_spec_replace_list=^foo/bar,string1/string2 64 | #cache_dir= 65 | #downstream_source_git_key=bar-distgit 66 | #downstream_source_git_branch= 67 | 68 | [mockbuild_driver] 69 | # options to be specified if build_driver is set to 70 | # dlrn.drivers.mockdriver.MockBuildDriver 71 | #install_after_build=1 72 | 73 | [kojibuild_driver] 74 | # options to be specified if build_driver is set to 75 | # dlrn.drivers.kojidriver.KojiBuildDriver 76 | #koji_exe=koji 77 | #krb_principal=user@EXAMPLE.COM 78 | #krb_keytab=/home/user/user.keytab 79 | #scratch_build=True 80 | #build_target=koji-target-build 81 | #arch=x86_64 82 | #use_rhpkg=False 83 | #fetch_mock_config=False 84 | #mock_base_packages=basesystem rpm-build python2-devel gcc make python-sqlalchemy python-webob ghostscript graphviz python-sphinx python-eventlet python-six python-pbr openstack-macros git yum-plugin-priorities rubygems python-setuptools_scm 85 | #mock_package_manager= 86 | #additional_koji_tags= 87 | 88 | [coprbuild_driver] 89 | # options to be specified if build_driver is set to 90 | # dlrn.drivers.coprdriver.CoprBuildDriver 91 | #coprid=account/repo 92 | -------------------------------------------------------------------------------- /dlrn/migrations/env.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | from __future__ import with_statement 14 | from alembic import context 15 | from logging.config import fileConfig 16 | from sqlalchemy import engine_from_config, pool 17 | import sys 18 | 19 | # this is the Alembic Config object, which provides 20 | # access to the values within the .ini file in use. 21 | config = context.config 22 | 23 | # Interpret the config file for Python logging. 24 | # This line sets up loggers basically. 25 | fileConfig(config.config_file_name) 26 | 27 | # add your model's MetaData object here 28 | # for 'autogenerate' support 29 | # from myapp import mymodel 30 | # target_metadata = mymodel.Base.metadata 31 | sys.path.append('dlrn') 32 | from db import Base # noqa 33 | target_metadata = Base.metadata 34 | 35 | # other values from the config, defined by the needs of env.py, 36 | # can be acquired: 37 | # my_important_option = config.get_main_option("my_important_option") 38 | # ... etc. 39 | 40 | 41 | def run_migrations_offline(): 42 | """Run migrations in 'offline' mode. 43 | 44 | This configures the context with just a URL 45 | and not an Engine, though an Engine is acceptable 46 | here as well. By skipping the Engine creation 47 | we don't even need a DBAPI to be available. 48 | 49 | Calls to context.execute() here emit the given string to the 50 | script output. 51 | 52 | """ 53 | url = config.get_main_option("sqlalchemy.url") 54 | context.configure( 55 | url=url, target_metadata=target_metadata, literal_binds=True) 56 | 57 | with context.begin_transaction(): 58 | context.run_migrations() 59 | 60 | 61 | def run_migrations_online(): 62 | """Run migrations in 'online' mode. 63 | 64 | In this scenario we need to create an Engine 65 | and associate a connection with the context. 66 | 67 | """ 68 | connectable = engine_from_config( 69 | config.get_section(config.config_ini_section), 70 | prefix='sqlalchemy.', 71 | poolclass=pool.NullPool) 72 | 73 | with connectable.connect() as connection: 74 | context.configure( 75 | connection=connection, 76 | target_metadata=target_metadata 77 | ) 78 | 79 | with context.begin_transaction(): 80 | context.run_migrations() 81 | 82 | 83 | if context.is_offline_mode(): 84 | run_migrations_offline() 85 | else: 86 | run_migrations_online() 87 | -------------------------------------------------------------------------------- /scripts/centos-stream-9.cfg: -------------------------------------------------------------------------------- 1 | config_opts['root'] = 'dlrn-centos9stream-x86_64' 2 | config_opts['target_arch'] = 'x86_64' 3 | config_opts['legal_host_arches'] = ('x86_64',) 4 | config_opts['chroot_setup_cmd'] = 'install basesystem rpm-build python3-devel gcc make redhat-rpm-config redhat-release which xz python3-six \ 5 | python3-pbr git-core sed bzip2 gzip gcc-c++ tar coreutils unzip shadow-utils diffutils cpio bash gawk info patch util-linux findutils grep \ 6 | python3-setuptools_scm' 7 | config_opts['dist'] = 'el9' # only useful for --resultdir variable subst 8 | config_opts['releasever'] = '9' 9 | config_opts['plugin_conf']['ccache_enable'] = False 10 | config_opts['package_manager'] = 'dnf' 11 | config_opts['isolation'] = 'simple' 12 | # With isolation=simple needs to explicitly set use_nspawn to False with mock-1.4 13 | config_opts['use_nspawn'] = False 14 | config_opts['extra_chroot_dirs'] = [ '/run/lock', ] 15 | config_opts['bootstrap_image'] = 'quay.io/centos/centos:stream9' 16 | config_opts['dnf_vars'] = { 'stream': '9-stream', 17 | 'contentdir': 'centos', 18 | } 19 | config_opts['dnf_install_command'] = 'install dnf dnf-plugins-core' 20 | config_opts['priorities.conf'] = """ 21 | [main] 22 | enabled = 1 23 | check_obsoletes = 1 24 | """ 25 | 26 | config_opts['dnf.conf'] = """ 27 | [main] 28 | keepcache=1 29 | debuglevel=2 30 | reposdir=/dev/null 31 | logfile=/var/log/yum.log 32 | retries=20 33 | obsoletes=1 34 | gpgcheck=0 35 | assumeyes=1 36 | plugins=1 37 | syslog_ident=mock 38 | syslog_device= 39 | mdpolicy=group:primary 40 | best=0 41 | protected_packages= 42 | module_platform_id=platform:el9 43 | user_agent={{ user_agent }} 44 | 45 | # repos 46 | 47 | [baseos] 48 | name=CentOS Stream $releasever - BaseOS 49 | metalink=https://mirrors.centos.org/metalink?repo=centos-baseos-9-stream&arch=$basearch&protocol=https,http 50 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial 51 | gpgcheck=0 52 | repo_gpgcheck=0 53 | metadata_expire=6h 54 | countme=1 55 | enabled=1 56 | 57 | [appstream] 58 | name=CentOS Stream $releasever - AppStream 59 | metalink=https://mirrors.centos.org/metalink?repo=centos-appstream-9-stream&arch=$basearch&protocol=https,http 60 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial 61 | gpgcheck=0 62 | repo_gpgcheck=0 63 | metadata_expire=6h 64 | countme=1 65 | enabled=1 66 | 67 | [crb] 68 | name=CentOS Stream $releasever - CRB 69 | metalink=https://mirrors.centos.org/metalink?repo=centos-crb-9-stream&arch=$basearch&protocol=https,http 70 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial 71 | gpgcheck=0 72 | repo_gpgcheck=0 73 | metadata_expire=6h 74 | countme=1 75 | enabled=1 76 | 77 | [highavailability] 78 | name=CentOS Stream $releasever - HighAvailability 79 | metalink=https://mirrors.centos.org/metalink?repo=centos-highavailability-9-stream&arch=$basearch&protocol=https,http 80 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial 81 | gpgcheck=0 82 | repo_gpgcheck=0 83 | metadata_expire=6h 84 | countme=1 85 | enabled=1 86 | """ 87 | -------------------------------------------------------------------------------- /scripts/recreate-promotion-symlinks.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | # This utility script will recreate all promotion symlinks from the database, 14 | # using the latest promotion for each. 15 | 16 | import argparse 17 | import os 18 | import sys 19 | 20 | from dlrn.db import Commit 21 | from dlrn.db import getSession 22 | from dlrn.db import Promotion 23 | from six.moves import configparser 24 | from sqlalchemy import desc 25 | 26 | 27 | def main(): 28 | parser = argparse.ArgumentParser() 29 | parser.add_argument('--config-file', 30 | help="Config file (required)", required=True) 31 | parser.add_argument('--noop', 32 | help="Preview actions but do not execute them", 33 | action="store_true") 34 | 35 | options = parser.parse_args(sys.argv[1:]) 36 | 37 | cp = configparser.RawConfigParser() 38 | cp.read(options.config_file) 39 | datadir = os.path.realpath(cp.get('DEFAULT', 'datadir')) 40 | session = getSession(cp.get('DEFAULT', 'database_connection')) 41 | 42 | # We need to find promotions, and for each promotion name create the 43 | # corresponding symlink 44 | promotions = session.query(Promotion.promotion_name).distinct() 45 | 46 | promotion_list = [] 47 | for promotion in promotions: 48 | promotion_list.append(promotion.promotion_name) 49 | 50 | # Find latest promotions for each promotion name, and re-do the symlinks 51 | for name in promotion_list: 52 | promotion = session.query(Promotion).\ 53 | order_by(desc(Promotion.timestamp)).\ 54 | filter(Promotion.promotion_name == name).first() 55 | 56 | commit = session.query(Commit).\ 57 | filter(Commit.id == promotion.commit_id).first() 58 | 59 | repo_dir = os.path.join(datadir, "repos", commit.getshardedcommitdir()) 60 | symlink_path = os.path.join(datadir, "repos", name) 61 | print("Going to symlink %s to %s" % (symlink_path, repo_dir)) 62 | 63 | if not options.noop: 64 | try: 65 | os.symlink(os.path.relpath(repo_dir, 66 | os.path.join(datadir, "repos")), 67 | symlink_path + "_") 68 | os.rename(symlink_path + "_", symlink_path) 69 | except Exception as e: 70 | print("Symlink creation failed: %s", e) 71 | 72 | 73 | if __name__ == '__main__': 74 | main() 75 | exit(0) 76 | -------------------------------------------------------------------------------- /dlrn/api/inputs/report_result.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | from pydantic import BaseModel 13 | from pydantic import root_validator 14 | from pydantic import StrictStr 15 | from typing import Optional 16 | 17 | from dlrn.api.utils import InvalidUsage 18 | 19 | 20 | class ReportResultInput(BaseModel): 21 | """Input class that validates request's arguments for report result. 22 | 23 | Either commit_hash+distro_hash or aggregate_hash must be provided 24 | :param int timestamp: CI execution timestamp 25 | :param bool success: Result success boolean 26 | :param str job_id: Result job_id identifier 27 | :param str url: URL where more information can be found 28 | :param str commit_hash: Result commit hash identifier 29 | :param str distro_hash: Result distro hash identifier 30 | :param str extended_hash: Result extended hash identifier 31 | :param str aggregate_hash: Result aggregate hash identifier 32 | :param str notes: Return metrics for package_name 33 | """ 34 | timestamp: int 35 | success: bool 36 | job_id: StrictStr 37 | url: str 38 | commit_hash: Optional[StrictStr] 39 | distro_hash: Optional[StrictStr] 40 | extended_hash: Optional[StrictStr] 41 | aggregate_hash: Optional[StrictStr] 42 | notes: Optional[StrictStr] = '' 43 | 44 | @root_validator 45 | def validate_hashes_logic(cls, values): 46 | if (not values.get('commit_hash') and not values.get('distro_hash') and 47 | not values.get('aggregate_hash')): 48 | raise InvalidUsage('"Either the aggregate hash or both commit_hash' 49 | ' and distro_hash must be provided"', 50 | status_code=400) 51 | 52 | if values.get('commit_hash') and not values.get('distro_hash'): 53 | raise InvalidUsage('If commit_hash is provided, distro_hash ' 54 | 'must be provided too', status_code=400) 55 | 56 | if values.get('distro_hash') and not values.get('commit_hash'): 57 | raise InvalidUsage('If distro_hash is provided, commit_hash ' 58 | 'must be provided too', status_code=400) 59 | 60 | if ((values.get('aggregate_hash') and values.get('distro_hash')) or 61 | (values.get('aggregate_hash') and values.get('commit_hash'))): 62 | raise InvalidUsage('aggregate_hash and commit/distro_hash cannot ' 63 | 'be combined', status_code=400) 64 | return values 65 | -------------------------------------------------------------------------------- /dlrn/tests/test_driver_local.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import mock 16 | import os 17 | import shutil 18 | import tempfile 19 | 20 | from six.moves import configparser 21 | 22 | from dlrn.config import ConfigOptions 23 | from dlrn.drivers.local import LocalDriver 24 | from dlrn.tests import base 25 | 26 | 27 | class TestDriverLocal(base.TestCase): 28 | def setUp(self): 29 | super(TestDriverLocal, self).setUp() 30 | config = configparser.RawConfigParser() 31 | config.read("projects.ini") 32 | config.set("DEFAULT", "pkginfo_driver", 33 | "dlrn.drivers.local.LocalDriver") 34 | self.config = ConfigOptions(config) 35 | self.config.datadir = tempfile.mkdtemp() 36 | self.base_dir = tempfile.mkdtemp() 37 | 38 | def tearDown(self): 39 | super(TestDriverLocal, self).tearDown() 40 | shutil.rmtree(self.config.datadir) 41 | shutil.rmtree(self.base_dir) 42 | 43 | @mock.patch('sh.git', create=True) 44 | def test_getinfo(self, sh_mock): 45 | driver = LocalDriver(cfg_options=self.config) 46 | package = { 47 | 'upstream': 'Unknown', 'name': 'hello-world', 48 | 'master-distgit': '/tmp/hello-world', 'source-branch': '1.0.0'} 49 | info, skipped = driver.getinfo( 50 | package=package, project="hello-world") 51 | self.assertEqual(len(info), 1) 52 | self.assertEqual(info[0].commit_hash, 'hello-world-1.0.0') 53 | self.assertEqual(info[0].commit_branch, '1.0.0') 54 | self.assertEqual(skipped, False) 55 | 56 | def test_getpackages(self): 57 | package = 'hello-world' 58 | distgit_dir = os.path.join(self.base_dir, package) 59 | os.mkdir(distgit_dir) 60 | os.mkdir(os.path.join(distgit_dir, '.git')) 61 | with open(os.path.join(distgit_dir, 'hello-world.spec'), 'w') as fp: 62 | fp.write('Version: 1.0.0') 63 | # We expect to call DLRN from the local distgit directory 64 | driver = LocalDriver(cfg_options=self.config) 65 | packages = driver.getpackages(src_dir=distgit_dir) 66 | self.assertEqual(len(packages), 1) 67 | expected_package = { 68 | 'maintainers': 'test@example.com', 69 | 'master-distgit': '%s/hello-world' % self.config.datadir, 70 | 'name': 'hello-world', 71 | 'source-branch': '1.0.0', 72 | 'upstream': 'Unknown'} 73 | self.assertDictEqual(packages[0], expected_package) 74 | -------------------------------------------------------------------------------- /scripts/redhat.cfg: -------------------------------------------------------------------------------- 1 | config_opts['root'] = 'dlrn-rhel8-x86_64' 2 | config_opts['target_arch'] = 'x86_64' 3 | config_opts['legal_host_arches'] = ('x86_64',) 4 | config_opts['chroot_setup_cmd'] = 'install basesystem rpm-build python3-devel gcc make python3-sqlalchemy python3-sphinx python3-eventlet python3-six python3-pbr openstack-macros git-core python3-setuptools_scm' 5 | # config_opts['module_enable'] = ['virt:8.0.0'] 6 | config_opts['dist'] = 'el8' # only useful for --resultdir variable subst 7 | config_opts['releasever'] = '8' 8 | config_opts['bootstrap_image'] = 'registry.access.redhat.com/ubi8/ubi' 9 | config_opts['plugin_conf']['ccache_enable'] = False 10 | config_opts['priorities.conf'] = """ 11 | [main] 12 | enabled = 1 13 | check_obsoletes = 1 14 | """ 15 | 16 | config_opts['yum.conf'] = """ 17 | [main] 18 | keepcache=1 19 | debuglevel=2 20 | reposdir=/dev/null 21 | logfile=/var/log/yum.log 22 | retries=20 23 | obsoletes=1 24 | gpgcheck=0 25 | assumeyes=1 26 | plugins=1 27 | syslog_ident=mock 28 | syslog_device= 29 | # best=0 is required as a workaround for bz#1677583 30 | best=0 31 | module_platform_id=platform:el8 32 | 33 | # repos 34 | 35 | # NOTE: the following repos are only available if you have a RHUI installation 36 | # in the cloud were you are running DLRN. If you want to use a different set 37 | # of base RHEL 8 repos, you will need to replace these and enable your own ones 38 | 39 | [rhui-custom-deps] 40 | name=Custom Repositories - deps 41 | mirrorlist=https://rhui-cds/pulp/mirror/protected/deps 42 | enabled=1 43 | gpgcheck=0 44 | sslverify=1 45 | sslcacert=/etc/pki/rhui/ca.crt 46 | sslclientcert=/etc/pki/rhui/product/content.crt 47 | sslclientkey=/etc/pki/rhui/key.pem 48 | 49 | [rhui-rhel-8-for-x86_64-appstream-rhui-rpms] 50 | name=Red Hat Enterprise Linux 8 for x86_64 - AppStream from RHUI (RPMs) 51 | mirrorlist=https://rhui-cds/pulp/mirror//content/dist/rhel8/rhui/$releasever/x86_64/appstream/os 52 | enabled=1 53 | gpgcheck=1 54 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release 55 | sslverify=1 56 | sslcacert=/etc/pki/rhui/ca.crt 57 | sslclientcert=/etc/pki/rhui/product/content.crt 58 | sslclientkey=/etc/pki/rhui/key.pem 59 | 60 | [rhui-rhel-8-for-x86_64-baseos-rhui-rpms] 61 | name=Red Hat Enterprise Linux 8 for x86_64 - BaseOS from RHUI (RPMs) 62 | mirrorlist=https://rhui-cds/pulp/mirror//content/dist/rhel8/rhui/$releasever/x86_64/baseos/os 63 | enabled=1 64 | gpgcheck=1 65 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release 66 | sslverify=1 67 | sslcacert=/etc/pki/rhui/ca.crt 68 | sslclientcert=/etc/pki/rhui/product/content.crt 69 | sslclientkey=/etc/pki/rhui/key.pem 70 | 71 | [rhui-rhel-8-for-x86_64-highavailability-rhui-rpms] 72 | name=Red Hat Enterprise Linux 8 for x86_64 - High Availability (RPMs) from RHUI 73 | mirrorlist=https://rhui-cds/pulp/mirror//content/dist/rhel8/rhui/$releasever/x86_64/highavailability/os 74 | enabled=1 75 | gpgcheck=1 76 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release 77 | sslverify=1 78 | sslcacert=/etc/pki/rhui/ca.crt 79 | sslclientcert=/etc/pki/rhui/product/content.crt 80 | sslclientkey=/etc/pki/rhui/key.pem 81 | """ 82 | -------------------------------------------------------------------------------- /dlrn/tests/test_prom_metrics.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import mock 14 | import os 15 | import tempfile 16 | 17 | from dlrn.api import app 18 | from dlrn.config import ConfigOptions 19 | from dlrn import db 20 | from dlrn.tests import base 21 | from dlrn import utils 22 | from six.moves import configparser 23 | 24 | 25 | def mocked_session(url): 26 | db_fd, filepath = tempfile.mkstemp() 27 | session = db.getSession("sqlite:///%s" % filepath) 28 | utils.loadYAML(session, './dlrn/tests/samples/commits_1.yaml') 29 | return session 30 | 31 | 32 | def mock_opt(config_file): 33 | cp = configparser.RawConfigParser() 34 | cp.read(config_file) 35 | co = ConfigOptions(cp) 36 | co.baseurl = 'http://localhost/worker' 37 | return co 38 | 39 | 40 | class DLRNPrometheusMetricsTestCase(base.TestCase): 41 | def setUp(self): 42 | super(DLRNPrometheusMetricsTestCase, self).setUp() 43 | self.db_fd, self.filepath = tempfile.mkstemp() 44 | app.config['DB_PATH'] = "sqlite:///%s" % self.filepath 45 | app.config['REPO_PATH'] = '/tmp' 46 | self.app = app.test_client() 47 | self.app.testing = True 48 | 49 | def tearDown(self): 50 | os.close(self.db_fd) 51 | os.unlink(self.filepath) 52 | super(DLRNPrometheusMetricsTestCase, self).tearDown() 53 | 54 | 55 | @mock.patch('dlrn.api.prom_metrics._get_config_options', side_effect=mock_opt) 56 | @mock.patch('dlrn.api.prom_metrics.getSession', side_effect=mocked_session) 57 | class TestBasic(DLRNPrometheusMetricsTestCase): 58 | def test_query(self, db_mock, co_mock): 59 | response = self.app.get('/metrics') 60 | self.assertEqual(response.status_code, 200) 61 | self.assertEqual(response.mimetype, 'text/plain') 62 | 63 | def test_successful_buidls(self, db_mock, co_mock): 64 | response = self.app.get('/metrics') 65 | self.assertEqual(response.status_code, 200) 66 | # commits_1.yaml has 13 commits in SUCCESS state, 67 | # 5 in RETRY and 5 in FAILED 68 | self.assertIn('dlrn_builds_succeeded_total{baseurl="http://' 69 | 'localhost/worker"} 15.0', response.data.decode()) 70 | self.assertIn('dlrn_builds_failed_total{baseurl=' 71 | '"http://localhost/worker"} 5.0', response.data.decode()) 72 | self.assertIn('dlrn_builds_retry_total{baseurl=' 73 | '"http://localhost/worker"} 5.0', response.data.decode()) 74 | self.assertIn('dlrn_builds_total{baseurl="http://' 75 | 'localhost/worker"} 25.0', response.data.decode()) 76 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributing 3 | ============ 4 | 5 | Setting up a development environment in an OpenStack VM using cloud-init 6 | ------------------------------------------------------------------------ 7 | 8 | The following cloud-config script can be passed as a --user-data argument to 9 | `nova boot`. This will result in a fully operational DLRN environment to 10 | hack on. 11 | 12 | .. code-block:: yaml 13 | 14 | #cloud-config 15 | disable_root: 0 16 | 17 | users: 18 | - default 19 | 20 | package_upgrade: true 21 | 22 | packages: 23 | - vim 24 | - git 25 | - policycoreutils-python-utils 26 | 27 | runcmd: 28 | - yum -y install epel-release 29 | - yum -y install puppet 30 | - git clone https://github.com/rdo-infra/puppet-dlrn /root/puppet-dlrn 31 | - cd /root/puppet-dlrn 32 | - puppet module build 33 | - puppet module install pkg/jpena-dlrn-*.tar.gz 34 | - cp /root/puppet-dlrn/examples/common.yaml /var/lib/hiera 35 | - puppet apply --debug /root/puppet-dlrn/examples/site.pp 2>&1 | tee /root/puppet.log 36 | 37 | final_message: "DLRN installed, after $UPTIME seconds." 38 | 39 | Setting up a development environment manually 40 | --------------------------------------------- 41 | 42 | Follow the instructions from the `Setup section `_ of `README.rst `_ to manually setup a development environment. 43 | 44 | Submitting pull requests 45 | ------------------------ 46 | 47 | Pull requests submitted through GitHub will be ignored. They should be sent 48 | to SoftwareFactory's Gerrit instead, using git-review. The usual workflow is: 49 | 50 | .. code-block:: bash 51 | 52 | $ sudo yum install git-review (you can also use pip install if needed) 53 | $ git clone https://github.com/softwarefactory-project/DLRN 54 | 55 | $ git add 56 | $ git commit 57 | $ git review 58 | 59 | New patches should: 60 | - Explain the changes in the commit message. 61 | - Include unit tests for the new code to prove that it works 62 | correctly and to prevent regressions. 63 | - For changes visible to end-users (e.g. new API) we need to update the necessary RST files under the doc/ folder. 64 | - Execute all CI checks without errors by executing the command tox. 65 | 66 | Once submitted, your change request will show up here: 67 | 68 | https://softwarefactory-project.io/r/#/q/project:DLRN+status:open 69 | 70 | At least the approval from one core reviewer is needed to merge a change request. 71 | 72 | Generating the documentation 73 | ---------------------------- 74 | 75 | Please note that the `RDO Packaging Documentation 76 | `_ also contains 77 | instructions for DLRN. 78 | 79 | The documentation is generated with `Sphinx `_. To generate 80 | the documentation, go to the documentation directory and run the make file: 81 | 82 | .. code-block:: bash 83 | 84 | $ cd DLRN/doc/source 85 | $ make html 86 | 87 | The output will be in DLRN/doc/build/html 88 | 89 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ==== 2 | DLRN 3 | ==== 4 | 5 | DLRN builds and maintains yum repositories following OpenStack 6 | upstream commit streams. (DLRN is not an acronym or an abbreviation, 7 | and it can be pronounced "dee el arr en".) 8 | 9 | Documentation is available at 10 | http://dlrn.readthedocs.org/en/latest/ 11 | 12 | Setup 13 | ----- 14 | 15 | .. code-block:: shell-session 16 | 17 | # yum install git createrepo python-virtualenv mock gcc \ 18 | redhat-rpm-config rpmdevtools httpd libffi-devel \ 19 | openssl-devel yum-utils openldap-devel krb5-devel 20 | 21 | Add the user you intend to run as to the mock group and login again. 22 | 23 | .. code-block:: shell-session 24 | 25 | $ git clone https://github.com/softwarefactory-project/DLRN.git 26 | 27 | If you want to serve the built packages and the status reports, enable the 28 | httpd service, and then either add a section in the server configuration to 29 | map a URL to the data directories, or create a symbolic link: 30 | 31 | .. code-block:: shell-session 32 | 33 | # systemctl start httpd 34 | # cd /var/www/html 35 | # ln -s /repos . 36 | 37 | Preparing 38 | --------- 39 | 40 | .. code-block:: shell-session 41 | 42 | $ cd DLRN 43 | $ virtualenv ../dlrn-venv 44 | $ . ../dlrn-venv/bin/activate 45 | $ pip install --upgrade pip 46 | $ pip install -r requirements.txt 47 | $ python setup.py develop 48 | 49 | 50 | Edit ``projects.ini`` if needed. 51 | 52 | Bootstrapping 53 | ------------- 54 | 55 | Some of the projects require others to build. As a result, use the 56 | special option ``--order`` to build in the order computed from the 57 | BuildRequires and Requires fields of the spec files when you bootstrap 58 | your repository. 59 | 60 | .. code-block:: shell-session 61 | 62 | $ dlrn --order 63 | 64 | When using this special option, a special variable ``repo_bootstrap`` 65 | is defined in the specs, with a value of 1. You can use this variable if 66 | needed, to break dependency loops between packages. For example: 67 | 68 | .. code-block:: spec 69 | 70 | %if 0%{?repo_bootstrap} == 0 71 | BuildRequires: package-with-circular-dependency 72 | %endif 73 | 74 | Running 75 | ------- 76 | 77 | Once all the packages have been built once, you can get back to build 78 | the packages in the order of the timestamps of the commits. 79 | 80 | .. code-block:: shell-session 81 | 82 | $ dlrn 83 | 84 | Troubleshooting 85 | --------------- 86 | 87 | If you interrupt dlrn during mock build you might get an error 88 | 89 | .. code-block:: shell-session 90 | 91 | OSError: [Errno 16] Device or resource busy: '/var/lib/mock/dlrn-fedora-x86_64/root/var/cache/yum' 92 | 93 | Solution is to clear left-over bind mount as root: 94 | 95 | .. code-block:: shell-session 96 | 97 | # umount /var/lib/mock/dlrn-fedora-x86_64/root/var/cache/yum 98 | 99 | Other requirements 100 | ------------------ 101 | 102 | If the git clone operation fails for a package, DLRN will try to remove 103 | the source directory using sudo. Please make sure the user running DLRN 104 | can run ``rm -rf /path/to/dlrn/data/*`` without being asked for a password, 105 | otherwise DLRN will fail to process new commits. 106 | -------------------------------------------------------------------------------- /scripts/centos-stream-10.cfg: -------------------------------------------------------------------------------- 1 | config_opts['root'] = 'dlrn-centos-stream-10-x86_64' 2 | config_opts['target_arch'] = 'x86_64' 3 | config_opts['legal_host_arches'] = ('x86_64',) 4 | config_opts['chroot_setup_cmd'] = 'install basesystem rpm-build python3-devel gcc make redhat-rpm-config redhat-release which xz python3-six \ 5 | python3-pbr git-core sed bzip2 gzip gcc-c++ tar coreutils unzip shadow-utils diffutils cpio bash gawk info patch util-linux findutils grep \ 6 | python3-setuptools_scm' 7 | config_opts['dist'] = 'el10' # only useful for --resultdir variable subst 8 | config_opts['releasever'] = '10' 9 | config_opts['plugin_conf']['ccache_enable'] = False 10 | config_opts['package_manager'] = 'dnf' 11 | config_opts['isolation'] = 'simple' 12 | config_opts['extra_chroot_dirs'] = [ '/run/lock', ] 13 | config_opts['use_bootstrap_container'] = True 14 | config_opts['use_bootstrap_image'] = True 15 | config_opts['bootstrap_image'] = 'quay.io/centos/centos:stream10-development' 16 | config_opts['dnf_vars'] = { 'stream': '10-stream', 17 | 'contentdir': 'centos', 18 | } 19 | config_opts['dnf_install_command'] = 'install dnf dnf-plugins-core' 20 | config_opts['priorities.conf'] = """ 21 | [main] 22 | enabled = 1 23 | check_obsoletes = 1 24 | """ 25 | 26 | config_opts['dnf.conf'] = """ 27 | [main] 28 | keepcache=1 29 | debuglevel=2 30 | reposdir=/dev/null 31 | logfile=/var/log/yum.log 32 | retries=20 33 | obsoletes=1 34 | gpgcheck=0 35 | assumeyes=1 36 | plugins=1 37 | syslog_ident=mock 38 | syslog_device= 39 | mdpolicy=group:primary 40 | best=0 41 | protected_packages= 42 | module_platform_id=platform:el10 43 | user_agent={{ user_agent }} 44 | 45 | # repos 46 | 47 | [baseos] 48 | name=CentOS Stream $releasever - BaseOS 49 | #metalink=https://mirrors.centos.org/metalink?repo=centos-baseos-10-stream&arch=$basearch&protocol=https,http 50 | baseurl=http://mirror.stream.centos.org/10-stream/BaseOS/$basearch/os/ 51 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial 52 | gpgcheck=0 53 | repo_gpgcheck=0 54 | metadata_expire=6h 55 | countme=1 56 | enabled=1 57 | 58 | [appstream] 59 | name=CentOS Stream $releasever - AppStream 60 | #metalink=https://mirrors.centos.org/metalink?repo=centos-appstream-10-stream&arch=$basearch&protocol=https,http 61 | baseurl=http://mirror.stream.centos.org/10-stream/AppStream/$basearch/os/ 62 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial 63 | gpgcheck=0 64 | repo_gpgcheck=0 65 | metadata_expire=6h 66 | countme=1 67 | enabled=1 68 | 69 | [crb] 70 | name=CentOS Stream $releasever - CRB 71 | #metalink=https://mirrors.centos.org/metalink?repo=centos-crb-10-stream&arch=$basearch&protocol=https,http 72 | baseurl=http://mirror.stream.centos.org/10-stream/CRB/$basearch/os/ 73 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial 74 | gpgcheck=0 75 | repo_gpgcheck=0 76 | metadata_expire=6h 77 | countme=1 78 | enabled=1 79 | 80 | [highavailability] 81 | name=CentOS Stream $releasever - HighAvailability 82 | #metalink=https://mirrors.centos.org/metalink?repo=centos-highavailability-10-stream&arch=$basearch&protocol=https,http 83 | baseurl=http://mirror.stream.centos.org/10-stream/HighAvailability/$basearch/os/ 84 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial 85 | gpgcheck=0 86 | repo_gpgcheck=0 87 | metadata_expire=6h 88 | countme=1 89 | enabled=1 90 | """ 91 | -------------------------------------------------------------------------------- /dlrn/templates/report.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {{ project_name }} Packaging By DLRN 6 | 7 | 8 | 9 | 10 | 11 |

12 | 13 | {{ reponame | capitalize }} - {{ target | capitalize }} ({{ src }}) 14 |

15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | {% if config_options.pkginfo_driver == "dlrn.drivers.downstream.DownstreamInfoDriver" %} 27 | 28 | {% endif %} 29 | 30 | 31 | 32 | {% for commit in commits %} 33 | 34 | 35 | 36 | 37 | 43 | 44 | 51 | 55 | 58 | {% if config_options.pkginfo_driver == "dlrn.drivers.downstream.DownstreamInfoDriver" %} 59 | {% if commit.versions_csv != None %} 60 | 61 | {% else %} 62 | 63 | {% endif %} 64 | {% endif %} 65 | 66 | {% endfor %} 67 | 68 |
Build Date TimeCommit Date TimeProject NameCommit HashComponentStatusRepositoryBuild LogImport Source
{{ commit.dt_build | strftime }}{{ commit.dt_commit | strftime }}{{ commit.project_name }} 38 | 39 | 40 | {{ commit.commit_hash }} 41 | 42 | {{ commit.component }} 45 | {% if commit.status == "SUCCESS" %} 46 | SUCCESS 47 | {% else %} 48 | FAILED 49 | {% endif %} 50 | 52 | 53 | repo 54 | 56 | build log 57 | {{ commit.versions_csv.replace("/versions.csv","") }}
69 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /dlrn/api/templates/votes_general.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | CI Votes 5 | 6 | 7 | 8 | 9 | 10 | 11 |

12 | 13 | {{ target | capitalize }} 14 |

15 | 16 |
17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | {% for repo in repodetail %} 29 | 30 | 37 | 43 | 44 | 52 | 60 | 61 | {% endfor %} 62 | 63 |
Repo hash (distro_commit)ComponentLast timestampSuccessful jobsFailed jobs
31 |
32 | 33 | 34 | 35 |
36 | 		38 |
39 | 40 | 41 |
42 | 		{{ repo.timestamp | strftime }} 45 |
46 | 47 | 48 | 49 | 50 |
51 | 		53 |
54 | 55 | 56 | 57 | 58 |
59 |
64 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/2a0313a8a7d6_change_user_usernames_column_length.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """change user/usernames column length 14 | 15 | Revision ID: 2a0313a8a7d6 16 | Revises: cab7697f6564 17 | Create Date: 2018-02-27 15:35:04.667089 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | 24 | # revision identifiers, used by Alembic. 25 | revision = '2a0313a8a7d6' 26 | down_revision = 'ade85b2396bc' 27 | branch_labels = None 28 | depends_on = None 29 | 30 | 31 | def upgrade(): 32 | with op.batch_alter_table("civotes") as batch_op: 33 | batch_op.drop_constraint('civ_user_fk', type_='foreignkey') 34 | 35 | with op.batch_alter_table("promotions") as batch_op: 36 | batch_op.drop_constraint('prom_user_fk', type_='foreignkey') 37 | 38 | with op.batch_alter_table("users") as batch_op: 39 | batch_op.alter_column('username', existing_type=sa.String(256), 40 | type_=sa.String(255)) 41 | 42 | with op.batch_alter_table("civotes") as batch_op: 43 | batch_op.alter_column('user', existing_type=sa.String(256), 44 | type_=sa.String(255)) 45 | batch_op.create_foreign_key( 46 | constraint_name="civ_user_fk", 47 | referent_table="users", 48 | local_cols=["user"], 49 | remote_cols=["username"]) 50 | 51 | with op.batch_alter_table("promotions") as batch_op: 52 | batch_op.alter_column('user', existing_type=sa.String(256), 53 | type_=sa.String(255)) 54 | batch_op.create_foreign_key( 55 | constraint_name="prom_user_fk", 56 | referent_table="users", 57 | local_cols=["user"], 58 | remote_cols=["username"]) 59 | 60 | 61 | def downgrade(): 62 | with op.batch_alter_table("civotes") as batch_op: 63 | batch_op.drop_constraint('civ_user_fk', type_='foreignkey') 64 | 65 | with op.batch_alter_table("promotions") as batch_op: 66 | batch_op.drop_constraint('prom_user_fk', type_='foreignkey') 67 | 68 | with op.batch_alter_table("users") as batch_op: 69 | batch_op.alter_column('username', existing_type=sa.String(255), 70 | type_=sa.String(256)) 71 | 72 | with op.batch_alter_table("civotes") as batch_op: 73 | batch_op.alter_column('user', existing_type=sa.String(255), 74 | type_=sa.String(256)) 75 | batch_op.create_foreign_key( 76 | constraint_name="civ_user_fk", 77 | referent_table="users", 78 | local_cols=["user"], 79 | remote_cols=["username"]) 80 | 81 | with op.batch_alter_table("promotions") as batch_op: 82 | batch_op.alter_column('user', existing_type=sa.String(255), 83 | type_=sa.String(256)) 84 | batch_op.create_foreign_key( 85 | constraint_name="prom_user_fk", 86 | referent_table="users", 87 | local_cols=["user"], 88 | remote_cols=["username"]) 89 | -------------------------------------------------------------------------------- /dlrn/templates/status_report.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {{ project_name }} Packaging By DLRN 6 | 7 | 8 | 9 | 10 | 11 |

12 | 13 | {{ reponame | capitalize }} - {{ target | capitalize }} ({{ src }}) 14 |

15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | {% for pkg in pkgs %} 28 | 29 | 30 | 33 | 36 | {% if pkg["last_build"].status == "SUCCESS" %} 37 | 41 | 42 | 43 | {% elif pkg["last_build"].status == "RETRY" %} 44 | 48 | 49 | 50 | {% else %} 51 | 55 | 63 | 70 | {% endif %} 71 | 72 | {% endfor %} 73 | 74 |
Project NameNVRTarball SHAStatusFirst failure after success {{ msg }}Number of days since last success
{{ pkg["name"] }} 31 | {{ pkg["last_build"].artifacts | getNVRfrompkgsrpm }} 32 | 34 | {{ pkg["last_build"].commit_hash }} 35 | 38 | 39 | SUCCESS 40 | 45 | 46 | RETRY 47 | 52 | 53 | FAILED 54 | 56 | {% if "first_failure" in pkg %} 57 | 58 | {{ pkg["first_failure"].commit_hash }} (build log) 59 | {% else %} 60 | ?????? 61 | {% endif %} 62 | 64 | {% if pkg["days"] == -1 %} 65 | Never 66 | {% else %} 67 | {{ pkg["days"] }} days 68 | {% endif %} 69 |
75 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /dlrn/api/inputs/promotions.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | from pydantic import BaseModel 13 | from pydantic import NonNegativeInt 14 | from pydantic import root_validator 15 | from pydantic import StrictStr 16 | from pydantic import validator 17 | from typing import Optional 18 | 19 | from dlrn.api.utils import InvalidUsage 20 | 21 | MAX_LIMIT = 100 22 | 23 | 24 | class BasePromotion(BaseModel): 25 | """Base class for Promotion related endpoints 26 | 27 | 28 | :param str commit_hash(optional) A reference to the commit 29 | :param str distro_hash(optional): A reference to the distro 30 | :param str extended_hash(optional): A reference to the extended commit 31 | """ 32 | commit_hash: Optional[StrictStr] = None 33 | distro_hash: Optional[StrictStr] = None 34 | extended_hash: Optional[StrictStr] = None 35 | 36 | 37 | class PromotionsInput(BasePromotion): 38 | """Input class that validates request's arguments for last_tested_repo 39 | 40 | :param str aggregate_hash(optional): A reference to the aggregate hash 41 | :param str promote_name(optional): Only report promotions for promote_name 42 | :param int offset(optional): Skip the first X promotions 43 | (only 100 are shown per query) 44 | :param int limit(optional): Maximum number of entries to return 45 | :param str component(optional): Only report promotions for this component 46 | """ 47 | aggregate_hash: Optional[StrictStr] = None 48 | promote_name: Optional[StrictStr] = None 49 | offset: Optional[NonNegativeInt] = None 50 | limit: Optional[NonNegativeInt] = None 51 | component: Optional[StrictStr] = None 52 | 53 | @validator('limit') 54 | @classmethod 55 | def validate_limit(cls, limit): 56 | # Make sure we do not exceed maximum 57 | if int(limit) > MAX_LIMIT: 58 | limit = MAX_LIMIT 59 | return int(limit) 60 | 61 | @root_validator 62 | def validate_distro_commit_hash(cls, values): 63 | if (values.get("distro_hash") and not values.get("commit_hash")) or \ 64 | (not values.get("distro_hash") and values.get("commit_hash")): 65 | raise InvalidUsage('Both commit_hash and distro_hash must be ' 66 | 'specified if one of them is.', 67 | status_code=400) 68 | return values 69 | 70 | 71 | class PromoteInput(BasePromotion): 72 | """Input class that validates request's arguments for promote 73 | 74 | :param str promote_name: Symlink name 75 | """ 76 | promote_name: StrictStr 77 | 78 | @validator('promote_name') 79 | @classmethod 80 | def validate_promote_name(cls, promote_name): 81 | if (promote_name == 'consistent' or promote_name == 'current'): 82 | raise InvalidUsage('Invalid promote_name %s' % promote_name, 83 | status_code=403) 84 | return promote_name 85 | 86 | @root_validator 87 | def validate_distro_commit_hash(cls, values): 88 | if not values.get("distro_hash") or not values.get("commit_hash"): 89 | raise InvalidUsage('Both commit_hash and distro_hash must be ' 90 | 'specified.', status_code=400) 91 | return values 92 | -------------------------------------------------------------------------------- /dlrn/api/prom_metrics.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | from dlrn.api import app 14 | from dlrn.config import ConfigOptions 15 | from dlrn.db import closeSession 16 | from dlrn.db import Commit 17 | from dlrn.db import getSession 18 | 19 | from flask import g 20 | from flask import Response 21 | 22 | from prometheus_client.core import CounterMetricFamily 23 | from prometheus_client.core import REGISTRY 24 | from prometheus_client import generate_latest 25 | from prometheus_client import Summary 26 | 27 | from six.moves import configparser 28 | 29 | 30 | # Create a metric to track time spent and requests made. 31 | REQUEST_TIME = Summary('dlrn_request_processing_seconds', 32 | 'Time spent processing request') 33 | 34 | 35 | def _get_db(): 36 | if 'db' not in g: 37 | g.db = getSession(app.config['DB_PATH']) 38 | return g.db 39 | 40 | 41 | def _get_config_options(config_file): 42 | cp = configparser.RawConfigParser() 43 | cp.read(config_file) 44 | return ConfigOptions(cp) 45 | 46 | 47 | class DLRNPromCollector(object): 48 | @REQUEST_TIME.time() 49 | def collect(self): 50 | config_options = _get_config_options(app.config['CONFIG_FILE']) 51 | c_success = CounterMetricFamily('dlrn_builds_succeeded', 52 | 'Total number of successful builds', 53 | labels=['baseurl']) 54 | c_failed = CounterMetricFamily('dlrn_builds_failed', 55 | 'Total number of failed builds', 56 | labels=['baseurl']) 57 | c_retry = CounterMetricFamily('dlrn_builds_retry', 58 | 'Total number of builds in retry state', 59 | labels=['baseurl']) 60 | c_overall = CounterMetricFamily('dlrn_builds', 61 | 'Total number of builds', 62 | labels=['baseurl']) 63 | 64 | # Find the commits count for each metric 65 | with app.app_context(): 66 | session = _get_db() 67 | successful_commits = session.query(Commit).filter( 68 | Commit.status == 'SUCCESS').count() 69 | failed_commits = session.query(Commit).filter( 70 | Commit.status == 'FAILED').count() 71 | retried_commits = session.query(Commit).filter( 72 | Commit.status == 'RETRY').count() 73 | all_commits = session.query(Commit).count() 74 | 75 | c_success.add_metric([config_options.baseurl], successful_commits) 76 | c_failed.add_metric([config_options.baseurl], failed_commits) 77 | c_retry.add_metric([config_options.baseurl], retried_commits) 78 | c_overall.add_metric([config_options.baseurl], all_commits) 79 | 80 | return [c_success, c_failed, c_retry, c_overall] 81 | 82 | 83 | REGISTRY.register(DLRNPromCollector()) 84 | 85 | 86 | @app.route('/metrics', methods=['GET']) 87 | def prom_metrics(): 88 | return Response(generate_latest(), mimetype='text/plain') 89 | 90 | 91 | @app.teardown_appcontext 92 | def teardown_db(exception=None): 93 | session = g.pop('db', None) 94 | if session is not None: 95 | closeSession(session) 96 | -------------------------------------------------------------------------------- /.zuul.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - job: 3 | name: dlrn-base 4 | parent: base 5 | pre-run: playbooks/prepare.yaml 6 | post-run: playbooks/retrieve-logs.yaml 7 | irrelevant-files: 8 | - ^doc/.*$ 9 | nodeset: 10 | nodes: 11 | - name: test-node 12 | label: cloud-centos-9-stream 13 | 14 | - job: 15 | name: dlrn-rpmbuild-centos-9 16 | parent: dlrn-base 17 | description: Run DLRN to build a package using Python 3 on Centos9 18 | run: playbooks/rpmbuild.yaml 19 | vars: 20 | python_version: py39 21 | tag: '' 22 | centos_version: 'centos9' 23 | nodeset: 24 | nodes: 25 | - name: cloud-centos-9-stream 26 | label: cloud-centos-9-stream 27 | 28 | - job: 29 | name: dlrn-api-functional 30 | parent: dlrn-base 31 | description: Run a DLRN functional test, using the API and dlrnapi_client 32 | run: playbooks/dlrn-api-functional.yaml 33 | post-run: playbooks/dlrn-api-functional-getlogs.yaml 34 | timeout: 3600 35 | required-projects: 36 | - DLRN 37 | - dlrnapi_client 38 | 39 | - job: 40 | name: dlrn-api-functional-centos-9 41 | parent: dlrn-api-functional 42 | nodeset: 43 | nodes: 44 | - name: cloud-centos-9-stream 45 | label: cloud-centos-9-stream 46 | 47 | - job: 48 | name: docs-build 49 | description: Build the DLRN docs 50 | run: playbooks/dlrndocbuild.yaml 51 | success-url: 'docs-html/index.html' 52 | required-projects: 53 | - DLRN 54 | files: 55 | - ^doc/.*$ 56 | nodeset: 57 | nodes: 58 | - name: container 59 | label: zuul-worker-f40 60 | 61 | - job: 62 | name: tox-py312-ci-config 63 | parent: tox-pre 64 | pre-run: playbooks/krb5-devel-install.yaml 65 | run: playbooks/tox_run.yaml 66 | vars: 67 | tox_envlist: py312 68 | 69 | - job: 70 | name: py3-bandit-SAST 71 | parent: tox 72 | vars: 73 | tox_envlist: bandit 74 | 75 | - job: 76 | name: pip-audit-cve 77 | parent: tox 78 | vars: 79 | tox_envlist: pip-audit 80 | voting: false 81 | 82 | - project: 83 | name: DLRN 84 | vars: 85 | rtd_webhook_id: '101768' 86 | rtd_project_name: 'dlrn' 87 | check: 88 | jobs: 89 | - tox-pep8: 90 | nodeset: 91 | nodes: 92 | - name: zuul-worker-f40 93 | label: zuul-worker-f40 94 | - tox-py312-ci-config: 95 | nodeset: 96 | nodes: 97 | - name: zuul-worker-f40 98 | label: zuul-worker-f40 99 | vars: 100 | tox_install_bindep: false 101 | - py3-bandit-SAST: 102 | nodeset: 103 | nodes: 104 | - name: zuul-worker-f40 105 | label: zuul-worker-f40 106 | - pip-audit-cve: 107 | nodeset: 108 | nodes: 109 | - name: zuul-worker-f40 110 | label: zuul-worker-f40 111 | - dlrn-rpmbuild-centos-9 112 | - dlrn-api-functional-centos-9 113 | - docs-build 114 | gate: 115 | jobs: 116 | - tox-pep8: 117 | nodeset: 118 | nodes: 119 | - name: zuul-worker-f40 120 | label: zuul-worker-f40 121 | - tox-py312-ci-config: 122 | nodeset: 123 | nodes: 124 | - name: zuul-worker-f40 125 | label: zuul-worker-f40 126 | - dlrn-rpmbuild-centos-9 127 | post: 128 | jobs: 129 | - publish-readthedocs 130 | release: 131 | jobs: 132 | - upload-pypi: 133 | vars: 134 | release_python: python3 135 | twine_python: python3 136 | - publish-readthedocs 137 | -------------------------------------------------------------------------------- /dlrn/tests/samples/versions.csv: -------------------------------------------------------------------------------- 1 | Project,Source Repo,Source Sha,Dist Repo,Dist Sha,Status,Last Success Timestamp,Component,Extended Sha,Pkg NVR 2 | openstack-ironic,https://opendev.org/openstack/ironic,06895641fb8a44caf4574919bd518f0de76cba3d,https://github.com/rdo-packages/ironic-distgit.git,a971ec78bdb0978676e19ab31490c45ff3ae5410,SUCCESS,None,baremetal,None,openstack-ironic-21.1.1-0.20230110104845.0689564.el9 3 | openstack-ironic-inspector,https://opendev.org/openstack/ironic-inspector,6c3cbd1511b39e5a713477ff659c7d62b23250a7,https://github.com/rdo-packages/ironic-inspector-distgit.git,820cb024ce0fbdd122fd88daa9cc8560417bf666,SUCCESS,1671133807,baremetal,None,openstack-ironic-inspector-11.1.1-0.20221215195007.6c3cbd1.el9 4 | openstack-ironic-python-agent,https://opendev.org/openstack/ironic-python-agent,72dd56920bb50160b388e417f80c774fa37f5a7a,https://github.com/rdo-packages/ironic-python-agent-distgit.git,f326b39f0ba9a428c9d1de6730b0f40195d734a8,SUCCESS,1671135579,baremetal,None,openstack-ironic-python-agent-9.1.1-0.20221215201939.72dd569.el9 5 | openstack-ironic-python-agent-builder,https://opendev.org/openstack/ironic-python-agent-builder,cb5efad2011da3cfed604fd7fd831ee32ff08e0f,https://github.com/rdo-packages/ironic-python-agent-builder-distgit.git,0d1fae34a9242cde711ed888a9e2355a8cef4966,SUCCESS,1672787935,baremetal,None,openstack-ironic-python-agent-builder-5.0.1-0.20230103231855.cb5efad.el9 6 | python-glean,https://opendev.org/opendev/glean,528f7216c64b459c2028a2ba81149548862b58b5,https://github.com/rdo-packages/glean-distgit.git,2c3515f3cd8a928aa8bc9f55b116ffa3254a0e9d,SUCCESS,1662796207,baremetal,None,python-glean-1.23.1-0.20220910075007.528f721.el9 7 | python-ironic-lib,https://opendev.org/openstack/ironic-lib,340a4b264a09c961aa5dfb7886e0e919daf1c051,https://github.com/rdo-packages/ironic-lib-distgit.git,fbe534a825e621d286dfe809e60e4abb8ba51085,SUCCESS,1663176597,baremetal,None,python-ironic-lib-5.3.0-0.20220914172957.340a4b2.el9 8 | python-networking-baremetal,https://opendev.org/openstack/networking-baremetal,c501fcb7093f0f759f1b0da159962424ddf1d62d,https://github.com/rdo-packages/networking-baremetal-distgit.git,1bf7775bfd51e11f7f7d1e0febf9f291d53b587d,SUCCESS,1663575828,baremetal,None,python-networking-baremetal-6.0.1-0.20220919082348.c501fcb.el9 9 | python-proliantutils,https://opendev.org/openstack/proliantutils,de9759c0d2d6242ba812c7ebf3a4a7613ec99754,https://github.com/rdo-packages/proliantutils-distgit.git,8ed06e4aff5ded1ffb5a3ff02e4481e3b19dcf76,SUCCESS,1669229336,baremetal,None,python-proliantutils-2.14.0-0.20221123184856.de9759c.el9 10 | python-dracclient,https://opendev.org/openstack/python-dracclient,ce67a01418a6abe24a2f79620769aea2f791c294,https://github.com/rdo-packages/dracclient-distgit.git,a71c78b0f9ce60ab4cb00c394fc32f4aaedf20d5,SUCCESS,1672916013,baremetal,None,python-dracclient-8.0.1-0.20230105105333.ce67a01.el9 11 | python-virtualbmc,https://opendev.org/openstack/virtualbmc,835fd3b9e4cbb33a297e979db93df55498ff6ce0,https://github.com/rdo-packages/virtualbmc-distgit.git,381c6c18db192df1d930cacb264bba30c4a754ef,SUCCESS,1663669129,baremetal,None,python-virtualbmc-2.2.2-0.20220920101849.835fd3b.el9 12 | openstack-ironic-staging-drivers,https://opendev.org/openstack/ironic-staging-drivers,6a2590ec6a079688e788d7d382fb9b6311e10cb2,https://github.com/rdo-packages/ironic-staging-drivers-distgit.git,8bd15b3bf548bc20a28ad20de38610e4cbfe85eb,SUCCESS,1673091523,baremetal,None,openstack-ironic-staging-drivers-0.17.1-0.20230107113843.6a2590e.el9 13 | python-sushy,https://opendev.org/openstack/sushy,6c521c5067f1042d161a7c9f04d3ed3480fd6d0e,https://github.com/rdo-packages/sushy-distgit.git,61f15bff8de06cb11d802576c5774ee7ac55248e,SUCCESS,1669628929,baremetal,None,python-sushy-4.3.2-0.20221128094849.6c521c5.el9 14 | openstack-nova,https://opendev.org/openstack/nova,c9de185ea1ac1e8d4435c5863b2ad7cefdb28c76,https://github.com/rdo-packages/nova-distgit.git,118992921c733bc0079e34dbde59cc8b3c1312dc,SUCCESS,1671296928,compute,None,openstack-nova-26.0.1-0.20221217170848.c9de185.el9 15 | -------------------------------------------------------------------------------- /dlrn/migrations/versions/f38ba3389b85_set_string_length.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Set string length 14 | 15 | Revision ID: f38ba3389b85 16 | Revises: 1268c799620f 17 | Create Date: 2017-01-12 18:42:25.593667 18 | 19 | """ 20 | 21 | from alembic import op 22 | import sqlalchemy as sa 23 | 24 | 25 | # revision identifiers, used by Alembic. 26 | revision = 'f38ba3389b85' 27 | down_revision = '1268c799620f' 28 | branch_labels = None 29 | depends_on = None 30 | 31 | 32 | def upgrade(): 33 | with op.batch_alter_table("commits") as batch_op: 34 | batch_op.alter_column('project_name', existing_type=sa.String(), 35 | type_=sa.String(256)) 36 | batch_op.alter_column('repo_dir', existing_type=sa.String(), 37 | type_=sa.String(1024)) 38 | batch_op.alter_column('distgit_dir', existing_type=sa.String(), 39 | type_=sa.String(1024)) 40 | batch_op.alter_column('commit_hash', existing_type=sa.String(), 41 | type_=sa.String(64)) 42 | batch_op.alter_column('distro_hash', existing_type=sa.String(), 43 | type_=sa.String(64)) 44 | batch_op.alter_column('distgit_dir', existing_type=sa.String(), 45 | type_=sa.String(1024)) 46 | batch_op.alter_column('commit_branch', existing_type=sa.String(), 47 | type_=sa.String(256)) 48 | batch_op.alter_column('status', existing_type=sa.String(), 49 | type_=sa.String(64)) 50 | batch_op.alter_column('rpms', existing_type=sa.String(), 51 | type_=sa.Text()) 52 | batch_op.alter_column('notes', existing_type=sa.String(), 53 | type_=sa.Text()) 54 | 55 | with op.batch_alter_table("projects") as batch_op: 56 | batch_op.alter_column('project_name', existing_type=sa.String(), 57 | type_=sa.String(256)) 58 | 59 | 60 | def downgrade(): 61 | with op.batch_alter_table("commits") as batch_op: 62 | batch_op.alter_column('project_name', existing_type=sa.String(256), 63 | type_=sa.String()) 64 | batch_op.alter_column('repo_dir', existing_type=sa.String(1024), 65 | type_=sa.String()) 66 | batch_op.alter_column('distgit_dir', existing_type=sa.String(1024), 67 | type_=sa.String()) 68 | batch_op.alter_column('commit_hash', existing_type=sa.String(64), 69 | type_=sa.String()) 70 | batch_op.alter_column('distro_hash', existing_type=sa.String(64), 71 | type_=sa.String()) 72 | batch_op.alter_column('distgit_dir', existing_type=sa.String(1024), 73 | type_=sa.String()) 74 | batch_op.alter_column('commit_branch', existing_type=sa.String(256), 75 | type_=sa.String()) 76 | batch_op.alter_column('status', existing_type=sa.String(64), 77 | type_=sa.String()) 78 | batch_op.alter_column('rpms', existing_type=sa.Text(), 79 | type_=sa.String()) 80 | batch_op.alter_column('notes', existing_type=sa.Text(), 81 | type_=sa.String()) 82 | 83 | with op.batch_alter_table("projects") as batch_op: 84 | batch_op.alter_column('project_name', existing_type=sa.String(256), 85 | type_=sa.String()) 86 | -------------------------------------------------------------------------------- /dlrn/tests/test_driver_copr.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import mock 16 | import os 17 | import shutil 18 | import tempfile 19 | 20 | from dlrn.config import ConfigOptions 21 | from dlrn.drivers.coprdriver import CoprBuildDriver 22 | from dlrn.tests import base 23 | from six.moves import configparser 24 | 25 | 26 | @mock.patch('sh.restorecon', create=True) 27 | @mock.patch('sh.env', create=True) 28 | class TestDriverCopr(base.TestCase): 29 | def setUp(self): 30 | super(TestDriverCopr, self).setUp() 31 | config = configparser.RawConfigParser() 32 | config.read("projects.ini") 33 | config.set("DEFAULT", "build_driver", 34 | "dlrn.drivers.coprdriver.CoprBuildDriver") 35 | config.set('coprbuild_driver', 'coprid', 'foo/repo') 36 | self.config = ConfigOptions(config) 37 | self.temp_dir = tempfile.mkdtemp() 38 | # Create fake src.rpm 39 | with open('%s/pkg.src.rpm' % self.temp_dir, 'a') as fp: 40 | fp.write('') 41 | # Create fake build and download logs 42 | with open("%s/coprbuild.log" % self.temp_dir, 'a') as fp: 43 | fp.write("Created builds: 1234") 44 | with open('%s/coprdownload.log' % self.temp_dir, 'a') as fp: 45 | fp.write('') 46 | # Create fake download file structure 47 | os.mkdir(os.path.join(self.temp_dir, '1234')) 48 | target_chroot = os.path.join( 49 | self.temp_dir, '1234', 'fedora-rawhide-i386') 50 | os.mkdir(target_chroot) 51 | with open('%s/state.log.gz' % target_chroot, 'a') as fp: 52 | fp.write('') 53 | with open('%s/pkg.rpm' % target_chroot, 'a') as fp: 54 | fp.write('') 55 | 56 | def tearDown(self): 57 | super(TestDriverCopr, self).tearDown() 58 | shutil.rmtree(self.temp_dir) 59 | 60 | def test_build_package(self, env_mock, rc_mock): 61 | driver = CoprBuildDriver(cfg_options=self.config) 62 | driver.build_package(output_directory=self.temp_dir) 63 | 64 | expected = [mock.call(['copr', 'build', 65 | self.config.coprid, 66 | '%s/pkg.src.rpm' % 67 | self.temp_dir], 68 | _err=driver._process_copr_output, 69 | _out=driver._process_copr_output, 70 | _cwd=self.temp_dir, 71 | _env=mock.ANY), 72 | mock.call(['copr', 'download-build', '-d', 73 | '%s/1234' % self.temp_dir, '1234'], 74 | _err=driver._process_copr_output, 75 | _out=driver._process_copr_output, 76 | _cwd=self.temp_dir, 77 | _env=mock.ANY)] 78 | # 1- copr build (handled by env_mock) 79 | # 2- copr download-build (handled by env_mock) 80 | # 3- restorecon (handled by rc_mock) 81 | self.assertEqual(env_mock.call_count, 2) 82 | self.assertEqual(rc_mock.call_count, 1) 83 | self.assertEqual(env_mock.call_args_list, expected) 84 | 85 | # Make sure output_dir is as expected 86 | content = os.listdir(self.temp_dir) 87 | self.assertIn('coprdownload.log', content) 88 | self.assertIn('state.log.gz', content) 89 | self.assertIn('pkg.rpm', content) 90 | self.assertIn('pkg.src.rpm', content) 91 | self.assertNotIn('1234', content) 92 | 93 | def test_driver_config(self, env_mock, rc_mock): 94 | self.assertEqual(self.config.coprid, 'foo/repo') 95 | -------------------------------------------------------------------------------- /dlrn/notifications.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import copy 14 | import jinja2 15 | import logging 16 | import os 17 | import sh 18 | import smtplib 19 | 20 | from dlrn.config import getConfigOptions 21 | from dlrn.reporting import get_commit_url 22 | from email.mime.text import MIMEText 23 | 24 | logger = logging.getLogger("dlrn-notifications") 25 | 26 | 27 | def submit_review(commit, packages, env_vars): 28 | config_options = getConfigOptions() 29 | datadir = os.path.realpath(config_options.datadir) 30 | scriptsdir = os.path.realpath(config_options.scriptsdir) 31 | yumrepodir = os.path.join("repos", commit.getshardedcommitdir()) 32 | 33 | project_name = commit.project_name 34 | 35 | for pkg in packages: 36 | if project_name == pkg['name']: 37 | break 38 | else: 39 | logger.error('Unable to find info for project' 40 | ' %s' % project) 41 | return 42 | 43 | url = (get_commit_url(commit, pkg) + commit.commit_hash) 44 | env_vars.append('GERRIT_URL=%s' % url) 45 | env_vars.append('GERRIT_LOG=%s/%s' % (config_options.baseurl, 46 | commit.getshardedcommitdir())) 47 | maintainers = ','.join(pkg['maintainers']) 48 | env_vars.append('GERRIT_MAINTAINERS=%s' % maintainers) 49 | env_vars.append('GERRIT_TOPIC=%s' % config_options.gerrit_topic) 50 | logger.info('Creating a gerrit review using ' 51 | 'GERRIT_URL=%s ' 52 | 'GERRIT_MAINTAINERS=%s ' % 53 | (url, maintainers)) 54 | 55 | run_cmd = [] 56 | if env_vars: 57 | for env_var in env_vars: 58 | run_cmd.append(env_var) 59 | 60 | run_cmd.extend([os.path.join(scriptsdir, "submit_review.sh"), 61 | project_name, os.path.join(datadir, yumrepodir), 62 | datadir, config_options.baseurl, 63 | os.path.realpath(commit.distgit_dir)]) 64 | sh.env(run_cmd, _timeout=300) 65 | 66 | 67 | def sendnotifymail(packages, commit): 68 | config_options = getConfigOptions() 69 | 70 | details = copy.copy( 71 | [package for package in packages 72 | if package["name"] == commit.project_name][0]) 73 | 74 | email_to = details['maintainers'] 75 | if not config_options.smtpserver: 76 | logger.info("Skipping notify email to %r" % email_to) 77 | return 78 | 79 | details["logurl"] = "%s/%s" % (config_options.baseurl, 80 | commit.getshardedcommitdir()) 81 | # Render the notification template 82 | jinja_env = jinja2.Environment(autoescape=True, 83 | loader=jinja2.FileSystemLoader( 84 | [config_options.templatedir])) 85 | jinja_template = jinja_env.get_template("notification_email.j2") 86 | error_body = jinja_template.render(details=details) 87 | 88 | msg = MIMEText(error_body) 89 | msg['Subject'] = '[dlrn] %s master package build failed' % \ 90 | commit.project_name 91 | 92 | email_from = 'no-reply@delorean.com' 93 | msg['From'] = email_from 94 | msg['To'] = "packagers" 95 | 96 | logger.info("Sending notify email to %r" % email_to) 97 | try: 98 | s = smtplib.SMTP(config_options.smtpserver) 99 | s.sendmail(email_from, email_to, msg.as_string()) 100 | s.quit() 101 | except smtplib.SMTPException as e: 102 | logger.error("An issue occured when sending" 103 | "notify email to %r (%s)" % (email_to, e)) 104 | finally: 105 | s.close() 106 | -------------------------------------------------------------------------------- /dlrn/tests/test_config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from dlrn.tests import base 16 | from six.moves import configparser 17 | 18 | from dlrn.config import ConfigOptions 19 | from dlrn.config import getConfigOptions 20 | import os 21 | 22 | 23 | class TestConfigOptions(base.TestCase): 24 | def setUp(self): 25 | super(TestConfigOptions, self).setUp() 26 | self.config = configparser.RawConfigParser() 27 | self.config.read("projects.ini") 28 | 29 | def test_without_gitrepo_driver(self): 30 | self.config.remove_section("gitrepo_driver") 31 | ConfigOptions(self.config) 32 | 33 | def test_with_gitrepo_driver(self): 34 | self.config.set("DEFAULT", "pkginfo_driver", 35 | "dlrn.drivers.gitrepo.GitRepoDriver") 36 | self.config.set("gitrepo_driver", "skip", "pkg1,pkg2") 37 | config = ConfigOptions(self.config) 38 | self.assertEqual(config.skip_dirs, ["pkg1", "pkg2"]) 39 | 40 | def test_without_rdoinfo_driver(self): 41 | self.config.remove_section("rdoinfo_driver") 42 | ConfigOptions(self.config) 43 | 44 | def test_with_rdoinfo_driver(self): 45 | self.config.set("DEFAULT", "pkginfo_driver", 46 | "dlrn.drivers.rdoinfo.RdoInfoDriver") 47 | self.config.set( 48 | "rdoinfo_driver", "repo", "https://test/test.git") 49 | config = ConfigOptions(self.config) 50 | self.assertEqual( 51 | config.rdoinfo_repo, 'https://test/test.git') 52 | 53 | def test_get_config_option(self): 54 | config = ConfigOptions(self.config) 55 | self.assertEqual(config, getConfigOptions()) 56 | 57 | def test_override(self): 58 | self.config.set("DEFAULT", "fallback_to_master", "1") 59 | self.config.set("DEFAULT", "workers", "7") 60 | config = ConfigOptions(self.config) 61 | # No changes 62 | self.assertEqual(config.fallback_to_master, True) 63 | self.assertEqual(config.workers, 7) 64 | 65 | overrides = ['DEFAULT.fallback_to_master=0', 66 | 'DEFAULT.workers=3'] 67 | config = ConfigOptions(self.config, overrides=overrides) 68 | # Overrides in effect 69 | self.assertEqual(config.fallback_to_master, False) 70 | self.assertEqual(config.workers, 3) 71 | 72 | def test_override_non_existing(self): 73 | overrides = ['DEFAULT.non_existing_option=0'] 74 | self.assertRaises(RuntimeError, ConfigOptions, 75 | self.config, overrides=overrides) 76 | 77 | def test_dynamic_dirs(self): 78 | config = ConfigOptions(self.config) 79 | self.assertEqual(config.scriptsdir, './scripts') 80 | self.assertEqual(config.configdir, './scripts') 81 | self.assertEqual(config.templatedir, './dlrn/templates') 82 | 83 | def test_detect_dirs(self): 84 | self.config = configparser.RawConfigParser() 85 | self.config.read("samples/projects.ini.detect") 86 | config = ConfigOptions(self.config) 87 | self.assertEqual( 88 | config.datadir, 89 | os.path.realpath(os.path.join( 90 | os.path.dirname(os.path.abspath(__file__)), 91 | "../../data"))) 92 | self.assertEqual( 93 | config.templatedir, 94 | os.path.realpath(os.path.join( 95 | os.path.dirname(os.path.abspath(__file__)), 96 | "../templates"))) 97 | self.assertEqual( 98 | config.scriptsdir, 99 | os.path.realpath(os.path.join( 100 | os.path.dirname(os.path.abspath(__file__)), 101 | "../../scripts"))) 102 | -------------------------------------------------------------------------------- /dlrn/tests/test_notifications.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import mock 14 | import os 15 | import shutil 16 | import tempfile 17 | 18 | from dlrn.config import ConfigOptions 19 | from dlrn import db 20 | from dlrn import notifications 21 | from dlrn.tests import base 22 | from six.moves import configparser 23 | 24 | 25 | class TestNotifications(base.TestCase): 26 | def setUp(self): 27 | super(TestNotifications, self).setUp() 28 | config = configparser.RawConfigParser() 29 | config.read("projects.ini") 30 | config.set('DEFAULT', 'datadir', tempfile.mkdtemp()) 31 | config.set('DEFAULT', 'scriptsdir', tempfile.mkdtemp()) 32 | config.set('DEFAULT', 'baseurl', "file://%s" % config.get('DEFAULT', 33 | 'datadir')) 34 | 35 | self.config = ConfigOptions(config) 36 | self.commit = db.Commit(dt_commit=123, project_name='foo', 37 | commit_hash='1c67b1ab8c6fe273d4e175a14f0df5' 38 | 'd3cbbd0edf', 39 | repo_dir='/home/dlrn/data/foo', 40 | distro_hash='c31d1b18eb5ab5aed6721fc4fad06c9' 41 | 'bd242490f', 42 | dt_distro=123, 43 | distgit_dir='/home/dlrn/data/foo_distro', 44 | commit_branch='master', dt_build=1441245153) 45 | self.packages = [{'upstream': 'https://github.com/openstack/foo', 46 | 'name': 'foo', 'maintainers': ['test@test.com'], 47 | 'master-distgit': 48 | 'https://github.com/rdo-packages/foo-distgit.git'}] 49 | 50 | def tearDown(self): 51 | super(TestNotifications, self).tearDown() 52 | shutil.rmtree(self.config.datadir) 53 | shutil.rmtree(self.config.scriptsdir) 54 | 55 | @mock.patch('sh.env', create=True) 56 | def test_submit_review(self, sh_mock): 57 | notifications.submit_review(self.commit, self.packages, ['FOO=BAR']) 58 | yumdir = os.path.join(self.config.datadir, 'repos', 59 | self.commit.getshardedcommitdir()) 60 | 61 | expected = [mock.call(['FOO=BAR', 'GERRIT_URL=https://github.com/' 62 | 'openstack/foo/commit/' + 63 | self.commit.commit_hash, 64 | 'GERRIT_LOG=%s/%s' % ( 65 | self.config.baseurl, 66 | self.commit.getshardedcommitdir() 67 | ), 68 | 'GERRIT_MAINTAINERS=test@test.com', 69 | 'GERRIT_TOPIC=rdo-FTBFS', 70 | os.path.join(self.config.scriptsdir, 71 | "submit_review.sh"), 72 | self.commit.project_name, yumdir, self.config.datadir, 73 | self.config.baseurl, 74 | os.path.realpath(self.commit.distgit_dir)], 75 | _timeout=300)] 76 | 77 | self.assertEqual(sh_mock.call_count, 1) 78 | self.assertEqual(sh_mock.call_args_list, expected) 79 | 80 | @mock.patch('smtplib.SMTP', create=True) 81 | def test_sendnotifymail_nomail(self, smtp_mock): 82 | notifications.sendnotifymail(self.packages, self.commit) 83 | # By default there is no smtpserver, so no calls 84 | self.assertEqual(smtp_mock.call_count, 0) 85 | 86 | @mock.patch('smtplib.SMTP', create=True) 87 | def test_sendnotifymail(self, smtp_mock): 88 | self.config.smtpserver = 'foo.example.com' 89 | notifications.sendnotifymail(self.packages, self.commit) 90 | self.assertEqual(smtp_mock.call_count, 1) 91 | -------------------------------------------------------------------------------- /scripts/build_srpm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | 3 | set -o pipefail 4 | 5 | # FIXME(hguemar): we need to document all parameters passed to this script 6 | shift # First parameter is TARGET, not needed here 7 | 8 | source $(dirname $0)/common-functions 9 | 10 | exec &> >(tee "${OUTPUT_DIRECTORY}/rpmbuild.log") 2>&1 11 | set -x 12 | 13 | for FILE in {test-,}requirements.txt 14 | do 15 | if [ -f ${FILE} ] 16 | then 17 | sed -i "s/; *python_version.*[!=<>]=\?.*//g" ${FILE} 18 | sed -i "s/; *sys_platform.*[!=<>]=\?.*//g" ${FILE} 19 | fi 20 | done 21 | 22 | cleanup_sdist 23 | 24 | detect_python 25 | setup_mock 26 | 27 | if [ -z "$DLRN_KEEP_SPEC_AS_IS" ]; then 28 | if [ "$DLRN_KEEP_VERSION" == "1" ]; then 29 | VERSION=$(grep -e '^Version:' ${DISTGIT_DIR}/*.spec | awk '{print $2}') 30 | fi 31 | 32 | # As a first step, calculate version and release 33 | detect_version_and_release 34 | 35 | # As a second step, generate tarball 36 | if [ -r setup.py -a ! -r metadata.json ]; then 37 | SOURCETYPE='tarball' 38 | /usr/bin/mock $MOCKOPTS --chroot "cd /var/tmp/pkgsrc && (([ -x /usr/bin/python3 ] && python3 setup.py sdist) || python setup.py sdist)" 39 | /usr/bin/mock $MOCKOPTS --copyout /var/tmp/pkgsrc/dist ${TOP_DIR}/dist 40 | elif [ -r *.gemspec ]; then 41 | SOURCETYPE='gem' 42 | /usr/bin/mock $MOCKOPTS --chroot "cd /var/tmp/pkgsrc && gem build $GEMSPEC" 43 | /usr/bin/mock $MOCKOPTS --copyout /var/tmp/pkgsrc/$PROJECT-$VERSION.gem ./$PROJECT-$VERSION.gem 44 | else 45 | SOURCETYPE='tarball' 46 | if [ -r metadata.json ]; then 47 | # Detect if this is an OpenStack puppet module 48 | # We know OpenStack puppet modules have a common style for metadata.json 49 | MODULE_NAME=$($PYTHON -c "import json; print(json.loads(open('metadata.json').read(-1))['name'])") 50 | if [[ "$MODULE_NAME" =~ openstack-* ]]; then 51 | TARNAME=$MODULE_NAME 52 | else 53 | TARNAME=$(git remote -v|head -1|awk '{print $2;}'|sed 's@.*/@@;s@\.git$@@') 54 | fi 55 | elif [ -r Modulefile ]; then 56 | TARNAME=$(git remote -v|head -1|awk '{print $2;}'|sed 's@.*/@@;s@\.git$@@') 57 | elif [ -r Kconfig -a -r Kbuild ]; then 58 | TARNAME=linux 59 | else 60 | TARNAME=${PROJECT_NAME} 61 | fi 62 | tar zcvf ${TOP_DIR}/${VERSION}.tar.gz --exclude=.git --transform="s@${PWD#/}@${TARNAME}-${version}@" --show-transformed-names $PWD 63 | mkdir -p ${TOP_DIR}/dist 64 | mv ${TOP_DIR}/${VERSION}.tar.gz ${TOP_DIR}/dist/ 65 | fi 66 | 67 | if [ "$SOURCETYPE" == 'gem' ]; then 68 | SOURCE=$(ls -l | grep '.gem$' | awk '{print $9}') 69 | SOURCEEXT='.gem' 70 | SOURCEPATH=$SOURCE 71 | else 72 | SOURCE=$(ls ${TOP_DIR}/dist | grep '.tar.gz') 73 | SOURCEEXT='.tar.gz' 74 | SOURCEPATH="${TOP_DIR}/dist/$SOURCE" 75 | fi 76 | SOURCEWITHREL=$(basename $SOURCE $SOURCEEXT)-$RELEASE$SOURCEEXT 77 | mv $SOURCEPATH ${TOP_DIR}/SOURCES/$SOURCEWITHREL 78 | fi 79 | 80 | cd ${DISTGIT_DIR} 81 | cp -a * ${TOP_DIR}/SOURCES/ 82 | cp *.spec ${TOP_DIR}/SPECS/ 83 | cd ${TOP_DIR}/SPECS/ 84 | 85 | if [ -z "$DLRN_KEEP_SPEC_AS_IS" ]; then 86 | grep -qc "^%define upstream_version.*" *.spec && \ 87 | sed -i -e "s/^%define upstream_version.*/%define upstream_version $UPSTREAMVERSION/" *.spec || \ 88 | sed -i -e "1i%define upstream_version $UPSTREAMVERSION\\" *.spec 89 | grep -qc "^%global dlrn .*" *.spec && \ 90 | sed -i -e "s/^%global dlrn .*/%global dlrn 1/" *.spec || \ 91 | sed -i -e "1i%global dlrn 1\\" *.spec 92 | grep -qc "^%global dlrn_nvr .*" *.spec && \ 93 | sed -i -e "s/^%global dlrn_nvr .*/%global dlrn_nvr $(basename $SOURCEWITHREL $SOURCEEXT)/" *.spec || \ 94 | sed -i -e "1i%global dlrn_nvr $(basename $SOURCEWITHREL $SOURCEEXT)\\" *.spec 95 | sed -i -e "s/UPSTREAMVERSION/$UPSTREAMVERSION/g" *.spec 96 | set_nvr_in_spec 97 | if [ "$DLRN_KEEP_TARBALL" != "1" ]; then 98 | sed -i -e "s/^\(Source\|Source0\):.*/\1: $SOURCEWITHREL/" *.spec 99 | fi 100 | if [ "$DLRN_KEEP_CHANGELOG" != "1" ]; then 101 | sed -i -e '/^%changelog.*/q' *.spec 102 | fi 103 | fi 104 | cat *.spec 105 | spectool -g -C ${TOP_DIR}/SOURCES *.spec 106 | /usr/bin/mock --buildsrpm ${MOCKOPTS} --spec *.spec --sources=${TOP_DIR}/SOURCES 107 | 108 | -------------------------------------------------------------------------------- /dlrn/drivers/local.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | # The LocalDriver provides the following: 14 | # 15 | # 1- A getpackages function based on finding all directories inside a specific 16 | # git repo, where each directory represents a package 17 | # 18 | # 2- A getinfo function based on a single-distgit repo paradigm 19 | 20 | import logging 21 | import os 22 | import sh 23 | import shutil 24 | 25 | from dlrn.db import Commit 26 | from dlrn.drivers.pkginfo import PkgInfoDriver 27 | from rdopkg.utils import specfile 28 | 29 | logging.basicConfig(level=logging.ERROR) 30 | logger = logging.getLogger("dlrn-local-driver") 31 | logger.setLevel(logging.INFO) 32 | 33 | 34 | class LocalDriver(PkgInfoDriver): 35 | DRIVER_CONFIG = { 36 | 'local_driver': { 37 | } 38 | } 39 | 40 | def _get_infos_from_pkg(self, packagepath): 41 | version = None 42 | package = None 43 | if not os.path.isdir(os.path.join(packagepath, '.git')): 44 | raise RuntimeError( 45 | 'The distgit directory must a be git repository') 46 | for pkgfile in os.listdir(packagepath): 47 | if pkgfile.endswith('.spec'): 48 | spec = specfile.Spec(fn=os.path.join(packagepath, pkgfile)) 49 | version = spec.get_tag('Version') 50 | package = pkgfile.replace('.spec', '') 51 | if not version: 52 | raise RuntimeError("No spec file or no version found") 53 | return package, version 54 | 55 | def getpackages(self, **kwargs): 56 | datadir = self.config_options.datadir 57 | # src_dir is only set for the test case. Normal behavior is 58 | # to look at the current directory. 59 | if 'src_dir' in kwargs: 60 | src_dir = kwargs['src_dir'] 61 | else: 62 | src_dir = os.getcwd() 63 | is_src_dir_distgit = [ 64 | f for f in os.listdir(src_dir) if f.endswith('.spec')] 65 | if not is_src_dir_distgit: 66 | return [] 67 | package, version = self._get_infos_from_pkg(src_dir) 68 | dest_dir = os.path.join(datadir, package) 69 | logger.info("Copy distgit source from %s to %s" % (src_dir, dest_dir)) 70 | if os.path.isdir(dest_dir): 71 | shutil.rmtree(dest_dir) 72 | shutil.copytree(src_dir, dest_dir) 73 | pkg_hash = {} 74 | pkg_hash['name'] = package 75 | pkg_hash['maintainers'] = 'test@example.com' 76 | pkg_hash['master-distgit'] = dest_dir 77 | pkg_hash['upstream'] = 'Unknown' 78 | logger.info( 79 | "Got version %s for %s from the spec" % (version, package)) 80 | pkg_hash['source-branch'] = version 81 | return [pkg_hash] 82 | 83 | def getinfo(self, **kwargs): 84 | project = kwargs.get('project') 85 | package = kwargs.get('package') 86 | datadir = self.config_options.datadir 87 | distro_dir = os.path.join(datadir, package['name']) 88 | 89 | # Get distro_hash from last commit in distgit directory 90 | git = sh.git.bake(_cwd=package['master-distgit'], _tty_out=False) 91 | repoinfo = str(git.log("--pretty=format:%H %ct", "-1", ".") 92 | ).strip().split(" ") 93 | distro_hash = repoinfo[0] 94 | dt_distro = repoinfo[1] 95 | 96 | project_toprocess = [] 97 | commit_hash = "%s-%s" % (project, package['source-branch']) 98 | commit = Commit( 99 | dt_commit=0, project_name=project, 100 | type='rpm', 101 | commit_hash=commit_hash, repo_dir=None, 102 | distro_hash=distro_hash, dt_distro=dt_distro, 103 | distgit_dir=distro_dir, 104 | commit_branch=package['source-branch'], 105 | dt_extended=0, extended_hash=None, component=None) 106 | project_toprocess.append(commit) 107 | 108 | return PkgInfoDriver.Info(project_toprocess, False) 109 | 110 | def preprocess(self, **kwargs): 111 | pass 112 | -------------------------------------------------------------------------------- /playbooks/prepare.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | tasks: 4 | - name: Load operating system specific variables 5 | include_vars: "{{ item }}" 6 | failed_when: false 7 | loop: 8 | - "family-{{ ansible_os_family | lower }}.yml" 9 | - "family-{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" 10 | - "{{ ansible_distribution | lower | replace(' ', '-') }}.yml" 11 | - "{{ ansible_distribution | lower | replace(' ', '-') }}-{{ ansible_distribution_major_version | lower }}.yml" 12 | - "{{ ansible_distribution | lower | replace(' ', '-') }}-{{ ansible_distribution_version.split('.')[0:2] | join('-') | lower }}.yml" 13 | - "{{ ansible_distribution | lower | replace(' ', '-') }}-{{ ansible_distribution_version.split('.')[0:3] | join('-') | lower }}.yml" 14 | tags: 15 | - always 16 | 17 | - name: Assure EPEL is enabled (CentOS 8) 18 | become: true 19 | shell: | 20 | yum-config-manager --enable epel || dnf install epel-release -y 21 | when: 22 | - ansible_distribution == "CentOS" 23 | - ansible_distribution_major_version | int == 8 or 24 | ansible_distribution_major_version | int == 9 25 | 26 | # NOTE(jpena) The Fedora image contains an exclude for python-virtualenv 27 | # in dnf.conf, because it installs it from PyPi. We do not need it in DLRN 28 | # jobs. 29 | - name: Remove excludes from /etc/dnf/dnf.conf (Fedora) 30 | lineinfile: 31 | path: /etc/dnf/dnf.conf 32 | state: absent 33 | regexp: '^exclude=' 34 | become: true 35 | when: ansible_distribution == "Fedora" 36 | 37 | - name: Install required dependencies 38 | package: 39 | name: "{{ system_packages }}" 40 | state: present 41 | become: true 42 | 43 | - import_role: 44 | name: ensure-tox 45 | vars: 46 | ensure_global_symlinks: true 47 | 48 | - name: Ensure pip is upgraded 49 | command: python3 -mpip install --upgrade pip setuptools wheel packaging 50 | become: true 51 | when: (ansible_distribution == 'CentOS' or ansible_distribution == 'RedHat' or ansible_distribution == 'Fedora') and ansible_distribution_major_version | int > 7 52 | 53 | - name: Install dnf-yum and distribution-gpg-keys (Fedora) 54 | package: 55 | name: 56 | - dnf-yum 57 | - distribution-gpg-keys 58 | state: present 59 | become: true 60 | when: ansible_distribution == "Fedora" 61 | 62 | # NOTE(jpena) The python3 job will build a CentOS package, so we need 63 | # to have the CentOS 7 GPG key in the place mock will expect it 64 | - name: Create symlink for RPM-GPG-KEY-CentOS-7 65 | file: 66 | src: /usr/share/distribution-gpg-keys/centos/RPM-GPG-KEY-CentOS-7 67 | dest: /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 68 | owner: root 69 | group: root 70 | state: link 71 | become: true 72 | when: (ansible_distribution == 'CentOS' or ansible_distribution == 'RedHat' or ansible_distribution == 'Fedora') and ansible_distribution_major_version | int > 7 73 | 74 | # NOTE(jpena) This is needed by mock when building on RHEL 75 | - name: Make sure /usr/share/rhn/RHN-ORG-TRUSTED-SSL-CERT exists (RHEL) 76 | shell: 77 | cmd: | 78 | touch /usr/share/rhn/RHN-ORG-TRUSTED-SSL-CERT 79 | become: true 80 | when: ansible_distribution == 'RedHat' and ansible_distribution_major_version | int > 7 81 | 82 | - name: Set up mock configuration 83 | shell: 84 | cmd: | 85 | cat << EOF | tee /etc/mock/site-defaults.cfg 86 | config_opts['plugin_conf']['tmpfs_enable'] = True 87 | config_opts['plugin_conf']['tmpfs_opts'] = {} 88 | config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 2048 89 | config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '4g' 90 | config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755' 91 | config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True 92 | config_opts['use_bootstrap'] = False 93 | config_opts['dnf_warning'] = False 94 | EOF 95 | become: true 96 | 97 | - name: Add user to mock group 98 | user: name="{{ ansible_user }}" 99 | groups=mock 100 | append=yes 101 | become: true 102 | 103 | - name: Install krb5-devel 104 | package: 105 | name: 106 | - krb5-devel 107 | state: present 108 | become: true 109 | -------------------------------------------------------------------------------- /dlrn/drivers/mockdriver.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | # BuildRPMDriver derived classes expose the following methods: 14 | # 15 | # build_package(). This method will perform the actual package build using 16 | # the driver-specific approach. 17 | 18 | from dlrn.config import setup_logging 19 | from dlrn.drivers.buildrpm import BuildRPMDriver 20 | 21 | import io 22 | import logging 23 | import os 24 | import re 25 | import sh 26 | 27 | logger = logging.getLogger("dlrn-build-mock") 28 | 29 | 30 | class MockBuildDriver(BuildRPMDriver): 31 | DRIVER_CONFIG = { 32 | 'mockbuild_driver': { 33 | 'install_after_build': {'type': 'boolean', 'default': True}, 34 | }, 35 | } 36 | 37 | # We are using this method to "tee" mock output to mock.log and stdout 38 | def _process_mock_output(self, line): 39 | if self.verbose_build: 40 | logger.info(line[:-1]) 41 | self.mock_fp.write(line) 42 | 43 | def __init__(self, *args, **kwargs): 44 | super(MockBuildDriver, self).__init__(*args, **kwargs) 45 | self.verbose_build = False 46 | setup_logging() 47 | 48 | def build_package(self, **kwargs): 49 | """Valid parameters: 50 | 51 | :param output_directory: directory where the SRPM is located, 52 | and the built packages will be. 53 | :param additional_mock_opts: string with additional options to 54 | be passed to mock. 55 | """ 56 | output_dir = kwargs.get('output_directory') 57 | additional_mock_opts = kwargs.get('additional_mock_opts') 58 | datadir = os.path.realpath(self.config_options.datadir) 59 | mock_config = os.environ.get('MOCK_CONFIG') 60 | install_after_build = self.config_options.install_after_build 61 | self.verbose_build = kwargs.get('verbose') 62 | 63 | # Find src.rpm 64 | for rpm in os.listdir(output_dir): 65 | if rpm.endswith(".src.rpm"): 66 | src_rpm = '%s/%s' % (output_dir, rpm) 67 | try: 68 | # And build package 69 | with io.open("%s/mock.log" % output_dir, 'a', 70 | encoding='utf-8', errors='replace') as self.mock_fp: 71 | try: 72 | mock_opts = ['-v', '-r', '%s/%s' % (datadir, mock_config), 73 | '--resultdir', output_dir] 74 | if additional_mock_opts: 75 | mock_opts += [additional_mock_opts] 76 | mock_opts += ['--rebuild', src_rpm] 77 | sh.env('/usr/bin/mock', *mock_opts, 78 | postinstall=install_after_build, 79 | _err=self._process_mock_output, 80 | _out=self._process_mock_output) 81 | except Exception as e: 82 | raise e 83 | 84 | if install_after_build: 85 | # Check for warning about built packages failing to install 86 | with open("%s/mock.log" % output_dir, 'r') as fp: 87 | mock_content = fp.readlines() 88 | warn_match = re.compile( 89 | r'\W*WARNING: Failed install built packages.*') 90 | for line in mock_content: 91 | m = warn_match.match(line) 92 | if m is not None: 93 | raise Exception('Failed to install built packages') 94 | 95 | # All went fine, create the $OUTPUT_DIRECTORY/installed file 96 | open('%s/installed' % output_dir, 'a').close() 97 | 98 | finally: 99 | with open("%s/mock.log" % output_dir, 'r') as fp: 100 | mock_content = fp.readlines() 101 | 102 | # Append mock output to rpmbuild.log 103 | with open('%s/rpmbuild.log' % output_dir, 'a') as fp: 104 | for line in mock_content: 105 | fp.write(line) 106 | 107 | # Finally run restorecon 108 | try: 109 | sh.restorecon('-Rv', output_dir) 110 | except Exception as e: 111 | logger.info('restorecon did not run correctly, %s' % e) 112 | --------------------------------------------------------------------------------